BASH Job Queue. This is a example of how to make a job queue in GNU Bash. It may not work for other shells as it relies on the bash built in `read`. You will need to see the man pages for your shell to determine if this will work for you.

queue.sh

#!/usr/bin/env bash

rep() {
  i=$1
  data=$2
  ## run the replicate ....
}

# make the files
START=$(mktemp -t start-XXXX) ## signals the workers are starting
FIFO=$(mktemp -t fifo-XXXX)   ## the queue
LOCK=$(mktemp -t lock-XXXX)   ## the lock file.

## mktemp makes a regular file. Delete that an make a fifo.
rm $FIFO
mkfifo $FIFO
echo $FIFO

## create a trap to cleanup on exit if we fail in the middle.
cleanup() {
  rm $FIFO
  rm $START
  rm $LOCK
}
trap cleanup 0

## This is the worker to read from the queue.
work() {
  ID=$1
  ## first open the fifo and lock for reading.
  exec 3<$FIFO
  exec 4<$LOCK
  ## signal the worker has started.
  flock 4 # obtain the lock
  echo $ID >> $START
  flock -u 4
  while true; do
    flock 4 # obtain the lock
    read -st .2 -u 3 data i ## read one line from fd 3 (the fifo)
    read_status=$?
    flock -u 4 ## release the lock
    ## check the line read.
    if [[ $read_status -eq 0 ]]; then
      # got a work item. do the work
      echo $ID got data=$data i=$i
      rep $i $data
    elif [[ $read_status -gt 128 ]]; then
      # a read_status > 128 means a timeout. on the FIFO read.
      # This means there should be a retry.
      continue
    else
      # a non-zero read status <= 128 means an EOF. actual status
      # is likely 1 but this is undocumented. In anycase the FIFO
      # is now closed on the write side an we can break out of the
      # loop.
      break
    fi
  done
  # clean up the fd(s)
  exec 3<&-
  exec 4<&-
  echo $ID "done working"
}

## Start the workers.
WORKERS=4
for ((i=1;i<=$WORKERS;i++)); do
  echo will start $i
  work $i &
done

## Open the fifo for writing. This lets the workers start. Otherwise
## they will block on waiting for the producer to make the first
## item which causes wierd race conditions among them.
exec 3>$FIFO

## Wait for them to actually open their files.
while true; do
  echo waiting $(wc -l $START)
  if [[ "$(wc -l $START | cut -d \  -f 1)" -eq $WORKERS ]]; then
    echo ok starting producer $(wc -l $START)
    break
  else
    sleep 1
  fi
done

## Produce the jobs to run. In this case run 10 replicates of each
## dataset in the list.
for data in {dataset-A,dataset-B,dataset-C,dataset-D}; do
  for i in {1..10}; do
    echo "sending $data $i"
    echo $data $i 1>&3
  done
done
exec 3<&-
trap '' 0
## It is safe to delete the files because the workers
## already opened them. Thus, only the names are going away
## the actual files will stay there until the workers
## all finish.
cleanup
## now wait for all the workers.
wait

references:

1. https://blog.garage-coding.com/2016/02/05/bash-fifo-jobqueue.html
2. http://wiki.osdev.org/Unix_Pipes
3. https://beej.us/guide/bgipc/output/html/multipage/fifos.html
4. http://tldp.org/LDP/Bash-Beginners-Guide/html/sect_08_02.html The file descriptor usage.
5. https://linux.die.net/man/1/flock . The flock man page

read [-ers] [-a aname] [-d delim] [-i text] [-n nchars] [-N nchars] [-p  prompt] 
     [-t timeout] [-u fd] [name ...]

  One  line is read from the standard input, or from the file descriptor fd sup‐
  plied as an argument to the -u option, and the first word is assigned  to  the
  first name, the second word to the second name, and so on, with leftover words
  and their intervening separators assigned to the  last  name.   If  there  are
  fewer  words  read  from  the input stream than names, the remaining names are
  assigned empty values.  The characters in IFS are used to split the line  into
  words using the same rules the shell uses for expansion (described above under
  Word Splitting).  The backslash character (\) may be used to remove  any  spe‐
  cial  meaning for the next character read and for line continuation.  Options,
  if supplied, have the following meanings:
  -a aname
         The words are assigned to sequential  indices  of  the  array  variable
         aname,  starting  at  0.   aname  is  unset  before  any new values are
         assigned.  Other name arguments are ignored.
  -d delim
         The first character of delim is  used  to  terminate  the  input  line,
         rather than newline.
  -e     If the standard input is coming from a terminal, readline (see READLINE
         above) is used to obtain the  line.   Readline  uses  the  current  (or
         default, if line editing was not previously active) editing settings.
  -i text
         If  readline  is  being  used to read the line, text is placed into the
         editing buffer before editing begins.
  -n nchars
         read returns after reading nchars characters rather than waiting for  a
         complete  line  of  input,  but  honor a delimiter if fewer than nchars
         characters are read before the delimiter.
  -N nchars
         read returns after reading exactly nchars characters rather than  wait‐
         ing  for  a  complete  line of input, unless EOF is encountered or read
         times out.  Delimiter characters  encountered  in  the  input  are  not
         treated  specially and do not cause read to return until nchars charac‐
         ters are read.
  -p prompt
         Display prompt on standard error, without a  trailing  newline,  before
         attempting to read any input.  The prompt is displayed only if input is
         coming from a terminal.
  -r     Backslash does not act as an escape character.  The backslash  is  con‐
         sidered  to  be  part  of the line.  In particular, a backslash-newline
         pair may not be used as a line continuation.
  -s     Silent mode.  If input is coming from a terminal,  characters  are  not
         echoed.
  -t timeout
         Cause  read  to time out and return failure if a complete line of input
         (or a specified number of characters) is not read within  timeout  sec‐
         onds.   timeout  may be a decimal number with a fractional portion fol‐
         lowing the decimal point.  This option is only  effective  if  read  is
         reading  input  from a terminal, pipe, or other special file; it has no
         effect when reading from regular files.  If read times out, read  saves
         any partial input read into the specified variable name.  If timeout is
         0, read returns immediately, without trying to read any data.  The exit
         status  is  0  if  input is available on the specified file descriptor,
         non-zero otherwise.  The exit status is greater than 128 if the timeout
         is exceeded.
  -u fd  Read input from file descriptor fd.

  If  no  names  are  supplied, the line read is assigned to the variable REPLY.
  The return code is zero, unless end-of-file is encountered, read times out (in
  which  case  the return code is greater than 128), a variable assignment error
  (such as assigning to a readonly variable) occurs, or an invalid file descrip‐
  tor is supplied as the argument to -u.