Process a set of data in N concurrent processes

bash_multiproc.sh

#!/usr/local/bin/bash
# Homebrew bash because macOS standard bash is too old for 'wait -n'.

max_jobs=10

data=`seq 1 20`

# Process to be run in background
function batch_job() {
    echo "Starting $1"
    sleep $(( 5 + ( ( RANDOM * 10 ) / 32767 ) ))
    echo "Finished $1"
}

# This could be 'while read x ; do' if you want to take stdin
for x in $data ; do
    # wait until there less than $max_jobs jobs running before starting a new one
    while true ; do
        j=`jobs | wc -l | cut -d ' ' -f 1`
        # echo "---- $j jobs running ----"
        if [[ $j -lt $max_jobs ]] ; then
            break
        fi
        # wait for a child to finish - requires Bash v. 4.3
        wait -n
        # RHEL 7.3 is only on 4.2, and macOS on 3.7
        #sleep 1
    done

    batch_job $x &
done

# wait for all children to finish
wait
echo "All done"