concurrency-in-bash.sh

#!/usr/bin/env bash

function example() {
    set -o errexit
    set -o nounset

    ## only needed for the example
    local worktime=0

    # how many concurrent operations to perform
    with-concurrency "$(getcores)"

    ## the `seq` would be replaced with whatever you're looping over
    for x in $(seq 20); do

        ## just for the example, not needed in your implementation
        local naptime
        naptime=$(randbetween 1 10)
        worktime=$((worktime + naptime))

        ## your job would go here
        takelock && (
            echo "doing job #${x}; working for ${naptime}s"
            sleep "${naptime}"
            releaselock
        ) &

    done
    wait

    echo "completed ${worktime}s of work"
}

## implementation
trap "cleanup" EXIT SIGABRT SIGINT
function cleanup() {
    set +o nounset
    if [[ -n ${__LOCKFILE} && -f ${__LOCKFILE} ]]; then
        set -o nounset
        rm -f "${__LOCKFILE}"
        for jobid in $(jobs -p); do
            ps "${jobid}" >/dev/null && kill "${jobid}"
        done
    fi
}

function debug() {
    echo 1>&2 "DEBUG: ${*}"
}

function tmpfile() {
    head -c 32 /dev/random | md5
}

function with-concurrency() {
    ## purposely NOT local so these can escape their scope
    __LOCKMAX="${1}"
    __LOCKFILE=${TMPDIR}/$(tmpfile)
    function readlock() {
        if [[ ! -f ${__LOCKFILE} ]]; then
            exit 1
        fi
        cat "${__LOCKFILE}"
    }
    function takelock() {
        if [[ ! -f ${__LOCKFILE} || ${__STOPPED} -eq 1 ]]; then
            exit 1
        fi

        while true; do
            if [[ ! $(readlock) -lt ${__LOCKMAX} ]]; then
                sleep 1
            else
                echo $(($(readlock) + 1)) >"${__LOCKFILE}"
                debug "taking lock"
                break
            fi
        done
    }
    function releaselock() {
        if [[ ! -f ${__LOCKFILE} ]]; then
            exit 1
        fi
        echo $(($(readlock) - 1)) >"${__LOCKFILE}"
        debug "released lock"
    }
    echo 0 >"${__LOCKFILE}"
}

function randbetween() {
    # macOS or FreeBSD only
    jot -r 1 "${1}" "${2}"
}

function getcores() {
    getconf _NPROCESSORS_ONLN 2>/dev/null || 8
}

time example "$(getcores)"