Running Jobs in Parallel

run_jobs_parallel.sh

# Runs the run_job function in parallel

run_job() {
    /path/to/script.ext <args(optional)> $1 > "logs/$1.log" 2>&1
}
export -f run_job

# limits.txt file has the args for the run_job function.
# -j4 => run 4 threads at max in parallel
# --eta => print the eta of the jobs on the console
cat limits.txt | parallel -j4 --eta run_job {}