A bash script to run scheduled jobs

run_scheduled_jobs.sh

#!/bin/bash

#set -o errexit # must be disabled
set -o nounset
#set -o pipefail # must be disabled

usage() {
    cat <<EOF
This script permits to execute jobs specified in a text file. Each line should contain a complete command to be executed.

Usage: $0 job_start job_end job_file max_concurrent_jobs max_errors job_timeout out_file

Ex:    $0 2023-02-15T23:00:00 2023-02-06T06:00:00 jobs.txt 5 10 300 res.txt
       $0 now 2023-02-06T06:00:00 jobs.txt 5 10 300 res.txt
       $0 2023-02-15T23:00:00 60 jobs.txt 5 10 300 res.txt
       $0 now 60 jobs.txt 5 10 300 res.txt

Details:
       - job_end indicate the moment we should stop creating new jobs (running jobs will not be interrupted when it is reached)
       - job_start can be set to 'now' so that the execution of jobs will start immediatelly
       - job_end can be set to an integer like 10 indicating the number of seconds the app should run for
       - max_errors: if this is reached the script will exit with error
       - job_timeout: maximum number of seconds a job should take to complete
       - out_file: wil contain details about of each job (start, approx_end and status)
       - operation can be aborted by pressing ctrl-c. The app will exit when currently running jobs finish.

Sample job file:

sleep 0.1 && echo OK1
sleep 0.1 && echo OK2
sleep 0.1 && echo OK3

It is also possible to do redirection of both stdout and stderr:

somecmd1 arg1 arg2 > res1.txt 2>&1
somecmd2 arg1 arg2 > res2.txt 2>&1
somecmd3 arg1 arg2 > res3.txt 2>&1

EOF
}

if [[ $# != 7 ]]
then
    usage
    exit 1
fi

job_start=$1
job_end=$2
job_file=$3
max_concurrent_jobs=$4
max_errors=$5
job_timeout=$6
out_file=$7

aborted=0

function handle_sigint {
    echo "Got SIGINT. Aborting."
    aborted=1
}

# Set the trap for SIGINT (ctrl-c) to call the signal handler function
trap handle_sigint SIGINT

if [[ "$job_start" == "now" ]]
then
    JOB_START=`date +%s -d now`
else
    JOB_START=`date -d "$job_start" +%s`
fi

if [[ "$job_end" =~ ^[1-9]+$ ]]
then
    JOB_END=$((JOB_START + job_end))
else
    JOB_END=`date -d "$job_end" +%s`
fi

if [[ $JOB_END -lt JOB_START ]]
then
    echo "Invalid time range: job_end=$job_end is before job_start=$job_start"
    exit 1
fi

NOW=`date +%s -d now`

SECS=$(expr $JOB_START - $NOW)

if  [[ "$job_start" == "now" ]]
then
    echo "Starting jobs"
elif [[ SECS -lt 0 ]]
then
    echo "job_start=$job_start already passed as now=$NOW. Aborting"
    exit 1
else
    echo "Sleeping for $SECS seconds"
    sleep $SECS
    echo "Sleeping done. Starting jobs"
fi

total_count=0
error_count=0

# open file descriptor
exec 9< $job_file

NOW=`date +%s -d now`

read -u 9 job

while [[ 1 ]]
do
    if [[ $aborted == 1 ]]
    then
        echo "Aborted"
        echo "Stats: total_count: $total_count, error_count: $error_count"
        exec 9<&-
        exit 0
    fi

    echo "Starting cycle. total_count=$total_count"
    job_count=0 
    while [[ "$job" != "" && $job_count -lt $max_concurrent_jobs ]]
    do
        echo "start '$job' in the background"
        bash -c "timeout $job_timeout $job" &

        # save pid of the newly created job
        pids[${job_count}]=$!

        # save job to add status to out_file
        jobs_res[${job_count}]="$job # start: `date '+%Y-%m-%dT%H:%M:%S' -d @$NOW`"

        job_count=$((job_count+1))
        total_count=$((total_count+1))

        read -u 9 job
        if [[ "$job" == "" ]]
        then
            break
        fi
    done

    echo "Waiting for job_count=$job_count"

    NOW=`date +%s`

    # wait for all pids
    for i in $(seq 0 $((job_count-1)))
    do
        pid=${pids[${i}]}
        job_res=${jobs_res[${i}]}

        echo waiting job idx=$i: $pid
        wait $pid
        status=$?
        if [[ $status == 130 ]]
        then
            # this is due to SIGINT (ctrl-c) so wait again.
            wait $pid
            status=$?
        fi

        echo "pid $pid status: $status"

        NOW=`date +%s`
        echo "$job_res approx_end: `date '+%Y-%m-%dT%H:%M:%S' -d @$NOW`, status: $status" >> $out_file

        if [[ $status != 0 ]]
        then
            error_count=$((error_count+1))
        fi
    done

    if [[ $error_count -ge $max_errors ]]
    then
        echo "Error limit ($error_count) reached. Aborting."
        echo "Stats: total_count: $total_count, error_count: $error_count"
        exec 9<&-
        exit 1
    fi

    if [[ "$job" == "" ]]
    then
        break
    fi

    NOW=`date +%s`
    if [[ $NOW -gt "$JOB_END" ]]
    then
        echo "Limit time reached"
        echo "Stats: total_count: $total_count, error_count: $error_count"
        exec 9<&-
        exit 0
    fi
done

# close file descriptor
exec 9<&-

echo Success
echo "Stats: total_count: $total_count, error_count: $error_count"

sample_usage.txt

$ cat jobs.txt 
sleep 1 && echo OK1 && date
sleep 2 && echo OK2 && date
sleep 3 && echo OK3 && date
sleep 4 && echo OK4 && date
sleep 5 && echo OK5 && date

$ ./run_scheduled_jobs.sh now 10 jobs.txt 3 10 15 res.txt
Starting jobs
Starting cycle. total_count=0
start 'sleep 1 && echo OK1 && date' in the background
start 'sleep 2 && echo OK2 && date' in the background
start 'sleep 3 && echo OK3 && date' in the background
Waiting for job_count=3
waiting job idx=0: 27750
OK1
Sat Feb 18 08:25:09 JST 2023
pid 27750 status: 0
waiting job idx=1: 27752
OK2
Sat Feb 18 08:25:10 JST 2023
pid 27752 status: 0
waiting job idx=2: 27755
OK3
Sat Feb 18 08:25:11 JST 2023
pid 27755 status: 0
Starting cycle. total_count=3
start 'sleep 4 && echo OK4 && date' in the background
start 'sleep 5 && echo OK5 && date' in the background
Waiting for job_count=2
waiting job idx=0: 27771
OK4
Sat Feb 18 08:25:15 JST 2023
pid 27771 status: 0
waiting job idx=1: 27773
OK5
Sat Feb 18 08:25:16 JST 2023
pid 27773 status: 0
Success
Stats: total_count: 5, error_count: 0

$ cat res.txt 
sleep 1 && echo OK1 && date # start: 2023-02-18T08:25:08 approx_end: 2023-02-18T08:25:09, status: 0
sleep 2 && echo OK2 && date # start: 2023-02-18T08:25:08 approx_end: 2023-02-18T08:25:10, status: 0
sleep 3 && echo OK3 && date # start: 2023-02-18T08:25:08 approx_end: 2023-02-18T08:25:11, status: 0
sleep 4 && echo OK4 && date # start: 2023-02-18T08:25:11 approx_end: 2023-02-18T08:25:15, status: 0
sleep 5 && echo OK5 && date # start: 2023-02-18T08:25:11 approx_end: 2023-02-18T08:25:16, status: 0