bash_supervisor.sh

#!/usr/bin/env bash

NAME="$1"

CONCURRENCY="$2"
MAX_MESSAGES="$3"

LOGFILE="var/log/worker-${NAME}.log"
PIDFILE="var/run/worker-${NAME}.pid"

if [[ $# -eq 2 ]] && [[ "$2" == "stop" ]] ; then 
	if [[ ! -f "$PIDFILE" ]] ; then
		echo "No pidfile at $PIDFILE - process is not running?"
		exit 0
	elif ! pkill -F "$PIDFILE" ; then
		echo "Error: No process with the id from $PIDFILE found." >&2
		exit 1
	fi

	echo "Terminated process with the id from $PIDFILE."
	exit 0
fi

echo $$ > "$PIDFILE"

CMD="bin/app worker:start $NAME --max-messages $MAX_MESSAGES -vvv"

mkdir -p `dirname $LOGFILE` `dirname $PIDFILE`

WORKER_ID=0

function shutdown() {
	log_line '[SHUTDOWN] Received signal, shutting down gracefully'

	jobs -p | xargs kill

	rm "$PIDFILE"

	exit 0
}

trap "shutdown" SIGINT SIGTERM

function log_timestamp() {
	date +'%Y-%m-%d %H:%M:%S'
}

function log_line_prefix() {
	echo "[$(log_timestamp)] [$NAME]"
}

function log_line() {
	echo -e "$(log_line_prefix) $@" | tee -a "$LOGFILE"
}


function process_count() {
	jobs -p | wc -l
}

function spawn_worker_processes() {
	local COUNT=$1

	shift 1

	INITIALLY_RUNNING=$(process_count)

	for i in $(seq 1 $COUNT) ; do
		((++WORKER_ID))

		WORKER_ID_STRING="[W#$(printf "%06d" "$WORKER_ID")]"

		(bash -c "$@" | sed "s/^/$WORKER_ID_STRING /" | tee -a "$LOGFILE") &

		local PROCESS_PID="$!"

		log_line "[WORKER:PROCESS:SPAWNED][P#$(( $INITIALLY_RUNNING + $i ))]$WORKER_ID_STRING pid: '$PROCESS_PID', running: $(process_count)"
	done

	sleep 1s
}

function spawn_missing_worker_processes() {
	local RUNNING=`process_count`
	local MISSING=$(( $CONCURRENCY - $RUNNING ))

	if [[ $MISSING -lt 0 ]] ; then
		log_line "[WARNING] There are $RUNNING jobs running, but concurrency is at $CONCURRENCY."
		return
	fi

	[[ $MISSING -gt 0 ]] || return

	log_line "[INFO] There are $RUNNING jobs running, spawning $MISSING jobs to achieve $CONCURRENCY target concurrency."

	spawn_worker_processes $MISSING "$CMD"
}


log_line "[WORKER:START] name: '$NAME', concurrency: $CONCURRENCY, cmd: '$CMD'"

spawn_worker_processes $CONCURRENCY "$CMD"

while true ; do
	wait -n 

	CODE=$?

	if [[ $CODE -eq 0 ]] ; then
		echo '[WORKER:PROCESS:FINISHED] Worker process terminated gracefully'
	else
		if [[ $CODE -gt 127 ]] ; then
			echo '[WARNING] Background job wait exited with code $CODE.'
			sleep 1s
		else
			echo '[WORKER:PROCESS:CRASH] exit_code: $CODE'
		fi
	fi

	spawn_missing_worker_processes
done