This script does a configurable intelligent exponential backoff with maximum. If the app runs for some time without any problem, the interval is resetted.

intelligent-exponential-backoff.sh

#!/bin/bash
# This script does a configurable intelligent exponential backoff with maximum. If the app runs for some time without any problem, the interval is resetted.
# Requires: bash, genius
# Security considerations: Like in almost any other shell scripts, some info may leak via /proc/. This should be rather negligible in this case. A local adversary may read sleep values, application run time, wait time and some WAIT_TIME_* variables.
#
# 1. Replace {{run-the-app}}.
# 2. Replace {{clean}} by some clean procedure, e.g. rm -f "$BASE_DIR/play.pid". It might be a good idea to add some notification or logging there.
# 3. Configure WAIT_TIME_* variables.
# 4. Adjust if needed.

WAIT_TIME_INITIAL=1
WAIT_TIME_MAX=120
WAIT_TIME_FACTOR=2
WAIT_TIME_CLEAR_THRESHOLD=3600


wait_time=$WAIT_TIME_INITIAL
last_start_time=$(date +%s)
while ! {{run-the-app}}; do
	run_time=$(($(date +%s) - $last_start_time))
	{{clean}}
	echo "App failed"
	echo "Run time: $run_time"
	if [ "$run_time" -ge "$WAIT_TIME_CLEAR_THRESHOLD" ]; then
		echo "Resetting wait time (from $wait_time to $WAIT_TIME_INITIAL)"
		wait_time=$WAIT_TIME_INITIAL
	else
		echo -n "Updating wait time $wait_time"
		wait_time=$(echo "
			min($WAIT_TIME_MAX, $WAIT_TIME_FACTOR*$wait_time)
		" | genius)
		echo " -> $wait_time"
	fi

	sleep $wait_time
	last_start_time=$(date +%s)
done