Retry a BASH command

retry.sh

#!/usr/bin/env bash

function err() { cat <<< "$@" 1>&2; }
function fatal() { cat <<< "$@" 1>&2; exit 1; }
function retry() {
  # Tries to run a cmd 5 times before failing 
  # If a command is successful, it will break out of attempt loop
  # Failed attempts are padding with the following exponential 
  # back-off strategy {4, 16, 64, 256, 1024} in seconds
  # @INPUTS "$@"" = cmd to run 
  # @CALLS fatal() if command cannot be run in 5 attempts
  local n=1
  local max=5
  local attempt=true # flag for while loop
  while $attempt; do
    # Attempt command and break out of attempt loop if successful 
    "$@" && attempt=false || {
      # Try again up to 5 times 
      if [[ $n -le $max ]]; then
        err "Command failed: $@"
        delay=$(( 4**$n ))
        err "Attempt: ${n}/${max}. Trying again in ${delay} seconds!\n"
        sleep $delay;
        ((n++))
      else
        fatal "Fatal: command failed after max retry attempts!"
      fi
    }
  done
}

# Examples
retry ls /fake/directory/does/not/exist
retry ping 123.com

retry.py

Here is a python implementation of the same bash function above:

#!/usr/bin/env python3

# Python standard library
from __future__ import print_function
from subprocess import CalledProcessError
import sys, os, subprocess, time


def err(*message, **kwargs):
    """Prints any provided args to standard error.
    kwargs can be provided to modify print functions 
    behavior.
    @param message <any>:
        Values printed to standard error
    @params kwargs <print()>
        Key words to modify print function behavior
    """
    print(*message, file=sys.stderr, **kwargs)


def fatal(*message, **kwargs):
    """Prints any provided args to standard error
    and exits with an exit code of 1.
    @param message <any>:
        Values printed to standard error
    @params kwargs <print()>
        Key words to modify print function behavior
    """
    err(*message, **kwargs)
    sys.exit(1)


def retry(times=5, exceptions=(Exception)):
    """
    Decorator to retry running a function. Retries the wrapped function 
    N times with an exponential backoff stragety. A tuple of Exceptions
    can be passed that trigger a retry attempt. When times is equal to 
    4 the back-off strategy will be {4, 16, 64, 256} seconds. Calls fatal
    if the function cannot be run within the defined number of times.
    @param times <int>: 
        The number of times to repeat the wrapped function,
        default: 5
    @param exceptions tuple(<Exception>):
        Tuple of Python Exceptions that will trigger a retry attempt,
        default: (Exception)
    @return <object func>:
        Calls fatal when func cannot be run after N times.
    """
    def decorator(func):
        def do(*args, **kwargs):
            # Begin the attempt loop and return if successful 
            attempt = 0
            delay = 1
            while attempt < times:
                try:
                    return func(*args, **kwargs)
                except exceptions as e:
                    # Default expection, Exception
                    err('Function failed: {0}'.format(func.__name__))
                    err('\t@reason: {0}'.format(e))
                    # Increase backoff: 4, 16, 64, 256, 1024, 4096...
                    attempt += 1
                    delay = 4**attempt
                    err("Attempt: {0}/{1}".format(attempt, times))
                    err("Trying again in {0} seconds!\n".format(delay))
                    time.sleep(delay)
            # Could not succesfully run the given function 
            # in its alloted number of tries, exit with error 
            fatal('Fatal: failed after max retry attempts!')
        return do
    return decorator


@retry(times=5, exceptions=(CalledProcessError))
def bash(cmd, interpreter='/bin/bash', strict=True, cwd=os.getcwd(), **kwargs):
    """
    Interface to run a process or bash command. Using subprocess.call_check()
    due to portability across most python versions. It was introduced in python 2.5
    and it is also interoperabie across all python 3 versions. 
    @param cmd <str>:
        Shell command to run
    @param interpreter <str>:
        Interpreter for command to run [default: bash]
    @pararm strict <bool>:
        Prefixes any command with 'set -euo pipefail' to ensure process fail with
        the expected exit-code  
    @params kwargs <check_call()>:
        Keyword arguments to modify subprocess.check_call() behavior
    @return exitcode <int>:
        Returns the exit code of the run command, failures return non-zero exit codes
    """
    prefix = ''  # permissive shell option
    if strict: 
        # Changes behavior of default shell
        # set -e: exit immediately upon error
        # set -u: treats unset variables as an error
        # set -o pipefail: exits if a error occurs in any point of a pipeline
        prefix = 'set -euo pipefail; '

    # Run bash command 
    exitcode = subprocess.check_call(strict + cmd, 
        shell=True, 
        executable=interpreter, 
        cwd=cwd, 
        **kwargs
    )
    
    return exitcode


if __name__ == '__main__':
    # Tests
    bash('ls -la /home/')
    bash('ls -la /fake/dne/path')