run_and_wait.sh

#!/usr/bin/env bash
#
# run_and_wait.sh
#
# Description:
#  This script will run a specified command and will wait until a
#  specified pattern is found in the specified log file or until
#  a specified time out period expires.
#
#  Returns 0 if pattern found within time out period and 1 otherwise.
#
# Notes:
# - The bash script template used to create the following was sourced from:
#   https://github.com/livibetter-backup/template.sh
#
# - The way to efficiently block until a string is matched in a file with a
#   a defined time out, was sourced from:
#   http://superuser.com/a/1052328
#
# - The script has the following external dependencies:
#   - basename (gnu version)
#   - dirname (gnu version)
#   - bash >= v4.1.2
#   - nohup  (gnu version)
#   - grep (gnu version)
#   - tail (gnu version)
#   - timeout (gnu version)
#   - date (gnu version)
#   - mkdir (gnu version)
#   - tee (gnu version)
#   - head (gnu version)

##############################################################################
# Common Initialization                                                      #
##############################################################################

# This script should not be sourced. Detect if user did do this and terminate
# gracefully. If the script is being sourced then the "return" will terminate
# the sourced script and return to the caller.
if [ "${BASH_SOURCE[0]}" != "${0}" ]; then
  echo "This script should not be sourced."
  return 1 2>/dev/null
fi

# Turn on trace level logging. (Useful while debugging but commented out by
# default.)
#set -o xtrace

# Available log levels declared as constants.
declare -r ERR_LOG_LEVEL=1
declare -r WARN_LOG_LEVEL=2
declare -r INFO_LOG_LEVEL=3
declare -r DEBUG_LOG_LEVEL=4

# Max number of arguments. (empty value = unlimited arguments)
declare -i script_max_args=7

# Useful variables with information about this script.
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
script_file_name="${script_dir}/$(basename "${BASH_SOURCE[0]}")"
script_name="$(basename "${script_file_name}" .sh)"

# Used to store arguments.
script_args=()

# Used to store option flags.
script_opts=()

# Used for returning value after calling script_opt.
script_opt_value=

# By default we show only error and warn messages.
declare -i script_log_level=${WARN_LOG_LEVEL}


##############################################################################
# Functions                                                                  #
##############################################################################

display_arg_error () {
  error_msg "The [$1] mandatory argument was not specified."
  error_msg "Try '${script_name} --help' for more information."
  exit 1
}


debug_msg () { log ${DEBUG_LOG_LEVEL} "`date +'%Y/%m/%d %H:%M:%S'` DEBUG: $1"; }
error_msg () { log ${ERR_LOG_LEVEL} "`date +'%Y/%m/%d %H:%M:%S'` ERROR: $1"; }
info_msg ()  { log ${INFO_LOG_LEVEL} "`date +'%Y/%m/%d %H:%M:%S'` INFO: $1"; }
warn_msg ()  { log ${WARN_LOG_LEVEL} "`date +'%Y/%m/%d %H:%M:%S'` WARNING: $1"; }


log () {
  if [ "${script_log_level}" -ge "$1" ]; then
    echo -e "$2" 
  fi
}


usage () {
  echo "
Usage: ${script_name} [optional args] mandatory args

Optional arguments:
  -h, --help                   display this help and exit
  -v, --log-level \"level\"      1=ERROR, 2=WARN(default), 3=INFO, 4=DEBUG
  -a, --command-args \"args\"    arguments for the command to execute

Mandatory arguments:
  -c, --command \"command\"      command to execute
  -p, --pattern \"pattern\"      grep pattern to search for in log file
  -l, --log-file \"file\"        log file to follow and inspect for pattern
  -o, --nohup-output \"dir\"     directory to host the nohup output file
  -t, --timeout \"NUM[SUFFIX]\"  Number[s|m|h|d]
                               s = seconds
                               m = minutes
                               h = hours
                               d = days
"
}


parse_options () {
  while (( $#>0 )); do
    opt="$1"
    arg="$2"

    case "${opt}" in
      -h|--help)
        usage
        exit 0
        ;;
      -v|--log-level)
        script_opt_set "LOGLEVEL" "${arg}" 1
        shift
        ;;
      -c|--command)
        script_opt_set "COMMAND" "${arg}" 1
        shift
        ;;
      -a|--command-args)
        script_opt_set "COMMANDARGS" "${arg}" 1
        shift
        ;;
      -p|--pattern)
        script_opt_set "PATTERN" "${arg}" 1
        shift
        ;;
      -l|--log-file)
        script_opt_set "LOGFILE" "${arg}" 1
        shift
        ;;
      -t|--timeout)
        script_opt_set "TIMEOUT" "${arg}" 1
        shift
        ;;
      -o|--save-output)
        script_opt_set "SAVEOUTPUT" "${arg}" 1
        shift
        ;;
      -*)
        error_msg "${script_name}: invalid option -- '${opt}'"
        error_msg "Try '${script_name} --help' for more information."
        exit 1
        ;;
      *)
        if [[ ! -z ${script_max_args} ]] && (( ${#script_args[@]} == script_max_args )); then
          error_msg "${script_name}: cannot accept any more arguments -- '${opt}'"
          error_msg "Try '${script_name} --help' for more information."
          exit 1
        else
          script_args=("${script_args[@]}" "${opt}")
        fi
        ;;
    esac
    shift
  done
}


main () {

  # Initialise variables we will be working with
  local my_cmd=""
  local my_cmd_pid=""
  local my_cmdargs=""
  local my_pattern=""
  local my_logfile=""
  local my_timeout=""
  local my_timeout_pid=""
  local my_saveoutput=""

  parse_options "$@"

  if script_opt "LOGLEVEL"; then
    script_log_level=${script_opt_value}
  fi

  info_msg "run_and_wait.sh"
  info_msg "Copyright (c) 2015 Algomi Limited"

  debug_msg "--Runtime arguments:------------------------------------------------------------"

  if script_opt "COMMAND"; then
    my_cmd=${script_opt_value}
    debug_msg "  --command = [${my_cmd}]"
  else
    display_arg_error "-c|--command"
  fi

  if script_opt "COMMANDARGS"; then
    my_cmdargs=${script_opt_value}
    debug_msg "  --command-args = [${my_cmdargs}]"
  else
    my_cmdargs=""
    debug_msg "  --command-args = [${my_cmdargs}]"
  fi

  if script_opt "PATTERN"; then
    my_pattern=${script_opt_value}
    debug_msg "  --pattern = [${my_pattern}]"
  else
    display_arg_error "-p|--pattern"
  fi

  if script_opt "LOGFILE"; then
    my_logfile=${script_opt_value}
    debug_msg "  --log-file = [${my_logfile}]"
  else
    display_arg_error "-l|--log-file"
  fi

  if script_opt "TIMEOUT"; then
    my_timeout=${script_opt_value}
    debug_msg "  --timeout = [${my_timeout}]"
  else
    display_arg_error "-t|--timeout"
  fi

  if script_opt "SAVEOUTPUT"; then
    # Silently try to create the nohup output folder
    mkdir -p "${script_opt_value}"

    # Base the nohup output log file name on the name of the command being
    # executed.
    my_saveoutput="${script_opt_value}/$(basename ${my_cmd})-nohup.log"

    # Ensure the nohup output log file exists:
    touch "${my_saveoutput}"

    debug_msg "  --save-output = [${my_saveoutput}]"
  else
    display_arg_error "-o|--nohup-output"
  fi


  debug_msg "--Executing command:------------------------------------------------------------"
  debug_msg "nohup ${my_cmd} ${my_cmdargs} >> \"${my_saveoutput}\" 2>&1 &"
  # By using nohup we disconnect the process from the terminal yet we still
  # save the nohup output to a specific log file.
  nohup "${my_cmd}" "${my_cmdargs}" >> "${my_saveoutput}" 2>&1 &
  my_cmd_pid=$!
  debug_msg "NOHUP PID IS [${my_cmd_pid}]"


  # Tail the output from the nohup log file to let user know of any issues
  # starting the process. This tail process will get killed when this script
  # finishes execution due to the use of the --pid argument.
  tail -q -n 0 --pid=$$ -F "${my_saveoutput}" &
  debug_msg "Tailing nohup log file. It's PID=[$!]. This process will terminate along with parent script whose PID=[$$]."


  debug_msg "--Log file monitoring:----------------------------------------------------------"
  debug_msg 'timeout ${my_timeout} bash -c ''((sleep 1; exec tail -q -n 0 --pid=$0 -F "$1" 2> /dev/null) & echo $! ; wait $! 2>/dev/null ) | (trap "kill \${my_tail_pid} 2>/dev/null" EXIT; my_tail_pid="`head -1`"; grep -q "$2")'' "${my_cmd_pid}" "${my_logfile}" "${my_pattern}" 2>/dev/null &'''

  # How does this work?
  # - In a bash sub shell, started in the background, we sleep for a second and
  #   then execute tail to monitor the application's log file.
  # - Via the arguments passed to it, tail has been configured to exit if the
  #   process whose log file it is monitoring dies.
  # - The above sub shell, is executed within another bash sub shell in which
  #   we identify the process id of the above sub shell and echo it to stdout.
  # - Lastly, in that sub shell we wait for the sub shell with tail running in
  #   it as a child process, to terminate and if it does terminate, we redirect
  #   any output from its stderr stream to /dev/null.
  # - The stdout output of the above sub shell is piped into another sub shell
  #   in which we setup a trap to watch for an EXIT event, use head -1 to read
  #   the process id of the tail sub shell and finally start a grep process
  #   to grep the stdout for the requested pattern. Grep will quit on the first
  #   match found. The EXIT trap will kill the process if of the tail sub shell
  #   if the sub shell running grep quits.
  # All of this is needed to tidy up the monitoring child processes for
  # tail'ing + grep'ing the application log file.
  # Logic of implementing the above sourced from: http://superuser.com/a/1052328
  timeout ${my_timeout} bash -c '((sleep 1; exec tail -q -n 0 --pid=$0 -F "$1" 2> /dev/null) & echo $! ; wait $! 2>/dev/null ) | (trap "kill \${my_tail_pid} 2>/dev/null" EXIT; my_tail_pid="`head -1`"; grep -q "$2")' "${my_cmd_pid}" "${my_logfile}" "${my_pattern}" 2>/dev/null &
  my_timeout_pid=$!
  debug_msg "TIMEOUT PID IS [${my_timeout_pid}]"

  # By setting a trap for CTRL-C, we can clean up the child processes spawned by
  # this script.
  trap 'echo "Interrupt signal caught. Cleaning up child processes: [${my_timeout_pid} ${my_cmd_pid}]." >> "${my_saveoutput}"; kill ${my_timeout_pid} ${my_cmd_pid} 2> /dev/null' SIGINT
  debug_msg "Waiting for application log monitoring child processes to exit..."
  wait ${my_timeout_pid}
  my_retval=$?
  trap - SIGINT

  # If the time out expires, then 'timeout' will exit with status 124 otherwise
  # it exits with the status of the executed command (which is grep in this
  # case).
  if [ ${my_retval} -eq 124 ]; then
    error_msg "Waited for [${my_timeout}] and the [${my_pattern}] pattern was not encountered in application's log file."
    exit 1
  else
    if [ ${my_retval} -ne 0 ]; then
      error_msg "An issue occurred whilst starting process. Check log files:"
      error_msg "  * nohup output log file: [${my_saveoutput}]"
      error_msg "  * application log file: [${my_logfile}]"
      error_msg "  * application's console log file (if applicable)"
      exit 1
    else
      info_msg "Success! Pattern was found."
      exit 0
    fi
  fi

}


##############################################################################
# Script Template Functions                                                  #
##############################################################################

# Stores options
# $1 - option name
# $2 - option value
# $3 - non-empty if value is not optional
script_opt_set () {
  if [[ ! -z "$3" ]] && [[ -z "$2" ]]; then
    error_msg "${script_name}: missing option value -- '${opt}'" >&2
    error_msg "Try '${script_name} --help' for more information." >&2
    exit 1
  fi
  # XXX should check duplication, but doesn't really matter
  script_opts=("${script_opts[@]}" "$1" "$2")
}

# Checks if an option is set, also set script_opt_value.
# Returns 0 if found, 1 otherwise.
script_opt () {
  local i opt needle="$1"
  for (( i=0; i<${#script_opts[@]}; i+=2 )); do
    opt="${script_opts[i]}"
    if [[ "${opt}" == "${needle}" ]]; then
      script_opt_value="${script_opts[i+1]}"
      return 0
    fi
  done
  script_opt_value=
  return 1
}


##############################################################################
# Main                                                                       #
##############################################################################

if [ "$#" -eq 0 ]; then
  usage
  exit 1
else
  main "$@"
fi