Bash utility AWS Run Command

remote_script.sh

#!/bin/bash

#================================================================
# HEADER
#================================================================
#% SYNOPSIS
#+    ${SCRIPT_NAME} --tagName "argument" --tagValue "argument" --scriptFile "argument" --command "argument" --timeout "argument" [--comment] "argument" [--help] [--version]
#%
#% DESCRIPTION
#%    Helper function for running external shell scripts on multiple instances with AWS SSM. 
#%    Script expect following prerequisities:
#%    - Target instance has AWS SSM installed
#%    - Target instance can be identified by AWS Tag:Value
#%    - Local instance has aws-cli installed in the $PATH
#%    
#%    Local instance must has sufficient permissions to run:
#%    - aws ec2 describe-instances
#%    - aws ssm send-command
#%    - aws ssm list-command-invocations
#%    - ssm get-command-invocation
#%
#% OPTIONS
#%    --tagName                 Tag name described in EC2 instance
#%    --tagValue                Value of Tag name attached to EC2 instance
#%    --scriptFile              One of the script files located in S3 bucket
#%    --command                 Execution of scriptFile with optional arguments and parameters
#%    --comment                 Add comment text. Easier identification of this command in SSM Run Command table
#%    --timeout                 Add timeout in seconds, between running the script and checking the response. Default value is 10 seconds
#%    --help                    Print this help
#%    --version                 Print script version
#%
#% EXAMPLES
#%    ${SCRIPT_NAME} --tagName "App" --tagValue "Nginx" --scriptFile "backup.sh" --command "backup.sh" --comment "Backup: Nginx logs" --timeout 10
#%
#================================================================
#- version 0.0.1
#================================================================
# END_OF_HEADER
#================================================================

# Check:
# Template for bash: https://stackoverflow.com/questions/14008125/shell-script-common-template
# Default parameters in bash: https://brianchildress.co/named-parameters-in-bash/
# Colored echo in bash: https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux#answer-53463162

SCRIPT_HEADSIZE=$(head -200 ${0} |grep -n "^# END_OF_HEADER" | cut -f1 -d:)
SCRIPT_NAME="$(basename ${0})"

run_help() { 
  head -${SCRIPT_HEADSIZE:-99} ${0} | grep -e "^#[%+-]" | sed -e "s/^#[%+-]//g" -e "s/\${SCRIPT_NAME}/${SCRIPT_NAME}/g" ; 
}
run_version() { 
  head -${SCRIPT_HEADSIZE:-99} ${0} | grep -e "^#-" | sed -e "s/^#-//g"
}

HELP=0
DIVIDER="-------------------------"

### ! CUSTOM VARIABLES ! ###
LOGS_BUCKET="<bucket_name>" # <namespace>-logs
SCRIPT_BUCKET="<bucket_name>" # <namespace>-scripts

cecho(){
    RED="\033[0;31m"
    GREEN='\033[0;32m'
    YELLOW='\033[1;33m'
    NC='\033[0m' # No Color
    printf "${!1}${2} ${NC}\n"
}

for i in "$@"
do
case $i in
    --tagName)
    shift
      if [ -z "$1" ]
      then
        cecho "RED" "Tag name cannot be empty"
        exit 1
      else
        TAG_NAME="$1"
      fi
    shift
    ;;
    --tagValue)
    shift
      if [ -z "$1" ]
      then
        cecho "RED" "Tag value cannot be empty"
        exit 1
      else
        TAG_VALUE="$1"
      fi
    shift
    ;;
    --scriptFile)
    shift
      if [ -z "$1" ]
      then
        cecho "RED" "Path to script file cannot be empty"
        exit 1
      else
        SCRIPT_FILE="$1"
      fi
    shift
    ;;
    --command)
    shift 
    COMMAND="$1"
    shift 
    ;;
    --comment)
    shift
    COMMENT="$1"
    shift
    ;;
    --timeout)
    shift
      if [ -z "$1" ]
      then
        TIMEOUT=2
      else
        TIMEOUT="$1"
      fi
    shift
    ;;
    --help)
    HELP=1
    run_help
    shift
    ;;
    --version)
    HELP=1
    run_version
    shift
    ;;
    -*)
    cecho "RED" "Error: Unknown option: $1" >&2
    run_help
    exit 1
esac
done

validate_response () {
  RESPONSE=$1
  COMMAND_TEXT=$2
  if [ "$RESPONSE" -ne 0 ]; then
      cecho "RED" "ERROR: $COMMAND_TEXT"
      exit "$RESPONSE"
  fi
}

run_command () {
  cecho "GREEN" "Running ${SCRIPT_FILE} on instances with tag ${TAG_NAME}:${TAG_VALUE}"

  #AWS=/var/lib/jenkins/.local/bin/aws
  AWS=aws

  COMMAND_ID=$(${AWS} ssm send-command \
      --document-name "AWS-RunRemoteScript" \
      --targets '{"Key":"tag:'${TAG_NAME}'","Values":["'"${TAG_VALUE}"'"]}' \
      --parameters '{"sourceType":["S3"],"sourceInfo":["{\"path\":\"https://s3-us-west-2.amazonaws.com/'"${SCRIPT_BUCKET}'"/'"${SCRIPT_FILE}"'\"}"],"executionTimeout":["10800"],"commandLine":["'"${COMMAND}"'"]}' \
      --comment "'${COMMENT}'" \
      --timeout-seconds 600 \
      --output-s3-bucket-name "${LOGS_BUCKET}" \
      --output-s3-key-prefix "${TAG_NAME}/${TAG_VALUE}" \
      --region us-west-2 \
      --output text \
      --max-errors "1" \
      --query Command.CommandId
  )

  RES=$?

  validate_response $RES "Problem with command: $COMMAND_ID"

  INSTANCE_IDS=$(
      ${AWS} ec2 describe-instances \
      --filter "Name=tag:$TAG_NAME,Values='$TAG_VALUE'" \
      --region us-west-2 \
      --query 'Reservations[*].Instances[*].InstanceId' \
      --output text
  )

  [[ -z ${INSTANCE_IDS} ]] && validate_response 1 "No instances found with tag ${TAG_NAME}:${TAG_VALUE}"

  echo "Timeout: ${TIMEOUT}"
  
  # sleep "${TIMEOUT}"
  # This is default timeout to wait until SSM service is ready
  sleep 3

  echo "Instance Ids: $INSTANCE_IDS"

  echo "Command launched with id $COMMAND_ID"
  echo "${DIVIDER}"

  INSTANCES=$( echo $INSTANCE_IDS | wc -w )
  while true; do
      finished=0
      for instance in $INSTANCE_IDS; do
          
          STATUS=$(${AWS} ssm list-command-invocations --command-id "${COMMAND_ID}" --instance-id "${instance}" --query CommandInvocations[*].Status --region us-west-2 --output text | tr '[A-Z]' '[a-z]' )
          NOW=$( date "+%Y/%m/%d %H:%M:%S" )
          echo "${NOW}, command-id: ${COMMAND_ID}, instance-id: ${instance}, status: ${STATUS}"
          echo "${DIVIDER}"

          # Fixes the Timeout issue
          # TODO: Remove timeout option?
          if [ -z "${STATUS}" ];then
            STATUS=pending
          fi

          case $STATUS in
              pending|inprogress|delayed) : ;;
              *) finished=$(( finished + 1 )) ;;
          esac
          
          sleep "${TIMEOUT}"
      done
      [ $finished -ge "${INSTANCES}" ] && break
  done

  ERRORS=0

  for instance in $INSTANCE_IDS; do
    STATUS=$( ${AWS} ssm get-command-invocation --command-id "${COMMAND_ID}" --instance-id "${instance}" --plugin-name "runShellScript" --query Status --output text --region us-west-2 )
    OUT_RESULT=$( ${AWS} ssm get-command-invocation --command-id "${COMMAND_ID}" --instance-id "${instance}" --plugin-name "runShellScript" --query StandardOutputContent --output text --region us-west-2 )
    ERR_RESULT=$( ${AWS} ssm get-command-invocation --command-id "${COMMAND_ID}" --instance-id "${instance}" --plugin-name "runShellScript" --query StandardErrorContent --output text --region us-west-2 )
    INSTANCE_NAME=$( ${AWS} ec2 describe-instances --instance-ids "${instance}" --region us-west-2 --query "Reservations[*].Instances[*].Tags[?Key=='Name'].Value" --output text)

    #echo "------------------------------------"
    cecho "GREEN"  "RESULTS for ID: $instance, Name: $INSTANCE_NAME"
    echo "STATUS $STATUS"
    if [ -n "$OUT_RESULT" ]; then
        echo "STDOUT:"
        echo "$OUT_RESULT"
        echo "${DIVIDER}"
        echo "Full output is available: https://${LOGS_BUCKET}/${TAG_NAME}/${TAG_VALUE}/${COMMAND_ID}/${instance}/awsrunShellScript/runShellScript/stdout"
    fi
    if [ -n "$ERR_RESULT" ] && [ "$STATUS" != "Success" ]; then
        echo "STDERR: $STATUS"
        echo "$ERR_RESULT"
        echo "${DIVIDER}"
        ERRORS=1
    fi
    if [ -z "$OUT_RESULT" ] && [ -z "$ERR_RESULT" ]; 
    then
        cecho "RED" "No output returned !"
        ERRORS=1
    fi

    if [ "${STATUS}" != 'Success' ]
    then
      cecho "RED" "There is a problem with command ${COMMAND_ID}, status: ${STATUS}, Instance Id: ${instance}, Name: ${INSTANCE_NAME}"
      ERRORS=1
    fi
  done

  if [ $ERRORS != 0 ];
  then
    cecho "RED" "FAILED !"
    exit 1
  else
    cecho "GREEN" "PASSED !"
    exit 0
  fi
}

missing_argument() {
  cecho "YELLOW" "\n$1 argument is missing"
}

if [ "${TAG_NAME}" ] && [ "${TAG_VALUE}" ] && [ "${SCRIPT_FILE}" ] && [ "${COMMAND}" ]
then
  [[ -z ${TIMEOUT} ]] && TIMEOUT="5"
  run_command
else
  if [ ${HELP} -ne 1 ]
  then
    [[ -z ${TAG_NAME} ]] && missing_tag_name=$( missing_argument "--tagName" )
    [[ -z ${TAG_VALUE} ]] && missing_tag_value=$( missing_argument "--tagValue" )
    [[ -z ${SCRIPT_FILE} ]] && missing_script_file=$( missing_argument "--scriptFile" )
    [[ -z ${COMMAND} ]] && missing_command=$( missing_argument "--command" )
    echo "Script is not running. Check: $missing_tag_name $missing_tag_value $missing_script_file $missing_command"
  fi
fi