Synchronously run commands on an SSM-managed EC2 instance

ec2-exec.sh

#!/bin/bash

which jq > /dev/null 2>&1
if [ "$?" -ne 0 ]; then
  >&2 echo Error: this script requires jq; exit 1
fi

which aws > /dev/null 2>&1
if [ "$?" -ne 0 ]; then
  >&2 echo Error: this script requires awscli; exit 1
fi

aws sts get-caller-identity > /dev/null 2>&1
if [ "$?" -ne 0 ]; then
  >&2 echo Error: Failure running aws sts get-caller-identity; exit 1
fi

OPTIND=1         # Reset in case getopts has been used previously in the shell.

# Initialize our own variables:
output_file=""
verbose=0

if [ $# -ne 2 ]; then
  >&2 echo "Usage: Call this script with $0 \"i-instance1 i-instance2\" \"command\""
  exit 1
fi

set -e

INSTANCE_LIST=$1
COMMAND=$2

#echo "Launching command on $INSTANCE_LIST"

commandOutput=$(aws ssm send-command \
--instance-ids $INSTANCE_LIST \
--document-name "AWS-RunShellScript" \
--parameters commands="$COMMAND")

commandId=$(echo $commandOutput | jq -r .Command.CommandId)

#echo "Command ID: $commandId"

# Wait ~2m for command to start running
for i in $(seq 24 -1 0); do
  should_continue=0
  # The command isn't running until all instances move off of Pending/Delayed status
  for server in $INSTANCE_LIST; do
    commandStatus=$(aws ssm get-command-invocation --instance-id $server --command-id $commandId | jq -r .StatusDetails)
    >&2 echo "# Command on $server is $commandStatus"
    case "$commandStatus" in
      "Pending")
      should_continue=0
      ;;
      "Delayed")
      should_continue=0
      ;;
      *)
      should_continue=1
    esac
  done

  if [ $should_continue -eq 1 ]; then break; fi
  if [ $i -eq 0 ]; then
    >&2 echo "ERROR: Command never finished starting. Giving up. Command ID: $commandId"
    exit 1
  fi
  sleep 4
done

# Wait for command to complete
while true; do
  should_continue=0
  # Wait while we're In Progress
  for server in $INSTANCE_LIST; do
    commandStatus=$(aws ssm get-command-invocation --instance-id $server --command-id $commandId | jq -r .StatusDetails)
    case "$commandStatus" in
      "InProgress")
      should_continue=0
      ;;
      *)
      should_continue=1
    esac
  done

  # Break when done or continue to wait
  if [ $should_continue -eq 1 ]; then break; fi
  >&2 echo -n .
  sleep 0.5
done

# Dump output and exit

exit_status=0
# Wait while we're In Progress
for server in $INSTANCE_LIST; do
  commandDetails=$(aws ssm get-command-invocation --instance-id $server --command-id $commandId)
  commandStatus=$(echo $commandDetails | jq -r .StatusDetails)
  echo ""
  echo "# Command on $server: $commandStatus"
  echo "#############################################"
  echo $commandDetails | jq -r .StandardOutputContent
  echo $commandDetails | jq -r .StandardErrorContent
  echo "#############################################"
  case "$commandStatus" in
    "Success")
    ;;
    *)
    exit_status=1
    >&2 echo "Error: Command failed on $server with status $commandStatus"
  esac
done

exit $exit_status