Docker service update wait bash script

docker-service-update-wait.md

Here is how this script can be used:

./docker-service-update-wait.sh redis-slave 60 2 "docker service update --image redis:3.2.6-alpine --stop-grace-period 10s --update-delay 5s redis-slave"

     Update command:  docker service update --image redis:3.2.6-alpine --stop-grace-period 10s --update-delay 5s redis-slave
      # of replicas:  3
    Timeout/replica:  60 sec
      Total timeout:  180 sec
    Stability delay:  2 sec
   Updating service:  redis-slave
 Remaining replicas:  3 2 1 done
     Stability test:  done

SUCCESS

docker-service-update-wait.sh

#!/usr/bin/env bash

if [[ "$#" -ne 4 ]]; then
  echo Expected arguments are missing: SERVICE_NAME TIMEOUT_PER_REPLICA_S STABILITY_DELAY_S UPDATE_CMD
  exit 1
fi

name=$1
timeout_per_replica=$(echo $2 | sed 's/[^0-9]//g')
stability_delay=$(echo $3 | sed 's/[^0-9]//g')
update_cmd=$4
delay=3

# find out which container IDs to ignore while upgrading
ignore_ids=$(docker service ps $name | grep "_ $name" | awk '{print "|"$1}' | xargs echo | awk '{gsub(/ /, ""); print}')
ignore_ids=${ignore_ids#|*}
function get_new_ids() {
  echo $(docker service ps $name | awk -v ids="$ignore_ids" '(NR > 1) && ($1 !~ ids) {print $1}' | wc -l)
}

n=$(get_new_ids)
replicas=$(docker service inspect $name | grep -i replicas | sed 's/[^0-9]//g')
timeout=$(($timeout_per_replica * $replicas))

echo ""
echo "     Update command:  $update_cmd"
echo "      # of replicas:  $replicas"
echo "    Timeout/replica:  $timeout_per_replica sec"
echo "      Total timeout:  $timeout sec"
echo "    Stability delay:  $stability_delay sec"
echo -n "   Updating service:  "
eval $update_cmd
exit_code=$?
if [ "$exit_code" -ne 0 ]; then
  echo "ERROR: cannot update service"
  exit $exit_code
fi

# test that there are at least $replicas more containers after update
m=$n
x=0
echo -n " Remaining replicas:  "
while [ "$m" -lt "$(($n+$replicas))" ]; do
  i=$(get_new_ids)
  if [ "$i" -gt "$m" ]; then
    echo -n "$(($n+$replicas-$i+1)) "
  fi
  m=$(get_new_ids)
  if [ "$(($x * $delay))" -gt "$stability_delay" ]; then
    if [ "$m" -eq $n ]; then
      echo "SKIPPED: it seems that there is nothing to update"
      exit 0
    fi
  fi
  if [ "$(($x * $delay))" -gt "$timeout" ]; then
    echo "ERROR: update is taking too long time (timeout reached)"
    exit 2
  fi
  x=$(($x+1))
  sleep $delay
done
echo "done"
echo -n "     Stability test:  "

# test that there are no more containers during small timeframe
k=$(get_new_ids)
if [ "$k" -ne "$(($n+$replicas))" ]; then
  echo "failed"
  exit 3
fi
sleep $stability_delay
k=$(get_new_ids)
if [ "$k" -ne "$(($n+$replicas))" ]; then
  echo "failed"
  exit 3
fi
echo "done"
echo ""

echo "SUCCESS"