Elastic Beanstalk worker with graceful shutdown

worker.config

files:
  /usr/local/bin/worker:
    mode: "000755"
    owner: root
    group: root
    content: |
      #!/bin/sh
      restartLimit=10
      timeLimit=300 # 5 minutes
      crashCount=0
      cd /var/app/current/
      while true; do
        node services/queue/worker.js &
        pid=$!
        echo $pid > /tmp/worker.pid
        wait $pid
        retval=$?
        if [ $retval -eq 0 ]; then
          exit 0
        else
          # Crash
          crashTime=`date +%s`
          if [ -z "$firstCrashTime" ]; then
            firstCrashTime=$crashTime
          fi
          timeDiff="$[$crashTime - $firstCrashTime]"

          if [ $timeDiff -lt $timeLimit ]; then
            crashCount=$[$crashCount + 1]
            if [ $crashCount -gt $restartLimit ]; then
              echo "Too many crashes, worker stopped"
              exit $retval
            fi
          else
            crashCount=1
            firstCrashTime=$crashTime
          fi

          echo "Worker crashed, restarting"
        fi
      done

  /etc/init.d/worker:
    mode: "000755"
    owner: root
    group: root
    content: |
      #!/bin/sh
      #
      # worker - stops and starts the worker
      #
      # chkconfig:   - 20 80
      # description: Worker with graceful shutdown

      # Source function library.
      . /etc/rc.d/init.d/functions

      exec="/usr/local/bin/worker"
      prog="worker"
      env="/tmp/env"
      log="/var/log/nodejs/nodejs.log"

      [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog

      lockfile=/var/lock/subsys/$prog
      pidfile=/tmp/${prog}.pid
      min_kill_timeout=5
      default_kill_timeout=60

      start() {
        echo -n "Starting $prog:"
        . $env
        daemon --user=games $exec >> $log 2>&1 &
        retval=$?
        echo
        pid=`cat $pidfile`
        [ $retval -eq 0 ] && touch $lockfile && echo "[Started $pid]"
        return $retval
      }

      stop() {
        kill_timeout=$default_kill_timeout
        if [ -f $env ]; then
          . $env
          if [ -n "$WORKER_SHUTDOWN_DELAY" ] && [ "$WORKER_SHUTDOWN_DELAY" -gt $min_kill_timeout ]; then
            kill_timeout="$WORKER_SHUTDOWN_DELAY"
          fi
        fi
        pid=`cat $pidfile`
        i="0"
        echo "Stopping $prog [$pid]:"
        kill $pid
        retval=$?
        while [ $i -lt $kill_timeout ]; do
          if [ -d "/proc/$pid" ]; then
            sleep 1
            i=$[$i+1]
          else
            rm -f $lockfile
            [ $retval -eq 0 ] && echo "[Stopped]"
            return $retval
          fi
        done
        echo "$prog did not shut down in the grace period, sending sigkill"
        pkill -9 $prog
        retval=$?
        rm -f $lockfile
        [ $retval -eq 0 ] && echo "[Stopped]"
        return $retval
      }

      restart() {
        stop
        start
      }

      reload() {
        restart
      }

      force_reload() {
        restart
      }

      rh_status() {
        if [ -f $lockfile ]; then
         echo "$prog is running!"
         return 0
        else
         echo "$prog is not running!"
         return 1
        fi
      }

      rh_status_q() {
        rh_status >/dev/null 2>&1
      }


      case "$1" in
        start)
          rh_status_q && exit 0
          $1
          ;;
        stop)
          rh_status_q || exit 0
          $1
          ;;
        restart)
          $1
          ;;
        reload)
          rh_status_q || exit 7
          $1
          ;;
        force-reload)
          force_reload
          ;;
        status)
          rh_status
          ;;
        condrestart|try-restart)
          rh_status_q || exit 0
          restart
          ;;
        *)
          echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
          exit 2
      esac
      exit $?

  /opt/elasticbeanstalk/hooks/appdeploy/post/60-gen-env.sh:
    mode: "000755"
    owner: root
    group: root
    content: |
      #!/bin/bash
      NODE_INSTALL_DIR="$(/opt/elasticbeanstalk/bin/get-config container -k nodejs_install_dir)"
      NODE_VERSION="$(/opt/elasticbeanstalk/bin/get-config optionsettings -n "aws:elasticbeanstalk:container:nodejs" -o "NodeVersion")"
      export NODE_PATH="${NODE_INSTALL_DIR}/node-v${NODE_VERSION}-linux-x64"
      export PATH="$NODE_PATH/bin:$PATH"

      ENVIRONMENT_CONFIG="$(/opt/elasticbeanstalk/bin/get-config environment)"
      ENV_VARS=`echo ${ENVIRONMENT_CONFIG} | sed 's/"\([^"]*\)":"\([^"]*\)",*/\1="\2" /g;s/^{//;s/}$//'`

      echo "export $ENV_VARS PATH=\"$PATH\"" > /tmp/env

  /opt/elasticbeanstalk/hooks/appdeploy/post/70-restart-worker.sh:
    mode: "000755"
    owner: root
    group: root
    content: |
      #!/bin/bash
      /etc/init.d/worker stop
      . /tmp/env
      if [ "$NODE_TYPE" != "api" ]; then
        # Start the worker when not a pure API node
        /etc/init.d/worker start
      fi


  /opt/elasticbeanstalk/hooks/configdeploy/post/80-gen-env.sh:
    mode: "120755"
    owner: root
    group: root
    content: "/opt/elasticbeanstalk/hooks/appdeploy/post/60-gen-env.sh"

  /opt/elasticbeanstalk/hooks/configdeploy/post/90-restart-worker.sh:
    mode: "120755"
    owner: root
    group: root
    content: "/opt/elasticbeanstalk/hooks/appdeploy/post/70-restart-worker.sh"

  /opt/elasticbeanstalk/hooks/restartappserver/post/80-gen-env.sh:
    mode: "120755"
    owner: root
    group: root
    content: "/opt/elasticbeanstalk/hooks/appdeploy/post/60-gen-env.sh"

  /opt/elasticbeanstalk/hooks/restartappserver/post/90-restart-worker.sh:
    mode: "120755"
    owner: root
    group: root
    content: "/opt/elasticbeanstalk/hooks/appdeploy/post/70-restart-worker.sh"