Elastic Beanstalk worker with graceful shutdown
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
files: | |
/usr/local/bin/worker: | |
mode: "000755" | |
owner: root | |
group: root | |
content: | | |
#!/bin/sh | |
restartLimit=10 | |
timeLimit=300 # 5 minutes | |
crashCount=0 | |
cd /var/app/current/ | |
while true; do | |
node services/queue/worker.js & | |
pid=$! | |
echo $pid > /tmp/worker.pid | |
wait $pid | |
retval=$? | |
if [ $retval -eq 0 ]; then | |
exit 0 | |
else | |
# Crash | |
crashTime=`date +%s` | |
if [ -z "$firstCrashTime" ]; then | |
firstCrashTime=$crashTime | |
fi | |
timeDiff="$[$crashTime - $firstCrashTime]" | |
if [ $timeDiff -lt $timeLimit ]; then | |
crashCount=$[$crashCount + 1] | |
if [ $crashCount -gt $restartLimit ]; then | |
echo "Too many crashes, worker stopped" | |
exit $retval | |
fi | |
else | |
crashCount=1 | |
firstCrashTime=$crashTime | |
fi | |
echo "Worker crashed, restarting" | |
fi | |
done | |
/etc/init.d/worker: | |
mode: "000755" | |
owner: root | |
group: root | |
content: | | |
#!/bin/sh | |
# | |
# worker - stops and starts the worker | |
# | |
# chkconfig: - 20 80 | |
# description: Worker with graceful shutdown | |
# Source function library. | |
. /etc/rc.d/init.d/functions | |
exec="/usr/local/bin/worker" | |
prog="worker" | |
env="/tmp/env" | |
log="/var/log/nodejs/nodejs.log" | |
[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog | |
lockfile=/var/lock/subsys/$prog | |
pidfile=/tmp/${prog}.pid | |
min_kill_timeout=5 | |
default_kill_timeout=60 | |
start() { | |
echo -n "Starting $prog:" | |
. $env | |
daemon --user=games $exec >> $log 2>&1 & | |
retval=$? | |
echo | |
pid=`cat $pidfile` | |
[ $retval -eq 0 ] && touch $lockfile && echo "[Started $pid]" | |
return $retval | |
} | |
stop() { | |
kill_timeout=$default_kill_timeout | |
if [ -f $env ]; then | |
. $env | |
if [ -n "$WORKER_SHUTDOWN_DELAY" ] && [ "$WORKER_SHUTDOWN_DELAY" -gt $min_kill_timeout ]; then | |
kill_timeout="$WORKER_SHUTDOWN_DELAY" | |
fi | |
fi | |
pid=`cat $pidfile` | |
i="0" | |
echo "Stopping $prog [$pid]:" | |
kill $pid | |
retval=$? | |
while [ $i -lt $kill_timeout ]; do | |
if [ -d "/proc/$pid" ]; then | |
sleep 1 | |
i=$[$i+1] | |
else | |
rm -f $lockfile | |
[ $retval -eq 0 ] && echo "[Stopped]" | |
return $retval | |
fi | |
done | |
echo "$prog did not shut down in the grace period, sending sigkill" | |
pkill -9 $prog | |
retval=$? | |
rm -f $lockfile | |
[ $retval -eq 0 ] && echo "[Stopped]" | |
return $retval | |
} | |
restart() { | |
stop | |
start | |
} | |
reload() { | |
restart | |
} | |
force_reload() { | |
restart | |
} | |
rh_status() { | |
if [ -f $lockfile ]; then | |
echo "$prog is running!" | |
return 0 | |
else | |
echo "$prog is not running!" | |
return 1 | |
fi | |
} | |
rh_status_q() { | |
rh_status >/dev/null 2>&1 | |
} | |
case "$1" in | |
start) | |
rh_status_q && exit 0 | |
$1 | |
;; | |
stop) | |
rh_status_q || exit 0 | |
$1 | |
;; | |
restart) | |
$1 | |
;; | |
reload) | |
rh_status_q || exit 7 | |
$1 | |
;; | |
force-reload) | |
force_reload | |
;; | |
status) | |
rh_status | |
;; | |
condrestart|try-restart) | |
rh_status_q || exit 0 | |
restart | |
;; | |
*) | |
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" | |
exit 2 | |
esac | |
exit $? | |
/opt/elasticbeanstalk/hooks/appdeploy/post/60-gen-env.sh: | |
mode: "000755" | |
owner: root | |
group: root | |
content: | | |
#!/bin/bash | |
NODE_INSTALL_DIR="$(/opt/elasticbeanstalk/bin/get-config container -k nodejs_install_dir)" | |
NODE_VERSION="$(/opt/elasticbeanstalk/bin/get-config optionsettings -n "aws:elasticbeanstalk:container:nodejs" -o "NodeVersion")" | |
export NODE_PATH="${NODE_INSTALL_DIR}/node-v${NODE_VERSION}-linux-x64" | |
export PATH="$NODE_PATH/bin:$PATH" | |
ENVIRONMENT_CONFIG="$(/opt/elasticbeanstalk/bin/get-config environment)" | |
ENV_VARS=`echo ${ENVIRONMENT_CONFIG} | sed 's/"\([^"]*\)":"\([^"]*\)",*/\1="\2" /g;s/^{//;s/}$//'` | |
echo "export $ENV_VARS PATH=\"$PATH\"" > /tmp/env | |
/opt/elasticbeanstalk/hooks/appdeploy/post/70-restart-worker.sh: | |
mode: "000755" | |
owner: root | |
group: root | |
content: | | |
#!/bin/bash | |
/etc/init.d/worker stop | |
. /tmp/env | |
if [ "$NODE_TYPE" != "api" ]; then | |
# Start the worker when not a pure API node | |
/etc/init.d/worker start | |
fi | |
/opt/elasticbeanstalk/hooks/configdeploy/post/80-gen-env.sh: | |
mode: "120755" | |
owner: root | |
group: root | |
content: "/opt/elasticbeanstalk/hooks/appdeploy/post/60-gen-env.sh" | |
/opt/elasticbeanstalk/hooks/configdeploy/post/90-restart-worker.sh: | |
mode: "120755" | |
owner: root | |
group: root | |
content: "/opt/elasticbeanstalk/hooks/appdeploy/post/70-restart-worker.sh" | |
/opt/elasticbeanstalk/hooks/restartappserver/post/80-gen-env.sh: | |
mode: "120755" | |
owner: root | |
group: root | |
content: "/opt/elasticbeanstalk/hooks/appdeploy/post/60-gen-env.sh" | |
/opt/elasticbeanstalk/hooks/restartappserver/post/90-restart-worker.sh: | |
mode: "120755" | |
owner: root | |
group: root | |
content: "/opt/elasticbeanstalk/hooks/appdeploy/post/70-restart-worker.sh" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment