Skip to content

Instantly share code, notes, and snippets.

@iancoffey
Last active August 29, 2015 14:19
Show Gist options
  • Save iancoffey/7f2ac99cfe6b47397812 to your computer and use it in GitHub Desktop.
Save iancoffey/7f2ac99cfe6b47397812 to your computer and use it in GitHub Desktop.
#!/usr/bin/bash
if [ "$1" != '--really' ]; then
echo "command must be run as: $0 --really"
exit 1
fi
# ensure the cluster has graceful_shutdown enabled
if ! /usr/bin/etcdctl get /deis/graceful_shutdown; then
exit 0
fi
# procedure requires the store-admin
ADMIN_RUNNING=$(docker inspect --format="{{ .State.Running }}" deis-store-admin)
if [ $? -eq 1 ] || [ "$ADMIN_RUNNING" == "false" ]; then
echo "deis-store-admin container is required for graceful shutdown"
exit 1
fi
set -e -x -o pipefail
CURRENT_STATUS=$(/usr/bin/docker exec deis-store-admin ceph health | awk '{print $1}')
OSD_HOSTS=($(/usr/bin/etcdctl ls /deis/store/hosts/| awk -F'/' '{print $5}'))
for HOST in "${OSD_HOSTS[@]}"
do
PUBLIC_IP=$(/usr/bin/etcdctl get /_coreos.com/fleet/machines/`cat /etc/machine-id`/object | /opt/bin/jq -r '.PublicIP')
if [ "$HOST" = "$PUBLIC_IP" ] ; then
OSD_ID=$(/usr/bin/etcdctl get /deis/store/osds/$PUBLIC_IP)
break
fi
done
# if we own an osd and its healthy, try to gracefully remove it
if [ -z "$OSD_ID" ] && [ "$CURRENT_STATUS" == "HEALTH_OK"]; then
/usr/bin/docker exec deis-store-admin ceph osd out $OSD_ID
sleep 30
TIMEWAITED=0
until /usr/bin/docker exec deis-store-admin ceph health | grep HEALTH_OK
do
if [ $TIMEWAITED -gt "600" ]
then
echo "ceph graceful removal timeout exceeded"
break
fi
echo "waiting" && sleep 5
TIMEWAITED=$((TIMEWAITED+5))
done
/usr/bin/docker stop deis-store-daemon
/usr/bin/docker exec deis-store-admin ceph osd crush remove osd.$OSD_ID
/usr/bin/docker exec deis-store-admin ceph auth del osd.$OSD_ID
/usr/bin/docker exec deis-store-admin ceph osd rm $OSD_ID
/usr/bin/etcdctl rm /deis/store/osds/$PUBLIC_IP
etcdctl rm /deis/store/hosts/$PUBLIC_IP && sleep 10
# remove ceph mon
/usr/bin/docker stop deis-store-monitor
/usr/bin/docker exec deis-store-admin ceph mon remove `hostname -f` # fixme
/usr/bin/docker stop deis-store-metadata
fi
NODE=$(curl -L http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`)
# remove from etcd cluster
if [ $NODE != 'null' ]; then
/usr/bin/curl -L -XDELETE http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`
fi
sudo systemctl stop fleet.service
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment