Skip to content

Instantly share code, notes, and snippets.

@iancoffey

iancoffey/gist:7bdc2d43318c61cd82b1

Created September 29, 2015 13:33
Show Gist options
  • Save iancoffey/7bdc2d43318c61cd82b1 to your computer and use it in GitHub Desktop.
Save iancoffey/7bdc2d43318c61cd82b1 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
core@ip-10-21-1-94 ~ $ deisctl install store-admin
deis-store-admin.service: loaded
core@ip-10-21-1-94 ~ $ deisctl start store-admin
deis-store-admin.service: active/running
core@ip-10-21-1-94 ~ $ /opt/bin/graceful-shutdown.sh --really
+ '[' '!' -z 9 ']'
+ '[' 5 -gt 3 ']'
++ docker inspect '--format={{ .State.Running }}' deis-store-admin
+ ADMIN_RUNNING=true
+ '[' 0 -eq 1 ']'
+ '[' true == false ']'
++ docker exec deis-store-admin ceph health
++ awk '{print $1}'
+ CURRENT_STATUS=HEALTH_OK
+ [[ HEALTH_OK == *\H\E\A\L\T\H\_\O\K* ]]
+ docker exec deis-store-admin ceph osd out 9
marked out osd.9.
+ sleep 30
+ TIMEWAITED=0
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 54 pgs degraded; 6 pgs recovering; 5 pgs recovery_wait; 21 pgs stuck unclean; recovery 303/1128 objects degraded (26.862%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 0 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=5
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 54 pgs degraded; 6 pgs recovering; 5 pgs recovery_wait; 21 pgs stuck unclean; recovery 303/1128 objects degraded (26.862%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 5 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=10
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 47 pgs degraded; 1 pgs recovering; 5 pgs recovery_wait; 19 pgs stuck unclean; recovery 280/1128 objects degraded (24.823%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 10 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=15
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 47 pgs degraded; 2 pgs recovering; 5 pgs recovery_wait; 19 pgs stuck unclean; recovery 278/1128 objects degraded (24.645%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 15 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=20
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 37 pgs degraded; 2 pgs recovering; 2 pgs recovery_wait; 16 pgs stuck unclean; recovery 229/1128 objects degraded (20.301%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 20 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=25
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 37 pgs degraded; 2 pgs recovering; 2 pgs recovery_wait; 16 pgs stuck unclean; recovery 229/1128 objects degraded (20.301%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 25 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=30
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 37 pgs degraded; 2 pgs recovering; 2 pgs recovery_wait; 16 pgs stuck unclean; recovery 229/1128 objects degraded (20.301%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 30 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=35
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 35 pgs degraded; 8 pgs recovering; 2 pgs recovery_wait; 16 pgs stuck unclean; recovery 196/1128 objects degraded (17.376%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 35 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=40
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 35 pgs degraded; 8 pgs recovering; 2 pgs recovery_wait; 16 pgs stuck unclean; recovery 196/1128 objects degraded (17.376%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 40 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=45
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 35 pgs degraded; 8 pgs recovering; 2 pgs recovery_wait; 16 pgs stuck unclean; recovery 196/1128 objects degraded (17.376%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 45 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=50
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 29 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 11 pgs stuck unclean; recovery 180/1128 objects degraded (15.957%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 50 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=55
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 29 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 11 pgs stuck unclean; recovery 180/1128 objects degraded (15.957%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 55 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=60
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 29 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 11 pgs stuck unclean; recovery 180/1128 objects degraded (15.957%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 60 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=65
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 23 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 9 pgs stuck unclean; recovery 138/1128 objects degraded (12.234%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 65 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=70
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 23 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 10 pgs stuck unclean; recovery 138/1128 objects degraded (12.234%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 70 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=75
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 23 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 10 pgs stuck unclean; recovery 138/1128 objects degraded (12.234%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 75 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=80
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 19 pgs degraded; 6 pgs recovering; 2 pgs recovery_wait; 8 pgs stuck unclean; recovery 114/1128 objects degraded (10.106%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 80 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=85
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 13 pgs degraded; 1 pgs recovering; 2 pgs recovery_wait; 5 pgs stuck unclean; recovery 87/1128 objects degraded (7.713%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 85 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=90
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 13 pgs degraded; 2 pgs recovering; 1 pgs recovery_wait; 5 pgs stuck unclean; recovery 87/1128 objects degraded (7.713%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 90 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=95
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 12 pgs degraded; 4 pgs recovering; 1 pgs recovery_wait; 5 pgs stuck unclean; recovery 75/1128 objects degraded (6.649%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 95 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=100
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 12 pgs degraded; 4 pgs recovering; 1 pgs recovery_wait; 5 pgs stuck unclean; recovery 75/1128 objects degraded (6.649%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 100 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=105
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 12 pgs degraded; 4 pgs recovering; 1 pgs recovery_wait; 5 pgs stuck unclean; recovery 75/1128 objects degraded (6.649%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 105 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=110
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 10 pgs degraded; 3 pgs recovering; 1 pgs recovery_wait; 3 pgs stuck unclean; recovery 63/1128 objects degraded (5.585%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 110 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=115
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 10 pgs degraded; 3 pgs recovering; 1 pgs recovery_wait; 3 pgs stuck unclean; recovery 63/1128 objects degraded (5.585%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 115 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=120
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_WARN 10 pgs degraded; 3 pgs recovering; 1 pgs recovery_wait; 3 pgs stuck unclean; recovery 63/1128 objects degraded (5.585%) == *\H\E\A\L\T\H\_\O\K* ]]
+ '[' 120 -gt 1200 ']'
+ echo waiting
waiting
+ sleep 5
+ TIMEWAITED=125
++ docker exec deis-store-admin ceph health
+ [[ HEALTH_OK == *\H\E\A\L\T\H\_\O\K* ]]
+ docker stop deis-store-daemon
deis-store-daemon
+ docker exec deis-store-admin ceph osd crush remove osd.9
removed item id 9 name 'osd.9' from crush map
+ docker exec deis-store-admin ceph auth del osd.9
updated
+ docker exec deis-store-admin ceph osd rm 9
removed osd.9
+ etcdctl rm /deis/store/osds/10.21.1.94
+ etcdctl rm /deis/store/hosts/10.21.1.94
+ sleep 10
+ docker stop deis-store-monitor
deis-store-monitor
++ hostname -f
+ docker exec deis-store-admin ceph mon remove ip-10-21-1-94.ec2.internal
2015-09-29 13:33:19.183873 7f8c6e0b9700 0 monclient: hunting for new mon
2015-09-29 13:33:19.183873 7f8c6e0b9700 0 monclient: hunting for new mon
removed mon.ip-10-21-1-94.ec2.internal at 10.21.1.94:6789/0, there are now 4 monitors
+ docker stop deis-store-metadata
deis-store-metadata
++ etcdctl member list
+++ cat /etc/machine-id
++ cut -d : -f 1
++ grep 5a6face934824d83bf439e49598fcd92
+ NODE=13db40d7186c2612
+ etcdctl member remove 13db40d7186c2612
Removed member 13db40d7186c2612 from cluster
+ curl https://discovery.etcd.io/bfddd8c173e0d663da240e86ce1dcfe0/13db40d7186c2612 -XDELETE
{"action":"delete","node":{"key":"/_etcd/registry/bfddd8c173e0d663da240e86ce1dcfe0/13db40d7186c2612","modifiedIndex":822851918,"createdIndex":822834420},"prevNode":{"key":"/_etcd/registry/bfddd8c173e0d663da240e86ce1dcfe0/13db40d7186c2612","value":"5a6face934824d83bf439e49598fcd92=http://10.21.1.94:2380","modifiedIndex":822834420,"createdIndex":822834420}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment