Skip to content

Instantly share code, notes, and snippets.

@yteraoka
Last active July 15, 2020 00:24
Show Gist options
  • Save yteraoka/68c03cd5024f8bfb68dbde5c7ae53dbb to your computer and use it in GitHub Desktop.
Save yteraoka/68c03cd5024f8bfb68dbde5c7ae53dbb to your computer and use it in GitHub Desktop.
#!/bin/bash
info(){
echo -e "\033[32m$(date +%Y-%m-%dT%H:%M:%S) $*\033[m"
}
usage(){
if [ $# -ge 1 ] ; then
echo $*
echo
fi
echo "Usage: $0 ip-111-222-333-444.ap-northeast-1.compute.internal" 1>&2
}
nodename=$1
echo $nodename | grep -q '^ip-'
if [ $? -ne 0 ] ; then
usage
exit 1
fi
kubectl get node $nodename
if [ $? -ne 0 ] ; then
usage "Unknown node: $nodename"
exit 1
fi
# aws:///ap-northeast-1a/i-0fc00637cc970e6f5
provider_id=$(kubectl get node -o json | jq -r ".items[] | select(.metadata.name == \"$nodename\") | .spec.providerID")
# i-0fc00637cc970e6f5
instance_id=$(basename $provider_id)
info "Shutting down ${nodename} (${instance_id})"
echo -n "Are you sure? (Y/n): "
read ans
case "$ans" in
n*|N*)
echo "Bye."
exit 2
esac
asgname=$(aws --output json ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].Tags[] | select(.Key == "aws:autoscaling:groupName") | .Value')
info "Autoscaling group name: ${asgname}"
info "Searching ${instance_id} in target groups"
target_group_arns=""
for target_group_arn in $(aws --output json elbv2 describe-target-groups --no-paginate | jq -r '.TargetGroups[].TargetGroupArn'); do
state=$(aws --output json elbv2 describe-target-health --target-group-arn $target_group_arn | jq -r ".TargetHealthDescriptions[] | select(.Target.Id == \"$instance_id\") | .TargetHealth.State")
if [ -n "$state" ] ; then
info "${instance_id} registerd in ${target_group_arn}, state: ${state}"
info "Deregistering ${instance_id} from target group ${target_group_arn}"
aws elbv2 deregister-targets --target-group-arn $target_group_arn --targets Id=${instance_id}
target_group_arns="$target_group_arns $target_group_arn"
fi
done
info "Draining node"
kubectl drain $nodename --ignore-daemonsets --delete-local-data
info "Deleting node"
kubectl delete node $nodename
num_nodes=$(aws --output json autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${asgname} | jq -r '.AutoScalingGroups[0].Instances | length')
info "Setting instance ${instance_id} unhealthy"
aws autoscaling set-instance-health --instance-id ${instance_id} --health-status Unhealthy
info "Terminating instance"
aws ec2 terminate-instances --instance-ids ${instance_id}
info "Waiting new instance ready"
tmpfile1=$(mktemp)
tmpfile2=$(mktemp)
trap "rm -f $tmpfile1 $tmpfile2" EXIT
while :; do
aws --output json autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${asgname} \
| jq -M -c '.AutoScalingGroups[0].Instances[] | [.InstanceId, .InstanceType, .LifecycleState, .HealthStatus]' > $tmpfile1
grep -q "\"${instance_id}\"" $tmpfile1
if [ $? -ne 0 ] ; then
# deleted node had been invisible
num_ready_nodes=$(cat $tmpfile1 | grep -c '"InService","Healthy"')
if [ $num_nodes -eq $num_ready_nodes ] ; then
echo
info "All EC2 instances are Ready"
break
fi
fi
if [ ! -f $tmpfile2 ] ; then
# first time
echo
date
cat $tmpfile1
echo
else
diff $tmpfile1 $tmpfile2 > /dev/null
if [ $? -ne 0 ] ; then
echo
date
cat $tmpfile1
echo
else
# nothing difference
echo -n .
fi
fi
mv $tmpfile1 $tmpfile2
sleep 5
done
info "Pods not in Running state"
kubectl get pods -A | grep -v Running
info "Cheking target group status"
for target_group_arn in $target_group_arns; do
echo "aws --output json elbv2 describe-target-health --target-group-arn $target_group_arn | jq -r \".TargetHealthDescriptions[]\""
aws --output json elbv2 describe-target-health --target-group-arn $target_group_arn | jq -r ".TargetHealthDescriptions[]"
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment