eks-node-replace.sh

#!/bin/bash

info(){
  echo -e "\033[32m$(date +%Y-%m-%dT%H:%M:%S) $*\033[m"
}

usage(){
  if [ $# -ge 1 ] ; then
    echo $*
    echo
  fi
  echo "Usage: $0 ip-111-222-333-444.ap-northeast-1.compute.internal" 1>&2
}

nodename=$1

echo $nodename | grep -q '^ip-'
if [ $? -ne 0 ] ; then
  usage
  exit 1
fi

kubectl get node $nodename
if [ $? -ne 0 ] ; then
  usage "Unknown node: $nodename"
  exit 1
fi

# aws:///ap-northeast-1a/i-0fc00637cc970e6f5
provider_id=$(kubectl get node -o json | jq -r ".items[] | select(.metadata.name == \"$nodename\") | .spec.providerID")

# i-0fc00637cc970e6f5
instance_id=$(basename $provider_id)

info "Shutting down ${nodename} (${instance_id})"
echo -n "Are you sure? (Y/n): "
read ans
case "$ans" in
  n*|N*)
    echo "Bye."
    exit 2
esac

asgname=$(aws --output json ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].Tags[] | select(.Key == "aws:autoscaling:groupName") | .Value')
info "Autoscaling group name: ${asgname}"

info "Searching ${instance_id} in target groups"

target_group_arns=""
for target_group_arn in $(aws --output json elbv2 describe-target-groups --no-paginate | jq -r '.TargetGroups[].TargetGroupArn'); do
  state=$(aws --output json elbv2 describe-target-health --target-group-arn $target_group_arn | jq -r ".TargetHealthDescriptions[] | select(.Target.Id == \"$instance_id\") | .TargetHealth.State")
  if [ -n "$state" ] ; then
    info "${instance_id} registerd in ${target_group_arn}, state: ${state}"
    info "Deregistering ${instance_id} from target group ${target_group_arn}"
    aws elbv2 deregister-targets --target-group-arn $target_group_arn --targets Id=${instance_id}
    target_group_arns="$target_group_arns $target_group_arn"
  fi
done

info "Draining node"
kubectl drain $nodename --ignore-daemonsets --delete-local-data

info "Deleting node"
kubectl delete node $nodename

num_nodes=$(aws --output json autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${asgname} | jq -r '.AutoScalingGroups[0].Instances | length')

info "Setting instance ${instance_id} unhealthy"
aws autoscaling set-instance-health --instance-id ${instance_id} --health-status Unhealthy

info "Terminating instance"
aws ec2 terminate-instances --instance-ids ${instance_id}


info "Waiting new instance ready"
tmpfile1=$(mktemp)
tmpfile2=$(mktemp)
trap "rm -f $tmpfile1 $tmpfile2" EXIT 
while :; do
  aws --output json autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${asgname} \
    | jq -M -c '.AutoScalingGroups[0].Instances[] | [.InstanceId, .InstanceType, .LifecycleState, .HealthStatus]' > $tmpfile1
  grep -q "\"${instance_id}\"" $tmpfile1
  if [ $? -ne 0 ] ; then
    # deleted node had been invisible
    num_ready_nodes=$(cat $tmpfile1 | grep -c '"InService","Healthy"')
    if [ $num_nodes -eq $num_ready_nodes ] ; then
      echo
      info "All EC2 instances are Ready"
      break
    fi
  fi
  if [ ! -f $tmpfile2 ] ; then
    # first time
    echo
    date
    cat $tmpfile1
    echo
  else
    diff $tmpfile1 $tmpfile2 > /dev/null
    if [ $? -ne 0 ] ; then
      echo
      date
      cat $tmpfile1
      echo
    else
      # nothing difference
      echo -n .
    fi
  fi
  mv $tmpfile1 $tmpfile2
  sleep 5
done

info "Pods not in Running state"
kubectl get pods -A | grep -v Running

info "Cheking target group status"
for target_group_arn in $target_group_arns; do
  echo "aws --output json elbv2 describe-target-health --target-group-arn $target_group_arn | jq -r \".TargetHealthDescriptions[]\""
  aws --output json elbv2 describe-target-health --target-group-arn $target_group_arn | jq -r ".TargetHealthDescriptions[]"
done