cycle-nodes

cycle-nodes

#!/bin/bash

set -e
# set -x

RED='\033[0;31m'
BLUE='\033[0;34m'
GREEN='\033[1;32m'
NC='\033[0m' # No Color

IFS=$'\n'
CONTEXT=`kubectl config current-context`
RETIRE_TIME=$(date +"%Y-%m-%dT%H-%M-%SZ")

log() {
    ts=$(date +"%Y-%m-%dT%H:%M:%SZ")
    printf "${1}${ts} ${2}${NC}\n"
}

if [[ ! -n "${CONTEXT}" ]]; then
        log $RED "Your kubectl is not set with a current-context. Please set the cluster you want to update."
        exit 1
fi

log $GREEN "Current Kubernetes cluster is $CONTEXT"

MASTERS=(`kubectl get nodes -l role=master -o json | jq '.items[] | .status .addresses[]  | select(.type=="Hostname") | .address' | sed 's/\"//g'`)
WORKERS=(`kubectl get nodes -l role=worker -o json | jq '.items[] | .status .addresses[]  | select(.type=="Hostname") | .address' | sed 's/\"//g'`)

log $NC "$CONTEXT cluster is made up of ${#MASTERS[@]} masters and ${#WORKERS[@]} workers"

log $BLUE "Labelling masters for retirement..."
for master in "${MASTERS[@]}"
do
  kubectl label node $master retiring-master=$RETIRE_TIME --overwrite=true
done

log $BLUE "Labelling workers for retirement..."
for worker in "${WORKERS[@]}"
do
  kubectl label node $worker retiring-worker=$RETIRE_TIME --overwrite=true
done


log $GREEN "Retiring workers in Kubernetes $CONTEXT"
for worker in "${WORKERS[@]}"
do
  log $BLUE "draining $worker..."
  kubectl drain $worker --ignore-daemonsets --force --delete-local-data
  sleep 30
  INSTANCE_ID=`aws ec2 describe-instances --filters "Name=network-interface.private-dns-name,Values=$worker" --profile=uw_dev | jq '.Reservations[].Instances[].InstanceId' | sed 's/\"//g'`
  log $BLUE "terminating $worker with instance-id $INSTANCE_ID..."
  TERMINATE=`aws ec2 terminate-instances --instance-ids=$INSTANCE_ID --profile=uw_dev`
  sleep 60
  ACTIVE_WORKERS=`kubectl get nodes -l role=worker | grep "Ready" | wc -l | xargs`
  while [[ $ACTIVE_WORKERS != ${#WORKERS[@]} ]]; do
    echo "Waiting for new worker to join $CONTEXT cluster ($ACTIVE_WORKERS active / ${#WORKERS[@]} desired)"
    sleep 5
    ACTIVE_WORKERS=`kubectl get nodes -l role=worker | grep "Ready" | wc -l | xargs`
  done
  REMAINING_WORKERS=(`kubectl get nodes -l retiring-worker=$RETIRE_TIME -o json | jq '.items[] | .status .addresses[]  | select(.type=="Hostname") | .address' | sed 's/\"//g'`)
  log $GREEN "Currently $ACTIVE_WORKERS active worker instances. We started with ${#WORKERS[@]}. ${#REMAINING_WORKERS[@]} still to retire."
done