Rolling restart of all nodes in an AKS cluster

aks-restart-nodes.sh

#!/bin/bash

# TODO: probably accept these as CLI parameters or something
resourceGroup='<my-resource-group>'
cluster='<my-cluster-name>'
region='<my-region>'

group="MC_${resourceGroup}_${cluster}_$region"

function wait_for_status() {
  node=$1
  reason=$2
  i=0
  while [[ $i -lt 30 ]]; do
    status=$(k get node $node -o "jsonpath={.status.conditions[?(.reason==\"$reason\")].type}")
    if [[ "$status" == "Ready" ]]; then
      echo "Not ready after $i iterations, breaking"
      break;
    else
      sleep 2s
      i=$(($i+1))
    fi
  done
  if [[ $i == 30 ]]; then
    echo "Error: Did not reach $reason state within 1 minute"
    exit 1
  fi
}

nodes=$(kubectl get nodes -o jsonpath={.items[*].metadata.name})
for node in $nodes; do
  echo "Draining $node..."
  kubectl drain "$node" --ignore-daemonsets

  echo "Initiating VM restart for $node..."
  az vm restart --resource-group "$group" --name "$node"

  echo "Waiting for $node to start back up..."
  wait_for_status $node KubeletNotReady
  wait_for_status $node KubeletReady

  echo "Re-enabling $node for scheduling"
  kubectl uncordon "$node"
done