A bash alternative to kubectl drain

drain.sh

#!/bin/bash

##
## This aims to be an alternative to `kubectl drain NODE`, overcoming
## some of its limitations. A GitHub issue was closed a while ago
## without any solution or alternatives.
##
## https://github.com/kubernetes/kubernetes/issues/48307
##
## Even though `kubectl drain` respects PDBs defined by the user,
## single Pod Deployments can experience downtime by `kubectl drain`
## because it does not respect the Deployment rollout strategy.
##
## This script implements a different, though more expensive, approach
## to drain a node by forcing a `kubectl rollout restart` on all
## Deployments/StatefulSets that have pods running on that node.
##
## Keep in mind this will cause ALL PODS from the affected sets
## to restart, even ones that are running in other nodes that are
## not being drained. This is only helpful for single Pod Deployments
## with strategy similar to:
##
##       strategy:
##         type: RollingUpdate
##         rollingUpdate:
##           maxSurge: 1
##           maxUnavailable: 0
##
## This script ignores all DaemonSets. Further more, it will not
## wait for all Pods to become READY. In other words, it won't
## wait until the node is completely drained before exiting. Keep
## watch over all affected pods after this script exits.
##

NODE_NAME=$1
ROLLOUT_CMD=$2

if [[ "$NODE_NAME" == "" ]]; then
  echo "
  USAGE: ./drain.sh <NODE_NAME> [ROLLOUT_CMD]

  Gracefully drain a Kubernetes node by forcing a rollout restart
  on all Deployments and StatefulSets that have pods running on
  that node.

  Examples:

    # Dry run, shows the kubectl commands that would be issued
    ./drain.sh NODE_NAME

    # Drains the node
    ./drain.sh NODE_NAME restart

  Pre-requisite: Kubernetes 1.15.x or later. Needs `rollout restart`.

  NODE_NAME: The name that appears in `kubectl get nodes`.

  ROLLOUT_CMD: The rollout sub-command to use, usually `restart`.
"
  exit 1
fi

function rollout_restart() {

  ## Deployment or StatefulSet.
  OBJTYPE=$1

  ## Loop through all deploy/sts in the cluster
  for dp in $(kubectl get $OBJTYPE -A --no-headers | awk '{print $1 "|" $2}'); do
    NAMESPACE=$(echo $dp | sed 's/|.*//')
    DEPLOY=$(echo $dp | sed 's/.*|//')
    
    ## Recover a pod list from the deploy/sts filtering the ones that match
    ## the node we want to drain.
    SELECTOR=$(kubectl get $OBJTYPE --no-headers -owide -n $NAMESPACE $DEPLOY -owide | awk '{print $8}')
    PODLIST=$(kubectl get pod -owide --no-headers -n $NAMESPACE -l "$SELECTOR" | grep $NODE_NAME | awk '{print $1}')

    if [[ "$PODLIST" != "" ]]; then
      
      ## dry run echo
      echo "kubectl rollout restart -n $NAMESPACE $OBJTYPE/$DEPLOY"
      
      if [[ "$ROLLOUT_CMD" != "" ]]; then
        echo ">> Rollout restart..."
        echo $PODLIST | sed 's/ /\n/g'
        set -x
        kubectl rollout $ROLLOUT_CMD -n $NAMESPACE $OBJTYPE/$DEPLOY
        set +x
        echo
      fi
    fi
  done
}

if [[ "$ROLLOUT_CMD" == "restart" ]]; then
  set -x
  kubectl cordon $NODE_NAME
  set +x
fi

rollout_restart deploy
rollout_restart sts