replace-nodes.sh

replace-nodes.sh

# Label used to determine nodes that should be replaced
LABEL="ami=old"

replace_nodes=$(kubectl get nodes --sort-by=.metadata.creationTimestamp --no-headers -o=name --selector=$LABEL)
echo "-- Replacing the following nodes:"
echo "$replace_nodes"

for node in $replace_nodes
  do
    echo "-- Checking $node"
    CP_ETCD=$(kubectl get $node | egrep "controlplane|control-plane|etcd")
    if [ -n "$CP_ETCD" ]
      then
        echo "-- controlplane or etcd node detected, skipping this node, please delete this node in Cluster Management"
        continue
    fi

    echo "-- Draining $node"
    kubectl drain $node --ignore-daemonsets --delete-emptydir-data --force
    sleep 5

    NEWEST_NODE=$(kubectl get nodes --sort-by=.metadata.creationTimestamp -o=jsonpath='{.items[-1].metadata.name}')
    echo "-- Deleting $node"
    kubectl delete $node
   
    echo "-- Waiting for new node to join"
    NEW_NODE=$(kubectl get nodes --sort-by=.metadata.creationTimestamp -o=jsonpath='{.items[-1].metadata.name}')
    while [ "$NEW_NODE" = "$NEWEST_NODE" ]
      do
        echo -n "."
        sleep 20
        NEW_NODE=$(kubectl get nodes --sort-by=.metadata.creationTimestamp -o=jsonpath='{.items[-1].metadata.name}')
    done

    echo "-- Node node joined the cluster: $NEW_NODE"
    check_nodes=true
    echo "-- Waiting for all nodes to be Ready"
    while [ $check_nodes == "true" ]
      do
        kubectl get nodes -o jsonpath='{range .items[*]} {.metadata.name} {" "} {.status.conditions[?(@.type=="Ready")].status} {"\n"} {end}' | grep "False"
        if [ $? -ne 0 ]
          then
            # All nodes are Ready
            check_nodes=false
        fi
      sleep 10
    done
done