Ceph Before Reboot Maintenance Script

ceph-before-reboot-script.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: ceph-before-reboot-script
  namespace: rook-ceph
data:
  status-check.sh: |-
    #!/bin/bash
    # preflightCheck checks for existence of "dependencies"
    preflightCheck() {
        if [ ! -f "/var/run/secrets/kubernetes.io/serviceaccount/token" ]; then
            echo "$(date) | No Kubernetes ServiceAccount token found."
            exit 1
        else
            KUBE_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
            export KUBE_TOKEN
        fi
    }
    # updateNodeRebootAnnotation sets the `ceph-reboot-check` annotation to `true` on `$NODE`
    updateNodeRebootAnnotation(){
        export PATCH="[{ \"op\": \"add\", \"path\": \"/metadata/annotations/ceph-before-reboot-check\", \"value\": \"true\" }]"
        TRIES=0
        until [ $TRIES -eq 10 ]; do
            if curl -sSk \
                --fail \
                -XPATCH \
                -H "Authorization: Bearer $KUBE_TOKEN" \
                -H "Accept: application/json" \
                -H "Content-Type:application/json-patch+json" \
                --data "$PATCH" \
                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE"; then
                echo "$(date) | Annotation \"ceph-reboot-check\" from node $NODE updated to \"true\". Reboot confirmed!"
                return 0
            else
                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
                (( TRIES++ ))
                /bin/sleep 5
            fi
        done
        return 1
    }
    # Check if noout option should be set
    checkForNoout() {
        # Fetch the annotations list of the node
        TRIES=0
        until [ $TRIES -eq 10 ]; do
            if NODE_ANNOTATIONS=$(curl -sSk \
                --fail \
                -H "Authorization: Bearer $KUBE_TOKEN" \
                -H "Accept: application/json" \
                "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes/$NODE" \
                | jq ".metadata.annotations") ; then
                echo "$(date) | Node annotations collected, looking for \"ceph-no-noout\" annotation"
                if [ $(echo "$NODE_ANNOTATIONS" | jq '."ceph-no-noout"') != "null" ] ; then
                    echo "$(date) | Node annotation \"ceph-no-noout\" exists, not setting Ceph noout flag"
                else
                    echo "$(date) | Node annotation \"ceph-no-noout\" doesn't exist, setting Ceph noout flag"
                    ceph osd set noout
                fi
                return 0
            else
                echo "$(date) | Kubernetes API server connection error, will retry in 5 seconds..."
                (( TRIES++ ))
                /bin/sleep 5
            fi
        done
        return 1
    }
    # checkCephClusterHealth checks `ceph health` command for `HEALTH_OK`
    checkCephClusterHealth(){
        echo "$(date) | Running ceph health command"
        if /usr/bin/ceph health | grep -q "HEALTH_OK"; then
            echo "$(date) | Ceph cluster health is: OKAY"
            return 0
        fi
        return 1
    }
    preflightCheck
    echo "$(date) | Running the rook toolbox config initiation script..."
    /usr/local/bin/toolbox.sh &
    TRIES=0
    until [ -f /etc/ceph/ceph.conf ]; do
        [ $TRIES -eq 10 ] && { echo "$(date) | No Ceph config found after 10 tries. Exiting ..."; exit 1; }
        echo "$(date) | Waiting for Ceph config (try $TRIES from 10) ..."
        (( TRIES++ ))
        sleep 3
    done
    while true; do
        if checkCephClusterHealth; then
            if checkForNoout; then
                if updateNodeRebootAnnotation; then
                    while true; do
                        echo "$(date) | Waiting for $NODE to reboot ..."
                        /bin/sleep 30
                    done
                    exit 0
                else
                    echo "$(date) | Failed updating annotation for $NODE. Exiting."
                    exit 1
                fi
            else
                echo "$(date) | Failed setting the Ceph osd noout flag. Exiting."
                exit 1
            fi
        fi
        echo "$(date) | Ceph cluster Health not HEALTH_OK currently. Checking again in 20 seconds ..."
        /bin/sleep 20
    done