local volume pv janitor

configmap-pv-janitor.yaml

{{- if .Values.enableLocalstoragePvJanitor }}
apiVersion: v1
kind: ConfigMap
metadata:
  name: rabbitmq-pv-janitor
  labels:
    {{- include "rabbitmq.labels" . | nindent 4 }}
data:
  rabbitmq-pv-janitor.sh: |
    # This is a hack for dealing with what happens when a node goes away and a new one comes up
    # The PVC will stay bound to a PV on a non-existing host which prevents the RabbitMQ pod from coming up because of the PVs affinity to the dead host
    # This script checks constantly if pending RabbitMQ pods have PVCs bound to PVs on dead hosts and will remove the PVC if necessary
    #!/usr/bin/env bash
    log() {
      echo "$(date +"[%F %T %Z]") $1"
    }
    trap_sigterm() {
      log "caught SIGTERM, exiting"
      exit 0
    }
    trap trap_sigterm term
    trap trap_sigterm int
    while true
    do
      NODES=($(kubectl get no -l kubernetes.io/role=rabbitmq --no-headers | grep Ready | awk {'print $1'}))
      if [ ${#NODES[@]} -lt 3 ]; then
        log "less than 3 nodes online, waiting for all nodes to be available..."
        # sleep a little less, some node died and pvcs probably need cleanup
        sleep 10
        continue
      fi
      PENDING_RABBITMQ_PODS=($(kubectl get po -l app.kubernetes.io/name=rabbitmq -n {{ .Release.Namespace }} | grep Pending | awk {'print $1'}))
      if [ ${#PENDING_RABBITMQ_PODS[@]} -gt 0 ]; then
        for POD in ${PENDING_RABBITMQ_PODS[@]}; do
          log "RabbitMQ pod [${POD}] is in pending state, checking if its PVC is bound to a PV on dead node"
          PV=$(kubectl get pv | grep "{{ .Release.Namespace }}/data-${POD}" | awk {'print $1'})
          if [ -z "${PV}" ]; then
            log "no PV with the binding [{{ .Release.Namespace }}/data-${POD}] found, assuming this is ok"
          else
            log "found PV [${PV}] checking if it is on a non-existing node"
            NODE=$(kubectl get pv ${PV} -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[].matchExpressions[].values[]}')
            if [ -z "${NODE}" ]; then
              log "PV [${PV}] has no node affinity, this shouldn't happen"
            else
              log "PV [${PV}] has node affinity [${NODE}], checking if node exists"
              if [[ " ${NODES[@]} " =~ " ${NODE} " ]]; then
                log "node [${NODE}] is Ready, nothing to do"
              else
                log "node [${NODE}] doesn't exist, deleting PVC [data-${POD}]"
                kubectl delete pvc data-${POD} -n {{ .Release.Namespace }}
                log "deleting pending pod [${POD}] for PVC to get recreated"
                kubectl delete pod ${POD} -n {{ .Release.Namespace }}
                log "done cleaning up PVC bound to PV on dead node"
              fi
            fi
          fi
        done
      fi
      sleep 20
    done
{{- end }}