A foreground script that constantly tries to hold a consul lock to make itself master for PXC

pxc_consul_master_lock.sh

#!/bin/bash

MYSQL_CMDLINE="mysql -nNE --connect-timeout=5"
AVAILABLE_WHEN_DONOR=1

function getSession() {

    local retry=0
    local __dummy=""
    
    while [ $retry -lt 10 ]; do

        echo "-- Looking for existing session."
        local __res=$(curl -s http://localhost:8500/v1/session/node/$HOSTNAME)
        if [ "x${__res:0:2}" == "xNo" ]; then

          # Consul offline.
          __dummy=""

        else

          # Received a list of sessions. Could be empty; could be good.

          # Is there a session for this node named 'pxcmaster'?
          local __sess=$(echo $__res | jq -r '.[] | select(.Name?=="pxcmaster").ID' | head -n 1)
          if [ "x$__sess" == "x" ]; then

            # Empty list. Make new session.
            local __data="{\"Name\": \"pxcmaster\", \"LockDelay\": \"15s\", \"Node\": \"$HOSTNAME\"}"
            __res=$(curl -s -X PUT -d "$__data" http://localhost:8500/v1/session/create)
            if [ "x${__res:0:2}" == "xNo" -o "x${__res:0:3}" == "xrpc" ]; then

              # Consul offline
              __dummy=""

            else

              echo "--- Created new session."
              SESSION=$(echo $__res | jq -r '.ID')
              break

            fi

          else

            # Got a session id
            echo "--- Found existing session."
            SESSION=$__sess
            break

          fi

        fi

        # If we didn't get a session, loopity.
		((retry++))

		echo "--- Not connected to Consul. Unable to get/create session. (Attempt $retry of 10. Sleep 30s between retry.)"
                echo "--- Error: $__res"

		if [ $retry -ge 10 ]; then

		  echo "-- Too many attempts. Exiting."
		  exit 1

		fi

		sleep 30
    
    done
}

getSession
echo "-- Session is $SESSION"

while true; do

  # Get Cluster Status
  WSREP_STATUS=($($MYSQL_CMDLINE -e "SHOW GLOBAL STATUS LIKE 'wsrep_%';" 2>/dev/null \
   | grep -A 1 -E 'wsrep_local_state$|wsrep_cluster_status$' | sed -n -e '2p'  -e '5p' | tr '\n' ' '))

  # Logic: If this node is Primary AND 
  #  ( Node in Sync state OR ( Node in Donor state AND Flag says we can be available during Donor state))
  if [[ ${WSREP_STATUS[1]} == 'Primary' && ( ${WSREP_STATUS[0]} -eq 4 || \
    ( ${WSREP_STATUS[0]} -eq 2 && $AVAILABLE_WHEN_DONOR -eq 1 ) ) ]]
  then

    echo "-- Primary State. Attempting to acquire lock."

    RESULT=$(curl -s -X PUT -d "$HOSTNAME" http://localhost:8500/v1/kv/pxcmaster?acquire=$SESSION)
    if [ "x$RESULT" == "xfalse" ]; then

      MASTER=$(curl -s http://localhost:8500/v1/kv/pxcmaster?raw=true)

      echo "-- Unable to acquire lock. Setting this node read_only. Master is '$MASTER'."
      $MYSQL_CMDLINE -e "SET GLOBAL read_only = 1"

    elif [ "x$RESULT" == "xtrue" ]; then

      echo "-- Acquired lock. Becoming master. Turning off read_only."
      $MYSQL_CMDLINE -e "SET GLOBAL read_only = 0"

    elif [ "x$RESULT" == "xNo cluster leader" ]; then

      echo "-- Consul appears offline on this node. Cannot determine anything. Sleeping 1 minute."
      # 45 + 15 (below) = 60s
      sleep 45

    else

      # Check for invalid session
      echo $RES | grep "invalid session" 2>&1 >/dev/null
      if [ $? -eq 0 ]; then
        getSession
        continue
      fi

      echo "-- Got '$RESULT'. Unknown result."

    fi

  else

    echo "-- Offline, Non-Primary or Donor State."

    # Did we have the lock before? If so, release it.
    RES=$(curl -s http://localhost:8500/v1/kv/pxcmaster?raw=true)
    if [ "x${RES:0:2}" == "xNo" ]; then
      # "No consul" error
      echo "-- Consul is unavailable. ($RES)"
    else
      CUR_SESSION=$(echo $RES | jq -r '.[].Session')
      if [ "$CUR_SESSION" == "$SESSION" ]; then
        echo "Releasing lock."
        RESULT=$(curl -s -X PUT -d "$HOSTNAME" http://localhost:8500/v1/kv/pxcmaster?release=$SESSION)
        echo "-- Got '$RESULT' from release."
      fi
    fi

  fi

  sleep 15

done