Substrate Block drift watcher

substrate_block_drift_watcher.sh

#!/bin/bash
#
# Bash script to be executed in the remote server to monitor block drift
#
# > Make a file executable
# chmod +x ./substrate_block_drift_watcher.sh
#
# > Positional arguments:
# 1st - blocks threshold
# 2nd - service name
# 3rd - node RPC port
#
# > Test and run with the following parameters e.g.:
# ./substrate_block_drift_watcher.sh 20 kusama-node-service 9933
#
# > Schedule a cron job to execute every minute
# https://www.digitalocean.com/community/tutorials/how-to-use-cron-to-automate-tasks-ubuntu-1804
#
# example:
# 5-55/1 * * * * /opt/substrate_block_drift_watcher/substrate_block_drift_watcher.sh 20 kusama-node-service 9933 >> /opt/substrate_block_drift_watcher/sbdw.log
#
#

if [ -z "$1" ]
then    
    echo "INFO: Blocks threshold is missing, default is 20. e.g. ./block_drift_monitor.sh 20 kusama-node 9933"
    BLOCKS_THRESHOLD=20
else
    BLOCKS_THRESHOLD=$1
fi

if [ -z "$2" ]
then
    echo "INFO: Node service name is missing, default is kusama-node. e.g. ./block_drift_monitor.sh 20 kusama-node 9933"
    NODE_SERVICE="kusama-node"
else
    NODE_SERVICE=$2
fi

if [ -z "$3" ]
then
    echo "INFO: Node RPC port is missing, default is 9933. e.g. ./block_drift_monitor.sh 20 kusama-node 9933"
    RPC_PORT=9933
else
    RPC_PORT=$3
fi

# Define filename to cache current block number
CURRENT_BLOCK_FILENAME="$(dirname $0)/$NODE_SERVICE.block"

# Verify if service node is active
sudo systemctl is-active --quiet $NODE_SERVICE.service
if [[ "$?" -ne 0 ]]; 
then
    echo "ERROR: Service $NODE_SERVICE is not active."
    exit
fi

# Verify if node is running on the RPC PORT specified
STATUS_CODE=$(curl --write-out %{http_code} --silent --output /dev/null \
  -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}' \
  'http://localhost:'$RPC_PORT'')

if [[ "$STATUS_CODE" -ne 200 ]]; 
then
    echo "ERROR: RPC port: $RPC_PORT fails to connect."
    exit
fi

# --- Fetch node health
# NOTE: system_health response example:
# {
#  "isSyncing": false,
#  "peers": 37,
#  "shouldHavePeers": true
# }
IS_SYNCING="$( curl --silent -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result.isSyncing' )"

# ---
# NOTE: Skip Monitoring if node is syncing old blocks
#
if [ "$IS_SYNCING" = true ]
then
  echo "INFO: Node is syncing -> SKIPPING monitor."
  exit
fi

# --- Fetch RPC `system_syncState`
# NOTE: system_health response example:
# {
#   "currentBlock": 11132625,
#   "highestBlock": 11132625,
#   "startingBlock": 10862594
# }
CURRENT_BLOCK_NUMBER="$( curl --silent -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "system_syncState", "params":[]}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result.currentBlock' )"
# ---

# --- Fetch Finalized block number
# Get Finalized head
BLOCK_HASH="$( curl --silent -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getFinalizedHead", "params":[]}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result' )"
BLOCK_HASH=$( echo "$BLOCK_HASH" | awk -F ' ' '{ printf $1 }' )

# Get Header
FINALIZED_BLOCK_NUMBER="$( curl --silent -H Content-Type:application/json \
  -d '{"id":1, "jsonrpc": "2.0", "method": "chain_getHeader", "params": ['$BLOCK_HASH']}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result.number' )"
# Note: To convert hex block number decimal
# we first need to emove "" and 0x from heximal number eg: "0xaa1047" -> aa1047
FINALIZED_BLOCK_NUMBER=${FINALIZED_BLOCK_NUMBER//\"/}
FINALIZED_BLOCK_NUMBER=${FINALIZED_BLOCK_NUMBER//0x/}
# Convert block number hex to decimal
FINALIZED_BLOCK_NUMBER=$(( 16#$FINALIZED_BLOCK_NUMBER ))
BLOCK_DRIFT=$(( $CURRENT_BLOCK_NUMBER-$FINALIZED_BLOCK_NUMBER ))
# ---
DATE=$(date '+%Y-%m-%d %H:%M:%S')
echo "$DATE [$NODE_SERVICE] 📏 Block drift ($BLOCK_DRIFT) watcher 👀"
# ---
if [ "$BLOCK_DRIFT" -gt "$BLOCKS_THRESHOLD" ] || [ "$CURRENT_BLOCK_NUMBER" -eq "$(< $CURRENT_BLOCK_FILENAME)" ]
then
  # restart node
  echo "$DATE [$NODE_SERVICE] ⚡ RESTARTING $NODE_SERVICE ⚡ "
  sudo systemctl restart $NODE_SERVICE.service
fi
# ---
# cache current block number
echo "$CURRENT_BLOCK_NUMBER" > $CURRENT_BLOCK_FILENAME

Logs output:

2023-03-27 13:38:01 [kusama-toto] 📏 Block drift (9) watcher 👀
2023-03-27 13:39:01 [kusama-toto] 📏 Block drift (9) watcher 👀
2023-03-27 13:40:01 [kusama-toto] 📏 Block drift (9) watcher 👀
2023-03-27 13:41:01 [kusama-toto] 📏 Block drift (46) watcher 👀
2023-03-27 13:41:01 [kusama-toto] ⚡ RESTARTING kusama-toto ⚡ 
2023-03-27 13:42:01 [kusama-toto] 📏 Block drift (3) watcher 👀
2023-03-27 13:43:02 [kusama-toto] 📏 Block drift (3) watcher 👀