Substrate Block Watcher (from source)

substrate_block_watcher.sh

#!/bin/bash
#
# Updated script from paulormart (Turboflakes)
# https://gist.github.com/paulormart
#
# Bash script to be executed in the remote server to monitor block drift
#
# > Make a file executable
# chmod +x ./substrate_block_watcher.sh
#
# > Positional arguments:
# 1st - blocks threshold
# 2nd - service name
# 3rd - node RPC port
#
# > Test and run with the following parameters e.g.:
# ./substrate_block_watcher.sh 20 node-service name 9944
#
# > Schedule a cron job to execute every minute
# https://www.digitalocean.com/community/tutorials/how-to-use-cron-to-automate-tasks-ubuntu-1804
#
# example:
# * * * * * /opt/substrate_block_watcher/substrate_block_watcher.sh 20 kusama-node-service 9933 >> /opt/substrate_block_watcher/sbdw.log
#

# Add a variable to store the previous block number file path
PREVIOUS_BLOCK_FILE="previous_block_number.txt"

if [ -z "$1" ]
then    
    echo "INFO: Blocks threshold is missing, default is 20. e.g. ./substrate_block_drift_watcher.sh 20 kusama-node 9933"
    BLOCKS_THRESHOLD=20
else
    BLOCKS_THRESHOLD=$1
fi

if [ -z "$2" ]
then
    echo "INFO: Node service name is missing, default is kusama-node. e.g. ./substrate_block_drift_watcher.sh 20 kusama-node 9933"
    NODE_SERVICE="kusama-node"
else
    NODE_SERVICE=$2
fi

if [ -z "$3" ]
then
    echo "INFO: Node RPC port is missing, default is 9933. e.g. ./substrate_block_drift_watcher.sh 20 kusama-node 9933"
    RPC_PORT=9933
else
    RPC_PORT=$3
fi

# Verify if service node is active
sudo systemctl is-active --quiet $NODE_SERVICE.service
if [[ "$?" -ne 0 ]]; 
then
    echo "ERROR: Service $NODE_SERVICE is not active."
    exit
fi

# Verify if node is running on the RPC PORT specified
STATUS_CODE=$(curl --write-out %{http_code} --silent --output /dev/null \
  -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}' \
  'http://localhost:'$RPC_PORT'')

if [[ "$STATUS_CODE" -ne 200 ]]; 
then
    echo "ERROR: RPC port: $RPC_PORT fails to connect."
    exit
fi

# --- Fetch node health
# NOTE: system_health response example:
# {
#  "isSyncing": false,
#  "peers": 37,
#  "shouldHavePeers": true
# }
IS_SYNCING="$( curl --silent -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result.isSyncing' )"

# ---
# NOTE: Skip Monitoring if node is syncing old blocks
#
if [ "$IS_SYNCING" = true ]
then
  echo "INFO: Node is syncing -> SKIPPING monitor."
  exit
fi

# --- Fetch RPC `system_syncState`
# NOTE: system_health response example:
# {
#   "currentBlock": 11132625,
#   "highestBlock": 11132625,
#   "startingBlock": 10862594
# }
CURRENT_BLOCK_NUMBER="$( curl --silent -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "system_syncState", "params":[]}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result.currentBlock' )"
# ---

# Read the previous block number from the file, if it exists
if [ -f "$PREVIOUS_BLOCK_FILE" ]; then
  PREVIOUS_BLOCK_NUMBER=$(cat $PREVIOUS_BLOCK_FILE)
else
  PREVIOUS_BLOCK_NUMBER=-1
fi

# Store the current block number in the file for the next execution
echo $CURRENT_BLOCK_NUMBER > $PREVIOUS_BLOCK_FILE

# --- Fetch Finalized block number
# Get Finalized head
BLOCK_HASH="$( curl --silent -H "Content-Type: application/json" \
  -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getFinalizedHead", "params":[]}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result' )"
BLOCK_HASH=$( echo "$BLOCK_HASH" | awk -F ' ' '{ printf $1 }' )

# Get Header
FINALIZED_BLOCK_NUMBER="$( curl --silent -H Content-Type:application/json \
  -d '{"id":1, "jsonrpc": "2.0", "method": "chain_getHeader", "params": ['$BLOCK_HASH']}' \
  'http://localhost:'$RPC_PORT'' \
  | jq '.result.number' )"
# Note: To convert hex block number decimal
# we first need to emove "" and 0x from heximal number eg: "0xaa1047" -> aa1047
FINALIZED_BLOCK_NUMBER=${FINALIZED_BLOCK_NUMBER//\"/}
FINALIZED_BLOCK_NUMBER=${FINALIZED_BLOCK_NUMBER//0x/}
# Convert block number hex to decimal
FINALIZED_BLOCK_NUMBER=$(( 16#$FINALIZED_BLOCK_NUMBER ))
BLOCK_DRIFT=$(( $CURRENT_BLOCK_NUMBER-$FINALIZED_BLOCK_NUMBER ))
# ---
DATE=$(date '+%Y-%m-%d %H:%M:%S')
echo "$DATE [$NODE_SERVICE] 📏 Highest Block ($CURRENT_BLOCK_NUMBER) : 📏 Block drift ($BLOCK_DRIFT) 👀"

# ---
if [ "$BLOCK_DRIFT" -gt "$BLOCKS_THRESHOLD" ] || [ "$CURRENT_BLOCK_NUMBER" -eq "$PREVIOUS_BLOCK_NUMBER" ]
then
  # restart node
  echo "$DATE [$NODE_SERVICE] ⚡ RESTARTING $NODE_SERVICE ⚡ "
  sudo systemctl restart $NODE_SERVICE.service
fi