Jormungandr Node Monitor

jormon.sh

#!/bin/bash
#
# Author: Michael Fazio (sandstone.io)
#
# This script monitors a Jormungandr node for "liveness" and executes a shutdown if the node is determined
# to be "stuck". A node is "stuck" if the time elapsed since last block exceeds the sync tolerance 
# threshold. The script does NOT perform a restart on the Jormungandr node. Instead we rely on process 
# managers such as systemd to perform restarts.

POLLING_INTERVAL_SECONDS=30
SYNC_TOLERANCE_SECONDS=240
REST_API="http://127.0.0.1:8443/api"

while true; do

    LAST_BLOCK=$(jcli rest v0 node stats get --output-format json --host $REST_API 2> /dev/null)
    LAST_BLOCK_HEIGHT=$(echo $LAST_BLOCK | jq -r .lastBlockHeight)
    LAST_BLOCK_DATE=$(echo $LAST_BLOCK | jq -r .lastBlockTime)
    LAST_BLOCK_TIME=$(date -d$LAST_BLOCK_DATE +%s 2> /dev/null)
    CURRENT_TIME=$(date +%s)
    DIFF_SECONDS=$((CURRENT_TIME - LAST_BLOCK_TIME))

    if ((LAST_BLOCK_TIME > 0)); then
        if ((DIFF_SECONDS > SYNC_TOLERANCE_SECONDS)); then
            echo "Jormungandr out-of-sync. Time difference of $DIFF_SECONDS seconds. Shutting down node..."
            jcli rest v0 shutdown get --host $REST_API
        else
            echo "Jormungandr synchronized. Time difference of $DIFF_SECONDS seconds. Last block height $LAST_BLOCK_HEIGHT."
        fi
    else
        echo "Jormungandr node is offline or bootstrapping..."
    fi

    sleep $POLLING_INTERVAL_SECONDS
done