Elixxir Monitor

elixxir_monitor.py

#! /usr/bin/env python3

"""elixxir_monitor.py: Monitors an elixxir node log for signs of activity and alerts when enough time elapses without activity"""

import time, subprocess, select, time, logging, signal, pushover

# Wait this long to send alert - default 5 minutes
alert_time = 60*5

logfile = "/opt/xxnetwork/node-logs/node.log"
search_string = '"took\|Updating"'

# Setup Pushover
po = pushover.Client("PUSHOVER CLIENT API TOKEN", api_token="PUSHOVER APP API TOKEN")

# Setup logging
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO)


# Alarm handler - this runs when the timeout has been reached
last_log = time.time()
def handler(signum, stack):
    _t_elapsed = time.time() - last_log
    _msg = "Alert - Elixxir node hasn't participated in a round for {} seconds".format(int(_t_elapsed))
    logging.error(_msg)
    po.send_message(_msg, title="Elixxir Alert")
    signal.alarm(alert_time)

# Set up alarm to trigger if we don't receive an alert in time
signal.signal(signal.SIGALRM, handler)


# Spawns a process to tail the log and grep for the search string
proc = subprocess.Popen('tail -F {} | grep --line-buffered {}'.format(logfile, search_string),
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)


try:
    # Wait for output from the process until it exits
    while not proc.poll():
        signal.alarm(alert_time)
        _l = proc.stdout.readline().decode('utf-8').strip()
        logging.debug("Log read: {}".format(_l))
        last_log = time.time()

finally:
    po.send_message("Elixxir monitor stopped")

Thanks for sharing! Make sure you have python-pushover installed: pip install python-pushover

Thanks for sharing! Make sure you have python-pushover installed: pip install python-pushover

Yes, thanks for that. the pushover package doesn't work.

It's quite easy to run this as a service, just follow this guide and use this service file. Make sure the user running it (for example root) also has python-pushover installed.