systemd WatchDog pinging HTTP healthcheck

http-watchdog-wrapper

#!/usr/bin/env python3
"""
https://gist.github.com/ulidtko/fdd7222bd7e85480fa201614fbc42faf
"""

import os
import subprocess
import sys
from time import sleep
from urllib.error import URLError
from urllib.request import urlopen

try:
    import sdnotify
    from setproctitle import setproctitle
except ImportError:
    print("apt install python3-sdnotify python3-setproctitle", file=sys.stderr)
    sys.exit(1)

WD_URL = os.getenv('WATCHDOG_URL') or "http://localhost/health"
WD_TIMEO = int(os.getenv('WATCHDOG_INTERVAL') or 15)

s2us = lambda s: int(s * 1e6)

def single_ping():
    try:
        with urlopen(WD_URL, timeout=WD_TIMEO / 4) as resp:
            return (resp.code == 200)
    except (TimeoutError, URLError):
        return False

systemd = sdnotify.SystemdNotifier()
systemd.notify("STATUS=http-watchdog-wrapper starting up")

wrapped_svc_cmd = sys.argv[1:]
svc = subprocess.Popen(wrapped_svc_cmd)
systemd.notify(f"MAINPID={svc.pid}")
systemd.notify(f"STATUS=wrapped process {svc.pid} spawned")

#-- poll 1: startup readiness
while True:
    sleep(WD_TIMEO / 2)
    if svc.poll() is not None: sys.exit(svc.returncode)
    if single_ping(): break
    systemd.notify(f"EXTEND_TIMEOUT_USEC={s2us(WD_TIMEO)}")
    systemd.notify("STATUS=starting up, no ping yet")

systemd.notify("READY=1")
systemd.notify(f"WATCHDOG_USEC={s2us(WD_TIMEO)}")
systemd.notify(f"STATUS=Running with watchdog, interval {WD_TIMEO}s")
setproctitle("http-watchdog-wrapper -- such bork, many health")

#-- poll 2: runtime
while True:
    sleep(WD_TIMEO / 2)
    if svc.poll() is not None: sys.exit(svc.returncode)
    systemd.notify("WATCHDOG=1" if single_ping() else "WATCHDOG=trigger")

#-- Ref: https://www.freedesktop.org/software/systemd/man/sd_notify.html

# ~/.config/systemd/user/dummy.service
"""
[Unit]
Description=Example service demonstrating http-watchdog-wrapper

[Service]
Type=notify
NotifyAccess=all

Environment=WATCHDOG_URL=http://localhost:8000/index.txt
Environment=WATCHDOG_TIMEO=4
ExecStart=/home/ulidtko/code/http-watchdog-wrapper \
          /home/ulidtko/dummy-service.sh

Restart=on-failure
"""

# dummy-service.sh
"""
#!/bin/bash

echo "Dummy service starting up..."
sleep 5
wd=$(mktemp -d); trap "rm -rvf $wd" EXIT
echo "Hello!" > "$wd"/index.txt

echo "Dummy service startup complete. Listening on 8000"
python3 -m http.server --directory $wd 8000
"""