keep an eye on things

healthcheck_stats.py

import argparse
import sys
import subprocess
from swift.common.utils import get_logger
from threading import Thread
import time
import socket
import os
import ssl


HTTP_ADDRESS = ('127.0.0.1', 8080)
HTTPS_ADDRESS = ('127.0.0.1', 443)

PROXY_LINE = """PROXY TCP4 127.0.0.1 127.0.0.1 8080 8080
"""
HTTP_MESSAGE = """
GET /healthcheck
Connection: close

""".lstrip()

MAX_RATE = 1.0
METRIC_BASE = 'sre.healthcheck.'

FNULL = open(os.devnull, 'w')


def check_curl(args):
    args = [
        'curl', '-gs', '--fail',
        '--resolve', '%s:443:127.0.0.1' % args.hostname,
        'https://%s:443/healthcheck' % args.hostname,
    ]
    subprocess.check_output(args, stderr=FNULL)


context = ssl.create_default_context()


def check_https(args):
    sock = context.wrap_socket(socket.socket(), server_hostname=args.hostname)
    sock.connect(HTTPS_ADDRESS)
    sock.send(HTTP_MESSAGE)
    msg = sock.recv(1024).strip()
    sock.close()
    assert msg.startswith('HTTP/1.1 200 OK')
    assert msg.endswith('OK')


def check_http_proxy(args):
    sock = socket.socket()
    sock.connect(HTTP_ADDRESS)
    sock.send(PROXY_LINE + HTTP_MESSAGE)
    msg = sock.recv(1024).strip()
    sock.close()
    assert msg.startswith('HTTP/1.1 200 OK')
    assert msg.endswith('OK')


def timing_stats(logger, args, metric, check):
    start_time = time.time()
    try:
        check(args)
    except Exception:
        logger.exception('woops')
        logger.increment(metric + '.error')
        return
    end_time = time.time()
    logger.timing(metric + '.timing', (end_time - start_time) * 1000)


def check_https_with_stats(logger, args):
    metric = METRIC_BASE + 'https'
    timing_stats(logger, args, metric, check_https)


def check_curl_with_stats(logger, args):
    metric = METRIC_BASE + 'curl'
    timing_stats(logger, args, metric, check_curl)


def check_http_proxy_with_stats(logger, args):
    metric = METRIC_BASE + 'http'
    timing_stats(logger, args, metric, check_http_proxy)


def monitor(f, logger, args):
    while True:
        next_check = time.time() + MAX_RATE
        f(logger, args)
        time.sleep(next_check - time.time())


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('hostname', nargs='?', default='saio')
    parser.add_argument('--statsd', default='127.0.0.1:8133')
    args = parser.parse_args()
    host, port = args.statsd.split(':', 1)
    logger = get_logger({
        'log_address': '/dev/log',
        'log_level': 'DEBUG',
        'log_statsd_host': host,
        'log_statsd_port': port,
    })
    for f in (check_curl_with_stats, check_https_with_stats):
        t = Thread(target=monitor, args=(f, logger, args))
        t.daemon = True
        t.start()
    monitor(check_http_proxy_with_stats, logger, args)


if __name__ == "__main__":
    sys.exit(main())