keep an eye on things

healthcheck_stats.py

import sys
import subprocess
from swift.common.utils import get_logger
from threading import Thread
import time
import socket
import os
import ssl


# HTTPS_URL = 'https://127.0.0.1:443'
HTTPS_URL = 'https://saio:443/healthcheck'
HTTP_ADDRESS = ('127.0.0.1', 8080)
HTTPS_ADDRESS = ('127.0.0.1', 443)

PROXY_LINE = """PROXY TCP4 127.0.0.1 127.0.0.1 8080 8080
"""
HTTP_MESSAGE = """
GET /healthcheck
Connection: close

""".lstrip()

CONF = {
    'log_address': '/dev/log',
    'log_level': 'DEBUG',
    # 'log_statsd_host': '127.0.0.1',
    'log_statsd_host': '192.168.8.8',
    # 'log_statsd_port': '8133',
    'log_statsd_port': '8125',
}

MAX_RATE = 1.0
METRIC_BASE = 'sre.healthcheck.'

FNULL = open(os.devnull, 'w')


def check_curl():
    subprocess.check_output(['curl', '-gsk', '--fail', HTTPS_URL],
                            stderr=FNULL)


def check_https():
    raw_sock = socket.socket()
    sock = ssl.wrap_socket(raw_sock)
    sock.connect(HTTPS_ADDRESS)
    sock.send(HTTP_MESSAGE)
    msg = sock.recv(1024).strip()
    sock.close()
    assert msg.startswith('HTTP/1.1 200 OK')
    assert msg.endswith('OK')


def check_http_proxy():
    sock = socket.socket()
    sock.connect(HTTP_ADDRESS)
    sock.send(PROXY_LINE + HTTP_MESSAGE)
    msg = sock.recv(1024).strip()
    sock.close()
    assert msg.startswith('HTTP/1.1 200 OK')
    assert msg.endswith('OK')


def timing_stats(logger, metric, check):
    start_time = time.time()
    try:
        check()
    except Exception:
        logger.exception('woops')
        logger.increment(metric + '.error')
        return
    end_time = time.time()
    logger.timing(metric + '.timing', (end_time - start_time) * 1000)


def check_https_with_stats(logger):
    metric = METRIC_BASE + 'https'
    timing_stats(logger, metric, check_https)


def check_curl_with_stats(logger):
    metric = METRIC_BASE + 'curl'
    timing_stats(logger, metric, check_curl)


def check_http_proxy_with_stats(logger):
    metric = METRIC_BASE + 'http'
    timing_stats(logger, metric, check_http_proxy)


def monitor(logger, f):
    while True:
        next_check = time.time() + MAX_RATE
        f(logger)
        time.sleep(next_check - time.time())


def main():
    logger = get_logger(CONF)
    for f in (check_curl_with_stats, check_https_with_stats):
        t = Thread(target=monitor, args=(logger, f))
        t.daemon = True
        t.start()
    monitor(logger, check_http_proxy_with_stats)


if __name__ == "__main__":
    sys.exit(main())