config.json

{
  "redis": {
    "host": "127.0.0.1",
    "port": 6379,
    "db": 0,
    "password": ""
  },  
  "aws": {
    "access_key_id": "<access>",
    "secret_access_key": "<secret>",
    "zone_id": "<id>"
  },  
  "number_checks": 5,
  "hosts": ["host1.example.org", "host2.example.org"],
  "app": {
    "domain": "domain.example.org",
    "domain_api": "domain-api.example.org"
  }
}

webcheck.py

import sys 
import argparse
import redis
import json
import requests
import boto3


def parse_arguments():
    parser = argparse.ArgumentParser(description='')
    parser.add_argument(
        "-c", "--config", dest="config", default='config.json', 
        help="Configuration file"
    )   
    
    return parser.parse_args()


def get_config(config_file):
    try:
        with open(config_file) as f:
            config = json.loads(f.read())
    except (IOError, json.decoder.JSONDecodeError) as e:
        print("[%s]: %s" % (e.__class__.__name__, e)) 
        sys.exit(1)

    return config


def redis_connect(config):
    host = config['redis']['host']
    port = config['redis']['port']
    db = config['redis']['db']
    pwd = config['redis']['password']

    return redis.StrictRedis(host=host, port=port, db=db, password=pwd)


def get_route53_client(config):
    return boto3.client(
            'route53', 
            aws_access_key_id=config['aws']['access_key_id'],
            aws_secret_access_key=config['aws']['secret_access_key']
        )


def get_dns_records(config):
    c53 = get_route53_client(config)
    try:
        response = c53.list_resource_record_sets(
            HostedZoneId=config['aws']['zone_id'],
            StartRecordName=config['app']['domain'],
            StartRecordType='A'
        )

        records = None
        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
            records = response['ResourceRecordSets']

    except Exception as e:
        print(e)
        sys.exit(1)

    return records


def update_record(config, record):
    c53 = get_route53_client(config)
    success = False

    change = {'Action': 'UPSERT', 'ResourceRecordSet': record}
    try:
        response = c53.change_resource_record_sets(
            HostedZoneId=config['aws']['zone_id'],
            ChangeBatch={'Changes': [change]}
        )

        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
            success = True

    except Exception as e:
        print(e)
        sys.exit(1)

    return success


def change_host_weight(config, host, weight):
    records = get_dns_records(config)
    record = None
    for rec in records:
        if rec['SetIdentifier'] == host:
            record = rec
            break

    if record:
        record['Weight'] = weight
        return update_record(config, record)

    return False


def exclude_host(config, _redis, host):
    success = False
    if change_host_weight(config, host, 0):
        # Add host into redis 'failed' set
        _redis.sadd('failed', host)
        success = True
        print("Host %s excluded successfully. Record weight set to 0." % host)
    else:
        print("There was an error excluding host %s." % host)

    return success


def include_host(config, _redis, host):
    success = False
    if change_host_weight(config, host, 1):
        # Add host from redis 'failed' set
        _redis.srem('failed', host)
        success = True
        print("Host %s included back successfully. Record weight set to 1." % host)
    else:
        print("There was an error including back host %s." % host)

    return success


def do_checks(config, _redis):
    hosts = config['hosts']
    for host in hosts:
        # check API
        headers = {'Host': config['app']['domain_api']}
        url = 'http://' + host + '/api'
        try:
            response = requests.get(url, headers=headers)
        except requests.exceptions.ConnectionError as e:
            print(e)
            sys.exit(1)

        if response.status_code == 200:
            _redis.lpush(host, 0)
        else:
            _redis.lpush(host, 1)

        # trim to last N checks
        number_checks = int(config['number_checks'])
        _redis.ltrim(host, 0, number_checks - 1)

        # get last N checks
        checks = _redis.lrange(host, 0, -1)
        checks = [ int(e) for e in checks ]

        if response.status_code != 200:
            if _redis.sismember('failed', host):
                continue
            if sum(checks) == number_checks:
                exclude_host(config, _redis, host)
        else:
            if _redis.sismember('failed', host):
                if sum(checks) == 0:
                    include_host(config, _redis, host)


def main():
    args = parse_arguments()
    config = get_config(args.config)
    _redis = redis_connect(config)
    do_checks(config, _redis)


if __name__ == '__main__':
    main()