logstash_query.py

#!/usr/bin/env python

import json
import os
import time

import requests

LOGSTASH_URL = 'http://logstash1001.eqiad.wmnet:9200'

AFTER = '2017-05-08T12:43:55.000Z'
BEFORE = '2017-05-08T13:43:55.000Z'

HOSTS = [
    'mw1261.eqiad.wmnet',
    'mw1262.eqiad.wmnet',
    'mw1263.eqiad.wmnet',
    'mw1264.eqiad.wmnet',
    'mw1265.eqiad.wmnet',
    'mwdebug1001.eqiad.wmnet',
    'mwdebug1002.eqiad.wmnet',
]

QUERY = '''
host: ({})
AND ((type:mediawiki
AND (channel:exception
OR channel:error))
OR type:hhvm)
'''.format(' OR '.join([h.split('.')[0] for h in HOSTS])).strip()

DATA = {
    "size": 0,
    "aggs": {
    "2": {
        "date_histogram": {
            "interval": "10s",
            "field": "@timestamp"
        }
    }},
    "query": {
        "bool": {
            "filter": [
                {
                    "range": {
                        "@timestamp": {
                            "lte": BEFORE,
                            "gte": AFTER
                        }
                    }
                },
                {
                    "query_string": {
                        "query": QUERY
                    }
                }
            ],
            "must_not": [
                {
                    "terms": {
                        "level": [
                            "INFO",
                            "DEBUG"
                        ]
                    }
                },
                {
                    "match": {
                        "message": {
                            "query": "SlowTimer",
                            "type": "phrase"
                        }
                    }
                },
                {
                    "match": {
                        "message": {
                            "query": "Invalid host name",
                            "type": "phrase"
                        }
                    }
                }
            ]
        }
    }
}


if __name__ == '__main__':
    headers = {'content-type': 'application/json'}
    r = requests.post(
        os.path.join(LOGSTASH_URL, 'logstash-*', '_search'),
        data=json.dumps(DATA),
        headers=headers)
    r.raise_for_status()
    # date +'%s' --date="2017-05-08T13:43:55Z"
    cutoff_ts = (1494251035 - 20) * 1000
    output = r.json()
    entries = output['aggregations']['2']['buckets']
    counts_before = [entry['doc_count'] for entry in entries
                     if entry['key'] < cutoff_ts]

    mean_before = float(sum(counts_before)) / max(1, len(counts_before))

    counts_after = [entry['doc_count'] for entry in entries
                    if entry['key'] >= cutoff_ts]

    mean_after = float(sum(counts_after)) / max(1, len(counts_after))

    # Check if there was a significant increase in the rate.
    target_error_rate = max(1.0, (mean_before * 10.0))

    over_threshold = mean_after > target_error_rate

    # print(json.dumps(output))
    print('Before: {} After: {} Target: {}'.format(
        mean_before, mean_after, target_error_rate))
    print('Over threshold: {}'.format(over_threshold))