gistfile1.txt

#!/usr/bin/env python
"""
Broker API log parser

https://help.datadoghq.com/hc/en-us/articles/209064386-How-to-collect-metrics-or-events-with-a-Custom-Log-Parser

"""

import datetime
import re
import time

ALERT_TYPES = {
    "FATAL": "error",
    "ERROR": "error",
    "WARN": "warning",
    "INFO": "info",
    "DEBUG": "info",
    "TRACE": "info",
}

# parse broker-api log for ERROR/Notification per order
LOG_PATTERN = re.compile('ERROR\s+Notification error for order.*endpoint: (.*) data')

URL_PATTERN = re.compile('(http[s]?)://([^/]*)')

DATE_FORMAT = '%Y-%m-%d %H:%M:%S'


def parse_brokerapi(log, line):
    """
    Parse log for broker notification errors
    :param log: logger object
    :param line: str, line from logger
    """
    matched = LOG_PATTERN.search(line)
    if matched:
        endpoint = matched.groups()[0]  # i.e. https://balance.1ticket.com/api/receiveOrder/....
        url_match = URL_PATTERN.search(endpoint)
        url_key = '.'.join(url_match.groups(0))  # I.e. 'https.balance.1ticket.com'
        log_line = matched.string

        str_date = log_line[:log_line.index(',')]
        timestamp = get_timestamp(str_date)

        # This will be logged to /var/log/datadog/collector.log
        log.info('Broker Notification Error: %s %s', timestamp, log_line)

        # For events
        # event = {'timestamp': timestamp,
        #          'msg_title': "Broker Notification Error at url {}".format(url_key),
        #          'msg_text': "Broker notification endpoint either rejected the notification or is down: "
        #                      "{}. Message {}".format(endpoint, log_line),
        #          'date': str_date,
        #          'auto_priority': 1,
        #          'alert_type': 'error'}
        # return [event]  # Can return dict, or list of dicts

        # For metrics
        metric_name = 'broker.notification.error.{}'.format(url_key)
        time_stamp = timestamp
        metric_value = 1.0
        metric_attributes = {'metric_type': 'counter', 'unit': 'request'}  # Can also include extras list
        # Can also return a list of lists/tuples
        return metric_name, time_stamp, metric_value, metric_attributes

    else:
        return None


def get_timestamp(str_date):
    """
    Convert str date to str timestamp
    :param str_date: str date time in date format
    :return: str, timestamp
    """
    date = datetime.datetime.strptime(str_date, DATE_FORMAT)
    date = time.mktime(date.timetuple())
    return int(date)


def test():
    # Set up the test logger
    import logging
    logging.basicConfig(level=logging.DEBUG)

    # Set up the test input and expected output
    test_input = '2017-02-25 02:01:09,450 root@cron2s [3781] ERROR    Notification error for order 571730, ' \
                 'code: 200 endpoint: https://balance.1ticket.com/api/receiveOrder/index.asp?username=api%2B ' \
                 '9620528646967@1ticket.com&password= 12345&source=fanxchange data: {"orderid": "EU94266", ' \
                 '"tixquantity": 2, "posid": "46795", "totalpayment": 70.2, "totalsale": 86.46, "venue": ' \
                 '"Madison Square Garden", "event": "WWE: Live - Road to WrestleMania", "instantdownload": ' \
                 'true, "token": 571730, "tixseats": "", "time": "03:00 pm", "date": "03-12-2017", "notes": null, ' \
                 '"tixsection": "227", "tixrow": "18"} response: Fail<br /> ' \
                 '[in /fanx-apis/apps/broker_api/workers/../../../lib/fanxchange/core/brokers.py:notify:916]'

    # Call the parse function
    actual = parse_brokerapi(logging, test_input)

    # Validate the results for event
    # actual = actual[0]
    # expected_msg_text_starts_with = 'Broker notification endpoint either rejected the notification or is down'
    # expected_msg_title = 'Broker Notification Error at url https://balance.1ticket.com/api/receiveOrder'
    # assert actual['msg_text'].startswith(expected_msg_text_starts_with), \
    #     "Expected to start with {}, got {}".format(expected_msg_text_starts_with, actual['msg_text'])
    # assert expected_msg_title == actual['msg_title'], "Expected {}, got {}" \
    #                                                   "".format(expected_msg_title, actual['msg_title'])

    # Validate the results for metric
    assert actual[0] == 'broker.notification.error.https.balance.1ticket.com', actual[0]
    assert actual[1] == 1488006069

    print 'test passes'

if __name__ == '__main__':
    # For local testing, callable as "python /path/to/parsers.py"
    test()