error-log-parse.py

#!/usr/bin/python -u
import os
import re
import json

LOG_PATH = '/var/log/datawarehouse/application.log'

logfile = open(LOG_PATH, 'r')
jsonfile = open('dwerrors.json', 'w')

linere = re.compile(r'\[(\d{4}-\d{2}-\d{2} \d{2}).*\] (INFO|ERROR|DEBUG) (.*) -- (.*)')
errorre = re.compile(r'.*(\[HiveCommandError\] Return Code \[-15\]|SSL SYSCALL|SSL error).*')

errors = [['date', 'error', 'count']]
ierrors = {}

for line in logfile.xreadlines():
    line = line.rstrip('\n')
    matchr = errorre.match(line)

    if not matchr: continue

    matchl = linere.match(line)

    if not matchl: continue

    _id = (matchl.group(1), matchr.group(1))
    if not _id in ierrors:
        ierrors[_id] = len(errors)
        errors.append([matchl.group(1) + ':00:00', matchr.group(1), 0])
    else:
        errors[ierrors[_id]][2] += 1

jsonfile.write(json.dumps(errors, indent=2))
jsonfile.close()
logfile.close()

get json here:

http://aimee.lon.spotify.net:9999/dwerrors.json