apachelogparse.py

import re
import sys

# note, currently doesn't handle:
# Jun 29 00:00:03 host tag: message repeated 3 times: [ 192.168.1.1 - - "POST /api/someendpoint/

r = re.compile(r'(\w{3,}\s+\d{2,}\s+\d{2,}:\d{2,}:\d{2,})\s+([\w-]+)\s+([\w-]+):\s+((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3},?\s?)+)\s+-\s+-\s+\"(GET|POST|PUT|PATCH|HEAD|DELETE) ((\/[\w_]+)+)\/\??(.*) HTTP\/\d\.\d" (\d{3,}) (\d+|-)')

anonymizers = (
        (re.compile(r'(\d+,?)+'), 'id'), # one or more optionally csv separated ids
        (re.compile(r'\w+:[\w\d\-_:\.]+'), 'guid'),
)

def anonymize_path(path):
        for anonymizer, repl in anonymizers:
                path = anonymizer.sub(repl, path)
        return path

for l in sys.stdin:
        m = r.search(l.strip())
        if m:
                groups = m.groups()
                print '%s - %s - %s' % (groups[1], groups[5], anonymize_path(groups[6]))
#       else:
#               print 'nomatch: %s' % l