splitlog.py

"""
Split a logfile by days
"""

import sys, os, re

filename = sys.argv[1]

out_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs')

if not os.path.exists(out_dir):
    os.makedirs(out_dir)

_months = {}
for m in 'Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' '):
    _months[m] = len(_months) + 1

_dates = [
    re.compile('(?P<y>\d{4})-(?P<m>\d{2})-(?P<d>\d{2})').search,
    re.compile('(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})').search,
    re.compile('(?P<y>\d{4})/(?P<m>\w{3})/(?P<d>\d{2})').search,
    re.compile('(?P<d>\d{2})/(?P<m>\w{3})/(?P<y>\d{4})').search,
]


def find_date(line):
    prefix = line[:32]

    for d in _dates:
        m = d(prefix)
        if m:
            day = int(m.group('d'))
            year = int(m.group('y'))

            try:
                month = int(m.group('m'))
            except:
                month = _months[m.group('m')]

            return year, month, day


lines = 0
buffer = []
current_date = None
current_file = None
with open(filename, 'r') as f:
    for line in f:
        lines += 1

        date = find_date(line)
        if not date:
            if current_date:
                print 'Using date %s for line %s' % (current_date, line),
            else:
                # Buffer line for when we find a date
                buffer.append(line)
                print '!!! Could not find date for %s' % line,
                continue
        elif date != current_date:
            if current_file:
                current_file.close()

            current_date = date
            print('====== New date %s ======' % (date,))

            current_file = open(os.path.join(out_dir, '%04d%02d%02d.log' % current_date), 'w')
            if buffer:
                print('Flushing %d lines from buffer' % len(buffer))
                for b in buffer:
                    current_file.write(b)
                buffer = []

        current_file.write(line)

if current_file:
    current_file.close()
    current_file = None

print('Split %d lines' % lines)