Python Structured logging aggregator

README.md

A very dummy parser for structured logging format into SQLite db.

logaggregator.py

import glob
import os

import peewee


def chunks(lst, n):
    """Yield successive n-or-less-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

DB = peewee.SqliteDatabase('all_logs.db')

class LogLine(peewee.Model):
    # Add/remove records as needed
    date = peewee.DateTimeField()
    controller = peewee.CharField(max_length=48)
    action = peewee.CharField(max_length=48)
    status = peewee.IntegerField()
    duration = peewee.IntegerField()
    view = peewee.IntegerField()
    db = peewee.IntegerField()
    guid = peewee.CharField(null=True)

    class Meta:
        database = DB

DB.connect()
DB.create_tables([LogLine])

counter = 0

print("START - Dumping")
for logfile in glob.glob('*/*/*.log'):

    row_counter = 0
    records = []
    print("  > START - Processing {}".format(logfile))

    for line in open(logfile):
        record = dict(map(lambda x: x.split('='), line.split()))
        record.setdefault('guid')
        records.append(record)
        counter += 1
        row_counter += 1

    print("  > END - Processing {}".format(logfile))
    print("  > START - Saving to DB")

    with DB.atomic() as txn:
        pages = chunks(records, 100)
        for key, page in enumerate(pages):
            print("\r    > Flushing {} page to DB".format(key+1), end="")
            LogLine.insert_many(page).execute()
        print("\n")
        print("\r    > Renaming logfile {}".format(logfile))
        os.rename(logfile, logfile + '.done')

    print("  > END - Saving to DB")
print("END - Dumping")

print("Saved {} records".format(counter))

requirements.txt

peewee==3.13.2