merge logs - honor timestamp at line start, used python-2.7

.gitignore

.mergeLogs.config
!.Merge_Logs

.Merge_Logs

this is project name keeper

.mergeLogs.config.example

[date-format]
dateReadFormat=%Y-%m-%d %H:%M:%S,%f
dateDumpFormat=%H:%M:%S,%f
[output]
colorCodes=32|32;1|33|33;1|34;1|35|35;1|36|37|42|43|44|45

howto

mergeLogs.py HOWTO
==================

1. cp .mergeLogs.config.example to .mergeLogs.config
2. (optional) edit .mergeLogs.config
3. run:
     ./mergeLogs.py log1 log2 log3 | less -SRN
   run with column removal:
     ./mergeLogs.py log1 log2 log3 | colrm 35 100 | less -SRN

mergeLogs.py

#!/usr/bin/python

from datetime import datetime


class MultiSourceContextManager:
    """
    Support for 'with' construction
    """

    def __init__(self, fileList):
        self.sourceBag = {log: None for log in fileList}

    def __enter__(self):
        for s in self.sourceBag:
            #print('opening: {0}'.format(s))
            self.sourceBag[s] = open(s, 'r')
        return self.sourceBag

    def __exit__(self, exc_type, exc_value, traceback):
        for (s, fp) in self.sourceBag.iteritems():
            #print('closing {0}'.format(s))
            fp.close()

class LogEntry:
    """
    Single log line container. Contains date and log name too.
    """

    def __init__(self, date, logLine, logName):
        self.date = date
        self.logLine = logLine
        self.logName = logName

    def buildId(self):
        return str(self.date) + self.logName

class Peekorator(object):
    """
    Lightweight peek decorator for iterable.
    """

    def __init__(self, generator):
        self.empty = False
        self.peek = None
        self.generator = generator
        try:
            self.peek = self.generator.next()
        except StopIteration:
            self.empty = True

    def __iter__(self):
        return self

    def next(self):
        """
        Return the self.peek element, or raise StopIteration
        if empty
        """
        if self.empty:
            raise StopIteration()
        to_return = self.peek
        try:
            self.peek = self.generator.next()
        except StopIteration:
            self.peek = None
            self.empty = True
        return to_return
    
class LogIterator:
    """
    Iterates through all lines in given log (file) and wraps items into LogEntry.
    """

    def __init__(self, logName, logIter, dateReadFormat, showStacktraces):
        self.logName = logName
        self.logIter = Peekorator(logIter)
        self.dateReadFormat = dateReadFormat
        self.showStacktraces = showStacktraces

    def __iter__(self):
        'Returns itself as an iterator object'
        return self

    def next(self):
        'Returns the next value till current is lower than high'
        # capture first line with parseable date prefix
        date = None
        while date == None:
            line = self.logIter.next()
            if line == None:
                raise StopIteration

            try:
                date = self.parseLogDate(line)
            except ValueError as e:
                pass

        # append consequent lines without date prefix
        lineWithContext = [line.strip()]
        
        if self.showStacktraces:
            while self.logIter.peek:
                try:
                    self.parseLogDate(self.logIter.peek)
                    break;
                except ValueError as e:
                    lineWithContext.append(self.logIter.peek.rstrip())
                    self.logIter.next()

        return LogEntry(date, lineWithContext, self.logName)

    def parseLogDate(self, line):
        return datetime.strptime(line[:23], self.dateReadFormat)


class OrderedLogsIterator:
    """
    Main processing logic - iterates through all given LogIterators 
    and always yields oldest LogItem among all.
    """

    def __init__(self, sourceBag, dateReadFormat, showStacktraces):
        self.sourceBagIters = {fname: LogIterator(fname, log, dateReadFormat, showStacktraces) for (fname, log) in sourceBag.iteritems()}
        self.boilerPlate = {}
        for (fname, logIter) in self.sourceBagIters.iteritems():
            #print(logIter)
            try:
                li = logIter.next()
                self.boilerPlate[li.buildId()] = li
            except StopIteration as e:
                pass

    def __iter__(self):
        'Returns itself as an iterator object'
        return self

    def next(self):
        'Returns the next value till current is lower than high'
        if len(self.boilerPlate) == 0:
            raise StopIteration
        else:
            oldestKey = sorted(self.boilerPlate)[0]
            li = self.boilerPlate.pop(oldestKey)
            try:
                nextLi = self.sourceBagIters[li.logName].next()
                self.boilerPlate[nextLi.buildId()] = nextLi
            except StopIteration as e:
                pass

            return li

    

if __name__ == '__main__':
    import sys
    import ConfigParser

    if len(sys.argv) < 3:
        sys.stderr.write('usage::{0} [-] <log1> <log2> [<log3> [<log4>...]]\n'.format(sys.argv[0]))
        sys.stderr.write('       - : if present then stacktraces will be visible\n')
        sys.exit(1)


    #2015-07-01 06:53:24,143
    # dateReadFormat = '%Y-%m-%d %H:%M:%S,%f'
    # dateDumpFormat = '%H:%M:%S,%f'

    #date_object = datetime.strptime('2015-07-01 06:53:24,143', dateReadFormat)
    # print(date_object)
    # print(date_object.strftime(dateDumpFormat))

    DATE_FORMAT_SECTION = 'date-format'
    config = ConfigParser.ConfigParser()
    config.readfp(open('.mergeLogs.config', 'r'))
    dateReadFormat = config.get(DATE_FORMAT_SECTION, 'dateReadFormat', '%Y-%m-%d %H:%M:%S,%f')
    dateDumpFormat = config.get(DATE_FORMAT_SECTION, 'dateDumpFormat', '%H:%M:%S,%f')

    sys.stderr.write('dateReadFormat: {0}\n'.format(dateReadFormat))
    sys.stderr.write('dateDumpFormat: {0}\n'.format(dateDumpFormat))

    OUTPUT_SECTION = 'output'
    colorCodes = config.get(OUTPUT_SECTION, 'colorCodes', '32|32;1|33|33;1|34;1|35|35;1|36|37|42|43|44|45').split('|')
    colorSize = len(colorCodes)

    params = sys.argv[1:]
    showStacktraces = False
    if params[0] == '-':
        showStacktraces = True
        params = params[1:]
    
    colorMap = {}
    logArgs = params
    for i in range(len(logArgs)):
        colorMap[logArgs[i]] = '\033[{0}m'.format(colorCodes[i%colorSize])

    def dumpLiWithContext(liValue):
        contextDate = li.date.strftime(dateDumpFormat)
        firstLine = True
        for linePart in li.logLine:
            if firstLine:
                noDatePart = linePart[23:]
                dumpDate = contextDate
                firstLine = False
            else:
                noDatePart = linePart
                dumpDate = contextDate + '?'
                    
            sys.stdout.write('{3}{0}--{1} {2}\033[0m\n'.format(
                li.logName, dumpDate, noDatePart, 
                colorMap[li.logName]))


    with MultiSourceContextManager(logArgs) as logs:
        oli = OrderedLogsIterator(logs, dateReadFormat, showStacktraces)
        for li in oli:
            dumpLiWithContext(li)