Script to parse windows event logs saved as csv

parse_event_log.py

import os, sys, re, operator
from datetime import datetime


def line_gen(input):
    for line in input:
        yield line

line_pattern = re.compile(r"""
    ^
    (?P<level>Information|Warning|Error),
    (?P<date>\d{2}-\d{2}-\d{4} \s \d{2}:\d{2}:\d{2}),
    (?P<source>[^,]+),
    (?P<event_id>\d+),
    (?P<category>[^,]+),
    (?P<message>.*)
""", re.VERBOSE)

def tokens_gen(input):

    # skip until first match
    match = False
    while not match:
        match = line_pattern.match( input.next() )

    while input:
        parsed_line = match.groupdict()
            
        # gather lines until next match
        line = input.next()
        match = line_pattern.match(line)
        while not match:
            parsed_line['message'] = parsed_line['message'] + line
            line = input.next()
            match = line_pattern.match(line)

        yield parsed_line

def message_gen(input, starttime=datetime.min, endtime=datetime.max):
    for line in input:
        msg = {
            'level': line['level'],
            'date': datetime.strptime(line['date'], '%d-%m-%Y %H:%M:%S'),
            'source': line['source'],
            'event_id': int(line['event_id']),
            'category': line['category'],
            'message': line['message']
        }
        if starttime < msg['date'] < endtime:
            yield msg

def exception_gen(input):
    for msg in [msg['message'] for msg in input if re.search('Stack trace', msg['message'])]:

        ex = {}

        for line in msg.split('\r'):
            parts = line.partition(':')
            if parts[1] and parts[2]:
                ex[parts[0].strip()] = parts[2].strip()

        yield ex

def filter_gen(logitems, filter):
    """returns a generator that filters logitems, so that only items with properties matching those specified in the filter argument is returned"""
    for logitem in logitems:
        if reduce( lambda equal, kv: equal and logitem.has_key(kv[0]) and (logitem[kv[0]] == kv[1]), filter.items(), True ):
            yield logitem


########################################################################################
#                                OUTPUT FUNCTIONS                                      #
########################################################################################

def group_exceptions(filename, starttime=datetime.min, endtime=datetime.max):
    """Prints a list of exceptions grouped by application"""

    file = open(filename)
    lines = line_gen(file)
    tokens = tokens_gen(lines)
    messages = message_gen(tokens, starttime, endtime)
    exceptions = exception_gen(messages)

    applications = {}

    for ex in exceptions:

        appname = ex['Application Virtual Path']

        if not appname in applications:
            applications[appname] = {}

        url = ex['Request path']
        if not url in applications[appname]:
            applications[appname][url] = {}

        type = ex['Exception type']
        if not type in applications[appname][url]:
            applications[appname][url][type] = 0

        applications[appname][url][type] += 1

    for a in applications:
        print '\n\n', a + ':  ',

        url_items = applications[a].items()

        total_cnt = reduce(operator.add, [x[0] for x in [y[1].values() for y in url_items]])

        print '(' + str(total_cnt) + ')'

        url_items.sort(lambda a,b: cmp(reduce(operator.add, b[1].values()), reduce(operator.add, a[1].values())))

        for (u,udict) in url_items:

            print '\t' + u + ' (', reduce(operator.add, udict.values()), '):'

            items = udict.items()
            items.sort(lambda a,b: cmp(b[1], a[1]))

            for (e,v) in items:
                print '\t\t' + e + ':', v

def exception_info(filename, filter, starttime=datetime.min, endtime=datetime.max):
    """Print detailed exceptionlog"""

    file = open(filename)
    lines = line_gen(file)
    tokens = tokens_gen(lines)
    messages = message_gen(tokens, starttime, endtime)
    exceptions = exception_gen(messages)

    gen = filter_gen(exceptions, filter)

    for ex in gen:
        print ("%s\t%s\t%s\t%s" % (ex['Event time'], ex['Application Virtual Path'], ex['Exception type'], ex['Exception message'])).replace('\n', '\\n')

def stacktraces(filename, filter, count, starttime=datetime.min, endtime=datetime.max):
    file = open(filename)
    lines = line_gen(file)
    tokens = tokens_gen(lines)
    messages = message_gen(tokens, starttime, endtime)
    exceptions = exception_gen(messages)

    gen = filter_gen(exceptions, filter)

    cnt = 0
    for ex in gen:
        if cnt > count:
            break
        cnt += 1

        print ("%s\n%s\n%s\n%s" % (ex['Event time'], ex['Request URL'], ex['Exception type'], ex['Exception message']))
        print ex['Stack trace']
        print '\n\n\n\n'

def all_info(filename, filter, starttime=datetime.min, endtime=datetime.max):
    file = open(filename)
    lines = line_gen(file)
    tokens = tokens_gen(lines)
    messages = message_gen(tokens, starttime, endtime)
    exceptions = exception_gen(messages)

    gen = filter_gen(exceptions, filter)

    for ex in gen:
        for (k,v) in ex.items():
            print "%s:\t%s" % (k, v)
        print "\n\n\n"


if __name__ == '__main__':
    #group_exceptions(sys.argv[1], datetime(2010, 9, 10))
    #exception_info(sys.argv[1], starttime=datetime(2010, 9, 10), filter={ 'Application Virtual Path': '/uplaylists' })
    #stacktraces(sys.argv[1], count=99999999, starttime=datetime(2010, 12, 17), filter={ 'Application Virtual Path': '/p3' })
    all_info(sys.argv[1], { 'Exception type': 'WebException' }, starttime=datetime(2010, 12, 18))




# Event code: 3005 
# Event message: An unhandled exception has occurred. 
# Event time: 12-08-2010 10:37:10 
# Event time (UTC): 12-08-2010 08:37:10 
# Event ID: 05cf854bb2ef4cc79fafdb30bb40f8d8 
# Event sequence: 1207 
# Event occurrence: 14 
# Event detail code: 0 
#  
# Application information: 
#     Application domain: /LM/W3SVC/1/ROOT/uplaylists-1-129260725154023010 
#     Trust level: Full 
#     Application Virtual Path: /uplaylists 
#     Application Path: D:\inetpub\wwwroot\uplaylists\ 
#     Machine name: TEMPOL01 
#  
# Process information: 
#     Process ID: 6588 
#     Process name: w3wp.exe 
#     Account name: NT AUTHORITY\NETWORK SERVICE 
#  
# Exception information: 
#     Exception type: InvalidOperationException 
#     Exception message: Sequence contains no elements 
#  
# Request information: 
#     Request URL: http://www.dr.dk/uplaylists/10/singleclip?i=3085&width=520&height=292&view=player.js 
#     Request path: /uplaylists/10/singleclip 
#     User host address: 172.18.64.205 
#     User:  
#     Is authenticated: False 
#     Authentication Type:  
#     Thread account name: NT AUTHORITY\NETWORK SERVICE 
#  
# Thread information: 
#     Thread ID: 6 
#     Thread account name: NT AUTHORITY\NETWORK SERVICE 
#     Is impersonating: False 
#     Stack trace:    at System.Data.Linq.SqlClient.SqlProvider.Execute(Expression query, QueryInfo queryInfo, IObjectReaderFactory factory, Object[] parentArgs, Object[] userArgs, ICompiledSubQuery[] subQueries, Object lastResult)
#    at System.Data.Linq.SqlClient.SqlProvider.ExecuteAll(Expression query, QueryInfo[] queryInfos, IObjectReaderFactory factory, Object[] userArguments, ICompiledSubQuery[] subQueries)
#    at System.Data.Linq.SqlClient.SqlProvider.System.Data.Linq.Provider.IProvider.Execute(Expression query)
#    at System.Data.Linq.DataQuery`1.System.Linq.IQueryProvider.Execute[S](Expression expression)
#    at System.Linq.Queryable.Single[TSource](IQueryable`1 source, Expression`1 predicate)
#    at DR.Ung.UplayerCMS.Repositories.ClipRepository.GetClipItemByID(Int32 siteID, Int32 clipID, Restriction restriction) in C:\code\UplayerCMS\UplayerCMS.Models\Repositories\ClipRepository.cs:line 93
#    at DR.Ung.UPlaylists.Controllers.UPlaylistsController.SingleClip(Int32 siteid, String view) in C:\code\UplayerCMS\UPlaylists\Controllers\UPlaylistsController.cs:line 254
#    at lambda_method(ExecutionScope , ControllerBase , Object[] )
#    at System.Web.Mvc.ReflectedActionDescriptor.Execute(ControllerContext controllerContext, IDictionary`2 parameters)
#    at System.Web.Mvc.ControllerActionInvoker.InvokeActionMethod(ControllerContext controllerContext, ActionDescriptor actionDescriptor, IDictionary`2 parameters)
#    at System.Web.Mvc.ControllerActionInvoker.<>c__DisplayClassd.<InvokeActionMethodWithFilters>b__a()
#    at System.Web.Mvc.ControllerActionInvoker.InvokeActionMethodFilter(IActionFilter filter, ActionExecutingContext preContext, Func`1 continuation)
#    at System.Web.Mvc.ControllerActionInvoker.InvokeActionMethodFilter(IActionFilter filter, ActionExecutingContext preContext, Func`1 continuation)
#    at System.Web.Mvc.ControllerActionInvoker.InvokeActionMethodWithFilters(ControllerContext controllerContext, IList`1 filters, ActionDescriptor actionDescriptor, IDictionary`2 parameters)
#    at System.Web.Mvc.ControllerActionInvoker.InvokeAction(ControllerContext controllerContext, String actionName)
#    at System.Web.Mvc.Controller.ExecuteCore()
#    at System.Web.Mvc.MvcHandler.<>c__DisplayClass8.<BeginProcessRequest>b__4()
#    at System.Web.Mvc.Async.AsyncResultWrapper.<>c__DisplayClass1.<MakeVoidDelegate>b__0()
#    at System.Web.Mvc.Async.AsyncResultWrapper.<>c__DisplayClass8`1.<BeginSynchronous>b__7(IAsyncResult _)
#    at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncResult`1.End()
#    at System.Web.Mvc.MvcHandler.EndProcessRequest(IAsyncResult asyncResult)
#    at System.Web.HttpApplication.CallHandlerExecutionStep.System.Web.HttpApplication.IExecutionStep.Execute()
#    at System.Web.HttpApplication.ExecuteStep(IExecutionStep step, Boolean& completedSynchronously)
# 
# Custom event details: 
#