log extraction

hlogreader.py

#!/usr/bin/env python

from __future__ import print_function
import argparse
import re
import os


class HlogReader:
    type_principal = 1
    type_rpc = 2

    def __init__(self, processtype, log, output):
        self._processtype = processtype
        self._log = log
        self._output = output
        self._combiner = {}
        self._aggregate = {}
        self._delimiter = "; "

    def process_line(self, line):
        """Processes a single log line using the specified process type.

        :param line:
        :return entry a processed line to write to the output file:
        """
        entry = None

        splitline = line.split('|')

        if self._processtype == HlogReader.type_principal and len(splitline) >= 4:

            if 'SecurityContextFilter' in splitline[3]:

                result = re.findall('#\[\[.*?\]\]', splitline[-1], re.IGNORECASE)

                if result:
                    name = result[0][13:-2]
                    method = result[1][10:-2]
                    uri = result[2][10:-2]

                    # date/time, user, method used, uri called
                    entry = splitline[1] + self._delimiter \
                        + name + self._delimiter \
                        + method + self._delimiter \
                        + uri

                    if name in self._aggregate.keys():
                        self._aggregate[name] += 1
                    else:
                        self._aggregate[name] = 1

        elif self._processtype == HlogReader.type_rpc and len(splitline) >= 4:

            guid = re.search('[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}', splitline[-1], re.IGNORECASE)

            if 'RpcRequest' in splitline[-1]:
                self._combiner[guid.group()] = [splitline[1], splitline[-1][(guid.end()+1):-1].replace(']','')]
            elif 'RpcReply' in splitline[-1] and guid.group() in self._combiner.keys():

                ms_found = re.search('[0-9]*ms', splitline[-1])

                if ms_found:
                    ms = ms_found.group()[:-2]

                    startdate = self._combiner[guid.group()][0]
                    method = self._combiner[guid.group()][1]

                    # start date/time, method called, milliseconds elapsed
                    entry = startdate + self._delimiter \
                        + method + self._delimiter \
                        + ms

                    if method in self._aggregate.keys():
                        self._aggregate[method] += 1
                    else:
                        self._aggregate[method] = 1

        return entry

    def process_logfile(self):
        """Processes the logfile for specific entries and writes the output.

        :return:
        """

        outname = 'user.out'
        if self._processtype == HlogReader.type_rpc:
            outname = 'rpc.out'

        with open(os.path.join(self._output, outname),'w') as outputfile:
            for fname in self._log:
                with open(fname,'r') as logfile:
                    print('Processing %s and writing output to %s.' % (fname, self._output))
                    for line in logfile:

                        entry = self.process_line(line)

                        if entry is not None:
                            outputfile.write(entry + "\n")

        # process aggregates
        print('Logfile processed')
        print('Totals:')
        for name in self._aggregate.keys():
            print('%s | %s' % (str(self._aggregate[name]).rjust(8), name))


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('type', help='Type of processing to perform')
    parser.add_argument('log', help='Log file or directory to parse')
    parser.add_argument('output', help='Output file to write the results to')

    args = parser.parse_args()

    processtype = HlogReader.type_principal
    if 'user' in args.type:
        processtype = HlogReader.type_principal
    elif 'rpc' in args.type:
        processtype = HlogReader.type_rpc
    else:
        print("Command argument unknown. Please use one of the following: user, rpc")
        exit()

    flist = []
    if os.path.isdir(args.log):
        for name in os.listdir(args.log):
            if os.path.isfile(os.path.join(args.log,name)):
                flist.append(os.path.join(args.log,name))
    else:
        flist.append(args.log)


    logreader = HlogReader(processtype, flist, args.output)
    logreader.process_logfile()