Skip to content

Instantly share code, notes, and snippets.

@jorisdevrede
Created December 21, 2017 13:54
Show Gist options
  • Save jorisdevrede/e155c51144c83f7027606e74c978f846 to your computer and use it in GitHub Desktop.
Save jorisdevrede/e155c51144c83f7027606e74c978f846 to your computer and use it in GitHub Desktop.
log extraction
#!/usr/bin/env python
from __future__ import print_function
import argparse
import re
import os
class HlogReader:
type_principal = 1
type_rpc = 2
def __init__(self, processtype, log, output):
self._processtype = processtype
self._log = log
self._output = output
self._combiner = {}
self._aggregate = {}
self._delimiter = "; "
def process_line(self, line):
"""Processes a single log line using the specified process type.
:param line:
:return entry a processed line to write to the output file:
"""
entry = None
splitline = line.split('|')
if self._processtype == HlogReader.type_principal and len(splitline) >= 4:
if 'SecurityContextFilter' in splitline[3]:
result = re.findall('#\[\[.*?\]\]', splitline[-1], re.IGNORECASE)
if result:
name = result[0][13:-2]
method = result[1][10:-2]
uri = result[2][10:-2]
# date/time, user, method used, uri called
entry = splitline[1] + self._delimiter \
+ name + self._delimiter \
+ method + self._delimiter \
+ uri
if name in self._aggregate.keys():
self._aggregate[name] += 1
else:
self._aggregate[name] = 1
elif self._processtype == HlogReader.type_rpc and len(splitline) >= 4:
guid = re.search('[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}', splitline[-1], re.IGNORECASE)
if 'RpcRequest' in splitline[-1]:
self._combiner[guid.group()] = [splitline[1], splitline[-1][(guid.end()+1):-1].replace(']','')]
elif 'RpcReply' in splitline[-1] and guid.group() in self._combiner.keys():
ms_found = re.search('[0-9]*ms', splitline[-1])
if ms_found:
ms = ms_found.group()[:-2]
startdate = self._combiner[guid.group()][0]
method = self._combiner[guid.group()][1]
# start date/time, method called, milliseconds elapsed
entry = startdate + self._delimiter \
+ method + self._delimiter \
+ ms
if method in self._aggregate.keys():
self._aggregate[method] += 1
else:
self._aggregate[method] = 1
return entry
def process_logfile(self):
"""Processes the logfile for specific entries and writes the output.
:return:
"""
outname = 'user.out'
if self._processtype == HlogReader.type_rpc:
outname = 'rpc.out'
with open(os.path.join(self._output, outname),'w') as outputfile:
for fname in self._log:
with open(fname,'r') as logfile:
print('Processing %s and writing output to %s.' % (fname, self._output))
for line in logfile:
entry = self.process_line(line)
if entry is not None:
outputfile.write(entry + "\n")
# process aggregates
print('Logfile processed')
print('Totals:')
for name in self._aggregate.keys():
print('%s | %s' % (str(self._aggregate[name]).rjust(8), name))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('type', help='Type of processing to perform')
parser.add_argument('log', help='Log file or directory to parse')
parser.add_argument('output', help='Output file to write the results to')
args = parser.parse_args()
processtype = HlogReader.type_principal
if 'user' in args.type:
processtype = HlogReader.type_principal
elif 'rpc' in args.type:
processtype = HlogReader.type_rpc
else:
print("Command argument unknown. Please use one of the following: user, rpc")
exit()
flist = []
if os.path.isdir(args.log):
for name in os.listdir(args.log):
if os.path.isfile(os.path.join(args.log,name)):
flist.append(os.path.join(args.log,name))
else:
flist.append(args.log)
logreader = HlogReader(processtype, flist, args.output)
logreader.process_logfile()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment