log-searcher.py

import sys
import pymongo
import zlib

class color:
   PURPLE = '\033[95m'
   CYAN = '\033[96m'
   DARKCYAN = '\033[36m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

# '2016-08-08T12:40:40.044-0400 I NETWORK  [initandlisten] connection accepted from 192.168.0.111:50968 #18 (10 connections now open)'
def parse_log_line(entry, line):
    data = line.split(' ')
    if len(data) < 2:
        return
    #print(entry)
    #print(data)
    p = {}
    p['host']=entry['hostname'].strip()+":"+str(entry['port'])
    p['logPath']=entry['logPath']
    p['groupId']=entry['groupId']
    p['ts'] = data[0]
    p['cat'] = data[1:3]
    p['cat2'] = data[3]
    p['msg'] = ' '.join(data[4:])
    return p

# given a connection to ops mgr app db
# create uncompressed version of host logs
# for searching
def uncompress(opsmgr_db):
    cursor = opsmgr_db["mmsdbautomationlog"]["mongoLogs"].find()
    while cursor.alive:
        entry = cursor.next();
        loglines = zlib.decompress( entry["compressedLogs"], 15+32 ).split("\n")
        # remove empty lines
        loglines = filter( None, loglines)
        for line in loglines:
            opsmgr_db["logsearch"]["logs"].insert( parse_log_line( entry, line ) )

def search(coll,term):
    results = coll.find( { "$text" : { "$search" : term.strip() } } )
    i = 0 
    try:
        while results.alive:
            entry = results.next()
            print "%s[%s %s] %s %s" % (color.BOLD,entry['host'], entry['ts'],color.END, entry['msg'])
            i = i + 1
    except: StopIteration

    print(str(i) + " results found")


connection = pymongo.MongoClient()
log_coll = connection["logsearch"]["logs"]
log_coll.drop()
log_coll.create_index([("msg", pymongo.TEXT)])
print("Uncompressing MongoDB host logs...")
uncompress( connection )
count = log_coll.count()
num_hosts = len(log_coll.distinct("host"))
print("Uncompress complete. Fount " + str(count) + " log lines from " + str(num_hosts) + " hosts.");
stop = False
while not stop:
    try:
        print "Enter a term to search:",
        search( log_coll, sys.stdin.readline() )
    except KeyboardInterrupt: 
        stop = True