logparse.py

#!/usr/bin/env python

# split logs in 10 min buckets

# per bucket, tel request time per netblock

# 

import json
import sys
from collections import defaultdict
from operator import itemgetter


MINTIME = 1000 # only consider timeslots with at least request time seconds per slot
MINPERC = 10 # only consider netblocks that have at least this amount of request time per slot

# http://stackoverflow.com/questions/5029934/python-defaultdict-of-defaultdict
tslots = defaultdict(lambda : defaultdict(float)) 

"""

 {"time":"2014-08-05T06:32:52+00:00", "remote_addr":"157.55.39.226", 
 "remote_user":"-", "host":"www.zonnebrillen.com", "request":"GET 
 /sportbrillen/hardloopbrillen/oakley-racing-jacket-silver-fire-iridiu
 m-polarized-black-iridium.html HTTP/1.1", "status":"200", 
 "body_bytes_sent":"25203", "referer":"-", "user_agent":"Mozilla/5.0 
 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 
 "request_time":"0.938", "handler":"phpfpm"}

"""

for rawline in open('access.log.1'):
    try:
        log = json.loads(rawline)
    except ValueError:
        print "Invalid json: ", rawline
        continue
    
    if log['handler'] != 'phpfpm':
        continue
    
    rtime = float(log['request_time'])
    
    
    # "2014-08-05T06:32:48+00:00" > '2014-08-05T06:3'
    timeslot = log['time'][:15]
    
    # 1.2.3.4 => 1.2
    network = log['remote_addr'].rsplit('.',2)[0]
    
    tslots[timeslot][network] += rtime

for slot, networks in sorted(tslots.iteritems(), key=itemgetter(0)) :

    totaltime = sum([v for k,v in networks.items()])

    print "%s has %d networks (totaltime: %s):" % (slot, len(networks), totaltime)
    for network, rtime in sorted(networks.iteritems(), key=itemgetter(1), reverse=True):
        quotum = 100 * rtime / totaltime
        if quotum > MINPERC and totaltime > MINTIME:
            print "\t%-7s: %5.1f (%2d%%)" % (network, rtime, quotum)