find requests per second

weather_log_parse.py

import argparse

import re
import random
import datetime
import os
import sys
import time


import numpy

regex = re.compile(r"""^\[
\d{2}\/[A-Za-z]{3}\/\d{4}:(\d{2}:\d{2}:\d{2})\s\+\d+\] # date
\spogoda\.yandex\.[a-z]+  # url
\s
\d+\.\d+\.\d+\.\d+        # ip address
\s\"
[A-Z]{3,4}                # type of http request
.*
HTTP\/\d\.\d\"\s
(\d+)                     # http resp code
.*$""", re.VERBOSE)

SEC_IN_DAY = 3600 * 24

def generate_log(path, lines):
    patterns = [
    '[10/Oct/2012:{0} +0400] pogoda.yandex.by 178.122.164.78 "GET / HTTP/1.1" {1} "-" "Opera/9.80 (Windows NT 5.1; U; Edition Yx 01; ru) Presto/2.10.229 Version/11.60" "-" 0.008 - 1170\n',
    '[10/Oct/2012:{0} +0400] pogoda.yandex.ru 212.193.232.183 "GET / HTTP/1.0" {1} "-" "-" "-" 0.011 - 1237\n',
    '[10/Oct/2012:{0} +0400] pogoda.yandex.ru 78.46.121.182 "GET /kemerovo/details HTTP/1.1" {1} "http://pogoda.yandex.ru" "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)" "-" 0.314 - 93275\n',
    '[10/Oct/2012:{0} +0400] pogoda.yandex.ru 93.158.129.212 "GET /astrahan/ HTTP/1.1" {1} "-" "Python-urllib/2.5" "-" 0.183 - 37160\n'
    ]
    
    with open(path, 'at+') as f:
        for i in range(lines):
            p = patterns[random.randint(0,3)] 
            sec = random.randint(0, SEC_IN_DAY)
            time_of_request = ('0' if sec < 36000 else '') + str(datetime.timedelta(seconds=sec))
            if len(time_of_request) != 8:
                import pdb;pdb.set_trace()
            resp_code = random.randint(200, 599)
            f.write(p.format(time_of_request, resp_code))

def read_log(path):
    seconds = numpy.zeros(SEC_IN_DAY, dtype='int')
    response_count = numpy.zeros(600, dtype='int')
    code_5xx_count = 0
    with open(path, 'rt') as f:
        for s in f:
            match = regex.match(s)
            event_time_strings = [int(x) for x in match.group(1).split(':')]
            event_time = 3600 * event_time_strings[0] + 60 * event_time_strings[1] + event_time_strings[2]
            code_index = int(match.group(2))
            seconds[event_time] += 1
            response_count[code_index] += 1
    code_5xx_count = sum(response_count[500:])
    return seconds, sum(response_count[500:])

def parse_args():
    parser = argparse.ArgumentParser(description='generate and parse log')
    group_lines = parser.add_mutually_exclusive_group()
    parser.add_argument('--generate', action='store_true', default=False, help="generates new file")
    parser.add_argument('--path', action='store', type=lambda x: os.path.abspath(x), help="path of file to write or to parse")
    group_lines.add_argument('--lines', action='store', type=int, help="how many lines should generate")
    group_lines.add_argument('--parse', action='store_true', default=False, help="parses log file")
    return parser.parse_args()




args = parse_args()

if args.generate:
    generate_log(args.path, args.lines)

if args.parse:
    start = time.monotonic()
    sec_count, resp_5xx_count = read_log(args.path)
    end = time.monotonic()
    print ("5xx response count: {0}".format(resp_5xx_count))
    print ('parse time: {0} seconds.'.format(end - start))