get most active ip addresses from log

get_active_ip.py

import argparse

import re
import random
import datetime
import os
import sys
import time

from random import randrange


regex = re.compile(r"""^\[
\d{2}\/[A-Za-z]{3}\/\d{4}:(\d{2}:\d{2}:\d{2})\s\+\d+\] # date
\spogoda\.yandex\.[a-z]+  # url
\s
(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})        # ip address
\s\"
([A-Z]{3,4})                # type of http request
.*
HTTP\/\d\.\d\"\s
(\d+)                     # http resp code
.*$""", re.VERBOSE)


SEC_IN_DAY = 3600 * 24 - 1


def generateIP():
    ip = ".".join([str(randrange(1,256)),str(randrange(1,256)),
    str(randrange(1,256)),str(randrange(1,256))])
    return ip


def generate_log(path, lines):
    patterns = [
    '[10/Oct/2012:{0} +0400] pogoda.yandex.by {1} "GET / HTTP/1.1" {2} "-" "Opera/9.80 (Windows NT 5.1; U; Edition Yx 01; ru) Presto/2.10.229 Version/11.60" "-" 0.008 - 1170\n',
    '[10/Oct/2012:{0} +0400] pogoda.yandex.ru {1} "GET / HTTP/1.0" {2} "-" "-" "-" 0.011 - 1237\n',
    '[10/Oct/2012:{0} +0400] pogoda.yandex.ru {1} "GET /kemerovo/details HTTP/1.1" {2} "http://pogoda.yandex.ru" "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)" "-" 0.314 - 93275\n',
    '[10/Oct/2012:{0} +0400] pogoda.yandex.ru {1} "GET /astrahan/ HTTP/1.1" {2} "-" "Python-urllib/2.5" "-" 0.183 - 37160\n'
    ]
    
    with open(path, 'at+') as f:
        for i in range(lines):
            p = patterns[random.randint(0,3)] 
            sec = random.randint(0, SEC_IN_DAY)
            time_of_request = ('0' if sec < 36000 else '') + str(datetime.timedelta(seconds=sec))
            if len(time_of_request) != 8:
                import pdb;pdb.set_trace()
            resp_code = random.randint(200, 599)
            ip = generateIP()
            f.write(p.format(time_of_request, ip, resp_code))


def read_log(path):
    ip_addresses = {}
    with open(path, 'rt') as f:
        for s in f:
            match = regex.match(s)
            ip = match.group(2)
            if ip in ip_addresses:
                ip_addresses[ip] += 1
            else:
                ip_addresses[ip] = 1
    return sorted(ip_addresses.items(), key=lambda x: x[1])


def parse_args():
    parser = argparse.ArgumentParser(description='generate and parse log')
    group_lines = parser.add_mutually_exclusive_group()
    parser.add_argument('--generate', action='store_true', default=False, help="generates new file")
    parser.add_argument('--path', action='store', type=lambda x: os.path.abspath(x), help="path of file to write or to parse")
    group_lines.add_argument('--lines', action='store', type=int, help="how many lines should generate")
    group_lines.add_argument('--parse', action='store_true', default=False, help="parses log file")
    return parser.parse_args()


args = parse_args()


if args.generate:
    generate_log(args.path, args.lines)


if args.parse:
    start = time.monotonic()
    ip_addresses = read_log(args.path)[:-11:-1]
    end = time.monotonic()

    print ("most active addresses: ")
    for ip, count in ip_addresses:
        print(count, ip)
    print ('parse time: {0} seconds.'.format(end - start))