[gae] deferred stats

gae_log_sum.py

"""
Parses GAE single-line logs and computes the sum of the time-related fields:
total time, cpu time and api time.
"""

from operator import add

def parse_file(url_path, file_path):

    def parse_line(line):
        # date, time, path, status, time_total, time_cpu, time_api, size, useragent
        parts = line.split(' ')
        strip_unit = lambda str, unit: int(str.split(unit)[0])
        ret = {
            'path': parts[2],
            'time_total': strip_unit(parts[4], 'ms'),
            'time_cpu': strip_unit(parts[5], 'cpu_ms'),
            'time_api': 0,
        }
        try:
            i = parts[6].index('api_cpu_ms')
            ret['time_api'] = int(parts[6][:i])
        except ValueError:
            pass
        return ret

    file = open(file_path)
    details = [d for d in (parse_line(line) for line in file) if d['path'] == url_path]
    sum_field = lambda k: reduce(add, [d[k] for d in details])

    str1 = "stats for '%s' from '%s': " % (url_path, file_path)
    str2 = "time: %sms; cpu: %sms; api: %sms" % (
        str(sum_field('time_total')).rjust(5),
        sum_field('time_cpu'),
        sum_field('time_api'),
    )
    print str1+str2

    file.close()

if __name__ == '__main__':
    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog [options] log_file1[ log_file2[ log_file3...]]]")
    parser.add_option("-p", "--path", dest="url_path",
        default='/_ah/queue/deferred',
        help="Only compute stats for this path; if not specified, defaults"
             " to '%default'")

    (options, args) = parser.parse_args()
    if not len(args):
        parser.error("no log file specified")

    for file_path in args:
        parse_file(options.url_path, file_path)