Parse the log output produced by IntegrationTestRegionReplicaPerf for latency summary data

parse_IntegrationTestRegionReplicaPerf.py

#!/usr/bin/env python

#
# parse the log output produced by IntegrationTestRegionReplicaPerf to
# collect the latency data from each client thread. Works on some build of
# 0.98 + region replicas.
#

import fileinput
import re

NEW_RUN = re.compile('.*Launching.*')
IS_NONREPLICA = re.compile('.* non-replica .*')
JOB_NUM = re.compile('.*(\d)/3$')
TEST_CLIENT = re.compile('.*TestClient-(\d).*')
FOUR_9S = re.compile('.*99\.99% = (.*)$')
THREE_9S = re.compile('.*99\.9% = (.*)$')
TWO_9S = re.compile('.*99% = (.*)$')
EIGHTY = re.compile('.*80% = (.*)$')
STDEV = re.compile('.*Standard Deviation = (.*)$')
MEAN = re.compile('.*Mean = (.*)$')

cur_job = None
isReplica = None
clients = None

for line in fileinput.input():
    line = line.strip()
    if not (NEW_RUN.match(line) or FOUR_9S.match(line) or THREE_9S.match(line) or TWO_9S.match(line) or EIGHTY.match(line) or STDEV.match(line) or MEAN.match(line)): continue
    if NEW_RUN.match(line):
        found_first_run = True
        if IS_NONREPLICA.match(line):
            isReplica = False
        else:
            isReplica = True
        cur_job = JOB_NUM.match(line).group(1)
        if clients:
            for (thread,run) in clients.iteritems():
                print "{} {} {} {} {} {} {} {} {}".format(run['replica'], run['run'], run['client'], run['stdev'], run['mean'], run['80pct'], run['99pct'], run['99.9pct'], run['99.99pct'])
        clients = {}
    if TEST_CLIENT.match(line):
        c = TEST_CLIENT.match(line).group(1)
        x = clients.get(c, None)
        if not x:
            clients[c] = {}
            clients[c]['client'] = c
            clients[c]['replica'] = isReplica
            clients[c]['run'] = cur_job
        if FOUR_9S.match(line):
            clients[c]['99.99pct'] = FOUR_9S.match(line).group(1)
            continue
        if THREE_9S.match(line):
            clients[c]['99.9pct'] = THREE_9S.match(line).group(1)
            continue
        if TWO_9S.match(line):
            clients[c]['99pct'] = TWO_9S.match(line).group(1)
            continue
        if EIGHTY.match(line):
            clients[c]['80pct'] = EIGHTY.match(line).group(1)
            continue
        if MEAN.match(line):
            clients[c]['mean'] = MEAN.match(line).group(1)
            continue
        if STDEV.match(line):
            clients[c]['stdev'] = STDEV.match(line).group(1)
            continue
if clients:
    for (thread,run) in clients.iteritems():
        print "{} {} {} {} {} {} {} {} {}".format(run['replica'], run['run'], run['client'], run['stdev'], run['mean'], run['80pct'], run['99pct'], run['99.9pct'], run['99.99pct'])