wrk_parser.py

import json
import re
import sys
from urlparse import urlparse
from datetime import datetime


START_BLOCK = 'Running'


def gen_json(wrk_block, datestamp, build_id):
    j = {
        'datestamp': datestamp,
        'build_id': build_id,
        'params': {},
        'thread_stats': {},
        'rps': '',
        'bandwidth': '',
        'gc': {}
    }
    for line in wrk_block:
        if line.startswith('Running'):
            _, runtime, _, _, endpoint = line.split()
            endpoint = urlparse(endpoint).path.replace('/', '')
            j['params'].update({
                'duration': runtime,
                'endpoint': endpoint
            })
        if line.endswith('connections'):
            threads, _, _, connections, _ = line.split()
            j['params'].update({
                'threads': threads,
                'connections': connections
            })
        if line.startswith('Thread Stats'):
            continue
        if line.startswith('Latency'):
            _, avg, stdev, _max, dif = line.split()
            j['thread_stats']['latency'] = {
                'avg': avg,
                'stdev': stdev,
                'max': _max,
                'dif': dif
            }
        if line.startswith('Req/Sec'):
            _, avg, stdev, _max, dif = line.split()
            j['thread_stats']['rps'] = {
                'avg': avg,
                'stdev': stdev,
                'max': _max,
                'dif': dif
            }
        if line.split()[1] == 'requests':
            reqs, _, _, runtime, bandwidth, _ = line.split()
            runtime = runtime.rstrip(',')
            j['thread_stats'].update({
                'reqs': reqs,
                'runtime': runtime,
                'bandwidth': bandwidth
            })
        if line.startswith('Socket errors:'):
            connect, read, write, timeout = re.findall('[0-9]+', line)
            j['thread_stats']['socket_errors'] = {
                'connect': connect,
                'read': read,
                'write': write,
                'timeout': timeout
            }
        if line.startswith('Non-2xx'):
            j['thread_stats']['failures'] = line.split()[-1]
        if line.startswith('Requests/sec'):
            j['rps'] = line.split()[-1]
        if line.startswith('Transfer/sec'):
            j['bandwidth'] = line.split()[-1]
        if line.startswith('GarbageCollector.PS-Scavenge'):
            count, total, avg = re.findall('([0-9]+m?s?)', line)
            j['gc']['scavenge'] = {
                'count': count,
                'total': total,
                'avg': avg
            }
        if line.startswith('GarbageCollector.PS-MarkSweep'):
            count, total, avg = re.findall('([0-9]+m?s?)', line)
            j['gc']['marksweep'] = {
                'count': count,
                'total': total,
                'avg': avg
            }

    return j


def write_output(wrk_block, datestamp, build_id):
    j = gen_json(wrk_block, datestamp, build_id)
    with open("{}-{}.json".format(build_id, j['params']['endpoint']), 'w') as fout:
        fout.write(json.dumps(j))


def main(build_id, name, datestamp):
    with open(fname, 'r') as f:
        wrk_block = []
        capturing = False
        for line in f:
            if line.startswith(START_BLOCK):
                if capturing:
                    write_output(wrk_block, datestamp, build_id)
                    wrk_block = []
                capturing = True
                wrk_block.append(line.strip())
            else:
                wrk_block.append(line.strip())

        write_output(wrk_block, datestamp, build_id)


if __name__ == '__main__':
    if len(sys.argv) != 2:
        sys.exit(1)
    
    fname = sys.argv[1]
    _, build_id, build_date, build_time, _ = fname.split('.')
    datestamp = datetime.strptime("{} {}".format(build_date, build_time), "%Y%m%d %H%M%S").isoformat()
    main(build_id, fname, datestamp)