HPX Output Performance Counter Analyzer

pfx_data_parser.py

#!/usr/bin/python

import re
import numpy
from os import path
from itertools import groupby

def parse_hpx_output(inputfile):
    matching_pattern = re.compile('/([a-z_]+){locality#(\d+)/total}/(?:(?:(count|time)/)([a-z/_-]+)|([a-z/_-]+)/(?:(count|time))),([0-9]+),([0-9.]+),\[[a-z]+\],([0-9.\+e]+)(?:,\[([a-z]+)?\])?')

    data = []

    with open(inputfile, 'r') as file_handle:
        for line in file_handle:
            match_result = matching_pattern.match(line)

            if match_result:
                data.append(list(match_result.groups()))

    for i in data:
        # Order is reversed for component, primary, and symbol counters
        if i[2] == None:
            i[2] = i[5]
            i[3] = i[4]

        # After the fix, remove the extra columns
        del i[4:6]

        # Distinguish between different agas counters
        if i[0] == 'agas':
            if any(part == i[3] for part in ['component', 'primary', 'symbol']):
                i[0] = 'agas_part'
            elif i[3].startswith('cache'):
                i[0] = 'agas_cache'
            else:
                i[0] = 'agas_service'

    # Need to sort since we messed up the keys
    data.sort(key=lambda x: x[0] + x[2])

    #for i in data:
    #    print i

    #groups = {}
    stats = {}

    # Group by category: parcel, agas, etc
    for category_key, category_data in groupby(data, lambda x: x[0]):
        #counters_by_name = {}
        stats_by_name = {}

        # Group by name: component, bind_gid, etc
        for names_key, names_data in groupby(category_data, lambda x: x[2]):
            gbyctr = ((elem[0:2] + elem[3:]) for elem in names_data)

            #counters_by_measurement = {}
            stats_by_measurement = {}

            # Group by measurement type: count, time
            for measurements_key, measurements_data in groupby(names_data, lambda x: x[3]):
                vals = list([elem[1],] + elem[4:] for elem in measurements_data)
                #counters_by_measurement[measurements_key] = vals
                stats_vals = [float(m[-2]) for m in vals]
                # Statistics: Mean, StDev, Min, Max, Unit
                if len(stats_vals) < 2:
                    stats_by_measurement[measurements_key] = (stats_vals[0], 'n/a', 'n/a', 'n/a', 'n/a')
                else:
                    stats_by_measurement[measurements_key] = (round(numpy.mean(stats_vals), 4), round(numpy.std(stats_vals), 4), round(numpy.min(stats_vals), 4), round(numpy.max(stats_vals), 4), vals[0][-1] or 'n/a')

            #counters_by_name[names_key] = counters_by_measurement
            stats_by_name[names_key] = stats_by_measurement

        #groups[category_key] = counters_by_name
        stats[category_key] = stats_by_name

    #print '\n****************************GROUPS********************************'
    #for i, j in groups.iteritems():
    #    print '\n', i, '===================='
    #    for k, l in j.iteritems():
    #        print k, '--------------------'
    #        for m, n in l.iteritems():
    #            print m, n

    print '\n****************************STATISTICS*****************************'
    for i, j in stats.iteritems():
        print '\n', i, '===================='
        for k, l in j.iteritems():
            print k, '--------------------'
            for m, n in l.iteritems():
                print m, n


    print '\n****************************WRITE TO OUTPUT************************'
    for i, j in stats.iteritems():
        for k, l in j.iteritems():
            filename = ('{0}_{1}_{2}.csv').format(path.join(path.dirname(inputfile), path.splitext(path.basename(inputfile))[0]), k, i)
            print ('Writing to {0} ....').format(filename)
            with open(filename, 'w') as cof:
                cof.write('Counter,Mean,StdDev,Min,Max,Unit\n')
                for m, n in l.iteritems():
                    cof.write(','.join([m,] + list(str(q) for q in n)) + '\n')
                    #print(','.join([m,] + list(str(q) for q in n)))

if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2:
        sys.stderr.write('Arguments not provided. Run as: ' + sys.argv[0] + ' <file1> [<file2> ...]\n')
        sys.exit(-1)

    for i in sys.argv[1:]:
        parse_hpx_output(sys.argv[1])