diff.py

import pprint
from math import sqrt
import scipy.stats

def parse(x):
    def c(x):
        try:
            return int(x.split('cycles')[0].replace(',',''))
        except:
            return None
    return [c(x) for x in x.split('median') if c(x) is not None]

def parsefile(x):
    with open(x) as f:
        txt = f.read()
    ret = []
    for bm in txt.split('Benchmarking'):
        if not bm:
            continue
        what, rest = bm.split('make', 1)
        what = what.strip()
        for line in rest.split('\n'):
            if 'median' not in line:
                continue
            action, rest2 = line.split('avg')
            action = action.replace('-', '').replace('.', '').strip()
            ret.append((what + ' ' + action, int(rest2.split('cycles')[0].split('median')[1].replace(',', ''))))
    return ret

old = {}
new = {}

for key, cycles in parsefile(f'old'):
    if key not in old:
        old[key] = []
    old[key].append(cycles)
for key, cycles in parsefile(f'new'):
    if key not in new:
        new[key] = []
    new[key].append(cycles)

def avg(xs):
    return sum(xs)/len(xs)

scores = []
for what in old:
    old_min = min(old[what])
    new_min = min(new[what])
    score = abs((new_min / old_min - 1)*100)
    scores.append((what, score, new_min/old_min-1,old[what], new[what]))
scores.sort(key=lambda x: -x[1])
for v in scores:
    if not any(x in v[0] for x in ('Signing', 'Generating', 'Verifying')):
        continue
    stats = scipy.stats.mannwhitneyu(v[3], v[4])
    diff = avg(v[4]) - avg(v[3])
    if stats.pvalue > 0.05:
        continue
    print()
    print(v[0])
    print(f"{diff/avg(v[3])*100:.3f}% difference with p={stats.pvalue:.5f}")
    print(' old:', v[3])
    print(' new:', v[4])