benchmark.py

#!/usr/bin/env python
from __future__ import print_function

from datetime import datetime
import os
import numpy as np
import sys
from tabulate import tabulate
from random import shuffle


alts = [
    "build/IntegrationTests/PropagationTests",
    "build/PropagationTests_unlikely"
]

def run(exe):
  start = datetime.now()
  os.system(exe)
  end = datetime.now()
  delta = end - start
  s = delta.total_seconds()
  return s


results =  [[] for x in range(len(alts))]
n=5
for i in range(n):
  print("Round", i+1, "/", n)

  exes = list(enumerate(alts))
  shuffle(exes)
  for idx, exe in exes:
      exe = os.path.join(os.path.dirname(__file__), exe)
      exe = "{} > /dev/null 2>&1".format(exe)
      print(idx, "Executing:", exe)
      s = run(exe)
      print("took {}s".format(s))
      results[idx].append(s)

averages = list(map(np.average, results))
stds = list(map(np.std, results))

mins = map(min, results)


print("\nResults:")

minAvg = min(averages)

relatives = map(lambda a: "{:.2f}%".format(a/minAvg * 100.), averages)

zipped = zip(alts, averages, stds, mins, relatives)
print(tabulate(zipped, headers=("exe", "average", "stdev", "min", "rel")))