Skip to content

Instantly share code, notes, and snippets.

@fingolfin
Created July 12, 2019 09:26
Show Gist options
  • Save fingolfin/b81101cc304d6062d195fa80fae32ab4 to your computer and use it in GitHub Desktop.
Save fingolfin/b81101cc304d6062d195fa80fae32ab4 to your computer and use it in GitHub Desktop.
GAP benchmark function
# Benchmark( func[, optrec] )
#
# func - a function taking no arguments
# optrec - an optional record with various options
#
# Measures how long executing the given function "func" takes.
# In order to improve accuracy, it invokes the function repeatedly.
# Before each repetition, the garbage collector is run, and
# (unless turned off by an option) the random number generators
# are reset.
# At the end, it outputs the average, median, and std deviation.
#
# Example:
# gap> Benchmark(function() Factors(x^293-1); end);
# .................................................
# Performed 49 iterations, taking 201 milliseconds.
# average: 4.1 +/- 0.11 (+/- 3%)
# median: 4.06396484375
# rec( avg := 4.09581, counter := 49, median := 4.06396, std := 0.109638, total := 200.695, var := 0.0120205 )
#
#
# The following options are currently implemented:
#
# minreps: the minimal number of repetitions (default: 5);
# the function will be executed at least that often,\
# unless some other condition (maxreps exceeded, maxtime exceeded)
# aborts the benchmark early.
# mintime: the minimal number of milliseconds that has to pass before
# benchmarking ends (default: 200)
# benchmarking will not stop before this time has passed,
# unless some other condition (maxreps exceeded, maxtime exceeded)
# aborts the benchmark early.
# maxreps: the maximal number of repetitions (default: infinity)
# once this is reached, benchmarking stops immediately.
# maxtime: the minimal number of milliseconds before benchmarking ends (default: 200)
# once this is reached, benchmarking stops immediately.
# showSummary:
# showProgress:
# resetRandom: whether to reset the random number generators before each repetition (default: true)
#
#
# TODO: allow passing a function that is executed before every test run.
# That function can reset other state, flush caches etc.
#
Benchmark := function(func, arg...)
local opt, getTime, timings, total, t, i, res;
opt := rec(
minreps := 5,
maxreps := infinity,
silent := false,
resetRandom := true,
mintime := 200,
maxtime := infinity,
showProgress := true,
showSummary := true,
);
if Length(arg) = 1 and IsRecord(arg[1]) then
for i in RecNames(arg[1]) do
opt.(i) := arg[1].(i);
od;
elif Length(arg) <> 0 then
Error("Usage: Benchmark( func[, optrec] )");
fi;
# force mintime and maxtime to be floats
opt.mintime := Float(opt.mintime);
opt.maxtime := Float(opt.maxtime);
# if available, use NanosecondsSinceEpoch
if IsBound(NanosecondsSinceEpoch) then
getTime := function() return NanosecondsSinceEpoch()/1000000.; end;
else
getTime := Runtime;
fi;
timings := [];
total := 0.0;
i := 0;
# repeat at most opt.maxreps times, and at least opt.minreps
# times, resp. at least till opt.mintime milliseconds passed
while i < opt.maxreps and (opt.maxtime = infinity or total < opt.maxtime) and
(i < opt.minreps or total < opt.mintime) do
i := i + 1;
if opt.resetRandom then
Reset(GlobalMersenneTwister);
Reset(GlobalRandomSource);
fi;
GASMAN("collect");
t := getTime();
func();
t := getTime() - t;
total := total + t;
Add(timings, t);
if opt.showProgress then
#Print(".\c");
Print(['\r', "|/-\\"[1+(i mod 4)],'\c']);
fi;
od;
if opt.showProgress then
#Print("\n");
Print("\r \c\r");
fi;
Sort(timings);
res := rec();
#res.timings := timings;
res.total := total;
res.counter := i;
res.avg := Sum(timings) * 1.0 / res.counter;
res.var := Sum(timings, t -> (t-res.avg)^2) / res.counter;
res.std := Sqrt(res.var);
if IsOddInt(res.counter) then
res.median := timings[(res.counter+1)/2];
else
res.median := (timings[(res.counter)/2] + timings[(res.counter+2)/2]) / 2.;
fi;
if opt.showSummary then
Print("Performed ", res.counter, " iterations, taking ", Round(res.total), " milliseconds.\n");
#Print("timings: ", timings, "\n");
Print("average: ", Round(100*res.avg)/100,
" +/- ", Round(100*res.std)/100,
" (+/- ", Round(100*res.std/res.avg), "%)",
"\n");
Print("median: ", res.median, "\n");
fi;
return res;
end;
Benchmark(function() Factors(x^293-1); end);
Benchmark(function() Factors(x^293-1); end, rec(maxreps:=10));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment