bench.csv

bench_approx_derivative.py

from scipy.optimize._numdiff import approx_derivative
from scipy.optimize import rosen
import numpy as np
import timeit
import functools
import subprocess
import sys


def bench(xk, method):
    approx_derivative(np.linalg.norm, xk, method)


def auto_timeit(stmt, number):
    # Warmup, throw result away
    timeit.timeit(stmt, number=1)
    t = timeit.timeit(stmt, number=number)

    #while t < 0.5:
    #    n *= 10
    #    t = timeit.timeit(stmt, setup, number=n)

    return t / number, number # normalise to time-per-run



branch = subprocess.check_output("git rev-parse --abbrev-ref HEAD", shell=True).decode('ascii').strip()
print(branch)
total_iter = 50000
for N in [1, 10, 100, 1000, 10000]:
    for method in ['2-point', '3-point', 'cs']:
        xk = np.ones(N)
        duration, trials = auto_timeit(functools.partial(bench, xk, method), number=total_iter // N)
        print(f"branch={branch} N={N}\tmethod={method}\ttime={duration:.8f}s\tloops={trials}")
        with open('foo/bench.csv', 'at') as f:
            f.write(','.join(map(str, (branch, method, N, duration))) + '\n')

bench_branch.sh

#!/usr/bin/env bash
set -euo pipefail
bench() {
	git checkout $1
	git rev-parse --abbrev-ref HEAD
	python3 dev.py python foo/bench_approx_derivative.py | sed -ne '/Installation OK/,$ p'
	sleep 1
}
echo 'branch,method,dim,time' > foo/bench.csv
for i in $(seq 1 20); do
	branches=( 'main' 'njo-numdiff-mem-usage' 'njo-numdiff-loop-exchange' 'njo-numdiff-sep-vars' )
	branches=( $(shuf -e "${branches[@]}") )
	for i in "${branches[@]}"
	do
		bench "$i"
	done
done

benchmark_analysis.ipynb