-gaius-benchmarks-.md

Gaius benchmarks

This is a benchmark driver for More balanced task DAG by tkf · Pull Request #83 · MasonProtter/Gaius.jl

Usage

make BENCHMARKS=bench_mat_vec_mul.jl all  # benchmark matrix-vector multiplication
make BENCHMARKS=bench_mat_mat_mul.jl all  # benchmark matrix-matrix multiplication
make all                                  # run all benchmarks

.gitignore

/backup
/benchmark
/build

bench_mat_mat_mul.jl

using BenchmarkTools
using Gaius

SUITE = BenchmarkGroup()
for n in [500, 800, 1200]
    A = randn(n, n)
    B = randn(n, n)
    C = similar(B)
    Gaius.mul!(C, A, B)  # warmup
    SUITE["n=$n"] = @benchmarkable Gaius.mul!($C, $A, $B)
end

bench_mat_vec_mul.jl

using BenchmarkTools
using Gaius

SUITE = BenchmarkGroup()
for n in [500, 1000, 1500, 2000]
    A = randn(n, n)
    B = randn(n)
    C = similar(B)
    Gaius.mul!(C, A, B)  # warmup
    SUITE["n=$n"] = @benchmarkable Gaius.mul!($C, $A, $B)
end

Makefile

JULIA ?= julia
JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no

BASELINE_REPO = $(HOME)/.julia/dev/_wt/Gaius/baseline
TARGET_REPO = $(HOME)/.julia/dev/_wt/Gaius/mid

REPO = $(TARGET_REPO)
BENCHMARK_LABEL = target
BUILD = build/$(BENCHMARK_LABEL)

BENCHMARKS = $(wildcard bench_*.jl)
BENCHMARK_TARGETS = $(patsubst %.jl, $(BUILD)/%/.done, $(BENCHMARKS))

.PHONY: all benchmark* clean backup

all: benchmark benchmark-baseline

benchmark: $(BENCHMARK_TARGETS)
$(BENCHMARK_TARGETS): $(BUILD)/%/.done:
	@mkdir -p $(BUILD)/$*
	git -C $(REPO) show --format='format:%H' --no-patch > $(BUILD)/$*/git-rev-sha1
	git -C $(REPO) show --format='format:%T' --no-patch > $(BUILD)/$*/git-tree-sha1
	JULIA_PROJECT=$(REPO) $(JULIA_CMD) run_benchmarks.jl $*.jl $(BUILD)/$*
	touch $@

benchmark-baseline:
	$(MAKE) REPO=$(BASELINE_REPO) BENCHMARK_LABEL=baseline benchmark

.PHONY: clean
clean:
	rm -rf build/*/

backup:
	mv build backup/build-$$(date +%Y-%m-%d-%H%M%S)

plot_speedup.jl

using BenchmarkTools
using DataFrames
using Glob
using PkgBenchmark
using VegaLite
using FileIO

function readsweep(resultdir)
    results = map(PkgBenchmark.readresults, sort!(readdir(glob"result-*.json", resultdir)))
    table =
        Iterators.map(results) do r
            nthreads = parse(Int, r.benchmarkconfig.env["JULIA_NUM_THREADS"])
            Iterators.map(leaves(r.benchmarkgroup)) do ((prob,), trial)
                @assert startswith(prob, "n=")
                n = parse(Int, prob[length("n=")+1:end])
                (; nthreads, n, trial)
            end
        end |>
        Iterators.flatten |>
        collect

    df = DataFrame(table)
    df[!, :time] = map(trial -> minimum(trial).time, df.trial)
    return df
end

df_baseline = readsweep(joinpath(@__DIR__, "build/baseline/bench_mat_vec_mul"))
df_target = readsweep(joinpath(@__DIR__, "build/target/bench_mat_vec_mul"))

df_raw = leftjoin(
    df_baseline,
    df_target;
    on = [:nthreads, :n],
    renamecols = "_baseline" => "_target",
)
df_raw[!, :speedup] = df_raw.time_baseline ./ df_raw.time_target
df_raw
#-

plt_speedup = @vlplot(
    data = select(df_raw, Not([:trial_baseline, :trial_target])),  # avoid serializing trials
    layer = [
        {
            mark = {type = :line, point = true},
            encoding = {
                x = {field = :nthreads},
                y = {field = :speedup, title = "Seedup (target vs baseline)"},
                color = {field = :n, type = :ordinal, title = "size(A, 1)"},
            },
        },
        {
            mark = {type = :rule},
            encoding = {
                y = {datum = 1},
            },
        },
    ],
)
save(joinpath(@__DIR__, "build", "speedup.png"), plt_speedup)
plt_speedup
#-

function speedup_wrt_singlethread(df, label)
    df = select(df, Not(:trial))
    df[!, :version] .= label
    return combine(groupby(df, [:n, :version])) do g
        t1 = only(g[g.nthreads.==1, :]).time
        (; g.nthreads, speedup = t1 ./ g.time)
    end
end

df_scaling = vcat(
    speedup_wrt_singlethread(df_baseline, :baseline),
    speedup_wrt_singlethread(df_target, :target),
)

plt_scaling = @vlplot(
    mark = {type = :line, point = true},
    x = :nthreads,
    y = {:speedup, title = "Seedup (vs nthreads=1)"},
    column = {:n, type = :ordinal, title = "size(A, 1)"},
    color = :version,
    data = df_scaling,
)
save(joinpath(@__DIR__, "build", "scaling.png"), plt_scaling)
plt_scaling
#-

run_benchmarks.jl

script, resultdir = ARGS

if length(ARGS) >= 3
    threads = include_string(@__MODULE__, ARGS[3])
else
    threads = 1:32
    # threads = [1, 32, 16, 8, 24, 28, 20, 12]
    # union!(threads, 1:32)
    # threads = [1, 8, 16, 24, 32]
end

using PkgBenchmark

resultdir = abspath(resultdir)
mkpath(resultdir)

for n in threads
    @info "Benchmarking with `JULIA_NUM_THREADS=$n`"
    resultname = "result-$n"
    resultfile = joinpath(resultdir, "$resultname.json")
    group = benchmarkpkg(
        @__DIR__,
        BenchmarkConfig(env = Dict(
            # "JULIA_EXCLUSIVE" => "1",
            "JULIA_NUM_THREADS" => string(n),
        )),
        resultfile = resultfile,
        script = abspath(script),
    )
    PkgBenchmark.export_markdown(joinpath(resultdir, "$resultname.md"), group)
end