Skip to content

Instantly share code, notes, and snippets.

@tkf
Last active Oct 19, 2021
Embed
What would you like to do?
/backup
/benchmark
/build
using BenchmarkTools
using Gaius
SUITE = BenchmarkGroup()
for n in [500, 800, 1200]
A = randn(n, n)
B = randn(n, n)
C = similar(B)
Gaius.mul!(C, A, B) # warmup
SUITE["n=$n"] = @benchmarkable Gaius.mul!($C, $A, $B)
end
using BenchmarkTools
using Gaius
SUITE = BenchmarkGroup()
for n in [500, 1000, 1500, 2000]
A = randn(n, n)
B = randn(n)
C = similar(B)
Gaius.mul!(C, A, B) # warmup
SUITE["n=$n"] = @benchmarkable Gaius.mul!($C, $A, $B)
end
JULIA ?= julia
JULIA_CMD ?= $(JULIA) --color=yes --startup-file=no
BASELINE_REPO = $(HOME)/.julia/dev/_wt/Gaius/baseline
TARGET_REPO = $(HOME)/.julia/dev/_wt/Gaius/mid
REPO = $(TARGET_REPO)
BENCHMARK_LABEL = target
BUILD = build/$(BENCHMARK_LABEL)
BENCHMARKS = $(wildcard bench_*.jl)
BENCHMARK_TARGETS = $(patsubst %.jl, $(BUILD)/%/.done, $(BENCHMARKS))
.PHONY: all benchmark* clean backup
all: benchmark benchmark-baseline
benchmark: $(BENCHMARK_TARGETS)
$(BENCHMARK_TARGETS): $(BUILD)/%/.done:
@mkdir -p $(BUILD)/$*
git -C $(REPO) show --format='format:%H' --no-patch > $(BUILD)/$*/git-rev-sha1
git -C $(REPO) show --format='format:%T' --no-patch > $(BUILD)/$*/git-tree-sha1
JULIA_PROJECT=$(REPO) $(JULIA_CMD) run_benchmarks.jl $*.jl $(BUILD)/$*
touch $@
benchmark-baseline:
$(MAKE) REPO=$(BASELINE_REPO) BENCHMARK_LABEL=baseline benchmark
.PHONY: clean
clean:
rm -rf build/*/
backup:
mv build backup/build-$$(date +%Y-%m-%d-%H%M%S)
using BenchmarkTools
using DataFrames
using Glob
using PkgBenchmark
using VegaLite
using FileIO
function readsweep(resultdir)
results = map(PkgBenchmark.readresults, sort!(readdir(glob"result-*.json", resultdir)))
table =
Iterators.map(results) do r
nthreads = parse(Int, r.benchmarkconfig.env["JULIA_NUM_THREADS"])
Iterators.map(leaves(r.benchmarkgroup)) do ((prob,), trial)
@assert startswith(prob, "n=")
n = parse(Int, prob[length("n=")+1:end])
(; nthreads, n, trial)
end
end |>
Iterators.flatten |>
collect
df = DataFrame(table)
df[!, :time] = map(trial -> minimum(trial).time, df.trial)
return df
end
df_baseline = readsweep(joinpath(@__DIR__, "build/baseline/bench_mat_vec_mul"))
df_target = readsweep(joinpath(@__DIR__, "build/target/bench_mat_vec_mul"))
df_raw = leftjoin(
df_baseline,
df_target;
on = [:nthreads, :n],
renamecols = "_baseline" => "_target",
)
df_raw[!, :speedup] = df_raw.time_baseline ./ df_raw.time_target
df_raw
#-
plt_speedup = @vlplot(
data = select(df_raw, Not([:trial_baseline, :trial_target])), # avoid serializing trials
layer = [
{
mark = {type = :line, point = true},
encoding = {
x = {field = :nthreads},
y = {field = :speedup, title = "Seedup (target vs baseline)"},
color = {field = :n, type = :ordinal, title = "size(A, 1)"},
},
},
{
mark = {type = :rule},
encoding = {
y = {datum = 1},
},
},
],
)
save(joinpath(@__DIR__, "build", "speedup.png"), plt_speedup)
plt_speedup
#-
function speedup_wrt_singlethread(df, label)
df = select(df, Not(:trial))
df[!, :version] .= label
return combine(groupby(df, [:n, :version])) do g
t1 = only(g[g.nthreads.==1, :]).time
(; g.nthreads, speedup = t1 ./ g.time)
end
end
df_scaling = vcat(
speedup_wrt_singlethread(df_baseline, :baseline),
speedup_wrt_singlethread(df_target, :target),
)
plt_scaling = @vlplot(
mark = {type = :line, point = true},
x = :nthreads,
y = {:speedup, title = "Seedup (vs nthreads=1)"},
column = {:n, type = :ordinal, title = "size(A, 1)"},
color = :version,
data = df_scaling,
)
save(joinpath(@__DIR__, "build", "scaling.png"), plt_scaling)
plt_scaling
#-
script, resultdir = ARGS
if length(ARGS) >= 3
threads = include_string(@__MODULE__, ARGS[3])
else
threads = 1:32
# threads = [1, 32, 16, 8, 24, 28, 20, 12]
# union!(threads, 1:32)
# threads = [1, 8, 16, 24, 32]
end
using PkgBenchmark
resultdir = abspath(resultdir)
mkpath(resultdir)
for n in threads
@info "Benchmarking with `JULIA_NUM_THREADS=$n`"
resultname = "result-$n"
resultfile = joinpath(resultdir, "$resultname.json")
group = benchmarkpkg(
@__DIR__,
BenchmarkConfig(env = Dict(
# "JULIA_EXCLUSIVE" => "1",
"JULIA_NUM_THREADS" => string(n),
)),
resultfile = resultfile,
script = abspath(script),
)
PkgBenchmark.export_markdown(joinpath(resultdir, "$resultname.md"), group)
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment