kurobakoの使い方メモ

kurobako-usage.md

$ cargo install kurobako
// or (only linux)
$ wget https://github.com/sile/kurobako/releases/download/0.0.15/kurobako-0.0.15.linux-amd64 -o kurobako && chmod +x kurobako
// or 
$ git clone git://github.com/sile/kurobako.git && cd kurobako && git checkout 0.0.14 && cargo install --path .

// 独自サンプラの場合
$ kurobako benchmark --problems (kurobako problem-suite sigopt auc) --solvers (kurobako solver command -- python3 /tmp/optuna_solver_example.py ) --budget 100 --iterations 10 | kurobako run > /tmp/sigopt-my-sampler.json

// TPEサンプラの場合
$ kurobako benchmark --problems (kurobako problem-suite sigopt auc) --solvers (kurobako solver optuna tpe) --budget 100 --iterations 10 | kurobako run > /tmp/sigopt-tpe-sampler.json

$ cat /tmp/sigopt-*.json | kurobako plot -o /tmp/results/ --errorbar
$ cat /tmp/sigopt-*.json | kurobako stats ranking

optuna_solver_example.py

import kurobako.problem
import kurobako.solver
import kurobako.solvers
import optuna

sampler = optuna.samplers.TPESampler()

print(kurobako.solvers.OptunaSolver.specification().to_message())

problem = kurobako.problem.ProblemSpec.from_message(input())

solver = kurobako.solvers.OptunaSolver(problem, sampler=sampler)
runner = kurobako.solver.SolverRunner(solver)
runner.run()