kurobako blog: basic-benchmark.sh

basic-benchmark.sh

# 1. Download kurobako binary.
$ curl -L https://github.com/sile/kurobako/releases/download/0.2.6/kurobako-0.2.6.linux-amd64 -o kurobako
$ chmod +x kurobako && sudo mv kurobako /usr/local/bin/
 
# 2. Download the data file for HPOBench (note that the file size is about 700MB).
$ curl -OL http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz
$ tar xf fcnet_tabular_benchmarks.tar.gz && cd fcnet_tabular_benchmarks/
 
# 3. Specify problems used in this benchmark.
#
# In this example, we use Protein Structure and Parkinsons Telemonitoring datasets.
$ kurobako problem hpobench fcnet_protein_structure_data.hdf5 > problems.json
$ kurobako problem hpobench fcnet_parkinsons_telemonitoring_data.hdf5 >> problems.json
$ cat problems.json
{"hpobench":{"dataset":"fcnet_protein_structure_data.hdf5"}}
{"hpobench":{"dataset":"fcnet_parkinsons_telemonitoring_data.hdf5"}}
 
# 4. Specify optimization algorithms ("solver" in Kurobako's terminology) used in this benchmark.
#
# In this example, we use the following four algorithms provided by Optuna:
# - RandomSampler without pruning
# - TPESampler without pruning
# - TPESampler with MedianPruner
# - TPESampler with SuccessiveHalvingPruner
$ kurobako solver --name 'RandomSampler' optuna --sampler RandomSampler --pruner NopPruner > solvers.json
$ kurobako solver --name 'TPESampler' optuna --pruner NopPruner >> solvers.json
$ kurobako solver --name 'TPESampler_with_MedianPruner' optuna --pruner MedianPruner --pruner-kwargs '{"warmup_steps": 4}' >> solvers.json
$ kurobako solver --name 'TPESampler_with_SuccessiveHalvingPruner' optuna --pruner SuccessiveHalvingPruner --pruner-kwargs '{"min_resource": 4}' >> solvers.json
$ cat solvers.json
{"name":"RandomSampler","optuna":{"sampler":"RandomSampler","pruner":"NopPruner"}}
{"name":"TPESampler","optuna":{"pruner":"NopPruner"}}
{"name":"TPESampler_with_MedianPruner","optuna":{"pruner":"MedianPruner","pruner_kwargs":"{\"warmup_steps\": 4}"}}
{"name":"TPESampler_with_SuccessiveHalvingPruner","optuna":{"pruner":"SuccessiveHalvingPruner","pruner_kwargs":"{\"min_resource\": 4}"}}
 
# 5. Run the benchmark.
$ kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) --repeats 30 --budget 80 > studies.json
$ cat studies.json | kurobako run --parallelism 10 > result.json
(ALL) [01:27:14] [STUDIES    240/240 100%] [ETA  0s] done
 
# 6. Generate the report of the benchmark result. 
$ cat result.json | kurobako report > report.md