`run_all.sh` runs the benchmark of `QMCSamplers` by kurobako. The performance of `QMCSampler` differs significantly when called from kurobako CUI vs when called using python script.

halton.py

from kurobako import solver
from kurobako.solver.optuna import OptunaSolverFactory
import optuna
from optuna.samplers import QMCSampler


def create_study(seed):
    sampler = QMCSampler(seed=seed, scramble=True, qmc_type="halton", warn_independent_sampling=False)
    return optuna.create_study(sampler=sampler)


if __name__ == "__main__":
    factory = OptunaSolverFactory(create_study, name="HaltonSampler_python")
    runner = solver.SolverRunner(factory)
    runner.run()

problems.json

{"hpobench":{"dataset":"../data/fcnet_protein_structure_data.hdf5"}}
{"hpobench":{"dataset":"../data/fcnet_parkinsons_telemonitoring_data.hdf5"}}
{"hpobench":{"dataset":"../data/fcnet_naval_propulsion_data.hdf5"}}
{"hpobench":{"dataset":"../data/fcnet_slice_localization_data.hdf5"}}

random_.py

# The file name is _random.py because random.py causes conflict in the namespace and leads to circular import.

from kurobako import solver
from kurobako.solver.optuna import OptunaSolverFactory
import optuna


def create_study(seed):
    sampler = optuna.samplers.RandomSampler()
    return optuna.create_study(sampler=sampler)


if __name__ == "__main__":
    factory = OptunaSolverFactory(create_study, name="RandomSampler_python")
    runner = solver.SolverRunner(factory)
    runner.run()

report.md

Benchmark Result Report

• Kurobako Version: 0.2.9
• Number of Solvers: 10
• Number of Problems: 4
• Metrics Precedence: best value -> AUC

Please refer to "A Strategy for Ranking Optimizers using Multiple Criteria" for the ranking strategy used in this report.

Table of Contents

1. Overall Results
2. Individual Results
3. Solvers
4. Problems
5. Studies

Overall Results

Solver	Borda	Firsts
HaltonSampler_python	13	3
MultivariateTPESampler_python	15	3
RandomSampler_python	17	2
SobolSampler_python	14	2
_HaltonSampler_NopPruner	0	0
_HaltonSampler_Scrambled_NopPruner	0	0
_RandomSampler_NopPruner	0	1
_SobolSampler_NopPruner	3	0
_SobolSampler_Scrambled_NopPruner	0	0
_TPESampler_NopPruner	23	4

Individual Results

(1) Problem: HPO-Bench-Naval

Ranking	Solver	Best (avg +- sd)	AUC (avg +- sd)	Elapsed (avg +- sd)
1	_TPESampler_NopPruner (study)	0.000033 +- 0.000004	0.016 +- 0.012	2.671 +- 0.147
1	RandomSampler_python (study)	0.000039 +- 0.000004	0.023 +- 0.030	33.599 +- 8.264
1	SobolSampler_python (study)	0.000040 +- 0.000012	0.013 +- 0.008	37.712 +- 6.227
1	HaltonSampler_python (study)	0.000039 +- 0.000004	0.016 +- 0.017	134.514 +- 30.717
1	MultivariateTPESampler_python (study)	0.000035 +- 0.000010	0.017 +- 0.015	37.450 +- 13.290
5	_SobolSampler_NopPruner (study)	0.000073 +- 0.000023	0.010 +- 0.005	0.290 +- 0.052
6	_SobolSampler_Scrambled_NopPruner (study)	0.000153 +- 0.000133	0.046 +- 0.034	0.397 +- 0.075
7	_HaltonSampler_NopPruner (study)	0.000191 +- 0.000065	0.056 +- 0.023	0.701 +- 0.053
7	_RandomSampler_NopPruner (study)	0.000119 +- 0.000045	0.181 +- 0.297	0.259 +- 0.020
7	_HaltonSampler_Scrambled_NopPruner (study)	0.000111 +- 0.000080	0.066 +- 0.030	1.175 +- 0.074

(2) Problem: HPO-Bench-Protein

Ranking	Solver	Best (avg +- sd)	AUC (avg +- sd)	Elapsed (avg +- sd)
1	_TPESampler_NopPruner (study)	0.222292 +- 0.002021	21.134 +- 0.428	2.605 +- 0.179
1	HaltonSampler_python (study)	0.226486 +- 0.004012	21.406 +- 0.493	102.396 +- 27.754
1	MultivariateTPESampler_python (study)	0.223906 +- 0.001617	20.432 +- 0.223	33.468 +- 9.718
2	SobolSampler_python (study)	0.226720 +- 0.002389	21.020 +- 0.294	36.265 +- 4.905
3	RandomSampler_python (study)	0.227897 +- 0.003933	21.295 +- 0.381	31.543 +- 9.787
6	_SobolSampler_NopPruner (study)	0.260289 +- 0.008488	24.538 +- 0.540	0.331 +- 0.062
6	_HaltonSampler_NopPruner (study)	0.248409 +- 0.011858	23.322 +- 1.252	0.734 +- 0.088
6	_RandomSampler_NopPruner (study)	0.248280 +- 0.010261	24.022 +- 1.434	0.280 +- 0.063
6	_HaltonSampler_Scrambled_NopPruner (study)	0.251172 +- 0.012329	23.951 +- 1.071	1.225 +- 0.151
6	_SobolSampler_Scrambled_NopPruner (study)	0.255692 +- 0.014936	23.961 +- 1.532	0.396 +- 0.056

(3) Problem: HPO-Bench-Slice

Ranking	Solver	Best (avg +- sd)	AUC (avg +- sd)	Elapsed (avg +- sd)
1	_TPESampler_NopPruner (study)	0.000195 +- 0.000028	0.035 +- 0.014	2.659 +- 0.129
2	_SobolSampler_NopPruner (study)	0.000413 +- 0.000139	0.047 +- 0.017	0.334 +- 0.076
2	RandomSampler_python (study)	0.000280 +- 0.000043	0.030 +- 0.005	36.207 +- 5.709
2	SobolSampler_python (study)	0.000279 +- 0.000025	0.032 +- 0.005	35.714 +- 5.751
2	_HaltonSampler_Scrambled_NopPruner (study)	0.000455 +- 0.000100	0.057 +- 0.018	1.200 +- 0.061
2	HaltonSampler_python (study)	0.000287 +- 0.000037	0.033 +- 0.006	142.358 +- 41.000
2	MultivariateTPESampler_python (study)	0.000348 +- 0.000162	0.037 +- 0.017	89.108 +- 56.204
4	_HaltonSampler_NopPruner (study)	0.000361 +- 0.000151	0.062 +- 0.011	0.691 +- 0.051
4	_RandomSampler_NopPruner (study)	0.000489 +- 0.000225	0.067 +- 0.035	0.275 +- 0.041
4	_SobolSampler_Scrambled_NopPruner (study)	0.000428 +- 0.000108	0.061 +- 0.022	0.390 +- 0.030

(4) Problem: HPO-Bench-Parkinson

Ranking	Solver	Best (avg +- sd)	AUC (avg +- sd)	Elapsed (avg +- sd)
1	_TPESampler_NopPruner (study)	0.007222 +- 0.000917	1.195 +- 0.370	2.690 +- 0.153
1	RandomSampler_python (study)	0.008132 +- 0.001297	1.005 +- 0.095	36.225 +- 7.699
1	SobolSampler_python (study)	0.008786 +- 0.001459	1.064 +- 0.112	35.611 +- 7.648
1	_RandomSampler_NopPruner (study)	0.014599 +- 0.005903	1.809 +- 0.604	0.271 +- 0.031
1	HaltonSampler_python (study)	0.008858 +- 0.002006	1.019 +- 0.244	135.309 +- 28.645
1	MultivariateTPESampler_python (study)	0.008708 +- 0.002181	0.942 +- 0.188	65.705 +- 20.257
3	_HaltonSampler_Scrambled_NopPruner (study)	0.014808 +- 0.004116	1.703 +- 0.618	1.207 +- 0.094
5	_SobolSampler_NopPruner (study)	0.013613 +- 0.004276	1.594 +- 0.320	0.312 +- 0.053
5	_SobolSampler_Scrambled_NopPruner (study)	0.012196 +- 0.001934	1.660 +- 0.427	0.401 +- 0.050
6	_HaltonSampler_NopPruner (study)	0.015017 +- 0.005061	1.856 +- 0.478	0.709 +- 0.036

Solvers

ID: d0c242b36aac85aa104576a6f510449113194235417c2e54488aeb75aae45b57

recipe:

{
  "command": {
    "path": "python3",
    "args": [
      "halton.py"
    ]
  }
}

specification:

{
  "name": "HaltonSampler_python",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: ef27564fe1ddc35c4028059d1f15df7517e08c4b9bac92adee34a029bb75d71f

recipe:

{
  "command": {
    "path": "python3",
    "args": [
      "tpe.py"
    ]
  }
}

specification:

{
  "name": "MultivariateTPESampler_python",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: 8062e475da01b160976c51fc01fddc1e422397f26ad860dd4a05148ccb92b5b6

recipe:

{
  "command": {
    "path": "python3",
    "args": [
      "random_.py"
    ]
  }
}

specification:

{
  "name": "RandomSampler_python",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: 8da15eb5d6d89960069f6b24b3256648bd641470ebdc98152c0a496e9b4f708e

recipe:

{
  "command": {
    "path": "python3",
    "args": [
      "sobol.py"
    ]
  }
}

specification:

{
  "name": "SobolSampler_python",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: a0a8ebdb9195ab1a4a61a89a0e638d5531cbd42efff393357fb2876a67c0d4a9

recipe:

{
  "name": "_HaltonSampler_NopPruner",
  "optuna": {
    "loglevel": "debug",
    "sampler": "QMCSampler",
    "sampler_kwargs": "{}",
    "pruner": "NopPruner",
    "pruner_kwargs": "{}"
  }
}

specification:

{
  "name": "_HaltonSampler_NopPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: aaf9fdc7071a2414a4fd00be14adac59ea045675a2f51a90a7a9e24598660880

recipe:

{
  "name": "_HaltonSampler_Scrambled_NopPruner",
  "optuna": {
    "loglevel": "debug",
    "sampler": "QMCSampler",
    "sampler_kwargs": "{\"scramble\":true}",
    "pruner": "NopPruner",
    "pruner_kwargs": "{}"
  }
}

specification:

{
  "name": "_HaltonSampler_Scrambled_NopPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: a6cf01ef67fb72912cc9949362e3eb221eb6d9aebd9c3fbf0b1795e1d86ff4ac

recipe:

{
  "name": "_RandomSampler_NopPruner",
  "optuna": {
    "loglevel": "debug",
    "sampler": "RandomSampler",
    "sampler_kwargs": "{}",
    "pruner": "NopPruner",
    "pruner_kwargs": "{}"
  }
}

specification:

{
  "name": "_RandomSampler_NopPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: 784b72151fc1cb2abcc1e10ac3850a367016c94c7235b7dc7d00dfda3657ab70

recipe:

{
  "name": "_SobolSampler_NopPruner",
  "optuna": {
    "loglevel": "debug",
    "sampler": "QMCSampler",
    "sampler_kwargs": "{\"qmc_type\":\"sobol\"}",
    "pruner": "NopPruner",
    "pruner_kwargs": "{}"
  }
}

specification:

{
  "name": "_SobolSampler_NopPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: c91be229b42e50d37a0f711c038496f61e47ef3af2cf0216b0647475b6c7ac46

recipe:

{
  "name": "_SobolSampler_Scrambled_NopPruner",
  "optuna": {
    "loglevel": "debug",
    "sampler": "QMCSampler",
    "sampler_kwargs": "{\"scramble\":true,\"qmc_type\":\"sobol\"}",
    "pruner": "NopPruner",
    "pruner_kwargs": "{}"
  }
}

specification:

{
  "name": "_SobolSampler_Scrambled_NopPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

ID: 3371bdcd6d1fc903733bc5d034ce94193d1e781126837c1067593e573ae6a246

recipe:

{
  "name": "_TPESampler_NopPruner",
  "optuna": {
    "loglevel": "debug",
    "sampler": "TPESampler",
    "sampler_kwargs": "{\"multivariate\":true,\"constant_liar\":true}",
    "pruner": "NopPruner",
    "pruner_kwargs": "{}"
  }
}

specification:

{
  "name": "_TPESampler_NopPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=2.9.0.dev0, kurobako-py=0.1.12"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "LOG_UNIFORM_DISCRETE",
    "CATEGORICAL",
    "CONDITIONAL",
    "MULTI_OBJECTIVE",
    "CONCURRENT"
  ]
}

Problems

ID: 459fb9ffc741d5ce2d04d9507d21249cb53e6a91379f02c5867abc173f7ad97d

recipe:

{
  "hpobench": {
    "dataset": "../data/fcnet_naval_propulsion_data.hdf5"
  }
}

specification:

{
  "name": "HPO-Bench-Naval",
  "attrs": {
    "github": "https://github.com/automl/nas_benchmarks",
    "paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
    "version": "kurobako_problems=0.1.13"
  },
  "params_domain": [
    {
      "name": "activation_fn_1",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "activation_fn_2",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "batch_size",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 4
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "init_lr",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "lr_schedule",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "cosine",
          "const"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    }
  ],
  "values_domain": [
    {
      "name": "Validation MSE",
      "range": {
        "type": "CONTINUOUS",
        "low": 0.0
      },
      "distribution": "UNIFORM"
    }
  ],
  "steps": 100
}

ID: d90a2389d3b8fd9455b2ccd692cdc0d0563fa56665d65b54c604de941dafab1b

recipe:

{
  "hpobench": {
    "dataset": "../data/fcnet_parkinsons_telemonitoring_data.hdf5"
  }
}

specification:

{
  "name": "HPO-Bench-Parkinson",
  "attrs": {
    "github": "https://github.com/automl/nas_benchmarks",
    "paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
    "version": "kurobako_problems=0.1.13"
  },
  "params_domain": [
    {
      "name": "activation_fn_1",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "activation_fn_2",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "batch_size",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 4
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "init_lr",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "lr_schedule",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "cosine",
          "const"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    }
  ],
  "values_domain": [
    {
      "name": "Validation MSE",
      "range": {
        "type": "CONTINUOUS",
        "low": 0.0
      },
      "distribution": "UNIFORM"
    }
  ],
  "steps": 100
}

ID: 8850cad4b2f6446b3a9c9fa28f063fd721a4ebf8763807d5c0a310cbb728bb7a

recipe:

{
  "hpobench": {
    "dataset": "../data/fcnet_protein_structure_data.hdf5"
  }
}

specification:

{
  "name": "HPO-Bench-Protein",
  "attrs": {
    "github": "https://github.com/automl/nas_benchmarks",
    "paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
    "version": "kurobako_problems=0.1.13"
  },
  "params_domain": [
    {
      "name": "activation_fn_1",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "activation_fn_2",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "batch_size",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 4
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "init_lr",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "lr_schedule",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "cosine",
          "const"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    }
  ],
  "values_domain": [
    {
      "name": "Validation MSE",
      "range": {
        "type": "CONTINUOUS",
        "low": 0.0
      },
      "distribution": "UNIFORM"
    }
  ],
  "steps": 100
}

ID: 8fe1066b16cf672e065bdc108b5a06c437f3f5f5b7a6e8a26905f51d32c0fcbc

recipe:

{
  "hpobench": {
    "dataset": "../data/fcnet_slice_localization_data.hdf5"
  }
}

specification:

{
  "name": "HPO-Bench-Slice",
  "attrs": {
    "github": "https://github.com/automl/nas_benchmarks",
    "paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
    "version": "kurobako_problems=0.1.13"
  },
  "params_domain": [
    {
      "name": "activation_fn_1",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "activation_fn_2",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "batch_size",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 4
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "dropout_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "init_lr",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "lr_schedule",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "cosine",
          "const"
        ]
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    },
    {
      "name": "n_units_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM"
    }
  ],
  "values_domain": [
    {
      "name": "Validation MSE",
      "range": {
        "type": "CONTINUOUS",
        "low": 0.0
      },
      "distribution": "UNIFORM"
    }
  ],
  "steps": 100
}

Studies

ID: f9c5e8adad26830e137ac12bd562a2e1808be529a4c12bceda6cabf3fb25c15d

• problem: HPO-Bench-Naval
• solver: HaltonSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 7d4b0b44cbda164a8a12395f7b1a6c9f36b5b1e4b5221769e63bca9deebdcb89

• problem: HPO-Bench-Naval
• solver: MultivariateTPESampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: c2c785125bef36f065405873c08a9b8b129873d4f9b07558df5297b800448792

• problem: HPO-Bench-Naval
• solver: RandomSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 0c6a15d2e4fc559cf6cf48eba04fec309199870ff33dd1b1a9c89edc16ee94ae

• problem: HPO-Bench-Naval
• solver: SobolSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 1606c14bf9bdf285f2b37546df95756c3f62956bae23f68de7eb17182a106e46

• problem: HPO-Bench-Naval
• solver: _HaltonSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 3be97d64ba577d5de08a3b0f3a15c35a26d9b994e1f01fefd1ebf5c099276935

• problem: HPO-Bench-Naval
• solver: _HaltonSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 6af62b41b4ce43dd846ddc2955ad77890062355a1841c3a6a22e41b92ee7f336

• problem: HPO-Bench-Naval
• solver: _RandomSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 543617e17b036fffc39e0eb95f250e7e3e9ae2a417cb3135dd95d2987d3df0a4

• problem: HPO-Bench-Naval
• solver: _SobolSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 35d6fba9ce7b23af421fa1836d1a0d9bdb514ca5b56a79ba4df9ac460f04ed9c

• problem: HPO-Bench-Naval
• solver: _SobolSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 7eef5aa16c7c9a0a07979fd4e3635e987184e5e497d1ce21dce3e93a32e3295c

• problem: HPO-Bench-Naval
• solver: _TPESampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 9463232796ab2e7a051d11ab25f1e731675567837c679c9bcd49546a354831d7

• problem: HPO-Bench-Parkinson
• solver: HaltonSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 5392d03ea03ebad48dc6150d540d4f9d8d6f86b5d0e30ceb1fc359f963c84e8a

• problem: HPO-Bench-Parkinson
• solver: MultivariateTPESampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: adbccf9de3a6fcd8fa116c74970909bf825bccb41cf1f4e9fcaeb8798bbf6ece

• problem: HPO-Bench-Parkinson
• solver: RandomSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 1a9d1af4f9e06d9ece9493618039b9e074dfdedb67b5c7dcd430cc354b288d59

• problem: HPO-Bench-Parkinson
• solver: SobolSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 5ab87e6209dca0edfd73e94976fd5f56b5b3bb1c3b11c786020e85eba6f3c36d

• problem: HPO-Bench-Parkinson
• solver: _HaltonSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: eef681a027879aa61404021f82f6b347d431c86d6035339af36b11f9ee619b43

• problem: HPO-Bench-Parkinson
• solver: _HaltonSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 2be32ecf1888347a453e6edcb8d4b0bb1164c296baa4fdab1f234bcba8b0bac2

• problem: HPO-Bench-Parkinson
• solver: _RandomSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 2a41fed1c3fd71b1b8c1da7209a0cdfdbb99d08546b58adb081920f76fd7de0e

• problem: HPO-Bench-Parkinson
• solver: _SobolSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: ac3f46a652ff234a73259e0508efc5db57045bd52d85f3613fe9ae32a232b001

• problem: HPO-Bench-Parkinson
• solver: _SobolSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 6319dbdf05726f7351415050c0f351e8afac7d666973bce394069a53d300c96e

• problem: HPO-Bench-Parkinson
• solver: _TPESampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 635ee61b522645fe02bee037eb4e6fc02eb80fb616b54b69cdb7f0c952f566bc

• problem: HPO-Bench-Protein
• solver: HaltonSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 377e85d73c64b916c9b47bb30419f56205c71ed4f431e366d865ca04a421cc5e

• problem: HPO-Bench-Protein
• solver: MultivariateTPESampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 04d303779723462e1d7f7cfd2219e45519cfd288e25210754989c86b5ff6278c

• problem: HPO-Bench-Protein
• solver: RandomSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 70f544343bcdb91a1f3d2ca54a4624cc685e6a8134c78032818595f7d28dcd9f

• problem: HPO-Bench-Protein
• solver: SobolSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: ca828e1d3b9e920bad27654c263f8a82ed24fbce75d8419b0481c8b7ba79065c

• problem: HPO-Bench-Protein
• solver: _HaltonSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: bfde025a6f342e2fc84902958eb6782a122269b139a4cec3986368fb4339eb86

• problem: HPO-Bench-Protein
• solver: _HaltonSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: eca0e345f40f97c622f9a8badea8e5de61a3f649c253cc22962fd8feaa29fec4

• problem: HPO-Bench-Protein
• solver: _RandomSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 2e92f70144f9232a4796e9d06ffdb35eb171a6f8b9ce808810ae8ea7efb6c8af

• problem: HPO-Bench-Protein
• solver: _SobolSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: cba7559289c4b4cb18a7338b03f4eb3c8c7bf6830cd5f6e73102db89ab8ceb62

• problem: HPO-Bench-Protein
• solver: _SobolSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: a9680bf4925ae9468224d06e8cac8e720e831ee716dd92528ccbe763938595fe

• problem: HPO-Bench-Protein
• solver: _TPESampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 1f0f5203b0107c28d83b0bba4a5ee897218db2517a3d9a843237ef765c7d9e7a

• problem: HPO-Bench-Slice
• solver: HaltonSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: ba09aeb09b6cfa66086cd7e13475514f1c5a61bfac79f406137016c81c02af82

• problem: HPO-Bench-Slice
• solver: MultivariateTPESampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 9589edc0aa098e6bbe3ef4d9296163eabf412ca17c650cc080b7b9665cb0b499

• problem: HPO-Bench-Slice
• solver: RandomSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: 09b3cd284c4c094f54924eb05ec403c43a31d52c317254835d30b5752b0fc6b2

• problem: HPO-Bench-Slice
• solver: SobolSampler_python
• budget: 100
• repeats: 10
• concurrency: 1

ID: fe644cb5b85c18e2d3e3ffafc360506cc9cc2979218f63db9d1d15cbba27a34f

• problem: HPO-Bench-Slice
• solver: _HaltonSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: bd0202502fddfd608218c70d6f598c4a90f4ea085383ef0ed5dc88675434bcfe

• problem: HPO-Bench-Slice
• solver: _HaltonSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: abfd801df316e1ed1d366d617c704d2086b05c69a079fc894e1607f492a2881b

• problem: HPO-Bench-Slice
• solver: _RandomSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: f4e26954ca9d24958b105fa90c6bf76e93ed8f740fe596774bdf21f94b27364c

• problem: HPO-Bench-Slice
• solver: _SobolSampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: b1c4c07cf71287f4bb2a1ffa94a3c55278bdfe37e98d37a184804e383c3c834b

• problem: HPO-Bench-Slice
• solver: _SobolSampler_Scrambled_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

ID: 0cb5e5136b2a4b364234b4fb655bd7aa0cfeaeac66c9f792f12b3b9c8a48fc29

• problem: HPO-Bench-Slice
• solver: _TPESampler_NopPruner
• budget: 100
• repeats: 10
• concurrency: 1

run_all.sh

# json files should not contain any spaces and newlines only allowed for new solvers/problems.
kurobako studies --solvers $(cat solvers.json) --problems $(cat problems.json) > studies.json --repeats 100 --budget 100
kurobako run --parallelism 8 < studies.json > ./results/results.json
cat ./results/results.json | kurobako report > ./results/report.md
cat ./results/results.json | kurobako plot curve --errorbar --ymin 0 --xmin 10 -o ./results/plots/

sobol.py

from kurobako import solver
from kurobako.solver.optuna import OptunaSolverFactory
import optuna
from optuna.samplers import QMCSampler


def create_study(seed):
    sampler = QMCSampler(seed=seed, qmc_type="sobol", scramble=True, warn_independent_sampling=False)
    return optuna.create_study(sampler=sampler)


if __name__ == "__main__":
    factory = OptunaSolverFactory(create_study, name="SobolSampler_python")
    runner = solver.SolverRunner(factory)
    runner.run()

solvers.json

{"name":"_TPESampler_NopPruner","optuna":{"loglevel":"debug","sampler":"TPESampler","sampler_kwargs":"{\"multivariate\":true,\"constant_liar\":true}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_RandomSampler_NopPruner","optuna":{"loglevel":"debug","sampler":"RandomSampler","sampler_kwargs":"{}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_HaltonSampler_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_HaltonSampler_Scrambled_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{\"scramble\":true}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_SobolSampler_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{\"qmc_type\":\"sobol\"}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_SobolSampler_Scrambled_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{\"scramble\":true,\"qmc_type\":\"sobol\"}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"command":{"path":"python3","args":["halton.py"]}}
{"command":{"path":"python3","args":["sobol.py"]}}
{"command":{"path":"python3","args":["tpe.py"]}}
{"command":{"path":"python3","args":["random_.py"]}}

solvers_copy.json

{"name":"_TPESampler_NopPruner","optuna":{"loglevel":"debug","sampler":"TPESampler","sampler_kwargs":"{\"multivariate\":true,\"constant_liar\":true}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_RandomSampler_NopPruner","optuna":{"loglevel":"debug","sampler":"RandomSampler","sampler_kwargs":"{}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_HaltonSampler_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_HaltonSampler_Scrambled_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{\"scramble\":true}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_SobolSampler_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{\"qmc_type\":\"sobol\"}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"name":"_SobolSampler_Scrambled_NopPruner","optuna":{"loglevel":"debug","sampler":"QMCSampler","sampler_kwargs":"{\"scramble\":true,\"qmc_type\":\"sobol\"}","pruner":"NopPruner","pruner_kwargs":"{}"}}
{"command":{"path":"python3","args":["./random.py"]}}
{"command":{"path":"python3","args":["./tpe.py"]}}
{"command":{"path":"python3","args":["./sobol.py"]}}
{"command":{"path":"python3","args":["./halton.py"]}}

tpe.py

from kurobako import solver
from kurobako.solver.optuna import OptunaSolverFactory
import optuna


def create_study(seed):
    sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True, constant_liar=True)
    return optuna.create_study(sampler=sampler)


if __name__ == "__main__":
    factory = OptunaSolverFactory(create_study, name="MultivariateTPESampler_python")
    runner = solver.SolverRunner(factory)
    runner.run()