- Report ID: 62ed67b3c37af40a5c8c830c9383cd41ec1fb4552587c897a8245edd944d289f
- Kurobako Version: 0.1.2
- Number of Solvers: 4
- Number of Problems: 2
- Metrics Precedence:
best value -> AUC -> elapsed time
Please refer to "A Strategy for Ranking Optimizers using Multiple Criteria" for the ranking strategy used in this report.
Solver | Borda | Firsts |
---|---|---|
RandomSampler | 0 | 0 |
TPESampler | 3 | 0 |
TPESampler with MedianPruner | 5 | 1 |
TPESampler with SuccessiveHalvingPruner | 4 | 1 |
(1) Problem: HPO-Bench-Protein
Ranking | Solver | Best (avg +- sd) | AUC (avg +- sd) | Elapsed (avg +- sd) |
---|---|---|---|---|
1 | TPESampler with SuccessiveHalvingPruner (study) | 0.228974 +- 0.006860 | 16.545 +- 0.579 | 19.918 +- 9.515 |
2 | TPESampler with MedianPruner (study) | 0.227354 +- 0.005347 | 16.828 +- 0.604 | 125.306 +- 10.155 |
3 | TPESampler (study) | 0.233030 +- 0.007815 | 17.797 +- 0.940 | 2.466 +- 0.187 |
4 | RandomSampler (study) | 0.251476 +- 0.015849 | 18.784 +- 1.067 | 0.490 +- 0.033 |
(2) Problem: HPO-Bench-Parkinson
Ranking | Solver | Best (avg +- sd) | AUC (avg +- sd) | Elapsed (avg +- sd) |
---|---|---|---|---|
1 | TPESampler with MedianPruner (study) | 0.010032 +- 0.004297 | 0.906 +- 0.295 | 126.191 +- 13.636 |
2 | TPESampler (study) | 0.010139 +- 0.002943 | 1.197 +- 0.390 | 2.427 +- 0.169 |
3 | TPESampler with SuccessiveHalvingPruner (study) | 0.012239 +- 0.003359 | 0.964 +- 0.282 | 21.887 +- 7.314 |
4 | RandomSampler (study) | 0.015099 +- 0.004315 | 1.445 +- 0.463 | 0.486 +- 0.028 |
recipe:
{
"name": "RandomSampler",
"optuna": {
"sampler": "random",
"pruner": "nop"
}
}
specification:
{
"name": "RandomSampler",
"attrs": {
"github": "https://github.com/optuna/optuna",
"paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
"version": "optuna=0.19.0, kurobako-py=0.1.0"
},
"capabilities": [
"UNIFORM_CONTINUOUS",
"UNIFORM_DISCRETE",
"LOG_UNIFORM_CONTINUOUS",
"CATEGORICAL",
"CONDITIONAL",
"CONCURRENT"
]
}
recipe:
{
"name": "TPESampler",
"optuna": {
"pruner": "nop"
}
}
specification:
{
"name": "TPESampler",
"attrs": {
"github": "https://github.com/optuna/optuna",
"paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
"version": "optuna=0.19.0, kurobako-py=0.1.0"
},
"capabilities": [
"UNIFORM_CONTINUOUS",
"UNIFORM_DISCRETE",
"LOG_UNIFORM_CONTINUOUS",
"CATEGORICAL",
"CONDITIONAL",
"CONCURRENT"
]
}
recipe:
{
"name": "TPESampler with MedianPruner",
"optuna": {
"median_warmup_steps": 4
}
}
specification:
{
"name": "TPESampler with MedianPruner",
"attrs": {
"github": "https://github.com/optuna/optuna",
"paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
"version": "optuna=0.19.0, kurobako-py=0.1.0"
},
"capabilities": [
"UNIFORM_CONTINUOUS",
"UNIFORM_DISCRETE",
"LOG_UNIFORM_CONTINUOUS",
"CATEGORICAL",
"CONDITIONAL",
"CONCURRENT"
]
}
recipe:
{
"name": "TPESampler with SuccessiveHalvingPruner",
"optuna": {
"pruner": "asha",
"asha_min_resource": 4
}
}
specification:
{
"name": "TPESampler with SuccessiveHalvingPruner",
"attrs": {
"github": "https://github.com/optuna/optuna",
"paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
"version": "optuna=0.19.0, kurobako-py=0.1.0"
},
"capabilities": [
"UNIFORM_CONTINUOUS",
"UNIFORM_DISCRETE",
"LOG_UNIFORM_CONTINUOUS",
"CATEGORICAL",
"CONDITIONAL",
"CONCURRENT"
]
}
recipe:
{
"hpobench": {
"dataset": "fcnet_parkinsons_telemonitoring_data.hdf5"
}
}
specification:
{
"name": "HPO-Bench-Parkinson",
"attrs": {
"github": "https://github.com/automl/nas_benchmarks",
"paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
"version": "kurobako_problems=0.1.2"
},
"params_domain": [
{
"name": "activation_fn_1",
"range": {
"type": "CATEGORICAL",
"choices": [
"tanh",
"relu"
]
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "activation_fn_2",
"range": {
"type": "CATEGORICAL",
"choices": [
"tanh",
"relu"
]
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "batch_size",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 4
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "dropout_1",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 3
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "dropout_2",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 3
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "init_lr",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 6
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "lr_schedule",
"range": {
"type": "CATEGORICAL",
"choices": [
"cosine",
"const"
]
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "n_units_1",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 6
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "n_units_2",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 6
},
"distribution": "UNIFORM",
"constraint": null
}
],
"values_domain": [
{
"name": "Validation MSE",
"range": {
"type": "CONTINUOUS",
"low": 0.0
},
"distribution": "UNIFORM",
"constraint": null
}
],
"steps": 100
}
recipe:
{
"hpobench": {
"dataset": "fcnet_protein_structure_data.hdf5"
}
}
specification:
{
"name": "HPO-Bench-Protein",
"attrs": {
"github": "https://github.com/automl/nas_benchmarks",
"paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
"version": "kurobako_problems=0.1.2"
},
"params_domain": [
{
"name": "activation_fn_1",
"range": {
"type": "CATEGORICAL",
"choices": [
"tanh",
"relu"
]
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "activation_fn_2",
"range": {
"type": "CATEGORICAL",
"choices": [
"tanh",
"relu"
]
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "batch_size",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 4
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "dropout_1",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 3
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "dropout_2",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 3
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "init_lr",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 6
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "lr_schedule",
"range": {
"type": "CATEGORICAL",
"choices": [
"cosine",
"const"
]
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "n_units_1",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 6
},
"distribution": "UNIFORM",
"constraint": null
},
{
"name": "n_units_2",
"range": {
"type": "DISCRETE",
"low": 0,
"high": 6
},
"distribution": "UNIFORM",
"constraint": null
}
],
"values_domain": [
{
"name": "Validation MSE",
"range": {
"type": "CONTINUOUS",
"low": 0.0
},
"distribution": "UNIFORM",
"constraint": null
}
],
"steps": 100
}
- problem: HPO-Bench-Parkinson
- solver: RandomSampler
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Parkinson
- solver: TPESampler
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Parkinson
- solver: TPESampler with MedianPruner
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Parkinson
- solver: TPESampler with SuccessiveHalvingPruner
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Protein
- solver: RandomSampler
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Protein
- solver: TPESampler
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Protein
- solver: TPESampler with MedianPruner
- budget: 80
- repeats: 30
- concurrency: 1
- problem: HPO-Bench-Protein
- solver: TPESampler with SuccessiveHalvingPruner
- budget: 80
- repeats: 30
- concurrency: 1