An example of kurokoba benchmark report.

example_report.md

Benchmark Result Report

• Report ID: 62ed67b3c37af40a5c8c830c9383cd41ec1fb4552587c897a8245edd944d289f
• Kurobako Version: 0.1.2
• Number of Solvers: 4
• Number of Problems: 2
• Metrics Precedence: best value -> AUC -> elapsed time

Please refer to "A Strategy for Ranking Optimizers using Multiple Criteria" for the ranking strategy used in this report.

Table of Contents

1. Overall Results
2. Individual Results
3. Solvers
4. Problems
5. Studies

Overall Results

Solver	Borda	Firsts
RandomSampler	0	0
TPESampler	3	0
TPESampler with MedianPruner	5	1
TPESampler with SuccessiveHalvingPruner	4	1

Individual Results

(1) Problem: HPO-Bench-Protein

Ranking	Solver	Best (avg +- sd)	AUC (avg +- sd)	Elapsed (avg +- sd)
1	TPESampler with SuccessiveHalvingPruner (study)	0.228974 +- 0.006860	16.545 +- 0.579	19.918 +- 9.515
2	TPESampler with MedianPruner (study)	0.227354 +- 0.005347	16.828 +- 0.604	125.306 +- 10.155
3	TPESampler (study)	0.233030 +- 0.007815	17.797 +- 0.940	2.466 +- 0.187
4	RandomSampler (study)	0.251476 +- 0.015849	18.784 +- 1.067	0.490 +- 0.033

(2) Problem: HPO-Bench-Parkinson

Ranking	Solver	Best (avg +- sd)	AUC (avg +- sd)	Elapsed (avg +- sd)
1	TPESampler with MedianPruner (study)	0.010032 +- 0.004297	0.906 +- 0.295	126.191 +- 13.636
2	TPESampler (study)	0.010139 +- 0.002943	1.197 +- 0.390	2.427 +- 0.169
3	TPESampler with SuccessiveHalvingPruner (study)	0.012239 +- 0.003359	0.964 +- 0.282	21.887 +- 7.314
4	RandomSampler (study)	0.015099 +- 0.004315	1.445 +- 0.463	0.486 +- 0.028

Solvers

ID: 5bbf512911e2fb8399759136a96da61083895cf701f4eb4b02b82b685e4f9823

recipe:

{
  "name": "RandomSampler",
  "optuna": {
    "sampler": "random",
    "pruner": "nop"
  }
}

specification:

{
  "name": "RandomSampler",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=0.19.0, kurobako-py=0.1.0"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "CATEGORICAL",
    "CONDITIONAL",
    "CONCURRENT"
  ]
}

ID: 6bead68572afd302421695fee7a215caff657b35b3500167f8f72d11e7a06331

recipe:

{
  "name": "TPESampler",
  "optuna": {
    "pruner": "nop"
  }
}

specification:

{
  "name": "TPESampler",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=0.19.0, kurobako-py=0.1.0"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "CATEGORICAL",
    "CONDITIONAL",
    "CONCURRENT"
  ]
}

ID: 9618c6b63a9af6753f480723eb758dcd11dfe253c08042b7b7c5c4f853004062

recipe:

{
  "name": "TPESampler with MedianPruner",
  "optuna": {
    "median_warmup_steps": 4
  }
}

specification:

{
  "name": "TPESampler with MedianPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=0.19.0, kurobako-py=0.1.0"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "CATEGORICAL",
    "CONDITIONAL",
    "CONCURRENT"
  ]
}

ID: 3a6863792684248c1e7a56d7e27b644c8ac09cc1025a375bd2a0687f7db9af04

recipe:

{
  "name": "TPESampler with SuccessiveHalvingPruner",
  "optuna": {
    "pruner": "asha",
    "asha_min_resource": 4
  }
}

specification:

{
  "name": "TPESampler with SuccessiveHalvingPruner",
  "attrs": {
    "github": "https://github.com/optuna/optuna",
    "paper": "Akiba, Takuya, et al. \"Optuna: A next-generation hyperparameter optimization framework.\" Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2019.",
    "version": "optuna=0.19.0, kurobako-py=0.1.0"
  },
  "capabilities": [
    "UNIFORM_CONTINUOUS",
    "UNIFORM_DISCRETE",
    "LOG_UNIFORM_CONTINUOUS",
    "CATEGORICAL",
    "CONDITIONAL",
    "CONCURRENT"
  ]
}

Problems

ID: 1d1cf159c9a05ae1ab2d37ee01e2d2b7e0e75c344f118b985afd4fb8215157e8

recipe:

{
  "hpobench": {
    "dataset": "fcnet_parkinsons_telemonitoring_data.hdf5"
  }
}

specification:

{
  "name": "HPO-Bench-Parkinson",
  "attrs": {
    "github": "https://github.com/automl/nas_benchmarks",
    "paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
    "version": "kurobako_problems=0.1.2"
  },
  "params_domain": [
    {
      "name": "activation_fn_1",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "activation_fn_2",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "batch_size",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 4
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "dropout_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "dropout_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "init_lr",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "lr_schedule",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "cosine",
          "const"
        ]
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "n_units_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "n_units_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM",
      "constraint": null
    }
  ],
  "values_domain": [
    {
      "name": "Validation MSE",
      "range": {
        "type": "CONTINUOUS",
        "low": 0.0
      },
      "distribution": "UNIFORM",
      "constraint": null
    }
  ],
  "steps": 100
}

ID: 007a182ddea33a47c9e27e1edb07ff9232478e0b863d85edaf9e499be5e770d8

recipe:

{
  "hpobench": {
    "dataset": "fcnet_protein_structure_data.hdf5"
  }
}

specification:

{
  "name": "HPO-Bench-Protein",
  "attrs": {
    "github": "https://github.com/automl/nas_benchmarks",
    "paper": "Klein, Aaron, and Frank Hutter. \"Tabular Benchmarks for Joint Architecture and Hyperparameter Optimization.\" arXiv preprint arXiv:1905.04970 (2019).",
    "version": "kurobako_problems=0.1.2"
  },
  "params_domain": [
    {
      "name": "activation_fn_1",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "activation_fn_2",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "tanh",
          "relu"
        ]
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "batch_size",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 4
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "dropout_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "dropout_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 3
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "init_lr",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "lr_schedule",
      "range": {
        "type": "CATEGORICAL",
        "choices": [
          "cosine",
          "const"
        ]
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "n_units_1",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM",
      "constraint": null
    },
    {
      "name": "n_units_2",
      "range": {
        "type": "DISCRETE",
        "low": 0,
        "high": 6
      },
      "distribution": "UNIFORM",
      "constraint": null
    }
  ],
  "values_domain": [
    {
      "name": "Validation MSE",
      "range": {
        "type": "CONTINUOUS",
        "low": 0.0
      },
      "distribution": "UNIFORM",
      "constraint": null
    }
  ],
  "steps": 100
}

Studies

ID: ca02d174780afb82db130e9b2feae8b8671552b42a82dbdba2a6b7122e838c4f

• problem: HPO-Bench-Parkinson
• solver: RandomSampler
• budget: 80
• repeats: 30
• concurrency: 1

ID: 53740ce3b9507abbc84d38b9f4ced0689e5ad009d54a0ee07c735964f0e09f26

• problem: HPO-Bench-Parkinson
• solver: TPESampler
• budget: 80
• repeats: 30
• concurrency: 1

ID: 4b5aaab282e7ad9b30d1ad0ccee7dc1a54733a481d3051d97fbce47e14e947e5

• problem: HPO-Bench-Parkinson
• solver: TPESampler with MedianPruner
• budget: 80
• repeats: 30
• concurrency: 1

ID: 5003170772452078f4cf0c207fc6601f18bc8cc8395e5a338a74157a244085d5

• problem: HPO-Bench-Parkinson
• solver: TPESampler with SuccessiveHalvingPruner
• budget: 80
• repeats: 30
• concurrency: 1

ID: a24912e3c978e0f49b782a04ac42183d5b011324791d6fd2407777483ba1573e

• problem: HPO-Bench-Protein
• solver: RandomSampler
• budget: 80
• repeats: 30
• concurrency: 1

ID: 072a57f416255d80ce490df266672d5fac8d200158df1f2d969a2db7526ca94f

• problem: HPO-Bench-Protein
• solver: TPESampler
• budget: 80
• repeats: 30
• concurrency: 1

ID: 05106a894aed96547cbc1dbd5cc01dc898f3589ce55d4100af1016977112e1c8

• problem: HPO-Bench-Protein
• solver: TPESampler with MedianPruner
• budget: 80
• repeats: 30
• concurrency: 1

ID: 54f1ecaeccb35d4e218fef1e194bd626b983ddf414231960020e65de283ab71b

• problem: HPO-Bench-Protein
• solver: TPESampler with SuccessiveHalvingPruner
• budget: 80
• repeats: 30
• concurrency: 1