For https://github.com/optuna/optuna/pull/4661

run_cmaes_benchmark.py

import argparse
import os
import subprocess


def run(args: argparse.Namespace) -> None:
    kurobako_cmd = os.path.join(args.path_to_kurobako, "kurobako")
    subprocess.run(f"{kurobako_cmd} --version", shell=True)

    if not (os.path.exists(args.data_dir) and os.path.isdir(args.data_dir)):
        raise ValueError(f"Data directory {args.data_dir} cannot be found.")

    os.makedirs(args.out_dir, exist_ok=True)
    study_json_fn = os.path.join(args.out_dir, "studies.json")
    solvers_filename = os.path.join(args.out_dir, "solvers.json")
    problems_filename = os.path.join(args.out_dir, "problems.json")

    # Ensure all files are empty.
    for filename in [study_json_fn, solvers_filename, problems_filename]:
        with open(filename, "w"):
            pass

    # Create HPO bench problem.
    datasets = [
        "fcnet_tabular_benchmarks/fcnet_naval_propulsion_data.hdf5",
        "fcnet_tabular_benchmarks/fcnet_parkinsons_telemonitoring_data.hdf5",
        "fcnet_tabular_benchmarks/fcnet_protein_structure_data.hdf5",
        "fcnet_tabular_benchmarks/fcnet_slice_localization_data.hdf5",
    ]
    for dataset in datasets:
        dataset = os.path.join(args.data_dir, dataset)
        cmd = (
            f'{kurobako_cmd} problem hpobench "{dataset}" | tee -a {problems_filename}'
        )
        subprocess.run(cmd, shell=True)

    # Create Optuna solvers
    for name, sampler, sampler_kwargs in [
        ("random", "RandomSampler", r"{}"),
        ("vanilla-cmaes", "CmaEsSampler", r"{}"),
        ("cmaes-with-margin", "CmaEsSampler", r"{\"with_margin\":true}"),
    ]:
        cmd = (
            f"{kurobako_cmd} solver --name {name} optuna --loglevel debug "
            f"--sampler {sampler} --sampler-kwargs {sampler_kwargs} "
            "--pruner NopPruner --pruner-kwargs {} "
            f"| tee -a {solvers_filename}"
        )
        subprocess.run(cmd, shell=True)

    # Create study.
    cmd = (
        f"{kurobako_cmd} studies --budget {args.budget} "
        f"--solvers $(cat {solvers_filename}) --problems $(cat {problems_filename}) "
        f"--repeats {args.n_runs} --seed {args.seed} --concurrency {args.n_concurrency} "
        f"> {study_json_fn}"
    )
    subprocess.run(cmd, shell=True, check=True)

    result_filename = os.path.join(args.out_dir, "results.json")
    cmd = (
        f"cat {study_json_fn} | {kurobako_cmd} run --parallelism {args.n_jobs} "
        f"> {result_filename}"
    )
    subprocess.run(cmd, shell=True)

    report_filename = os.path.join(args.out_dir, "report.md")
    cmd = f"cat {result_filename} | {kurobako_cmd} report > {report_filename}"
    subprocess.run(cmd, shell=True)

    cmd = (
        f"cat {result_filename} | docker run -v $(pwd)/{args.out_dir}/images:/images/ "
        f"--rm -i sile/kurobako plot curve --errorbar --xmin 10"
    )
    subprocess.run(cmd, shell=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--path-to-kurobako", type=str, default="")
    parser.add_argument("--budget", type=int, default=2000)
    parser.add_argument("--n-runs", type=int, default=10)
    parser.add_argument("--n-jobs", type=int, default=10)
    parser.add_argument("--n-concurrency", type=int, default=1)
    parser.add_argument("--seed", type=int, default=0)
    parser.add_argument("--data-dir", type=str, default="tmp/data")
    parser.add_argument("--out-dir", type=str, default="tmp/benchmark_report")
    args = parser.parse_args()

    run(args)

Problem Name	Before (master)	After (PR)
Slice
Protein
Parkinson
Naval