Benchmark script for `-Zno-embed-metadata`

benchmark.py

# /// script
# dependencies = [
#   "pandas>=2",
# ]
# ///

# Run e.g. with `uv run benchmark.py`

import dataclasses
import datetime
import os
import shutil
import subprocess
import time
from collections import defaultdict
from pathlib import Path
from typing import List, Tuple
import pandas as pd


@dataclasses.dataclass
class BenchmarkInput:
    cargo: Path
    rustc: Path
    release: bool
    embed_metadata: bool
    workspace: Path


@dataclasses.dataclass
class BenchmarkResult:
    duration: datetime.timedelta
    target_size: int


def run_benchmark(input: BenchmarkInput) -> BenchmarkResult:
    target_dir = input.workspace.absolute() / "target"
    shutil.rmtree(target_dir, ignore_errors=True)

    args = [str(input.cargo.absolute()), "build"]
    if input.release:
        args.append("--release")
    if not input.embed_metadata:
        args.append("-Zno-embed-metadata")
    env = os.environ.copy()
    env["RUSTC"] = str(input.rustc.absolute())

    print(input)
    start = time.time()
    output = subprocess.run(
        args,
        env=env,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=input.workspace,
    )
    if output.returncode != 0:
        stdout = output.stdout.decode("utf-8")
        stderr = output.stderr.decode("utf-8")
        raise Exception(
            f"Command {args} ended with exit code {output.returncode}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
        )

    duration = datetime.timedelta(seconds=time.time() - start)
    result = BenchmarkResult(
        duration=duration, target_size=get_directory_size(target_dir)
    )
    print(f"{result}\n")
    return result


def get_directory_size(path: Path) -> int:
    output = subprocess.check_output(["du", "-s", str(path)])
    for line in output.decode("utf-8").splitlines():
        line = line.strip()
        return int(line.split()[0])
    assert False, "No output found"


def rustup_resolve_bin(toolchain: str, bin: str) -> Path:
    return Path(
        subprocess.check_output(["rustup", f"+{toolchain}", "which", bin])
        .decode("utf-8")
        .strip()
    )


def run_benchmarks(
    inputs: List[BenchmarkInput],
) -> List[Tuple[BenchmarkInput, BenchmarkResult]]:
    results = []
    for input in inputs:
        result = run_benchmark(input)
        results.append((input, result))
    return results


results_path = Path("out.csv")
prev_results = None
if results_path.is_file():
    prev_results = pd.read_csv(results_path)

benchmarks = []
# Put your benchmark crates here
crates = ["hyperqueue", "cargo"]
for crate in crates:
    for release in (False, True):
        for embed_metadata in (False, True):
            input = BenchmarkInput(
                cargo=rustup_resolve_bin("nightly", "cargo"),
                rustc=rustup_resolve_bin("nightly", "rustc"),
                release=release,
                embed_metadata=embed_metadata,
                workspace=Path(crate),
            )
            if prev_results is not None:
                if (
                    len(
                        prev_results[
                            (prev_results["embed-metadata"] == embed_metadata)
                            & (prev_results["release"] == release)
                            & (prev_results["benchmark"] == str(input.workspace))
                        ]
                    )
                ) > 0:
                    print(f"Skipping {input}")
                    continue

            benchmarks.append(input)

results = run_benchmarks(benchmarks)

df = defaultdict(list)
for input, result in results:
    print(input)
    print(result)
    df["embed-metadata"].append(input.embed_metadata)
    df["release"].append(input.release)
    df["benchmark"].append(str(input.workspace))
    df["duration"].append(result.duration.total_seconds())
    df["target-size"].append(result.target_size)

df = pd.DataFrame(df)
if prev_results is not None:
    df = pd.concat((prev_results, df))
df.to_csv("out.csv", index=False)

Looks like there is a mistake on lines 43-44.

    if input.embed_metadata:
        pass

It should instead be:

    if not input.embed_metadata:
        args.append("-Zno-embed-metadata")

Oops, that was probably some debugging leftover. Thanks!