Evaluation benchmark for benchmark with different batchsizes

00_eval.txt

# Batch size: 1000
## Query all packages
elapsed: 24.68 s, heap size: 2610.18 MiB
## Evaluating packages in batches
elapsed: 15.04 s, heap size: 1217.18 MiB
elapsed: 18.31 s, heap size: 1377.18 MiB
elapsed: 9.13 s, heap size: 673.18 MiB
elapsed: 11.24 s, heap size: 769.18 MiB
elapsed: 16.69 s, heap size: 1169.18 MiB
elapsed: 3.50 s, heap size: 384.31 MiB
elapsed: 3.12 s, heap size: 384.31 MiB
elapsed: 3.40 s, heap size: 384.31 MiB
elapsed: 3.83 s, heap size: 384.31 MiB
elapsed: 3.29 s, heap size: 384.31 MiB
elapsed: 10.04 s, heap size: 672.68 MiB
elapsed: 13.84 s, heap size: 1121.18 MiB
elapsed: 8.33 s, heap size: 641.18 MiB
elapsed: 13.48 s, heap size: 881.18 MiB
elapsed: 10.16 s, heap size: 865.18 MiB
elapsed: 12.03 s, heap size: 913.18 MiB
elapsed: 12.38 s, heap size: 849.18 MiB
elapsed: 23.17 s, heap size: 481.18 MiB
elapsed: 15.50 s, heap size: 672.68 MiB
elapsed: 8.07 s, heap size: 432.43 MiB
elapsed: 6.32 s, heap size: 432.43 MiB
elapsed: 9.17 s, heap size: 465.18 MiB
elapsed: 18.98 s, heap size: 897.18 MiB
elapsed: 12.32 s, heap size: 865.18 MiB
elapsed: 11.39 s, heap size: 945.18 MiB
elapsed: 13.37 s, heap size: 753.18 MiB
elapsed: 11.20 s, heap size: 705.18 MiB
elapsed: 3.13 s, heap size: 384.25 MiB
elapsed: 3.06 s, heap size: 384.31 MiB
elapsed: 2.98 s, heap size: 384.31 MiB
elapsed: 16.03 s, heap size: 897.18 MiB
elapsed: 11.64 s, heap size: 801.18 MiB
elapsed: 7.56 s, heap size: 464.68 MiB
elapsed: 7.13 s, heap size: 528.68 MiB
elapsed: 6.10 s, heap size: 432.68 MiB
elapsed: 7.68 s, heap size: 624.68 MiB
elapsed: 17.42 s, heap size: 544.68 MiB
elapsed: 11.08 s, heap size: 496.68 MiB
elapsed: 6.15 s, heap size: 432.68 MiB
elapsed: 11.21 s, heap size: 608.68 MiB
elapsed: 7.94 s, heap size: 448.68 MiB
elapsed: 7.29 s, heap size: 592.68 MiB
elapsed: 5.28 s, heap size: 400.43 MiB
elapsed: 7.35 s, heap size: 592.68 MiB
elapsed: 6.56 s, heap size: 496.68 MiB
elapsed: 13.77 s, heap size: 1121.18 MiB
elapsed: 4.33 s, heap size: 384.25 MiB
elapsed: 11.03 s, heap size: 784.68 MiB
elapsed: 3.91 s, heap size: 384.25 MiB
elapsed: 2.30 s, heap size: 384.25 MiB
elapsed: 3.77 s, heap size: 384.25 MiB
elapsed: 2.28 s, heap size: 384.25 MiB
elapsed: 11.91 s, heap size: 1057.18 MiB
elapsed: 13.30 s, heap size: 929.18 MiB
elapsed: 15.09 s, heap size: 1121.18 MiB
elapsed: 9.03 s, heap size: 496.68 MiB
elapsed: 6.60 s, heap size: 512.68 MiB
elapsed: 18.44 s, heap size: 705.18 MiB
elapsed: 10.54 s, heap size: 849.18 MiB

Total elapsed: 9.92 min


# Batch size: 4000
## Query all packages
elapsed: 27.81 s, heap size: 2610.18 MiB
## Evaluating packages in batches
elapsed: 33.82 s, heap size: 2369.18 MiB
elapsed: 24.12 s, heap size: 1521.18 MiB
elapsed: 20.90 s, heap size: 1697.18 MiB
elapsed: 22.12 s, heap size: 1793.18 MiB
elapsed: 19.50 s, heap size: 1441.18 MiB
elapsed: 35.08 s, heap size: 1313.18 MiB
elapsed: 24.20 s, heap size: 1425.18 MiB
elapsed: 17.53 s, heap size: 1361.18 MiB
elapsed: 14.11 s, heap size: 849.18 MiB
elapsed: 17.97 s, heap size: 1041.18 MiB
elapsed: 12.73 s, heap size: 977.18 MiB
elapsed: 23.20 s, heap size: 1553.18 MiB
elapsed: 5.88 s, heap size: 400.43 MiB
elapsed: 29.77 s, heap size: 2017.18 MiB
elapsed: 20.60 s, heap size: 1265.18 MiB

Total elapsed: 5.83 min


# Batch size: 10000
## Query all packages
elapsed: 25.12 s, heap size: 2610.18 MiB
## Evaluating packages in batches
elapsed: 43.95 s, heap size: 3041.18 MiB
elapsed: 48.64 s, heap size: 2978.18 MiB
elapsed: 38.98 s, heap size: 2225.18 MiB
elapsed: 26.52 s, heap size: 1953.18 MiB
elapsed: 27.96 s, heap size: 2113.18 MiB
elapsed: 45.68 s, heap size: 2545.18 MiB

Total elapsed: 4.29 min


# Batch size: 40000
## Query all packages
elapsed: 23.34 s, heap size: 2594.18 MiB
## Evaluating packages in batches
elapsed: 117.78 s, heap size: 6932.18 MiB
elapsed: 54.00 s, heap size: 3842.18 MiB

Total elapsed: 3.26 min


# Batch size: 100000
## Query all packages
elapsed: 29.31 s, heap size: 2610.18 MiB
## Evaluating packages in batches
elapsed: 154.43 s, heap size: 9140.19 MiB

Total elapsed: 3.08 min

eval-benchmark.py

import json
import os
import subprocess
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from timeit import default_timer as timer
from typing import Any


def print_stats(stat_path: Path, time: float) -> None:
    stats = json.loads(stat_path.read_text())
    heap_size = stats["gc"]["heapSize"]
    print(f"elapsed: {time:.2f} s, heap size: {heap_size / 1024 / 1024:.2f} MiB")


def batch(iterable, n=1):
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx : min(ndx + n, l)]


@dataclass
class Package:
    name: str
    path: str


def nix_env(file: str, extra_args: list[str], tmpdir: Path) -> dict[str, Any]:
    env = os.environ.copy()
    stats_path = tmpdir / "stats.json"
    env["NIX_SHOW_STATS_PATH"] = str(stats_path)
    env["NIX_SHOW_STATS"] = "1"
    packages_path = tmpdir / "packages.json"
    start = timer()
    subprocess.run(
        [
            "nix-env",
            "-f",
            file,
            "-qaP",
            "--json",
            "--show-trace",
        ]
        + extra_args,
        check=True,
        stdout=open(packages_path, "w"),
        cwd=tmpdir,
        env=env,
    )
    end = timer()
    print_stats(stats_path, end - start)
    return json.loads(packages_path.read_text())


def escape_attr(attr: str) -> str:
    index = attr.rfind(".")
    if index == -1:
        return attr
    return f'{attr[:index]}."{attr[index+1:]}"'


def eval_batch(packages: list[str], tmpdir: Path) -> dict[str, Any]:
    subset_expr = tmpdir / "subset.nix"
    quoted_subset = [f"pkgs.{escape_attr(p)}\n" for p in packages]
    subset_expr.write_text(
        f"""
with import <nixpkgs> {{}}; [ 
{ ''.join(quoted_subset) } 
]"""
    )
    return nix_env(str(subset_expr), ["--out-path"], tmpdir)


def main():
    batch_size = int(os.environ.get("BATCH_SIZE", 40000))
    for batch_size in [1000, 4000, 10000, 40000, 100000]:
        print(f"# Batch size: {batch_size}")
        with TemporaryDirectory() as _tmpdir:
            tmpdir = Path(_tmpdir)

            start = timer()
            print("## Query all packages")
            packages = nix_env("<nixpkgs>", [], tmpdir)

            print("## Evaluating packages in batches")
            for subset in batch(list(packages.keys()), batch_size):
                eval_batch(subset, tmpdir)
            end = timer()
            print(f"Total elapsed: {(end - start)/ 60:.2f} min")


if __name__ == "__main__":
    main()