skatkov/README.md

## README.md

      
    Raw
  

              README.md
            
          
    RDoc RBS Member Lookup Benchmark

Benchmarks the two quadratic lookups replaced by parser-local Hash indexes in
RDoc::Parser::RBS:

Looking up methods in a growing Context#method_list.
Looking up attributes in a growing Context#attributes array.

The optimized parser keeps those ordered arrays unchanged. It builds one Hash
per context from each authoritative array and updates the indexes as RBS members
and aliases are added, reducing lookup work from quadratic to linear.
Usage

Ruby 3.2 or newer, RBS 4.0 or newer, Git, and an RDoc worktree at ../rdoc
are required.
./benchmark.rb
./benchmark.rb --micro-only
./benchmark.rb --declarations 5000 --runs 7
./benchmark.rb --rdoc /path/to/rdoc --base-ref master
The runner creates a temporary baseline worktree from master and compares it
with the selected RDoc working tree. Each variant runs in a fresh process, with
alternating execution order and one warmup before the measured runs.
Two workloads are measured independently:

One RBS class containing 2,000 unique methods.
One RBS class containing 2,000 unique attributes.

Parser-only timings isolate RBS parsing. End-to-end timings run the complete
Aliki pipeline with rdoc --dry-run, including parsing, store completion,
rendering, and search-index generation.
The benchmark verifies expected object counts and compares a structural SHA-256
covering class order, methods, attributes, signatures, and aliases. Reports also
record resolved commits, the optimized working-tree SHA-256, Ruby, and RBS
versions. Each run writes to a unique ignored tmp/run-* directory.
Reference Results

Five measured runs with 2,000 declarations per workload:


Parser-only workload
Baseline
Optimized
Speedup


RBS methods
0.1610s
0.0353s
4.6x


RBS attributes
0.1108s
0.0253s
4.4x


End-to-end workload
Baseline
Optimized
Change


RBS methods
0.722s
0.587s
-18.7%


RBS attributes
0.630s
0.569s
-9.8%


Environment and provenance:

Ruby 4.0.6, RBS 4.2.0.
Baseline commit 67b858164009135654110c96c26d77d4864d2006.
Optimized RDoc diff SHA-256 10088d4771cf98d8f6c90a88d5cdbefca8caea784b54115430d50293e7be0212.
Full report: tmp/run-20260829-1166718-mjvfgd/report.json.


## benchmark.rb
#!/usr/bin/env ruby
# frozen_string_literal: true

require "fileutils"
require "digest"
require "json"
require "optparse"
require "rbconfig"
require "rbs"
require "tmpdir"

ROOT = __dir__
SCRIPT = File.expand_path(__FILE__)
TMP_ROOT = File.join(ROOT, "tmp")
DEFAULT_RDOC = File.expand_path("../rdoc", ROOT)

def elapsed
  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
  yield
  Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
end

def parser_setup(file)
  options = RDoc::Options.new
  options.quiet = true
  store = RDoc::Store.new(options)
  top_level = store.add_file(file)
  [options, store, top_level, RDoc::Stats.new(store, 0)]
end

def child_benchmark
  require "rdoc"
  require "rdoc/parser"

  workload = ARGV.fetch(0)
  input = ARGV.fetch(1)
  count = Integer(ARGV.fetch(2))
  options, store, top_level, stats = parser_setup(input)

  abort "Unknown child workload: #{workload}" unless %w[rbs_methods rbs_attributes].include?(workload)

  content = File.binread(input)
  GC.start
  seconds = elapsed { RDoc::Parser::RBS.new(top_level, content, options, stats).scan }

  class_modules = store.all_classes_and_modules.sort_by(&:full_name)
  structure = [
    top_level.classes_or_modules.map(&:full_name),
    class_modules.map do |class_module|
      [
        class_module.full_name,
        class_module.method_list.map do |method|
          alias_name = method.is_alias_for.name if method.is_alias_for.respond_to?(:name)
          [method.name, method.singleton, alias_name, method.type_signature_lines]
        end,
        class_module.attributes.map do |attribute|
          [attribute.name, attribute.singleton, attribute.rw, attribute.type_signature_lines]
        end
      ]
    end
  ]
  result = {
    workload: workload,
    declarations: count,
    elapsed_seconds: seconds,
    locations: top_level.classes_or_modules.size,
    classes: class_modules.size,
    methods: class_modules.sum { |class_module| class_module.method_list.size },
    attributes: class_modules.sum { |class_module| class_module.attributes.size },
    structure_sha256: Digest::SHA256.hexdigest(Marshal.dump(structure))
  }
  puts JSON.generate(result)
end

if ENV["RDOC_SET_BENCHMARK_CHILD"]
  child_benchmark
  exit
end

def run!(*command, **options)
  return if system(*command, **options)

  abort "Command failed: #{command.join(" ")}"
end

def median(values)
  sorted = values.sort
  middle = sorted.length / 2
  return sorted[middle] if sorted.length.odd?

  (sorted[middle - 1] + sorted[middle]) / 2
end

def speedup(baseline, optimized)
  baseline / optimized
end

def child_result(rdoc, workload, input, declarations)
  env = {"RDOC_SET_BENCHMARK_CHILD" => "1"}
  command = [RbConfig.ruby, "-I#{File.join(rdoc, "lib")}", SCRIPT, workload, input, declarations.to_s]
  output = IO.popen(env, command, err: [:child, :out], &:read)
  abort "#{workload} benchmark failed:\n#{output}" unless $?.success?
  JSON.parse(output)
end

def command_output(*command)
  output = IO.popen(command, err: [:child, :out], &:read)
  abort "Command failed: #{command.join(" ")}\n#{output}" unless $?.success?
  output
end

def working_tree_sha256(rdoc)
  digest = Digest::SHA256.new
  digest << command_output("git", "-C", rdoc, "diff", "--binary", "HEAD")
  untracked = command_output("git", "-C", rdoc, "ls-files", "--others", "--exclude-standard", "-z").split("\0").sort
  untracked.each do |relative_path|
    digest << relative_path << "\0" << File.binread(File.join(rdoc, relative_path))
  end
  digest.hexdigest
end

def validate_workload!(workload, sample, declarations)
  expected = case workload
  when "rbs_methods"
    [1, 1, declarations, 0]
  when "rbs_attributes"
    [1, 1, 0, declarations]
  end
  actual = sample.values_at("locations", "classes", "methods", "attributes")
  abort "#{workload} produced #{actual.inspect}, expected #{expected.inspect}" unless actual == expected
end

def compare_micro(workload, paths, input, declarations, runs)
  paths.each_value { |rdoc| child_result(rdoc, workload, input, declarations) }
  results = {"baseline" => [], "optimized" => []}
  samples = {}

  runs.times do |index|
    order = index.even? ? %w[baseline optimized] : %w[optimized baseline]
    order.each do |variant|
      sample = child_result(paths.fetch(variant), workload, input, declarations)
      results.fetch(variant) << sample.fetch("elapsed_seconds")
      samples[variant] = sample
    end
  end

  comparable = samples.fetch("baseline").values_at("classes", "methods", "attributes", "structure_sha256") ==
    samples.fetch("optimized").values_at("classes", "methods", "attributes", "structure_sha256")
  abort "#{workload} parser results differ" unless comparable
  samples.each_value { |sample| validate_workload!(workload, sample, declarations) }

  {
    "workload" => workload,
    "declarations" => declarations,
    "baseline_seconds" => median(results.fetch("baseline")),
    "optimized_seconds" => median(results.fetch("optimized")),
    "structure_sha256" => samples.fetch("baseline").fetch("structure_sha256"),
    "results_match" => true
  }
end

def wall_time(rdoc, inputs, chdir)
  command = [
    RbConfig.ruby,
    "-I#{File.join(rdoc, "lib")}",
    File.join(rdoc, "exe/rdoc"),
    "--dry-run",
    "--quiet",
    *inputs
  ]
  elapsed do
    run!(*command, chdir: chdir, out: File::NULL, err: File::NULL)
  end
end

def compare_end_to_end(workload, paths, inputs, chdir, runs)
  paths.each_value { |rdoc| wall_time(rdoc, inputs, chdir) }
  results = {"baseline" => [], "optimized" => []}

  runs.times do |index|
    order = index.even? ? %w[baseline optimized] : %w[optimized baseline]
    order.each do |variant|
      results.fetch(variant) << wall_time(paths.fetch(variant), inputs, chdir)
    end
  end

  {
    "workload" => workload,
    "baseline_seconds" => median(results.fetch("baseline")),
    "optimized_seconds" => median(results.fetch("optimized"))
  }
end

def write_inputs(root, declarations)
  FileUtils.mkdir_p(root)
  methods_rbs = File.join(root, "methods.rbs")
  attributes_rbs = File.join(root, "attributes.rbs")

  File.write(methods_rbs, "class Bench\n" + declarations.times.map { |i| "  def m#{i}: () -> void" }.join("\n") + "\nend\n")
  File.write(attributes_rbs, "class Bench\n" + declarations.times.map { |i| "  attr_reader a#{i}: String" }.join("\n") + "\nend\n")

  {
    "rbs_methods" => methods_rbs,
    "rbs_attributes" => attributes_rbs
  }
end

def print_report(report)
  puts
  puts "Parser-only"
  puts "| Workload | Declarations | Baseline | Optimized | Speedup |"
  puts "| --- | ---: | ---: | ---: | ---: |"
  report.fetch("micro").each do |result|
    puts "| #{result.fetch("workload")} | #{result.fetch("declarations")} | #{format("%.4fs", result.fetch("baseline_seconds"))} | #{format("%.4fs", result.fetch("optimized_seconds"))} | #{format("%.1fx", speedup(result.fetch("baseline_seconds"), result.fetch("optimized_seconds")))} |"
  end

  return unless report["end_to_end"]

  puts
  puts "End-to-end `rdoc --dry-run`"
  puts "| Workload | Baseline | Optimized | Change |"
  puts "| --- | ---: | ---: | ---: |"
  report.fetch("end_to_end").each do |result|
    change = (result.fetch("optimized_seconds") / result.fetch("baseline_seconds") - 1) * 100
    puts "| #{result.fetch("workload")} | #{format("%.3fs", result.fetch("baseline_seconds"))} | #{format("%.3fs", result.fetch("optimized_seconds"))} | #{format("%+.1f%%", change)} |"
  end
end

options = {
  rdoc: DEFAULT_RDOC,
  base_ref: "master",
  declarations: 2_000,
  runs: 5,
  end_to_end: true
}
OptionParser.new do |parser|
  parser.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options]"
  parser.on("--rdoc PATH", "Optimized RDoc worktree (default: ../rdoc)") { |path| options[:rdoc] = File.expand_path(path) }
  parser.on("--base-ref REF", "Baseline Git ref (default: master)") { |ref| options[:base_ref] = ref }
  parser.on("--declarations COUNT", Integer, "Members per workload (default: 2000)") { |count| options[:declarations] = count }
  parser.on("--runs COUNT", Integer, "Measured runs per variant (default: 5)") { |count| options[:runs] = count }
  parser.on("--micro-only", "Skip end-to-end documentation generation") { options[:end_to_end] = false }
  parser.on("-h", "--help", "Show this help") { puts parser; exit }
end.parse!

abort "--declarations must be positive" unless options.fetch(:declarations).positive?
abort "--runs must be positive" unless options.fetch(:runs).positive?
rdoc = options.fetch(:rdoc)
abort "RDoc worktree does not exist: #{rdoc}" unless File.file?(File.join(rdoc, "lib/rdoc.rb"))

FileUtils.mkdir_p(TMP_ROOT)
run_root = Dir.mktmpdir("run-", TMP_ROOT)
baseline = File.join(run_root, "baseline-rdoc")
baseline_added = false

begin
  run!("git", "-C", rdoc, "worktree", "add", "--detach", baseline, options.fetch(:base_ref))
  baseline_added = true
  inputs = write_inputs(File.join(run_root, "inputs"), options.fetch(:declarations))
  paths = {"baseline" => baseline, "optimized" => rdoc}
  micro = %w[rbs_methods rbs_attributes].map do |workload|
    compare_micro(workload, paths, inputs.fetch(workload), options.fetch(:declarations), options.fetch(:runs))
  end

  report = {
    "ruby" => RUBY_DESCRIPTION,
    "rbs" => RBS::VERSION,
    "base_ref" => options.fetch(:base_ref),
    "base_commit" => command_output("git", "-C", baseline, "rev-parse", "HEAD^{commit}").strip,
    "optimized_commit" => command_output("git", "-C", rdoc, "rev-parse", "HEAD").strip,
    "optimized_worktree_sha256" => working_tree_sha256(rdoc),
    "runs" => options.fetch(:runs),
    "micro" => micro
  }

  if options.fetch(:end_to_end)
    report["end_to_end"] = [
      compare_end_to_end("rbs_methods", paths, [inputs.fetch("rbs_methods")], ROOT, options.fetch(:runs)),
      compare_end_to_end("rbs_attributes", paths, [inputs.fetch("rbs_attributes")], ROOT, options.fetch(:runs))
    ]
  end

  report_path = File.join(run_root, "report.json")
  File.write(report_path, JSON.pretty_generate(report))
  print_report(report)
  puts
  puts "JSON report: #{report_path}"
ensure
  if baseline_added
    run!("git", "-C", rdoc, "worktree", "remove", baseline)
  else
    FileUtils.rm_rf(baseline)
  end
end

## report.json
{
  "ruby": "ruby 4.0.6 (2026-07-14 revision 03b6d3f889) +PRISM [x86_64-linux]",
  "rbs": "4.2.0",
  "base_ref": "master",
  "base_commit": "67b858164009135654110c96c26d77d4864d2006",
  "optimized_commit": "67b858164009135654110c96c26d77d4864d2006",
  "optimized_worktree_sha256": "10088d4771cf98d8f6c90a88d5cdbefca8caea784b54115430d50293e7be0212",
  "runs": 5,
  "micro": [
    {
      "workload": "rbs_methods",
      "declarations": 2000,
      "baseline_seconds": 0.16101996799989138,
      "optimized_seconds": 0.03532826800073963,
      "structure_sha256": "cfa2fd70f5a24f6307250017bfa5a2fc6366087b8c3dcb8561be5beda5dad0de",
      "results_match": true
    },
    {
      "workload": "rbs_attributes",
      "declarations": 2000,
      "baseline_seconds": 0.11075896100373939,
      "optimized_seconds": 0.02530622499762103,
      "structure_sha256": "ffdd6b6ba7e6028a9cfa421899fc99c623fd853a52614406b1da2d1967565e0d",
      "results_match": true
    }
  ],
  "end_to_end": [
    {
      "workload": "rbs_methods",
      "baseline_seconds": 0.7218424330058042,
      "optimized_seconds": 0.5866772239969578
    },
    {
      "workload": "rbs_attributes",
      "baseline_seconds": 0.6301159760041628,
      "optimized_seconds": 0.5686040639993735
    }
  ]
}
Parser-only workload	Baseline	Optimized	Speedup
RBS methods	0.1610s	0.0353s	4.6x
RBS attributes	0.1108s	0.0253s	4.4x
End-to-end workload	Baseline	Optimized	Change
RBS methods	0.722s	0.587s	-18.7%
RBS attributes	0.630s	0.569s	-9.8%
	#!/usr/bin/env ruby
	# frozen_string_literal: true

	require "fileutils"
	require "digest"
	require "json"
	require "optparse"
	require "rbconfig"
	require "rbs"
	require "tmpdir"

	ROOT = __dir__
	SCRIPT = File.expand_path(__FILE__)
	TMP_ROOT = File.join(ROOT, "tmp")
	DEFAULT_RDOC = File.expand_path("../rdoc", ROOT)

	def elapsed
	started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
	yield
	Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
	end

	def parser_setup(file)
	options = RDoc::Options.new
	options.quiet = true
	store = RDoc::Store.new(options)
	top_level = store.add_file(file)
	[options, store, top_level, RDoc::Stats.new(store, 0)]
	end

	def child_benchmark
	require "rdoc"
	require "rdoc/parser"

	workload = ARGV.fetch(0)
	input = ARGV.fetch(1)
	count = Integer(ARGV.fetch(2))
	options, store, top_level, stats = parser_setup(input)

	abort "Unknown child workload: #{workload}" unless %w[rbs_methods rbs_attributes].include?(workload)

	content = File.binread(input)
	GC.start
	seconds = elapsed { RDoc::Parser::RBS.new(top_level, content, options, stats).scan }

	class_modules = store.all_classes_and_modules.sort_by(&:full_name)
	structure = [
	top_level.classes_or_modules.map(&:full_name),
	class_modules.map do \|class_module\|
	[
	class_module.full_name,
	class_module.method_list.map do \|method\|
	alias_name = method.is_alias_for.name if method.is_alias_for.respond_to?(:name)
	[method.name, method.singleton, alias_name, method.type_signature_lines]
	end,
	class_module.attributes.map do \|attribute\|
	[attribute.name, attribute.singleton, attribute.rw, attribute.type_signature_lines]
	end
	]
	end
	]
	result = {
	workload: workload,
	declarations: count,
	elapsed_seconds: seconds,
	locations: top_level.classes_or_modules.size,
	classes: class_modules.size,
	methods: class_modules.sum { \|class_module\| class_module.method_list.size },
	attributes: class_modules.sum { \|class_module\| class_module.attributes.size },
	structure_sha256: Digest::SHA256.hexdigest(Marshal.dump(structure))
	}
	puts JSON.generate(result)
	end

	if ENV["RDOC_SET_BENCHMARK_CHILD"]
	child_benchmark
	exit
	end

	def run!(command, *options)
	return if system(command, *options)

	abort "Command failed: #{command.join(" ")}"
	end

	def median(values)
	sorted = values.sort
	middle = sorted.length / 2
	return sorted[middle] if sorted.length.odd?

	(sorted[middle - 1] + sorted[middle]) / 2
	end

	def speedup(baseline, optimized)
	baseline / optimized
	end

	def child_result(rdoc, workload, input, declarations)
	env = {"RDOC_SET_BENCHMARK_CHILD" => "1"}
	command = [RbConfig.ruby, "-I#{File.join(rdoc, "lib")}", SCRIPT, workload, input, declarations.to_s]
	output = IO.popen(env, command, err: [:child, :out], &:read)
	abort "#{workload} benchmark failed:\n#{output}" unless $?.success?
	JSON.parse(output)
	end

	def command_output(*command)
	output = IO.popen(command, err: [:child, :out], &:read)
	abort "Command failed: #{command.join(" ")}\n#{output}" unless $?.success?
	output
	end

	def working_tree_sha256(rdoc)
	digest = Digest::SHA256.new
	digest << command_output("git", "-C", rdoc, "diff", "--binary", "HEAD")
	untracked = command_output("git", "-C", rdoc, "ls-files", "--others", "--exclude-standard", "-z").split("\0").sort
	untracked.each do \|relative_path\|
	digest << relative_path << "\0" << File.binread(File.join(rdoc, relative_path))
	end
	digest.hexdigest
	end

	def validate_workload!(workload, sample, declarations)
	expected = case workload
	when "rbs_methods"
	[1, 1, declarations, 0]
	when "rbs_attributes"
	[1, 1, 0, declarations]
	end
	actual = sample.values_at("locations", "classes", "methods", "attributes")
	abort "#{workload} produced #{actual.inspect}, expected #{expected.inspect}" unless actual == expected
	end

	def compare_micro(workload, paths, input, declarations, runs)
	paths.each_value { \|rdoc\| child_result(rdoc, workload, input, declarations) }
	results = {"baseline" => [], "optimized" => []}
	samples = {}

	runs.times do \|index\|
	order = index.even? ? %w[baseline optimized] : %w[optimized baseline]
	order.each do \|variant\|
	sample = child_result(paths.fetch(variant), workload, input, declarations)
	results.fetch(variant) << sample.fetch("elapsed_seconds")
	samples[variant] = sample
	end
	end

	comparable = samples.fetch("baseline").values_at("classes", "methods", "attributes", "structure_sha256") ==
	samples.fetch("optimized").values_at("classes", "methods", "attributes", "structure_sha256")
	abort "#{workload} parser results differ" unless comparable
	samples.each_value { \|sample\| validate_workload!(workload, sample, declarations) }

	{
	"workload" => workload,
	"declarations" => declarations,
	"baseline_seconds" => median(results.fetch("baseline")),
	"optimized_seconds" => median(results.fetch("optimized")),
	"structure_sha256" => samples.fetch("baseline").fetch("structure_sha256"),
	"results_match" => true
	}
	end

	def wall_time(rdoc, inputs, chdir)
	command = [
	RbConfig.ruby,
	"-I#{File.join(rdoc, "lib")}",
	File.join(rdoc, "exe/rdoc"),
	"--dry-run",
	"--quiet",
	*inputs
	]
	elapsed do
	run!(*command, chdir: chdir, out: File::NULL, err: File::NULL)
	end
	end

	def compare_end_to_end(workload, paths, inputs, chdir, runs)
	paths.each_value { \|rdoc\| wall_time(rdoc, inputs, chdir) }
	results = {"baseline" => [], "optimized" => []}

	runs.times do \|index\|
	order = index.even? ? %w[baseline optimized] : %w[optimized baseline]
	order.each do \|variant\|
	results.fetch(variant) << wall_time(paths.fetch(variant), inputs, chdir)
	end
	end

	{
	"workload" => workload,
	"baseline_seconds" => median(results.fetch("baseline")),
	"optimized_seconds" => median(results.fetch("optimized"))
	}
	end

	def write_inputs(root, declarations)
	FileUtils.mkdir_p(root)
	methods_rbs = File.join(root, "methods.rbs")
	attributes_rbs = File.join(root, "attributes.rbs")

	File.write(methods_rbs, "class Bench\n" + declarations.times.map { \|i\| " def m#{i}: () -> void" }.join("\n") + "\nend\n")
	File.write(attributes_rbs, "class Bench\n" + declarations.times.map { \|i\| " attr_reader a#{i}: String" }.join("\n") + "\nend\n")

	{
	"rbs_methods" => methods_rbs,
	"rbs_attributes" => attributes_rbs
	}
	end

	def print_report(report)
	puts
	puts "Parser-only"
	puts "\| Workload \| Declarations \| Baseline \| Optimized \| Speedup \|"
	puts "\| --- \| ---: \| ---: \| ---: \| ---: \|"
	report.fetch("micro").each do \|result\|
	puts "\| #{result.fetch("workload")} \| #{result.fetch("declarations")} \| #{format("%.4fs", result.fetch("baseline_seconds"))} \| #{format("%.4fs", result.fetch("optimized_seconds"))} \| #{format("%.1fx", speedup(result.fetch("baseline_seconds"), result.fetch("optimized_seconds")))} \|"
	end

	return unless report["end_to_end"]

	puts
	puts "End-to-end `rdoc --dry-run`"
	puts "\| Workload \| Baseline \| Optimized \| Change \|"
	puts "\| --- \| ---: \| ---: \| ---: \|"
	report.fetch("end_to_end").each do \|result\|
	change = (result.fetch("optimized_seconds") / result.fetch("baseline_seconds") - 1) * 100
	puts "\| #{result.fetch("workload")} \| #{format("%.3fs", result.fetch("baseline_seconds"))} \| #{format("%.3fs", result.fetch("optimized_seconds"))} \| #{format("%+.1f%%", change)} \|"
	end
	end

	options = {
	rdoc: DEFAULT_RDOC,
	base_ref: "master",
	declarations: 2_000,
	runs: 5,
	end_to_end: true
	}
	OptionParser.new do \|parser\|
	parser.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options]"
	parser.on("--rdoc PATH", "Optimized RDoc worktree (default: ../rdoc)") { \|path\| options[:rdoc] = File.expand_path(path) }
	parser.on("--base-ref REF", "Baseline Git ref (default: master)") { \|ref\| options[:base_ref] = ref }
	parser.on("--declarations COUNT", Integer, "Members per workload (default: 2000)") { \|count\| options[:declarations] = count }
	parser.on("--runs COUNT", Integer, "Measured runs per variant (default: 5)") { \|count\| options[:runs] = count }
	parser.on("--micro-only", "Skip end-to-end documentation generation") { options[:end_to_end] = false }
	parser.on("-h", "--help", "Show this help") { puts parser; exit }
	end.parse!

	abort "--declarations must be positive" unless options.fetch(:declarations).positive?
	abort "--runs must be positive" unless options.fetch(:runs).positive?
	rdoc = options.fetch(:rdoc)
	abort "RDoc worktree does not exist: #{rdoc}" unless File.file?(File.join(rdoc, "lib/rdoc.rb"))

	FileUtils.mkdir_p(TMP_ROOT)
	run_root = Dir.mktmpdir("run-", TMP_ROOT)
	baseline = File.join(run_root, "baseline-rdoc")
	baseline_added = false

	begin
	run!("git", "-C", rdoc, "worktree", "add", "--detach", baseline, options.fetch(:base_ref))
	baseline_added = true
	inputs = write_inputs(File.join(run_root, "inputs"), options.fetch(:declarations))
	paths = {"baseline" => baseline, "optimized" => rdoc}
	micro = %w[rbs_methods rbs_attributes].map do \|workload\|
	compare_micro(workload, paths, inputs.fetch(workload), options.fetch(:declarations), options.fetch(:runs))
	end

	report = {
	"ruby" => RUBY_DESCRIPTION,
	"rbs" => RBS::VERSION,
	"base_ref" => options.fetch(:base_ref),
	"base_commit" => command_output("git", "-C", baseline, "rev-parse", "HEAD^{commit}").strip,
	"optimized_commit" => command_output("git", "-C", rdoc, "rev-parse", "HEAD").strip,
	"optimized_worktree_sha256" => working_tree_sha256(rdoc),
	"runs" => options.fetch(:runs),
	"micro" => micro
	}

	if options.fetch(:end_to_end)
	report["end_to_end"] = [
	compare_end_to_end("rbs_methods", paths, [inputs.fetch("rbs_methods")], ROOT, options.fetch(:runs)),
	compare_end_to_end("rbs_attributes", paths, [inputs.fetch("rbs_attributes")], ROOT, options.fetch(:runs))
	]
	end

	report_path = File.join(run_root, "report.json")
	File.write(report_path, JSON.pretty_generate(report))
	print_report(report)
	puts
	puts "JSON report: #{report_path}"
	ensure
	if baseline_added
	run!("git", "-C", rdoc, "worktree", "remove", baseline)
	else
	FileUtils.rm_rf(baseline)
	end
	end
	{
	"ruby": "ruby 4.0.6 (2026-07-14 revision 03b6d3f889) +PRISM [x86_64-linux]",
	"rbs": "4.2.0",
	"base_ref": "master",
	"base_commit": "67b858164009135654110c96c26d77d4864d2006",
	"optimized_commit": "67b858164009135654110c96c26d77d4864d2006",
	"optimized_worktree_sha256": "10088d4771cf98d8f6c90a88d5cdbefca8caea784b54115430d50293e7be0212",
	"runs": 5,
	"micro": [
	{
	"workload": "rbs_methods",
	"declarations": 2000,
	"baseline_seconds": 0.16101996799989138,
	"optimized_seconds": 0.03532826800073963,
	"structure_sha256": "cfa2fd70f5a24f6307250017bfa5a2fc6366087b8c3dcb8561be5beda5dad0de",
	"results_match": true
	},
	{
	"workload": "rbs_attributes",
	"declarations": 2000,
	"baseline_seconds": 0.11075896100373939,
	"optimized_seconds": 0.02530622499762103,
	"structure_sha256": "ffdd6b6ba7e6028a9cfa421899fc99c623fd853a52614406b1da2d1967565e0d",
	"results_match": true
	}
	],
	"end_to_end": [
	{
	"workload": "rbs_methods",
	"baseline_seconds": 0.7218424330058042,
	"optimized_seconds": 0.5866772239969578
	},
	{
	"workload": "rbs_attributes",
	"baseline_seconds": 0.6301159760041628,
	"optimized_seconds": 0.5686040639993735
	}
	]
	}