pr1767_bench.py

#!/usr/bin/env python3

import subprocess
import os
import sys
import time
from openpyxl import Workbook
from openpyxl.styles import Font, Alignment

def run_command(cmd, log_file=None):
    """Run a shell command and optionally append output to log file"""
    if log_file:
        with open(log_file, 'a') as f:
            subprocess.run(cmd, shell=True, check=True, stdout=f, stderr=subprocess.STDOUT)
    else:
        subprocess.run(cmd, shell=True, check=True)

def build_bench_at_commit(commit, suffix):
    """Build benchmark binary at specific commit"""
    benchbin = f"pr1767_bench_{suffix}"

    print(f"Building secp #1767 '{suffix}' (commit {commit})...")

    run_command(f"git checkout -q {commit}")
    run_command("rm -rf build")

    log_file = f"{suffix}.log"
    run_command("cmake -B build", log_file)
    run_command("cmake --build build --target bench_ecmult --parallel 4", log_file)
    run_command(f"cp -f ./build/bin/bench_ecmult {benchbin}")

def run_bench(suffix):
    """Run benchmark binary with environment variable"""
    benchbin = f"pr1767_bench_{suffix}"
    print(f"Running benchmark binary {benchbin}...")

    env = os.environ.copy()
    env['SECP256K1_BENCH_ITERS'] = '100000'

    time.sleep(2)  # brief pause before running
    with open(f"pr1767_bench_{suffix}.txt", 'w') as f:
        subprocess.run(f"./{benchbin} pippenger_wnaf", shell=True, check=True, stdout=f, env=env)

def parse_benchmark_file(filename):
    """Parse benchmark result file and return dict of benchmark_name -> avg_time"""
    results = {}

    with open(filename, 'r') as f:
        lines = f.readlines()

    # Skip first 2 lines (header and column names)
    for line in lines[2:]:
        line = line.strip()
        if not line:
            continue

        # Split by comma and strip whitespace
        parts = [p.strip() for p in line.split(',')]
        if len(parts) == 4:
            benchmark_name = parts[0]
            avg_time = float(parts[2])
            results[benchmark_name] = avg_time

    return results

def compare_benchmarks():
    """Compare variant benchmarks against baseline and generate Excel report"""
    print("\nParsing benchmark results...")

    # Parse all benchmark files
    baseline = parse_benchmark_file("pr1767_bench_baseline.txt")
    variant1 = parse_benchmark_file("pr1767_bench_variant1.txt")
    variant2 = parse_benchmark_file("pr1767_bench_variant2.txt")
    variant3 = parse_benchmark_file("pr1767_bench_variant3.txt")

    # Prepare Excel output
    output_file = "pr1767_bench_cmp.xlsx"
    print(f"Generating comparison report: {output_file}")

    wb = Workbook()
    ws = wb.active
    ws.title = "Benchmark Comparison"

    red_font = Font(color="FF0000")
    wrap_alignment = Alignment(wrap_text=True)

    # Set column widths
    ws.column_dimensions['A'].width = 22  # fits "ecmult_multi_32767p_g"
    for col in ['B', 'C', 'D', 'E', 'F', 'G', 'H']:
        ws.column_dimensions[col].width = 10  # fits "Variant" or "Baseline"

    # Write header
    headers = [
        'Benchmark',
        'Baseline Avg (us)',
        'Variant1 Avg (us)',
        'Variant1 vs Baseline (%)',
        'Variant2 Avg (us)',
        'Variant2 vs Baseline (%)',
        'Variant3 Avg (us)',
        'Variant3 vs Baseline (%)'
    ]
    ws.append(headers)

    # Apply text wrapping to header row
    for cell in ws[1]:
        cell.alignment = wrap_alignment

    # Track percentages for averaging
    variant1_pcts = []
    variant2_pcts = []
    variant3_pcts = []

    # Write data for each benchmark
    for benchmark_name in baseline.keys():
        baseline_time = baseline[benchmark_name]
        variant1_time = variant1[benchmark_name]
        variant2_time = variant2[benchmark_name]
        variant3_time = variant3[benchmark_name]

        # Calculate percentage change (negative means improvement/faster)
        variant1_pct = ((variant1_time - baseline_time) / baseline_time) * 100
        variant2_pct = ((variant2_time - baseline_time) / baseline_time) * 100
        variant3_pct = ((variant3_time - baseline_time) / baseline_time) * 100

        variant1_pcts.append(variant1_pct)
        variant2_pcts.append(variant2_pct)
        variant3_pcts.append(variant3_pct)

        row = [
            benchmark_name,
            f"{baseline_time:.2f}",
            f"{variant1_time:.2f}",
            f"{variant1_pct:+.2f}%",
            f"{variant2_time:.2f}",
            f"{variant2_pct:+.2f}%",
            f"{variant3_time:.2f}",
            f"{variant3_pct:+.2f}%"
        ]
        ws.append(row)

        # Apply red font to negative percentage cells (columns D, F, H)
        current_row = ws.max_row
        for col, pct in [(4, variant1_pct), (6, variant2_pct), (8, variant3_pct)]:
            if pct < 0:
                ws.cell(row=current_row, column=col).font = red_font

    # Add average row
    avg1 = sum(variant1_pcts) / len(variant1_pcts)
    avg2 = sum(variant2_pcts) / len(variant2_pcts)
    avg3 = sum(variant3_pcts) / len(variant3_pcts)

    avg_row = ['Average', '', '', f"{avg1:+.2f}%", '', f"{avg2:+.2f}%", '', f"{avg3:+.2f}%"]
    ws.append(['', '', '', '', '', '', '', ''])
    ws.append(avg_row)

    # Apply red font to negative average percentages
    current_row = ws.max_row
    for col, pct in [(4, avg1), (6, avg2), (8, avg3)]:
        if pct < 0:
            ws.cell(row=current_row, column=col).font = red_font

    wb.save(output_file)
    print(f"Comparison report saved to {output_file}")
    print("\nSummary:")
    print("  - Negative % = Performance improvement (faster)")
    print("  - Positive % = Performance degradation (slower)")
    print("  - Near 0% = No significant change")

def main():
    run_command("git checkout master")
    run_command("git fetch upstream pull/1767/head:pr1767")
    run_command("git checkout pr1767")

    # Build variants
    build_bench_at_commit("d0f3123c0c2d4643a0191af6e7c5e18032666605", "baseline")
    build_bench_at_commit("2ccc5e02776b8ae3483adf342b741abd99e48931", "variant1")
    build_bench_at_commit("867fe34dd4f642fb966d5ee652e0d77a9adafa41", "variant2")
    build_bench_at_commit("0fc73f397bc6a10b723b89d6f2a0dc52e7621748", "variant3")
    time.sleep(10)

    # Run benchmarks
    run_bench("baseline")
    run_bench("variant1")
    run_bench("variant2")
    run_bench("variant3")

    # Compare results and generate CSV report
    compare_benchmarks()

if __name__ == "__main__":
    try:
        main()
    except subprocess.CalledProcessError as e:
        print(f"Error: Command failed with exit code {e.returncode}")
        sys.exit(1)
    except FileNotFoundError as e:
        print(f"Error: File not found - {e}")
        sys.exit(1)