Rust micro-benchmarks helper. Runs benchmarks multiple times taking best times & compares to a control. Requires stable and nightly toolchains installed, and `cargo +stable install benchcmp`.

rustbench

#!/usr/bin/env bash
# shellcheck disable=SC2086

set -euo pipefail

if [ "${1:-}" == "--help" ] || [ "${1:-}" == "-h" ]; then
  echo "Micro-benchmarks helper using: cargo +nightly bench"
  echo "  Setup a control / starting values to compare against"
  echo "  $ rustbench --control"
  echo
  echo '  With a control recorded, re-evaluate and compare:'
  echo '  $ rustbench'
  echo
  echo '  Can also add cargo-benchcmp arguments to change the comparison output'
  echo '  $ rustbench --threshold 2'
  echo
  echo '  Can parse args to cargo'
  echo '  $ rustbench --control -- --all-features'
  exit 0
fi

control="0"
if [ "${1:-}" == "--control" ]; then
  control="1"
  shift
fi

benchcmp_args=""
while [ "${1:---}" != "--" ]; do
  benchcmp_args="$benchcmp_args $1"
  shift
done

if [ "${1:-}" == "--" ]; then shift; fi
cargo_args=$*

# number of repeats to make picking the best score
runs=3

# @arg filename of output
function bench_to {
  file=$1
  echo '' > "$file"

  for run in $(seq 1 $runs); do
    echo -e "\\e[33mrunning bench > $file ($run/${runs}: pick best result)\\e[39m" >&2
    cargo +nightly bench $cargo_args \
      | tee -a "$file" \
      | grep --color=never 'bench:'

    if [ "$run" != "1" ]; then
      # sort scores (with best at top) & remove each 2nd line (the worse one)
      rm -f target/benchtmp || true
      grep 'bench:' "$file" \
        | grep -v 'ignored' \
        | sort \
        | while read -r line || [[ -n "$line" ]]; do
            read -r line2 || [[ -n "$line2" ]]
            score1=$(echo "$line" | awk '{print $5}' | sed 's/,//g')
            score2=$(echo "$line2" | awk '{print $5}' | sed 's/,//g')
            if [[ $score2 -lt $score1 ]]; then
              echo "$line2" >> target/benchtmp
            else
              echo "$line" >> target/benchtmp
            fi
          done
      mv target/benchtmp "$file"
    fi
  done
}

if [ "$control" == "1" ]; then
  runs=5
  mkdir -p target
  bench_to target/control
  exit 0
fi

if [ ! -f target/control ]; then
  echo 'target/control missing, will not perform a benchmark comparison' >&2
  echo "  run \`rustbench --control\` to generate from the current source" >&2
  exit 1
else
  mkdir -p target
  bench_to target/change
  echo ''
  cargo +stable benchcmp target/control target/change $benchcmp_args
fi