Go Benchstat Diff

go-benchstat.sh

#!/bin/bash

set -e
set -u
set -o pipefail

command -v benchstat  >/dev/null 2>&1 || { echo >&2 "I need Benchstat!";  exit 1; }
command -v git        >/dev/null 2>&1 || { echo >&2 "I need Git!";        exit 1; }

BRANCH_TARGET="master"
BRANCH_CURRENT="current"

THRESHOLD_FACTOR_TIME_OP=1.25
THRESHOLD_FACTOR_ALLOC_OP=1.25
THRESHOLD_FACTOR_ALLOCS_OP=3.25

clear() {
  rm -f $BRANCH_TARGET.out $BRANCH_CURRENT.out
}

end() {
  clear

  if [ "$1" = true ] ;
  then
    echo 'Done!'
    exit 0
  else
    echo 'Fail!'
    exit 1
  fi
}

check() {
  echo "Checking..."
  git update-index -q --refresh
  if test "$(git diff-index --name-only HEAD --)"; then
      echo "You must commit your changes first in order to use the script!"
      exit 1
  fi
}

bench() {
  echo "Benchmarking..."
  CHECK=`git rev-parse --abbrev-ref HEAD`

  echo "Checkout: $BRANCH_TARGET"
  git checkout $BRANCH_TARGET

  echo "Benchmarking: $BRANCH_TARGET"
  D1=$(go test -bench=. -run=^a ./...)

  echo "Checkout: $CHECK"
  git checkout ${CHECK}

  echo "Benchmarking: $CHECK"
  D2=$(go test -bench=. -run=^a ./...)

  cat <<< "$D1" > "$BRANCH_TARGET.out"
  cat <<< "$D2" > "$BRANCH_CURRENT.out"
}

compare() {
  echo "Comparing..."

	OP=0
	SUCCESS=true
	while read -r p;
	do
		case "$p" in
			*"time/op"*)
		    OP=1
		    continue
      ;;
			*"alloc/op"*)
			  OP=2
			  continue
      ;;
			*"allocs/op"*)
			  OP=3
			  continue
      ;;
		esac

    OLD=`echo "$p" | awk '{print $2}' | cut -d '.' -f 1 | sed 's/ns$//'`
    NEW=`echo "$p" | awk '{print $5}' | cut -d '.' -f 1 | sed 's/ns$//'`

    if [[ -z "$OLD" || -z "$NEW" ]]
    then
          continue
    fi

    if [[ "$OLD" == "$NEW" ]]
    then
          continue
    fi

    if [[ $OP == 1 ]] ;   #in case of: `XXX.XXXns XXX.XXXns`
    then
      RES=$(bc -l <<< "$OLD * $THRESHOLD_FACTOR_TIME_OP")
      if [ `echo "$RES > $NEW" | bc` -eq 1 ]; then
         echo "New 'time/ops' is exceeeds '$OLD * $THRESHOLD_FACTOR_TIME_OP' threshold. New is: $NEW"
         SUCCESS=false
      fi
    elif [[ $OP == 2 ]] ; #in case of: `XXX.XXXB XXX.XXXB`
    then
      RES=$(bc -l <<< "$OLD * $THRESHOLD_FACTOR_ALLOC_OP")
      if [ `echo "$RES > $NEW"| bc` -eq 1 ]; then
         echo "New 'alloc/ops' is exceeeds '$OLD * $THRESHOLD_FACTOR_ALLOC_OP' threshold. New is: $NEW"
         SUCCESS=false
      fi
    elif [[ $OP == 3 ]] ; #in case of: `XXX.XXX XXX.XXX`
    then
      RES=$(bc -l <<< "$OLD * $THRESHOLD_FACTOR_ALLOCS_OP")
      if [ `echo "$RES > $NEW" | bc` -eq 1 ]; then
         echo "New 'allocs/ops' is exceeeds '$OLD * $THRESHOLD_FACTOR_ALLOCS_OP' threshold. New is: $NEW"
         SUCCESS=false
      fi
    fi

	done < <(benchstat $BRANCH_TARGET.out $BRANCH_CURRENT.out)
}

clear
check
bench
compare
end $SUCCESS