Starship Speed Tester

benchmark_starship.sh

#!/bin/bash

# Downloads, builds, installs, and runs timing tests for two starship and two git
# programs, using a small repo (starship), a medium repo (rust-lang) and a large
# repo (chromium) as test cases. Also has an option on Linux to use `sudo` to run
# tests with cold caches. This can be done either by waiting for the interactive
# dialog or by setting the envar `RUN_COLD_STARSHIP_TESTS` to any non-empty value

# Unfortunately, I do not know what packages need to be installed on your computer
# before you can successfully use this script. It's provided more as a starting
# point than a plug-and-play setup. Sorry.

# Need to install at least:
# libCURL development packages
# C/C++ build tools
# hyperfine

# According to https://gist.github.com/samrocketman/8671036
# apt-get install libcurl4-gnutls-dev libexpat1-dev gettext libz-dev libssl-dev asciidoc xmlto

# Change this variable to control where code/repos/results are placed
TEST_LOCATION=/tmp/git-speed-tests

# Number of CPUs to build with
NCPUS=$(nproc --all)
#NCPUS=4

#===========================================================

git_loc="$TEST_LOCATION/src/gitsrc"
libgit2_loc="$TEST_LOCATION/src/libgit2src"
starship_loc="$TEST_LOCATION/src/starshipsrc"
spoc_loc="$TEST_LOCATION/src/spocsrc"

git_url='https://github.com/git/git'
libgit2_url='https://github.com/libgit2/libgit2.git'
starship_url='https://github.com/starship/starship.git'
spoc_url='https://github.com/matchai/starship-poc.git'
# Can we name this the "Starship Proof Of Concept Kit" for a nice acronym?

results_dir="$TEST_LOCATION/results"

#===========================================================
set -euo pipefail

echo "WARNING: Do not expect any of the files created by this script to be permanent."
echo "For example, the script hard-resets and cleans the repos it clones in order to"
echo "ensure the repo state is clean--any changes made to the repos will be lost!"

if_ne_then_mkdir(){
  if [ ! -d "$1" ]; then
      mkdir -p "$1"
  fi
}

exit_success(){
    echo "Benchmarks complete. You can find the results at $results_dir"
    exit 0
}

if_ne_then_mkdir "$TEST_LOCATION"
if_ne_then_mkdir "$TEST_LOCATION/src"
if_ne_then_mkdir "$results_dir"
if_ne_then_mkdir "$results_dir/git"
if_ne_then_mkdir "$results_dir/libgit2"
if_ne_then_mkdir "$results_dir/starship"
if_ne_then_mkdir "$results_dir/spoc"

section_message(){
  echo ""
  echo ""
  echo ""
  echo "======================================================"
  echo -e "$@"
}

section_message "Checking out repositories for candidates...\n"

if [ ! -d "$git_loc" ]; then
    git clone "$git_url" "$git_loc"
fi
if [ ! -d "$libgit2_loc" ]; then
git clone "$libgit2_url" "$libgit2_loc"
fi
if [ ! -d "$starship_loc" ]; then
git clone "$starship_url" "$starship_loc"
fi
if [ ! -d "$spoc_loc" ]; then
git clone "$spoc_url" "$spoc_loc"
fi

cd "$git_loc"
git checkout "tags/v2.26.0"

cd "$libgit2_loc"
git checkout "tags/v1.0.0"

cd "$starship_loc"
git checkout "tags/v0.40.1"

# Spoc has very few commits--assume we're fine there

echo "Code checkout complete!"

#===========================================================

section_message "Building code for candidates...\n"
cd "$git_loc"
# Minimze deps by only building the git CLI tool
make -j $NCPUS $MAKEFLAGS git   

cd "$libgit2_loc"
if [ ! -d build ]; then
    mkdir build
fi
cd build
cmake -DUSE_ICONV=OFF -DUSE_SSH=OFF -DUSE_HTTPS=OFF -DTHREADSAFE=ON\
      -DBUILD_SHARED_LIBS=OFF -DUSE_BUNDLED_ZLIB=ON  -DREGEX_BACKEND=builtin\
      -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release ..
make -j $NCPUS $MAKEFLAGS

cd "$starship_loc"
cargo build -j $NCPUS --release

cd "$spoc_loc"
cargo build -j $NCPUS --release

echo "Code build complete!"

git_bin="$git_loc/git"
lg2_bin="$libgit2_loc/build/examples/lg2"
starship_bin="$starship_loc/target/release/starship"
spoc_bin="$spoc_loc/target/release/starship-poc"

#============================================================

section_message "Checking out benchmark repos...\n"

# Test against three repos: large (chromium), medium (rust), and
# small (starship). Test in clean state by force-resetting repo,
# then in dirty by altering/creating/deleting a bunch of files

chromium_url='https://github.com/chromium/chromium.git'
chromium_loc="$TEST_LOCATION/testrepo/chromium"
rustlang_url='https://github.com/rust-lang/rust.git'
rustlang_loc="$TEST_LOCATION/testrepo/rust-lang"
starship_test_loc="$TEST_LOCATION/testrepo/starship"

if [ ! -d "$chromium_loc" ]; then
git clone "$chromium_url" "$chromium_loc"
fi
if [ ! -d "$rustlang_loc" ]; then
git clone "$rustlang_url" "$rustlang_loc"
fi
if [ ! -d "$starship_test_loc" ]; then
git clone "$starship_url" "$starship_test_loc"
fi

#===========================================================

section_message "Benchmarking small repo...\n"

git_nuke(){
  git reset --hard HEAD
  git clean -xfd
}

cd "$starship_test_loc"
git_nuke
$git_bin status

hyperfine --warmup 3 "$starship_bin prompt" --export-json "$results_dir/starship/small-clean.json"
hyperfine --warmup 3 "$spoc_bin prompt" --export-json "$results_dir/spoc/small-clean.json"
hyperfine --warmup 3 "$git_bin status" --export-json "$results_dir/git/small-clean.json"
hyperfine --warmup 3 "$lg2_bin status" --export-json "$results_dir/libgit2/small-clean.json"

#===========================================================

section_message "Benchmarking medium repo...\n"

cd "$rustlang_loc"
git_nuke
$git_bin status

hyperfine --warmup 3 "$starship_bin prompt" --export-json "$results_dir/starship/med-clean.json"
hyperfine --warmup 3 "$spoc_bin prompt" --export-json "$results_dir/spoc/med-clean.json"
hyperfine --warmup 3 "$git_bin status" --export-json "$results_dir/git/med-clean.json"
hyperfine --warmup 3 "$lg2_bin status" --export-json "$results_dir/libgit2/med-clean.json"

#===========================================================

section_message "Benchmarking large repo...\n"

cd "$chromium_loc"
git_nuke
$git_bin status

hyperfine --warmup 3 "$starship_bin prompt" --export-json "$results_dir/starship/large-clean.json"
hyperfine --warmup 3 "$spoc_bin prompt" --export-json "$results_dir/spoc/large-clean.json"
hyperfine --warmup 3 "$git_bin status" --export-json "$results_dir/git/large-clean.json"
hyperfine --warmup 3 "$lg2_bin status" --export-json "$results_dir/libgit2/large-clean.json"

#===========================================================

prepare_cmd='sync; echo 3 | sudo tee /proc/sys/vm/drop_caches'

echo "I would like to be able to flush your caches in order to do cold benchmarking runs."
echo "I will need sudo permissions in order to do this. If you do not wish to grant"
echo "them, you may Ctrl+C now and I will skip these benchmarks."
echo ""
echo "The command I will run to flush caches is:"
echo -e "\t$prepare_cmd"
echo ""
echo "Would you like to run these benchmarks? [y/N]"
read response
if [ "$response" = "y" ] || [ ! -z RUN_COLD_STARSHIP_TESTS ]; then
    sudo -v && havesudo="true" || havesudo="false"
else
    exit_success
fi

if [ "$havesudo" = "false" ]; then
    echo "Could not acquire sudo"
    exit_success
fi

#===========================================================

section_message "Benchmarking small repo...\n"

cd "$starship_test_loc"
git_nuke

hyperfine --prepare "$prepare_cmd" "$starship_bin prompt" --export-json "$results_dir/starship/small-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$spoc_bin prompt" --export-json "$results_dir/spoc/small-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$git_bin status" --export-json "$results_dir/git/small-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$lg2_bin status" --export-json "$results_dir/libgit2/small-clean-cold.json"
 
#===========================================================

section_message "Benchmarking medium repo...\n"

cd "$rustlang_loc"
git_nuke

hyperfine --prepare "$prepare_cmd" "$starship_bin prompt" --export-json "$results_dir/starship/med-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$spoc_bin prompt" --export-json "$results_dir/spoc/med-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$git_bin status" --export-json "$results_dir/git/med-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$lg2_bin status" --export-json "$results_dir/libgit2/med-clean-cold.json"

#===========================================================

section_message "Benchmarking large repo...\n"

cd "$chromium_loc"
git_nuke

hyperfine --prepare "$prepare_cmd" "$starship_bin prompt" --export-json "$results_dir/starship/large-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$spoc_bin prompt" --export-json "$results_dir/spoc/large-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$git_bin status" --export-json "$results_dir/git/large-clean-cold.json"
git_nuke
hyperfine --prepare "$prepare_cmd" "$lg2_bin status" --export-json "$results_dir/libgit2/large-clean-cold.json"

exit_success