Measure CI runs

measure-ci.sh

#!/usr/bin/env bash

# Method: Go through the "Code Quality Check" workflow and sample all the builds across pages
# https://github.com/SironaMedical/services-frontend/actions/workflows/pull-request.yml?query=is%3Asuccess

# Build caching PR: https://github.com/SironaMedical/services-frontend/pull/3915
# Merged Dec 18, 2023

# Old, pre-optimization GHA run IDs from 12/11/2023 to 12/14/2023
old_runs=(
  7212305471
  7186988869
  7171353735
  7170349376
  7127777922
  7118721929
  7197386025
  7197174853
  7196629155
  7195368501
  7190087687
  7186988869
  7178358890
  7178086578
  7175028507
  7172379811
  7172676730
)
# Random sample of recent successful runs from 1/19/2024 to 2/21/2024
new_runs=(
  7225083772
  7976016536
  7976046834
  7917142862
  7880430552
  7888185888
  7879357053
  7824866966
  7815731473
  7807444550
  7788228231
  7760331075
  7728510837
  7703423513
  7701436030
  7670042653
  7660546187
  7656977220
  7633499170
  7614661635
  7589489237
)

function queryGithub() {
  arr=("$@")
  sum=0
  count=0
  for runId in "${arr[@]}"
  do
    echo -n "  run/$runId dur: "
    pick_num=2
    avg_job_duration=$(\
      # The below will get the billable time for each 8-core job and will average the time by the
      # number of jobs
      # gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
      #   /repos/SironaMedical/services-frontend/actions/runs/$runId/timing \
      #   | jq '.billable.UBUNTU_8_CORE | {average_duration: (.job_runs | map(.duration_ms/1000) | add), jobs: .jobs}' \
      #   | jq '.average_duration/.jobs | floor'

      # Picks two of the longest running 8-core jobs and averages them for the average workflow duration
      gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
        /repos/SironaMedical/services-frontend/actions/runs/$runId/timing \
        | jq ".billable.UBUNTU_8_CORE.job_runs | map(.duration_ms) | sort | reverse | [limit($pick_num; .[])] | add / length / 1000"
      
      # 8-core jobs consist of: 3 unit test shards, 3 UAT shards (worklist, reporter, viewer), and
      # any re-run attempts for those jobs.
      
      # Why pick the top two? The previous algorithm averaged all the 8-core runs together, which is
      # not ideal because:
      # 1. it affects a workflow's average if many manual re-run attempts were made
      # 2. it skews the average lower for more recent UAT runs since the worklist UAT test was
      #    introduced and adds to the total runtime
      
      # By picking the top two longest running jobs, we're effectively eliminating the worklist UAT
      # change from the averaged result and normalizing the workflow statistics for an
      # apples-to-apples comparison.
    )
    echo "$(($avg_job_duration))s"
    sum=$(($sum+$avg_job_duration))
    count=$(($count+1))
  done
  average=$(($sum/$count))
  echo "Count:   $count"
  echo "Average: $(($average))s"
}

echo -e "\nRuns from pre-CI optimization"
queryGithub "${old_runs[@]}"
pre_average=$average
echo -e "\nRuns after build caching"
queryGithub "${new_runs[@]}"
post_average=$average

diff=$(($pre_average - $post_average))
echo ""
echo -e "Raw Diff: ${diff}s"
echo -e "  % Diff: $( jq -n $diff/$pre_average*100 )%"

exit 0

################################################################################
# SCRATCH
################################################################################

# Billable duration is the total runtime of each job which run in parallel
gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
  /repos/SironaMedical/services-frontend/actions/runs/7917142862/timing \
  | jq ".billable | [.UBUNTU_8_CORE.total_ms/1000,.UBUNTU.total_ms/1000] | add"

# 
gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
  /repos/SironaMedical/services-frontend/actions/runs/7212305471/timing \
  | jq ".billable.UBUNTU_8_CORE.job_runs | map(.duration_ms) | sort | reverse | [limit(2; .[])] | add / length"

# Run duration is the total wall time CI as a whole took to complete
gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
  /repos/SironaMedical/services-frontend/actions/runs/7917142862/timing \
  | jq ".run_duration_ms/1000"