taskcluster list coverage aggregation mozilla-central hook results

tc-list-coverage.sh

#!/bin/bash

# This is a script to help analyze the relative time that searchfox runs
# happened relative to the coverage aggregation runs.
#
# It uses the "taskcluster" client-shell binary tool from
# https://github.com/taskcluster/taskcluster/tree/main/clients/client-shell
# and the "coreapi" command line tool referenced by
# https://treeherder.mozilla.org/docs/ and installable via
# `pip3 install coreapi` to do REST API stuff from inside bash.
#
# Yeah, inside bash.
#
# I know.
#
# But it allows for iterative command line experimentation, I guess?  In any
# event, if this is ever needed on an ongoing basis this should absolutely be
# folded into something that is less excitingly cobbled together.

#set -x # Show commands
set -eu # Errors/undefined vars are fatal
set -o pipefail # Check all commands in a pipeline

export TASKCLUSTER_ROOT_URL=https://firefox-ci-tc.services.mozilla.com

## Ask Treeherder for the more recent searchfox jobs
# Get the treeherder schema / metadata into place:
coreapi get https://treeherder.mozilla.org/docs/ > /dev/null

export SOURCE_REPO=https://hg.mozilla.org/mozilla-central

# Get searchfox jobs for the last 8 days and find their revision
echo "### Searchfox rev artifact creator times"
typeset -A SF_REVS_PRESENT
typeset -A SF_REV_COMPLETED

# Note: This will return jobs from trees other than mozilla-central right now...
# so we filter it out in the loop by looking up the task def... but it seems
# like we really should be able to filter this here, but it's not jumping out at
# me in the treeherder API.  (Maybe an integer id needs to be looked up?)
#
# Previously used for date: $(date -d "-14 days" -Iseconds -u)
# changed because of 500 errors that went away by removing the TZ.

SF_JOBS=$(coreapi action jobs list -p start_time__gt=$(date -d "-14 days" -u +"%FT%T") -p job_type_name=searchfox-linux64-searchfox/debug)
SF_JOB_REVS=$(jq -M -r '(.job_property_names | index("push_revision")) as $idx_rev | (.job_property_names | index("task_id")) as $idx_taskid | .results[] | .[$idx_rev], .[$idx_taskid]' <<< "$SF_JOBS")
ART_CSV=""
# In order to be able to manipulate the array, we need to ensure that there is
# no pipeline, because a pipeline will result in a sub-shell being created
# which cannot modify the root bash shell's variables!
while read -r sf_rev; read -r sf_taskid; do
  SF_REVS_PRESENT[${sf_rev}]=${sf_taskid}
  COMPLETED_SF=$(taskcluster api queue status $sf_taskid | jq -M -r '.status.runs[0].resolved')
  SF_DEF_INFO=$(taskcluster task def $sf_taskid)
  SF_SOURCE_REPO=$(jq -M -r '.payload.env.MOZ_SOURCE_REPO' <<< "$SF_DEF_INFO")
  
  # Filter out searchfox jobs that aren't from our desired repo.
  if [[ ${SF_SOURCE_REPO} != ${SOURCE_REPO} ]]; then
    continue
  fi
  
  SF_REV_COMPLETED[${sf_rev}]=${COMPLETED_SF}
  #echo "Set '${sf_rev}' to ${SF_REVS_PRESENT[${sf_rev}]}"
  ARTIFACT_NAME=project.relman.code-coverage.production.repo.mozilla-central.${sf_rev}
  
  echo "- ${sf_rev} artifact ${ARTIFACT_NAME}"
  echo "  - $COMPLETED_SF - indexing completed in job $sf_taskid"
  
  ARTIFACT_INFO=$(taskcluster api index findTask ${ARTIFACT_NAME} 2>/dev/null || true)
  # The artifact may not exist.
  if [[ ${ARTIFACT_INFO} == "" ]]; then
    echo "  - No such artifact!"
    continue
  fi
  ARTIFACT_TASKID=$(jq -M -r '.taskId' <<< "$ARTIFACT_INFO")
  ART_STATUS_INFO=$(taskcluster api queue status $ARTIFACT_TASKID)
  COMPLETED_ART=$(jq -M -r '.status.runs[0].resolved' <<< "$ART_STATUS_INFO")
  STARTED_ART=$(jq -M -r '.status.runs[0].started' <<< "$ART_STATUS_INFO")
  ART_DURATION_MINS=$(jq -M -r '((.status.runs[0].resolved | sub("\\.[0-9]+Z$"; "Z") | fromdate) - (.status.runs[0].started | sub("\\.[0-9]+Z$"; "Z") | fromdate)) / 60' <<< "$ART_STATUS_INFO")
  ART_CSV+="${ART_DURATION_MINS},"
  
  echo "  - $COMPLETED_ART - artifact task completed in job $ARTIFACT_TASKID started at $STARTED_ART"
done <<< "$SF_JOB_REVS"
echo "Artifact run times: ${ART_CSV:0:-1}"

## Helper to constrain hook results by repo and revision that was covered by a searchfox indexing run.
function hook_results_checker {
  HOOK_ID=$1
  echo ""
  echo "### Hook Results for ${HOOK_ID}"
  RUN_CSV=""
  # Get the last hook firings and sort and filter by the creation time so we get only things for the last week.
  FIRED_TASKS=$(taskcluster api hooks listLastFires project-relman ${HOOK_ID} | \
    jq -M -r '.lastFires | sort_by(.taskCreateTime) | .[] |  select((.taskCreateTime | sub("\\.[0-9]+Z$"; "Z") | fromdate) > (now - (14 * 24 * 60 * 60))) | .taskId')
  while read -r taskId; do
    TASKDEF=$(taskcluster task def $taskId)
    #echo "${TASKDEF}"
    if [[ $(jq -M -r '.payload.env.REPOSITORY' <<< "$TASKDEF") == "${SOURCE_REPO}" ]]; then
      REV=$(jq -M -r '.payload.env.REVISION' <<< "$TASKDEF")
      #echo "Checking revision '$REV' with lookup result ${SF_REVS_PRESENT[${REV}]:-0}"
      # Check that this was a searchfox revision and only do more if it was.
      if [[ ${SF_REVS_PRESENT[${REV}]:-""} != "" ]]; then
        COV_INFO=$(taskcluster api queue status $taskId)
        SF_TASKID=${SF_REVS_PRESENT[${REV}]}
        COMPLETED_SF=${SF_REV_COMPLETED[${REV}]}
        COMPLETED_COV=$(jq -M -r '.status.runs[0].resolved' <<< "$COV_INFO")
        COV_DURATION_MINS=$(jq -M -r '((.status.runs[0].resolved | sub("\\.[0-9]+Z$"; "Z") | fromdate) - (.status.runs[0].started | sub("\\.[0-9]+Z$"; "Z") | fromdate)) / 60' <<< "$COV_INFO")
        RUN_CSV+="${COV_DURATION_MINS},"
        echo "- rev $REV"
        echo "  - $COMPLETED_SF - indexing completed in job $SF_TASKID"
        echo "  - $COMPLETED_COV - aggregation completed in job $taskId after $COV_DURATION_MINS minutes"
      fi
    fi
  done <<< "$FIRED_TASKS"
  echo "Run times: ${RUN_CSV:0:-1}"
}

function hook_runs_lister {
  HOOK_ID=$1
  echo ""
  echo "### Hook Recent Runs for ${HOOK_ID}"
  # Get the last hook firings and sort and filter by the creation time so we get only things for the last week.
  taskcluster api hooks listLastFires project-relman ${HOOK_ID} | \
  jq -M -r '.lastFires | sort_by(.taskCreateTime) | .[] |  select((.taskCreateTime | sub("\\.[0-9]+Z$"; "Z") | fromdate) > (now - (7 * 24 * 60 * 60))) | .taskId' | \
  while read -r taskId; do
    #TASKDEF=$(taskcluster task def $taskId)
    #echo "${TASKDEF}"
    STATUS_INFO=$(taskcluster api queue status $taskId)
    COMPLETED_COV=$(jq -M -r '.status.runs[0].resolved' <<< "$STATUS_INFO")
    STARTED_COV=$(jq -M -r '.status.runs[0].started' <<< "$STATUS_INFO")
    COV_DURATION_MINS=$(jq -M -r '((.status.runs[0].resolved | sub("\\.[0-9]+Z$"; "Z") | fromdate) - (.status.runs[0].started | sub("\\.[0-9]+Z$"; "Z") | fromdate)) / 60' <<< "$STATUS_INFO")
    echo "- $COMPLETED_COV completed after starting at $STARTED_COV for task $taskId after $COV_DURATION_MINS minutes"
  done
}

## Disabling the cron hook lookup because it doesn't have revision info inside it.
## Special handling would be needed, etc.  Also, it looks like it just triggers
## other tasks, which is why I implemented the artifact lookups in searchfox.
#hook_results_checker "code-coverage-cron-production"
## But instead let's just show who ran when.
hook_runs_lister "code-coverage-cron-production"

hook_results_checker "code-coverage-repo-production"