Count how many times a test occurs before the first failure, to help track down tests that cause other tests that come after them to fail

flake-culprit.sh

#!/bin/bash

#
# Example Usage:
#
# ./flake-culprits.sh pull-kubernetes-e2e-kind 250
#

job=$1
max=$2

# Find all tests that were performed before the first failing test
find_potential_culprits() {
  curl -s $1 > /tmp/build-log.txt
  cat /tmp/build-log.txt | grep -E -m1 -B99999999 '^{"msg":"FAILED' | grep -E '^{"msg":"PASSED' | grep -oP '\[.*][^"]*' | sort | uniq
}

# Create some javascript that will be used to extract build log urls
cat << EOF > /tmp/build-log-url-extractor.js
for (var i = 0; i < allBuilds.items.length; i++) {
  var item = allBuilds.items[i];
  if (item.spec.job === "pull-kubernetes-e2e-kind" && !!item.status.build_id && item.status.state === "failure") {
    var pulls = item.spec.refs.pulls;
    for (var j = 0; j < pulls.length; j++) {
      var p = pulls[j];
      console.log("https://storage.googleapis.com/kubernetes-jenkins/pr-logs/pull/" + p.number + "/" + item.spec.job + "/" + item.status.build_id + "/build-log.txt");
    }
  }
}
EOF

# Extract the build log urls
echo "Finding build logs for $job jobs..."
curl -s "https://prow.k8s.io/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" > /tmp/prowjobs.js
cat /tmp/prowjobs.js /tmp/build-log-url-extractor.js | node | head -n $max > /tmp/build-log-urls.txt

# Retrieve each build log and find potential culprits
total=$(cat /tmp/build-log-urls.txt | wc -l)
current=0
rm /tmp/potential-culprits.txt 2&> /dev/null
while read buildLogUrl; do 
  current=$((current + 1))
  echo "[$current/$total] $buildLogUrl"
  find_potential_culprits $buildLogUrl >> /tmp/potential-culprits.txt
  if [ $current -eq $max ]; then
    break;
  fi
done < /tmp/build-log-urls.txt

# Construct a report sorted by frequency descending
cat /tmp/potential-culprits.txt | sort | uniq -c | sort -k1,1nr -k2b > /tmp/potential-culprits-report.txt
cat /tmp/potential-culprits-report.txt