Last active
May 15, 2020 01:15
-
-
Save brianpursley/f7272e48154934931c5a306ddd25700b to your computer and use it in GitHub Desktop.
Count how many times a test occurs before the first failure, to help track down tests that cause other tests that come after them to fail
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# | |
# Example Usage: | |
# | |
# ./flake-culprits.sh pull-kubernetes-e2e-kind 250 | |
# | |
job=$1 | |
max=$2 | |
# Find all tests that were performed before the first failing test | |
find_potential_culprits() { | |
curl -s $1 > /tmp/build-log.txt | |
cat /tmp/build-log.txt | grep -E -m1 -B99999999 '^{"msg":"FAILED' | grep -E '^{"msg":"PASSED' | grep -oP '\[.*][^"]*' | sort | uniq | |
} | |
# Create some javascript that will be used to extract build log urls | |
cat << EOF > /tmp/build-log-url-extractor.js | |
for (var i = 0; i < allBuilds.items.length; i++) { | |
var item = allBuilds.items[i]; | |
if (item.spec.job === "pull-kubernetes-e2e-kind" && !!item.status.build_id && item.status.state === "failure") { | |
var pulls = item.spec.refs.pulls; | |
for (var j = 0; j < pulls.length; j++) { | |
var p = pulls[j]; | |
console.log("https://storage.googleapis.com/kubernetes-jenkins/pr-logs/pull/" + p.number + "/" + item.spec.job + "/" + item.status.build_id + "/build-log.txt"); | |
} | |
} | |
} | |
EOF | |
# Extract the build log urls | |
echo "Finding build logs for $job jobs..." | |
curl -s "https://prow.k8s.io/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" > /tmp/prowjobs.js | |
cat /tmp/prowjobs.js /tmp/build-log-url-extractor.js | node | head -n $max > /tmp/build-log-urls.txt | |
# Retrieve each build log and find potential culprits | |
total=$(cat /tmp/build-log-urls.txt | wc -l) | |
current=0 | |
rm /tmp/potential-culprits.txt 2&> /dev/null | |
while read buildLogUrl; do | |
current=$((current + 1)) | |
echo "[$current/$total] $buildLogUrl" | |
find_potential_culprits $buildLogUrl >> /tmp/potential-culprits.txt | |
if [ $current -eq $max ]; then | |
break; | |
fi | |
done < /tmp/build-log-urls.txt | |
# Construct a report sorted by frequency descending | |
cat /tmp/potential-culprits.txt | sort | uniq -c | sort -k1,1nr -k2b > /tmp/potential-culprits-report.txt | |
cat /tmp/potential-culprits-report.txt |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment