Skip to content

Instantly share code, notes, and snippets.

@sj26
Last active July 19, 2021 22:49
Show Gist options
  • Save sj26/64d77fd2ffded415c2051f8fa46da9fa to your computer and use it in GitHub Desktop.
Save sj26/64d77fd2ffded415c2051f8fa46da9fa to your computer and use it in GitHub Desktop.
#!/bin/bash
set -euo pipefail
echo -e "--- :database: Preparing databases"
bin/rake db:test:prepare db:job_log_chunks:reset
echo -e "+++ :rspec: Running \033[33mspecs\033[0m :cow::bell:"
bin/rake "knapsack_pro:rspec[--options .rspec.ci --seed ${BUILDKITE_SEED:-${BUILDKITE_BUILD_NUMBER:-$RANDOM}}]"
if [[ "$BUILDKITE_RETRY_COUNT" -gt 0 ]]; then
echo -e "--- Verifying retried set of specs"
.buildkite/scripts/rspec_retry_drift_detector
fi
#!/bin/bash
set -euo pipefail
bin/rspec --dry-run --format json --out tmp/rspec.json
#!/usr/bin/env ruby
# frozen_string_literal: true
# This script presumes there is an rspec report from a recently completed run
# in tmp, will look up the current build to find which job was retried,
# download the retried job's rspec report, and compare the reports.
#
# (Only run on a retry job, where retry count > 0)
require "bundler/setup"
require "http"
require "json"
# Make sure we know the job we're running in, and it has a report
job_id = ENV.fetch("BUILDKITE_JOB_ID")
puts "=> Job ID: #{job_id}"
puts
unless File.size? "tmp/rspec-#{job_id}.json"
puts "Missing rspec report for this job, #{job_id}"
exit 1
end
puts "Looking up current build details..."
# Needs a token to fetch the current build
# (A BUILDKITE_RETRIED_JOB_ID would be nice)
buildkite_token = ENV.fetch("BUILDKITE_API_TOKEN")
organization_slug = ENV.fetch("BUILDKITE_ORGANIZATION_SLUG")
pipeline_slug = ENV.fetch("BUILDKITE_PIPELINE_SLUG")
build_number = ENV.fetch("BUILDKITE_BUILD_NUMBER")
response = HTTP.auth("Bearer #{buildkite_token}").get("https://api.buildkite.com/v2/organizations/#{organization_slug}/pipelines/#{pipeline_slug}/builds/#{build_number}?include_retried_jobs=true")
unless response.status.success?
puts
puts "Unable to fetch build from API:"
puts response.inspect
puts response.body
exit 1
end
build = response.parse
build_id = build.fetch("id")
puts "=> Build ID: #{build.fetch("id")}"
puts
# Find the full set of retried jobs in order
puts "Tracing retried jobs..."
retried_jobs = []
retry_job_id = job_id
while retried_job = build["jobs"].find { |job| job["retried_in_job_id"] == retry_job_id }
retried_job_id = retried_job["id"]
retried_jobs.prepend retried_job
puts "=> Retried from Job ID: #{retried_job_id}"
retry_job_id = retried_job_id
end
puts
# Look for the first retried job which output an rspec report
puts "Finding the first rspec report..."
unless retried_job = retried_jobs.find { |job| system "buildkite-agent", "artifact", "download", "--step", job["id"], "--include-retried-jobs", "tmp/rspec-#{job["id"]}.json", "." }
puts "No earlier retry has an rspec report, so presuming this retry was unrelated to rspec issues."
exit
end
retried_job_id = retried_job["id"]
puts "Found original rspec report in #{retried_job_id}"
puts
# Compare the examples in both reports and make sure they match
#
# We don't care too much about order, as long as the same set of specs are
# run, so sort them on the way through.
puts "Comparing rspec reports for discrepancies..."
report = JSON.parse(File.read("tmp/rspec-#{job_id}.json"))
retried_report = JSON.parse(File.read("tmp/rspec-#{retried_job_id}.json"))
examples = report["examples"].map { |e| e["id"] }.sort
retried_examples = retried_report["examples"].map { |e| e["id"] }.sort
if examples.empty?
puts "+++ RSpec examples do not match across retried and retry job"
puts
puts "Retry job did not run any examples!"
exit 1
elsif examples != retried_examples
puts "+++ RSpec examples do not match across retried and retry job"
puts
# Output a pretty diff
File.write("tmp/rspec-#{job_id}.txt", examples.join("\n") << "\n")
File.write("tmp/rspec-#{retried_job_id}.txt", retried_examples.join("\n") << "\n")
system "diff", "-u", "tmp/rspec-#{retried_job_id}.txt", "tmp/rspec-#{job_id}.txt"
exit 1
else
puts "Looks okay! #{examples.count} matching examples run across both jobs."
end
#!/usr/bin/env ruby
# frozen_string_literal: true
require "bundler/setup"
require "json"
require "set"
# Compare a full rspec report and a collection of partial rspec reports to make
# sure they cover the same set of examples. This script should be runnable
# without being on Buildkite.
#
# If run locally, output some hints to help folks run a comparison.
#
# See .buildkite/steps/rspec_verify
puts "Comparing the full rspec suite against the pieces which were run."
puts
full_path = "tmp/rspec.json"
unless File.exists? full_path
puts "Full report is missing."
puts
puts "Generate one with:"
puts
puts " bin/rspec --dry-run --format json --out #{full_path}"
puts
exit 2
end
full = JSON.parse(File.read(full_path))
full_examples = full.fetch("examples").map { |example| example.fetch("id") }.to_set
puts "#{full_examples.size} examples in the full suite"
pieces_paths = Dir["tmp/rspec-*.json"]
unless pieces_paths.any?
puts "Pieces are misssing."
puts
puts "Find a buildkite build:"
puts
puts " https://buildkite.com/buildkite/buildkite/builds/last?state=finished"
puts
puts "then download its pieces with with:"
puts
puts %{ bk artifact download --build BUILD-UUID "tmp/rspec-*.json"}
puts
exit 2
end
pieces = pieces_paths.each_with_object({}) do |path, hash|
hash[path[%r{tmp/rspec-(.*?)\.json}, 1]] = JSON.parse(File.read(path))
end
pieces_examples = pieces.transform_values { |piece| piece.fetch("examples").map { |example| example.fetch("id") }.to_set }
all_pieces_examples = pieces_examples.values.sum(Set.new)
puts "#{all_pieces_examples.size} examples across all pieces"
if full_examples == all_pieces_examples
puts
puts "Examples match! ☑️"
else
puts
puts "Examples do not match."
only_in_full = full_examples - all_pieces_examples
if only_in_full.any?
puts
puts "Only in full suite:"
only_in_full.each do |example|
puts "- #{example}"
end
end
pieces_examples.each do |piece_name, piece_examples|
only_in_piece = piece_examples - full_examples
if only_in_piece.any?
puts
# Can we link to the job by its uuid?
if ENV["BUILDKITE"] && ENV["BUILDKITE_BUILD_URL"] && piece_name =~ /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
link_name = piece_name
link_url = "#{ENV["BUILDKITE_BUILD_URL"]}\##{piece_name}"
link = "\e]1339;url=#{link_url.gsub(";", "%3b")}#{";content=#{link_name.gsub(";", "%3b")}"}\a"
puts "Only in #{link}:"
else
puts "Only in #{piece_name}:"
end
only_in_piece.each do |example|
puts "- #{example}"
end
end
end
puts
puts "We're not running the whole suite of specs, so we cannot ship this code confidently to production. Something might be wrong in knapsack or rspec configuration."
puts
puts "You can safely _Rebuild_ this whole build as an immediate fix, but please also tell Sam or make sure this failure is investigated."
exit 1
end
#!/bin/bash
set -euo pipefail
echo "--- :buildkite: Downloading rspec reports..."
buildkite-agent artifact download "tmp/rspec.json" .
buildkite-agent artifact download "tmp/rspec-*.json" .
echo "--- 📑 Checking for full coverage..."
script/rspec_verify
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment