rspec

#!/bin/bash

set -euo pipefail

echo -e "--- :database: Preparing databases"
      
bin/rake db:test:prepare db:job_log_chunks:reset

echo -e "+++ :rspec: Running \033[33mspecs\033[0m :cow::bell:"

bin/rake "knapsack_pro:rspec[--options .rspec.ci --seed ${BUILDKITE_SEED:-${BUILDKITE_BUILD_NUMBER:-$RANDOM}}]"

if [[ "$BUILDKITE_RETRY_COUNT" -gt 0 ]]; then
  echo -e "--- Verifying retried set of specs"
  .buildkite/scripts/rspec_retry_drift_detector
fi

rspec_discover

#!/bin/bash

set -euo pipefail

bin/rspec --dry-run --format json --out tmp/rspec.json

rspec_retry_drift_detector.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

# This script presumes there is an rspec report from a recently completed run
# in tmp, will look up the current build to find which job was retried,
# download the retried job's rspec report, and compare the reports.
#
# (Only run on a retry job, where retry count > 0)

require "bundler/setup"

require "http"
require "json"

# Make sure we know the job we're running in, and it has a report
job_id = ENV.fetch("BUILDKITE_JOB_ID")

puts "=> Job ID: #{job_id}"
puts

unless File.size? "tmp/rspec-#{job_id}.json"
  puts "Missing rspec report for this job, #{job_id}"
  exit 1
end

puts "Looking up current build details..."

# Needs a token to fetch the current build
# (A BUILDKITE_RETRIED_JOB_ID would be nice)
buildkite_token = ENV.fetch("BUILDKITE_API_TOKEN")

organization_slug = ENV.fetch("BUILDKITE_ORGANIZATION_SLUG")
pipeline_slug = ENV.fetch("BUILDKITE_PIPELINE_SLUG")
build_number = ENV.fetch("BUILDKITE_BUILD_NUMBER")

response = HTTP.auth("Bearer #{buildkite_token}").get("https://api.buildkite.com/v2/organizations/#{organization_slug}/pipelines/#{pipeline_slug}/builds/#{build_number}?include_retried_jobs=true")
unless response.status.success?
  puts
  puts "Unable to fetch build from API:"
  puts response.inspect
  puts response.body
  exit 1
end

build = response.parse
build_id = build.fetch("id")
puts "=> Build ID: #{build.fetch("id")}"
puts

# Find the full set of retried jobs in order
puts "Tracing retried jobs..."
retried_jobs = []
retry_job_id = job_id
while retried_job = build["jobs"].find { |job| job["retried_in_job_id"] == retry_job_id }
  retried_job_id = retried_job["id"]
  retried_jobs.prepend retried_job
  puts "=> Retried from Job ID: #{retried_job_id}"
  retry_job_id = retried_job_id
end
puts

# Look for the first retried job which output an rspec report
puts "Finding the first rspec report..."
unless retried_job = retried_jobs.find { |job| system "buildkite-agent", "artifact", "download", "--step", job["id"], "--include-retried-jobs", "tmp/rspec-#{job["id"]}.json", "." }
  puts "No earlier retry has an rspec report, so presuming this retry was unrelated to rspec issues."
  exit
end

retried_job_id = retried_job["id"]
puts "Found original rspec report in #{retried_job_id}"
puts

# Compare the examples in both reports and make sure they match
#
# We don't care too much about order, as long as the same set of specs are
# run, so sort them on the way through.
puts "Comparing rspec reports for discrepancies..."

report = JSON.parse(File.read("tmp/rspec-#{job_id}.json"))
retried_report = JSON.parse(File.read("tmp/rspec-#{retried_job_id}.json"))

examples = report["examples"].map { |e| e["id"] }.sort
retried_examples = retried_report["examples"].map { |e| e["id"] }.sort

if examples.empty?
  puts "+++ RSpec examples do not match across retried and retry job"
  puts
  puts "Retry job did not run any examples!"
  exit 1
elsif examples != retried_examples
  puts "+++ RSpec examples do not match across retried and retry job"
  puts

  # Output a pretty diff
  File.write("tmp/rspec-#{job_id}.txt", examples.join("\n") << "\n")
  File.write("tmp/rspec-#{retried_job_id}.txt", retried_examples.join("\n") << "\n")
  system "diff", "-u", "tmp/rspec-#{retried_job_id}.txt", "tmp/rspec-#{job_id}.txt"

  exit 1
else
  puts "Looks okay! #{examples.count} matching examples run across both jobs."
end

rspec_verify.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

require "bundler/setup"

require "json"
require "set"

# Compare a full rspec report and a collection of partial rspec reports to make
# sure they cover the same set of examples. This script should be runnable
# without being on Buildkite.
#
# If run locally, output some hints to help folks run a comparison.
#
# See .buildkite/steps/rspec_verify

puts "Comparing the full rspec suite against the pieces which were run."
puts

full_path = "tmp/rspec.json"

unless File.exists? full_path
  puts "Full report is missing."
  puts
  puts "Generate one with:"
  puts
  puts "  bin/rspec --dry-run --format json --out #{full_path}"
  puts

  exit 2
end

full = JSON.parse(File.read(full_path))
full_examples = full.fetch("examples").map { |example| example.fetch("id") }.to_set
puts "#{full_examples.size} examples in the full suite"

pieces_paths = Dir["tmp/rspec-*.json"]

unless pieces_paths.any?
  puts "Pieces are misssing."
  puts
  puts "Find a buildkite build:"
  puts
  puts "  https://buildkite.com/buildkite/buildkite/builds/last?state=finished"
  puts
  puts "then download its pieces with with:"
  puts
  puts %{  bk artifact download --build BUILD-UUID "tmp/rspec-*.json"}
  puts

  exit 2
end

pieces = pieces_paths.each_with_object({}) do |path, hash|
  hash[path[%r{tmp/rspec-(.*?)\.json}, 1]] = JSON.parse(File.read(path))
end
pieces_examples = pieces.transform_values { |piece| piece.fetch("examples").map { |example| example.fetch("id") }.to_set }
all_pieces_examples = pieces_examples.values.sum(Set.new)
puts "#{all_pieces_examples.size} examples across all pieces"

if full_examples == all_pieces_examples
  puts
  puts "Examples match! ☑️"
else
  puts
  puts "Examples do not match."

  only_in_full = full_examples - all_pieces_examples
  if only_in_full.any?
    puts
    puts "Only in full suite:"
    only_in_full.each do |example|
      puts "- #{example}"
    end
  end

  pieces_examples.each do |piece_name, piece_examples|
    only_in_piece = piece_examples - full_examples
    if only_in_piece.any?
      puts

      # Can we link to the job by its uuid?
      if ENV["BUILDKITE"] && ENV["BUILDKITE_BUILD_URL"] && piece_name =~ /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
        link_name = piece_name
        link_url = "#{ENV["BUILDKITE_BUILD_URL"]}\##{piece_name}"
        link = "\e]1339;url=#{link_url.gsub(";", "%3b")}#{";content=#{link_name.gsub(";", "%3b")}"}\a"

        puts "Only in #{link}:"
      else
        puts "Only in #{piece_name}:"
      end

      only_in_piece.each do |example|
        puts "- #{example}"
      end
    end
  end

  puts
  puts "We're not running the whole suite of specs, so we cannot ship this code confidently to production. Something might be wrong in knapsack or rspec configuration."
  puts
  puts "You can safely _Rebuild_ this whole build as an immediate fix, but please also tell Sam or make sure this failure is investigated."

  exit 1
end

rspec_verify_wrapper

#!/bin/bash

set -euo pipefail

echo "--- :buildkite: Downloading rspec reports..."

buildkite-agent artifact download "tmp/rspec.json" .
buildkite-agent artifact download "tmp/rspec-*.json" .

echo "--- 📑 Checking for full coverage..."

script/rspec_verify