Run RSpec in a loop, noting what tests were run before the first failure

reduce_rspec_failure_context.rb

require 'json'
require 'pp'
require 'sequel'

module CaptureOutput
  module_function

  def capture_stdout(&block)
    Private.capture(STDOUT, &block)
  end

  def capture_stderr(&block)
    Private.capture(STDERR, &block)
  end

  module Private
    module_function

    def capture(stream)
      old_stream = stream.clone
      pipe_r, pipe_w = IO.pipe
      pipe_r.sync = true
      output = ""
      reader = Thread.new do
        begin
          loop do
            output << pipe_r.readpartial(1024)
          end
        rescue EOFError
          # Raised on pipe_w.close below
        end
      end
      stream.reopen(pipe_w)
      yield
    ensure
      stream.reopen(old_stream)
      pipe_w.close
      reader.join
      return output
    end
  end
end

class SpecRun
  class SpecsDidNotRunError < StandardError; end

  def initialize
    @results = nil
  end

  def call
    json_output = nil
    CaptureOutput.capture_stderr do
      CaptureOutput.capture_stdout do
        system({'RAILS_ENV' => 'test'}, *%w{bundle exec rake db:reset})
      end
      json_output = CaptureOutput.capture_stdout do
        system(*%w{bundle exec rspec --format json})
      end
    end
    @results = JSON.load(json_output)
  end

  def count_to_failure
    locations_to_failure.count if failed?
  end

  def locations_to_failure
    must_run_first!
    locations = []
    if failed?
      examples.find do |ex|
        locations << ex['file_path'] + ':' + ex['line_number'].to_s
        ex['status'] != 'passed'
      end
    end
    locations
  end

  def failures
    must_run_first!
    @failures ||= examples.select{|ex| ex['status'] != 'passed' }
  end

  def failed?
    must_run_first!
    failures.any?
  end

  private

  attr_reader :results

  def examples
    results['examples']
  end

  def must_run_first!
    if @results.nil?
      raise SpecsDidNotRunError, "You must run the specs first"
    end
  end
end

DB = Sequel.connect('sqlite://rspec-failure-reducer.db')

DB.create_table?(:attempts) do
  primary_key :id
  Integer :count_to_failure
  String :locations_to_failure_as_json, text: true
  DateTime :created_at
end

attempts = DB[:attempts]

loop do
  run = SpecRun.new
  run.call
  attempts.insert(
    count_to_failure: run.count_to_failure,
    locations_to_failure_as_json: JSON.dump(run.locations_to_failure),
    created_at: Time.now,
  )
  print(run.failed? ? "(F:#{run.count_to_failure})" : '.')
end
puts