Ruby Test Runner

test_runner.rb

require 'byebug'
require 'json'
require 'time'

#####
# This script is a quick and dirty way to check the success rate of an individual test file.
#
# Purpose:
# run a test file many times and figure out how flaky it is, reporting the success rate and logging any errors
#
# Usage:
# [~/projects/churn] ruby ../my_utils/test_runner.rb
###

PROJ_PATH = ENV['PROJ_PATH'] || File.expand_path(File.join(File.dirname(__FILE__),'../../churn'))
TEST_TARGET = ENV['SPEC_FILE'] || 'spec/features/churn_site_spec.rb'
RUNS = 30

class Runner

  def initialize
    @results = []
    @errors = []
  end

  def check_percentage
    Dir.chdir(PROJ_PATH) do
      collect_runs
      output_results
    end
  rescue => error
    puts "well that wasn't expected... debug it"
    debugger
  end

  private

  # run X times, record success / failures
  def collect_runs
    RUNS.times do
      output = `bundle exec rspec #{TEST_TARGET}`
      status = $?
      @results << status.success?
      @errors << output unless status.success?
    end
  end

  # output success failures and percentage
  def output_results
    puts @errors
    puts '*'*50
    puts @results
    puts "success rate: #{success_rate}"
  end

  def success_rate
     ((@results.select{ |res| res }.size.to_f / @results.size.to_f) * 100).round(2)
  end
  
end

Runner.new.check_percentage