Circle CI runner

circle_runner.rb

require 'byebug'
require 'json'
require 'time'

#####
# A quick and dirty script to compare the success rate of branches on CircleCI.
#
# This script will run a project and banch X number of times and calculate it's success / failure rate.
# Inititally developed to track and fix a flaky test suite, it should be easily modifiable to help with
# any CircleCI project that includes flaky jobs.
#
# execute:
# CIRCLE_TOKEN=YOUR_TOKEN ruby circle_stats/runner.rb
#
# How to use...
#
# 1. branch from master
# 2. run this script to capture current baseline
# 3. look at the failures that caused less than 100% success
# 4. use the single test runner and other methods to fix the flaky tests
# 5. check in and push fixes to the branch and run this script again...
# 6. repeat until you acheive your desired success rate.
#
# NOTE: I currently run 20 runs as I am trying to achieve greater than 95% success rate, and 20 runs seems to do that well.
# Also, CircleCI goes through periods where there system is experiencing failure,
# I would halt using the script if you notice a cluster of CircleCI
# failures and use it again when things return to normal.
# Over my usage during a good period of time CircleCI looks like for complex parallel tests
# could account for 2% of the failures on it's own
###
CIRCLE_TOKEN = ENV['CIRCLE_TOKEN'] || 'XYZ'
PROJECT = ENV['CIRCLE_PROJECT'] || 'churn_site'
ORG = ENV['CIRCLE_ORG'] || 'danmayer'
BRANCH = ENV['CIRCLE_BRANCH'] || 'capybara_fix'
RUNS = (ENV['CIRCLE_RUNS'] || 20).to_i
# failure rate is calculated for first job, other jobs used to ensure full workflow completes
JOBS = (ENV['CIRCLE_JOBS'] || 'test,some_slow_job').split(',')

class CircleStats

  def initialize
    @stats = []
    @last_built_at = nil
  end

  def run_build
    @last_built_at = Time.now.utc.to_i
    cmd = "curl -X POST --header 'Content-Type: application/json' -d '{\"branch\": \"#{BRANCH}\" }'  'https://circleci.com/api/v1.1/project/github/#{ORG}/#{PROJECT}/build?circle-token=#{CIRCLE_TOKEN}'"
    # puts cmd
    puts `#{cmd}`
  end

  def capture_build
    # limit is current project total workflow jobs
    cmd = "curl 'https://circleci.com/api/v1.1/project/github/#{ORG}/#{PROJECT}/tree/#{BRANCH}?circle-token=#{CIRCLE_TOKEN}&limit=7'"
    data = `#{cmd}`
    data = JSON.parse(data)
    data.reject!{ |i| i['stop_time'].nil? || Time.iso8601(i['stop_time']).to_i < (@last_built_at + 10) }

    tests_jobs = data.select{ |job| JOBS.include?(job['workflows']['job_name']) }
    metric_job = data.select{ |job| job['workflows']['job_name']==JOBS.first }.first
    
    raise "one of the jobs isn't running yet" unless test_jbs.length == JOBS.length
    tests_jobs.each do |job|
       raise "still running #{job}" unless (job['outcome']=='success' || job['outcome']=='failed')
    end
    
    outcome = metric_job['outcome']
    puts "recording: #{outcome}"
    @stats << outcome
  rescue => err
    puts "waiting, #{err}"
    sleep(15)
    retry
  end
  
  def run_builds
    RUNS.times do |i|
      run_build
      sleep(10) # let it get started
      capture_build
      sleep(45) # hack as other jobs sometimes lag, like docker build
      break if failed_max?
    end
  end

  def failed_max?
    @stats.select{ |s| s=='failed'}.length >= 2
  end

  def total_failure?
    @stats.select{ |s| s=='failed'}.length == @stats.length
  end

  def calc_stats
    puts @stats.join(', ')
    fail_rate = if failed_max?
                  'failed twice, > 10% failure rate'
                elsif  total_failure?
                  'total failure, 100% failure rate, broken'
                else
                  (@stats.select{ |s| s=='failed' }.length.to_f) / @stats.length.to_f
                end
    puts "fail rate: #{fail_rate}"
  end
end

stats = CircleStats.new
stats.run_builds
stats.calc_stats