simplecov merging race condition

dumb_test.rb

#!/usr/bin/env ruby

# This demonstrates a race condition in simplecov:HEAD that occurs when you
# have several processes merging results at the same time (e.g. test-queue,
# parallel_test). The bug manifests itself as either missing coverage, or
# outright failures.
#
# Usage:
#   ruby dumb_test.rb
#
#   # with oj
#   ENABLE_OJ=1 ruby dumb_test.rb
#
#   # make the race condition worse
#   NUM_PROCESSES=10 UNSYNCHRONIZED_CALLS=5 ruby dumb_test.rb
#
# Basically there are 3 main problems:
# 1. `SimpleCov.result` doesn't cache the `@result`, so the default
#    `at_exit` behavior causes it to store and merge 3 times.
# 2. `SimpleCov::ResultMerger.resultset` calls `.stored_data` twice
# 3. `SimpleCov::ResultMerger.merged_result` doesn't synchronize or
#    cache `.resultset`, so a concurrent `.store_result` call can
#    causes us to read an empty file
#
# This can cause the formatter to miss out on coverage data in our
# formatters and/or get the wrong values for covered percentages.
#
# Furthermore, if you use OJ, `JSON.parse("") -> nil`, which means
# `.resultset` can be nil, causing exceptions as seen in
# https://github.com/colszowka/simplecov/issues/406.

NUM_PROCESSES        = ENV.fetch("NUM_PROCESSES", "5").to_i
ENABLE_OJ            = ENV.fetch("ENABLE_OJ", "0").to_i
UNSYNCHRONIZED_CALLS = ENV.fetch("UNSYNCHRONIZED_CALLS", "2").to_i

if ENABLE_OJ == 1
  require "oj"
  require 'oj_mimic_json'
end

require "simplecov"
require "simplecov/result_merger"

File.delete("coverage/.resultset.json") if File.exist?("coverage/.resultset.json")

SimpleCov::ResultMerger.singleton_class.prepend(Module.new {
  def stored_data
    return unless File.exist?(resultset_path)
    data = File.read(resultset_path)

    if data.nil? || data.length < 2
      puts "WARNING: bad .resultset.json! #{data.inspect}"
      return
    end
    data
  end
})

pids = NUM_PROCESSES.times.map do |i|
  fork do
    data = {
      i => {
        "coverage" => Hash[
          1800.times.map { |j| ["file_#{j}.rb", 100.times.map { rand < 0.5 ? nil : 1 }] }
        ],
        "timestamp" => Time.now.to_i
      }
    }
    3.times do
      SimpleCov::ResultMerger.store_result(data)
      UNSYNCHRONIZED_CALLS.times do
        puts "resultset is empty!" if SimpleCov::ResultMerger.resultset.size == 0
      end
    end
    puts "done #{i}"
  end
end

pids.each do |pid|
  Process.wait pid
end