CircleCI 2.0 with parallelism & simplecov for Rails

config.yml

# Ruby CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-ruby/ for more details

defaults: &defaults
  working_directory: ~/split_app
  parallelism: 2

  docker:
    - image: circleci/ruby:2.5.0-node-browsers
      environment:
        PGHOST: localhost
        PGUSER: split
        RAILS_ENV: test

    - image: circleci/postgres:9.6
      environment:
        POSTGRES_USER: split
        POSTGRES_DB: split_test
        POSTGRES_PASSWORD: ""

version: 2
jobs:
  test:
    <<: *defaults
    steps:
      - checkout

      - save_cache:
          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}
          paths:
            - ~/split_app

      - restore_cache:
          keys:
            - gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
            - gem-cache-{{ arch }}-{{ .Branch }}
            - gem-cache
      - run: bundle install --path vendor/bundle
      - save_cache:
          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
          paths:
            - ~/split_app/vendor/bundle

      - restore_cache:
          keys:
            - yarn-cache-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
            - yarn-cache-{{ arch }}-{{ .Branch }}
            - yarn-cache
      - run:
          name: Yarn Install
          command: yarn install --cache-folder ~/.cache/yarn
      - save_cache:
          key: yarn-cache-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}
          paths:
            - ~/split_app/.cache/yarn

      - run: cp .env.test .env
      - run: bundle exec rubocop
      - run: bundle exec brakeman -z
      - run: dockerize -wait tcp://localhost:5432 -timeout 1m
      - run: bundle exec rake db:schema:load

      - run: 
          name: Parallel Rspec
          environment:
            - RAILS_ENV: test
            - RACK_ENV: test
          command: |
            mkdir -p /tmp/rspec
            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
            bundle exec rspec --profile 10 \
                              --format RspecJunitFormatter \
                              --out /tmp/rspec/rspec.xml \
                              --format progress \
                              -- \
                              $TEST_FILES
      - store_test_results:
          path: /tmp/rspec

      - run:
          name: Stash Coverage Results
          command: |
            mkdir coverage_results
            cp -R coverage/.resultset.json coverage_results/.resultset-${CIRCLE_NODE_INDEX}.json
      - persist_to_workspace:
          root: .
          paths:
            - coverage_results

  coverage:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .

      - restore_cache:
          key: v2-repo-{{ .Environment.CIRCLE_SHA1 }}

      - restore_cache:
          keys:
            - gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
            - gem-cache-{{ arch }}-{{ .Branch }}
            - gem-cache

      - run: bundle install --path vendor/bundle

      - run:
          name: Merge and check coverage
          command: |
            RUN_COVERAGE=true bundle exec rake simplecov:report_coverage
      - store_artifacts:
          path: ~/split_app/coverage
          destination: coverage

workflows:
  version: 2
  build_and_test:
    jobs:
        - test
        - coverage:
            requires: 
              - test

rails_helper.rb

# frozen_string_literal: true

# spec/rails_helper.rb
require 'active_support/inflector'
require 'simplecov'

SimpleCov.start 'rails' do
  add_filter '/spec/'
  add_filter '/config/'
  add_filter '/vendor/'
  Dir['app/*'].each do |dir|
    add_group File.basename(dir).humanize, dir
  end
  minimum_coverage 100 unless ENV['CIRCLE_JOB']
end

# ...

simplecov_helper.rb

# frozen_string_literal: true

# spec/simplecov_helper.rb
require 'active_support/inflector'
require "simplecov"

class SimpleCovHelper
  def self.report_coverage(base_dir: "./coverage_results")
    SimpleCov.start 'rails' do
      skip_check_coverage = ENV.fetch("SKIP_COVERAGE_CHECK", "false")

      add_filter '/spec/'
      add_filter '/config/'
      add_filter '/vendor/'

      Dir['app/*'].each do |dir|
        add_group File.basename(dir).humanize, dir
      end

      minimum_coverage(100) unless skip_check_coverage
      merge_timeout(3600)
    end
    new(base_dir: base_dir).merge_results
  end

  attr_reader :base_dir

  def initialize(base_dir:)
    @base_dir = base_dir
  end

  def all_results
    Dir["#{base_dir}/.resultset*.json"]
  end

  def merge_results
    results = all_results.map { |file| SimpleCov::Result.from_hash(JSON.parse(File.read(file))) }
    SimpleCov::ResultMerger.merge_results(*results).tap do |result|
      SimpleCov::ResultMerger.store_result(result)
    end
  end
end

simplecov_parallel.rake

# frozen_string_literal: true

# lib/tasks/simplecov_parallel.rake
if Rails.env.test?
  require_relative "../../spec/simplecov_helper"
  namespace :simplecov do
    desc "merge_results"
    task report_coverage: :environment do
      SimpleCovHelper.report_coverage
    end
  end
end

The gist of what's happening here is since CircleCI 2.0 wih parallelism(2x) will only run approximately 50% of the test per instance, we end up with failed SimpleCov coverage. We therefore use a combination of CircleCI workflows + custom SimpleCov rake task + SimpleCov merge helper to merge the partial coverage results into one.

Hey trev, thanks for the gist.
Just wanted to confirm if this generates the html file in the end or just merges the jsons? I am struggling to get my desired results out of it. Thanks in advance.

Got it working, had to tweak a bit for my usecase. And we should use parallelism 1 for coverage step.

Anyway, issue now is that i dont see option to rerun the build or rerun without cache. As i used to see on simple circle builds, i think it is something to do with workflows. It seems using workflows there is no option to rerun without cache. We can only rerun whole workflow. This is really odd

Here is a gist i wrote more detailed and covering another approach. May help anyone looking into same issue.

Only just saw this @ziaulrehman40, glad you got it all sorted 👍

For others running into similar issues:

simplecov 0.19.0 introduced breaking changes in this commit/this PR and the script here won't fully work anymore. The symptom was an empty (0% coverage) report as the result.

We were able to fix it by using the following merge_results code instead:

def merge_results
    results = []
    all_results.each do |result_file_name|
      puts "Processing #{result_file_name}"
      results += SimpleCov::Result.from_hash(JSON.parse(File.read(result_file_name)))
    end

    SimpleCov::ResultMerger.merge_results(*results).tap do |result|
      SimpleCov::ResultMerger.store_result(result)
    end
end

This issue on the simplecov repo was very helpful and special thanks to @mobilutz!

@FGoessler you can make it even tidier by using SimpleCov.collate.

Here's our latest simplecov_helper.rb:

# frozen_string_literal: true

require 'active_support/inflector'
require 'active_model/type'
require 'simplecov'
require 'parallel_tests'

module SimpleCovHelper
  def self.process_coverage
    skip_check_coverage = ActiveModel::Type::Boolean.new.cast(
      ENV.fetch('SKIP_COVERAGE_CHECK', 'false'),
    )

    SimpleCov.start 'rails' do
      enable_coverage :branch
      coverage_criterion :branch

      add_filter '/config/'
      add_filter '/spec/'
      add_filter '/vendor/'

      Dir['app/*'].each do |dir|
        add_group File.basename(dir).humanize, dir
      end

      minimum_coverage(line: 100, branch: 87.89) unless skip_check_coverage
      merge_timeout 3600
    end
  end

  def self.merge_coverage_results(base_dir: './coverage_results', file_pattern: '.resultset*.json')
    SimpleCov.collate(Dir["#{base_dir}/#{file_pattern}"])
  end
end

and the lib/tasks/simplecov_parallel.rake:

# frozen_string_literal: true

if Rails.env.test?
  require_relative '../../spec/simplecov_helper'
  namespace :simplecov do
    desc 'Merge coverage results'
    task merge_coverage_results: :environment do
      SimpleCovHelper.merge_coverage_results
    end

    desc 'Process coverage results'
    task process_coverage: :environment do
      SimpleCovHelper.process_coverage
    end
  end
end

@FGoessler Since you added a merge_coverage_results to your rake file, what does your new config file look like?

Thanks @trev :), I referred this and implemented for Github action using artifacts