Code Coverage with Calabash

01_launch.rb

AfterConfiguration do |config|
  CodeCoverage.clean_up_code_coverage_archive_folder
  CodeCoverage.generate_lcov_baseline_info_file
end
 
Before do |scenario|
  $calabash_launcher = Calabash::Cucumber::Launcher.launcher
 
  CodeCoverage.clean_up_last_run_files
 
#... snip ...
end
 
After do |scenario|
  begin
    CodeCoverage.flush
  rescue Errno::ECONNREFUSED
    CodeCoverage.generate_failed_coverage_file(scenario)
    raise
  end
 
  unless $calabash_launcher.calabash_no_stop?
    calabash_exit
    if $calabash_launcher.active?
      $calabash_launcher.stop
    end
  end
 
  if scenario.passed?
    CodeCoverage.generate_lcov_info_file(scenario)
  else
    CodeCoverage.generate_failed_coverage_file(scenario)
  end
end
 
at_exit do
  $calabash_launcher = Calabash::Cucumber::Launcher.launcher
 
  if $calabash_launcher.simulator_target?
    $calabash_launcher.simulator_launcher.stop unless $calabash_launcher.calabash_no_stop?
  end
 
  CodeCoverage.combine_lcov_info_files
  CodeCoverage.generate_lcov_reports_from_info_file
end

application.rb

class Application
  extend Calabash::Cucumber::Operations
  
  # Flush the code coverage
  def self.flush_code_coverage
    backdoor('flushGCov:', '')
  end
end

code_coverage.rb

class CodeCoverage
  @app_name = 'Foo'

  @build_path = CODE_COVERAGE_BUILD_PARENT_PATH ||= ''
  @code_coverage_folder_path = File.join(@build_path, 'CodeCoverage/')
  @baseline_info_filename = "#{@app_name}_base.info"
  @combined_info_filename = "#{@app_name}_combined.info"
  @project_intermediates = CODE_COVERAGE_INTERMEDIATES_DIR ||= nil

  # Check to see if we are set up to collect code coverage
  # @return [Boolean] True if code coverage is on (CODE_COVERAGE_BUILD_PARENT_PATH was set to something), false if not
  def self.on?
    return !@build_path.empty?
  end

  # If this is a code coverage build, we need to remove the old code coverage files between each scenario
  def self.clean_up_last_run_files
    if self.on?
      calabash_info("[CODE COVERAGE] Deleting .gcda files from #{@project_intermediates}")
      `cd #{@project_intermediates} && find . -name "*.gcda" -print0 | xargs -0 rm`
    end
  end

  # Clean up the folder that contains old .info files and reports
  def self.clean_up_code_coverage_archive_folder
    if self.on?
      Dir.chdir(@build_path)

      calabash_info("[CODE COVERAGE] Deleting the #{@code_coverage_folder_path} directory.")
      FileUtils.rmtree(@code_coverage_folder_path) if Dir.exists?(@code_coverage_folder_path)
    end
  end

  # Tell the app to output coverage files
  def self.flush
    # Flush the code coverage
    if self.on?
      calabash_info('[CODE COVERAGE] Flushing code coverage.')
      Application.flush_code_coverage
    end
  end

  # Generate the baseline file (before any of the tests run)
  # @param [String] path_to_info_files The path to where the .info files will be stored
  def self.generate_lcov_baseline_info_file(path_to_info_files=@code_coverage_folder_path)
    baseline_info_file_path = File.join(path_to_info_files, @baseline_info_filename)

    if self.on?
      FileUtils.mkpath(path_to_info_files)

      calabash_info("[CODE COVERAGE] Generating lcov baseline at #{baseline_info_file_path}.")
      `lcov --capture --initial --directory #{@project_intermediates} --output-file #{baseline_info_file_path}`
    end
  end

  # Get a sanitized filename without weird characters
  # @param [String] file_name File name you want to sanitize
  # @return [String] Sanitized file name
  def self.sanitize_filename(file_name)
    file_name = file_name.gsub(/[^\w\.\-]/, '_')
    file_name = file_name.gsub(/\.{2,}/, '.')
    return file_name
  end

  # Generate an info file for the specified scenario
  # @param [Object] scenario See https://github.com/cucumber/cucumber/wiki/Hooks
  def self.generate_lcov_info_file(scenario)
    if self.on?
      calabash_info('[CODE COVERAGE] Deleting the ObjectiveC.gcda files because the files cause errors.')
      `cd #{@project_intermediates} && find . -name "ObjectiveC.gcda" -print0 | xargs -0 rm`

      # Craft the filename string. Use the scenario feature and scenario name,
      # but convert spaces to underscores and double-periods into single periods.
      info_filename_string = "#{@app_name}_#{scenario.feature.name}__#{scenario.name}.info"
      info_filename_string=self.sanitize_filename(info_filename_string)
      info_file_path = File.join(@code_coverage_folder_path, info_filename_string)

      calabash_info("[CODE COVERAGE] Generating lcov info file at #{info_file_path}.")
      `lcov --capture --directory #{@project_intermediates} --output-file #{info_file_path}`
    end
  end

  # Combine the lcov info files into a single file for reporting
  # @param [String] path_to_info_files The path to where the .info files are stored
  def self.combine_lcov_info_files(path_to_info_files=@code_coverage_folder_path)
    combined_info_file_path = File.join(path_to_info_files, @combined_info_filename)

    if self.on?
      info_files = Dir["#{path_to_info_files}/*.info"]

      index_of_existing_combined_info_file = info_files.index{|s| s.include?(@combined_info_filename)}

      unless index_of_existing_combined_info_file.nil?
        log_warning("[CODE COVERAGE] combine_lcov_info_files found an existing #{@combined_info_filename} in #{path_to_info_files}. We will ignore it from the set to combine and override it.")
        info_files.delete_at(index_of_existing_combined_info_file)
      end

      if info_files.empty?
        raise '[CODE COVERAGE] combine_lcov_info_files could not find any coverage files to combine.'
      else
        calabash_info("[CODE COVERAGE] Combining #{info_files.count} info files into a single file at #{path_to_info_files}.")

        info_files_str = '--add-tracefile '
        info_files_str+= info_files.join(' --add-tracefile ')
        `lcov #{info_files_str} --output-file #{combined_info_file_path}`
      end
    end
  end

  # Generate a nice html report from the specified info file
  # @param [String] info_file (Optional) The specified info file is used to generate reports
  # @param [String] dest_dir (Optional) The destination directory to save the files to
  def self.generate_lcov_reports_from_info_file(info_file=nil, dest_dir="#{@code_coverage_folder_path}")
    report_folder_path = File.join(dest_dir, 'reports')
    info_file = File.join(@code_coverage_folder_path, @combined_info_filename) if info_file.nil?

    if self.on?
      unless File.exists?(info_file)
        raise "[CODE COVERAGE] generate_lcov_reports expected to find #{info_file} at #{dest_dir} but it was not there."
      end

      if Dir.exists?(report_folder_path)
        calabash_info("[CODE COVERAGE] Deleting the #{report_folder_path} directory which was there from a previous run.")
        FileUtils.rmtree(report_folder_path)
      end

      calabash_info("[CODE COVERAGE] Generating code coverage report at #{report_folder_path}.")

      `cd #{dest_dir} && genhtml --ignore-errors source #{info_file} --legend --title "#{@app_name} Code Coverage Report" --output-directory=#{report_folder_path}`
    end
  end

  # Generate a .fail file so we know which test cases to rerun
  # @param [Object] scenario See https://github.com/cucumber/cucumber/wiki/Hooks
  def self.generate_failed_coverage_file(scenario)
    if self.on?
      # Craft the filename string. Use the scenario feature and scenario name,
      # but convert spaces to underscores and double-periods into single periods.
      info_filename_string = "#{@app_name}_#{scenario.feature.name}__#{scenario.name}.failed"
      info_filename_string=self.sanitize_filename(info_filename_string)
      info_file_path = File.join(@code_coverage_folder_path, info_filename_string)

      `touch "#{info_file_path}"`
    end
  end
end

env.rb

# This should be set to the parent directory containing the build folders
CODE_COVERAGE_BUILD_PARENT_PATH = ENV['CODE_COVERAGE_BUILD_PARENT_PATH']
if CODE_COVERAGE_BUILD_PARENT_PATH
  # NOTE: The following lines will get you the path to your intermediates dir as of XCode 7.2 if you are using a workspace. 
  # If you are using projects instead, you will want to modify this section.
  
  xc_workspace_path=Dir["#{CODE_COVERAGE_BUILD_PARENT_PATH}/*.xcworkspace"].first
  schema='YourSchemaName'
  intermediates_dir=`xcodebuild -showBuildSettings -workspace #{xc_workspace_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'`
  CODE_COVERAGE_INTERMEDIATES_DIR=intermediates_dir.strip
end

Using this code, if you pass in a value for CODE_COVERAGE_BUILD_PARENT_PATH, the system will assume you want to collect code coverage. A CodeCoverage folder will be created under your Workspace directory with all the .info files (from successful tests) and .failed files (empty files I decided to generate from failed tests), as well as a report directory with a roll-up.

Note that you will need to replace the intermediates_dir line in env.rb with one that successfully matches your intermediates directory. We have an xcworkspace, but you may just be using a single project.

TODO: I'm going to investigate using Slather instead of lcov to more easily merge coverage between Unit Tests and Calabash tests.

Thanks, Teresa.
I have updated the env.rb and changed the schema variable. While I do:
cucumber CODE_COVERAGE_BUILD_PARENT_PATH=/Desktop/My-App/
I see and error:
xcodebuild: error: If you specify a workspace then you must also specify a scheme. Use -list to see the schemes in this workspace.

Any thoughts?

In '01_launch.rb', the below lines are missing.

require 'calabash-cucumber/launcher'

module Calabash::Launcher
@@launcher = nil

def self.launcher
@@launcher ||= Calabash::Cucumber::Launcher.new
end

def self.launcher=(launcher)
@@launcher = launcher
end
end

Before do |scenario|
launcher = Calabash::Launcher.launcher
options = {
# Add launch options here.
}
launcher.relaunch(options)
launcher.calabash_notify(self)
end

My experience show that this will cause a connection failure between client and server.
Is there a reason behind the omission?

@nishabe because this is meant to supplement the code you already have in there :)

With regard to the intermediates_dir line, I suspect you are not using a workspace with projects inside it, but rather you are using only a project. You will need to change that line to one that successfully matches your intermediates directory. I've updated the comments inline and below to indicate this.

Can you try this?

intermediates_dir=xcodebuild -showBuildSettings -project #{xcodeproj_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'

@TeresaP, thanks for the reply.
After making few more changes, it works now.
This is how my env.rb looks:

require "calabash-cucumber/cucumber"

# This should be set to the parent directory containing the build folders
CODE_COVERAGE_BUILD_PARENT_PATH = ENV['CODE_COVERAGE_BUILD_PARENT_PATH']
if CODE_COVERAGE_BUILD_PARENT_PATH
  # NOTE: The following lines will get you the path to your intermediates dir as of XCode 7.2 if you are using a workspace. 
  # If you are using projects instead, you will want to modify this section.

  schema='Minion-cal'  
  # NOTE: If you are using a workspace
  #xc_workspace_path=Dir["#{CODE_COVERAGE_BUILD_PARENT_PATH}/*.xcworkspace"].first
  #intermediates_dir=`xcodebuild -showBuildSettings -workspace #{xc_workspace_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'`

  # NOTE: If you are using a project
  xcodeproj_path=Dir["#{CODE_COVERAGE_BUILD_PARENT_PATH}/*.xcodeproj"].first
  intermediates_dir=`xcodebuild -showBuildSettings -project #{xcodeproj_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'`


  CODE_COVERAGE_INTERMEDIATES_DIR=intermediates_dir.strip
end

There are some more observations, that I want to share with you.

• I have replaced the function log_message with puts.
• Commented unmount_device
  I also see that, when the number of steps are small (around 13) everything works fine. But when it is around 18, the report generation fails.
  Here is the error that I am getting:

1 scenario (1 passed)
18 steps (18 passed)
1m17.809s
[CODE COVERAGE] Combining 2 info files into a single file at /Users/x08m/Desktop/Minion/CodeCoverage/.
lcov: ERROR: no valid records found in tracefile /Users/me/Desktop/Minion/CodeCoverage//Minion_App_launch__Successful_Applaunch.info
/Users/me/Desktop/Minion/features/support/code_coverage.rb:119:in `generate_lcov_reports_from_info_file': [CODE COVERAGE] generate_lcov_reports expected to find /Users/x08m/Desktop/Minion/CodeCoverage/Minion_combined.info at /Users/me/Desktop/Minion/CodeCoverage/ but it was not there. (RuntimeError)
    from /Users/me/Desktop/Minion/features/support/01_launch.rb:71:in `block in <top (required)>'

There is an extra / that I see in the path, which may be the issue.

Here is my sample application https://github.com/nishabe/Minion/tree/master in case you want to reproduce the issue.

@nishabe thanks for your feedback! unmount_device and log_message were a couple of internal functions I forgot to take out, so thank you :).

Teresa, where is calabash_info is defined? I get 'undefined method `calabash_info' for CodeCoverage:Class (NoMethodError)'.
Do you have any suggestions on the error scenario that I mentioned earlier?