Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Code Coverage with Calabash
AfterConfiguration do |config|
CodeCoverage.clean_up_code_coverage_archive_folder
CodeCoverage.generate_lcov_baseline_info_file
end
Before do |scenario|
$calabash_launcher = Calabash::Cucumber::Launcher.launcher
CodeCoverage.clean_up_last_run_files
#... snip ...
end
After do |scenario|
begin
CodeCoverage.flush
rescue Errno::ECONNREFUSED
CodeCoverage.generate_failed_coverage_file(scenario)
raise
end
unless $calabash_launcher.calabash_no_stop?
calabash_exit
if $calabash_launcher.active?
$calabash_launcher.stop
end
end
if scenario.passed?
CodeCoverage.generate_lcov_info_file(scenario)
else
CodeCoverage.generate_failed_coverage_file(scenario)
end
end
at_exit do
$calabash_launcher = Calabash::Cucumber::Launcher.launcher
if $calabash_launcher.simulator_target?
$calabash_launcher.simulator_launcher.stop unless $calabash_launcher.calabash_no_stop?
end
CodeCoverage.combine_lcov_info_files
CodeCoverage.generate_lcov_reports_from_info_file
end
class Application
extend Calabash::Cucumber::Operations
# Flush the code coverage
def self.flush_code_coverage
backdoor('flushGCov:', '')
end
end
class CodeCoverage
@app_name = 'Foo'
@build_path = CODE_COVERAGE_BUILD_PARENT_PATH ||= ''
@code_coverage_folder_path = File.join(@build_path, 'CodeCoverage/')
@baseline_info_filename = "#{@app_name}_base.info"
@combined_info_filename = "#{@app_name}_combined.info"
@project_intermediates = CODE_COVERAGE_INTERMEDIATES_DIR ||= nil
# Check to see if we are set up to collect code coverage
# @return [Boolean] True if code coverage is on (CODE_COVERAGE_BUILD_PARENT_PATH was set to something), false if not
def self.on?
return !@build_path.empty?
end
# If this is a code coverage build, we need to remove the old code coverage files between each scenario
def self.clean_up_last_run_files
if self.on?
calabash_info("[CODE COVERAGE] Deleting .gcda files from #{@project_intermediates}")
`cd #{@project_intermediates} && find . -name "*.gcda" -print0 | xargs -0 rm`
end
end
# Clean up the folder that contains old .info files and reports
def self.clean_up_code_coverage_archive_folder
if self.on?
Dir.chdir(@build_path)
calabash_info("[CODE COVERAGE] Deleting the #{@code_coverage_folder_path} directory.")
FileUtils.rmtree(@code_coverage_folder_path) if Dir.exists?(@code_coverage_folder_path)
end
end
# Tell the app to output coverage files
def self.flush
# Flush the code coverage
if self.on?
calabash_info('[CODE COVERAGE] Flushing code coverage.')
Application.flush_code_coverage
end
end
# Generate the baseline file (before any of the tests run)
# @param [String] path_to_info_files The path to where the .info files will be stored
def self.generate_lcov_baseline_info_file(path_to_info_files=@code_coverage_folder_path)
baseline_info_file_path = File.join(path_to_info_files, @baseline_info_filename)
if self.on?
FileUtils.mkpath(path_to_info_files)
calabash_info("[CODE COVERAGE] Generating lcov baseline at #{baseline_info_file_path}.")
`lcov --capture --initial --directory #{@project_intermediates} --output-file #{baseline_info_file_path}`
end
end
# Get a sanitized filename without weird characters
# @param [String] file_name File name you want to sanitize
# @return [String] Sanitized file name
def self.sanitize_filename(file_name)
file_name = file_name.gsub(/[^\w\.\-]/, '_')
file_name = file_name.gsub(/\.{2,}/, '.')
return file_name
end
# Generate an info file for the specified scenario
# @param [Object] scenario See https://github.com/cucumber/cucumber/wiki/Hooks
def self.generate_lcov_info_file(scenario)
if self.on?
calabash_info('[CODE COVERAGE] Deleting the ObjectiveC.gcda files because the files cause errors.')
`cd #{@project_intermediates} && find . -name "ObjectiveC.gcda" -print0 | xargs -0 rm`
# Craft the filename string. Use the scenario feature and scenario name,
# but convert spaces to underscores and double-periods into single periods.
info_filename_string = "#{@app_name}_#{scenario.feature.name}__#{scenario.name}.info"
info_filename_string=self.sanitize_filename(info_filename_string)
info_file_path = File.join(@code_coverage_folder_path, info_filename_string)
calabash_info("[CODE COVERAGE] Generating lcov info file at #{info_file_path}.")
`lcov --capture --directory #{@project_intermediates} --output-file #{info_file_path}`
end
end
# Combine the lcov info files into a single file for reporting
# @param [String] path_to_info_files The path to where the .info files are stored
def self.combine_lcov_info_files(path_to_info_files=@code_coverage_folder_path)
combined_info_file_path = File.join(path_to_info_files, @combined_info_filename)
if self.on?
info_files = Dir["#{path_to_info_files}/*.info"]
index_of_existing_combined_info_file = info_files.index{|s| s.include?(@combined_info_filename)}
unless index_of_existing_combined_info_file.nil?
log_warning("[CODE COVERAGE] combine_lcov_info_files found an existing #{@combined_info_filename} in #{path_to_info_files}. We will ignore it from the set to combine and override it.")
info_files.delete_at(index_of_existing_combined_info_file)
end
if info_files.empty?
raise '[CODE COVERAGE] combine_lcov_info_files could not find any coverage files to combine.'
else
calabash_info("[CODE COVERAGE] Combining #{info_files.count} info files into a single file at #{path_to_info_files}.")
info_files_str = '--add-tracefile '
info_files_str+= info_files.join(' --add-tracefile ')
`lcov #{info_files_str} --output-file #{combined_info_file_path}`
end
end
end
# Generate a nice html report from the specified info file
# @param [String] info_file (Optional) The specified info file is used to generate reports
# @param [String] dest_dir (Optional) The destination directory to save the files to
def self.generate_lcov_reports_from_info_file(info_file=nil, dest_dir="#{@code_coverage_folder_path}")
report_folder_path = File.join(dest_dir, 'reports')
info_file = File.join(@code_coverage_folder_path, @combined_info_filename) if info_file.nil?
if self.on?
unless File.exists?(info_file)
raise "[CODE COVERAGE] generate_lcov_reports expected to find #{info_file} at #{dest_dir} but it was not there."
end
if Dir.exists?(report_folder_path)
calabash_info("[CODE COVERAGE] Deleting the #{report_folder_path} directory which was there from a previous run.")
FileUtils.rmtree(report_folder_path)
end
calabash_info("[CODE COVERAGE] Generating code coverage report at #{report_folder_path}.")
`cd #{dest_dir} && genhtml --ignore-errors source #{info_file} --legend --title "#{@app_name} Code Coverage Report" --output-directory=#{report_folder_path}`
end
end
# Generate a .fail file so we know which test cases to rerun
# @param [Object] scenario See https://github.com/cucumber/cucumber/wiki/Hooks
def self.generate_failed_coverage_file(scenario)
if self.on?
# Craft the filename string. Use the scenario feature and scenario name,
# but convert spaces to underscores and double-periods into single periods.
info_filename_string = "#{@app_name}_#{scenario.feature.name}__#{scenario.name}.failed"
info_filename_string=self.sanitize_filename(info_filename_string)
info_file_path = File.join(@code_coverage_folder_path, info_filename_string)
`touch "#{info_file_path}"`
end
end
end
# This should be set to the parent directory containing the build folders
CODE_COVERAGE_BUILD_PARENT_PATH = ENV['CODE_COVERAGE_BUILD_PARENT_PATH']
if CODE_COVERAGE_BUILD_PARENT_PATH
# NOTE: The following lines will get you the path to your intermediates dir as of XCode 7.2 if you are using a workspace.
# If you are using projects instead, you will want to modify this section.
xc_workspace_path=Dir["#{CODE_COVERAGE_BUILD_PARENT_PATH}/*.xcworkspace"].first
schema='YourSchemaName'
intermediates_dir=`xcodebuild -showBuildSettings -workspace #{xc_workspace_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'`
CODE_COVERAGE_INTERMEDIATES_DIR=intermediates_dir.strip
end
Owner

TeresaP commented Mar 18, 2016

Using this code, if you pass in a value for CODE_COVERAGE_BUILD_PARENT_PATH, the system will assume you want to collect code coverage. A CodeCoverage folder will be created under your Workspace directory with all the .info files (from successful tests) and .failed files (empty files I decided to generate from failed tests), as well as a report directory with a roll-up.

Note that you will need to replace the intermediates_dir line in env.rb with one that successfully matches your intermediates directory. We have an xcworkspace, but you may just be using a single project.

Owner

TeresaP commented Mar 21, 2016

TODO: I'm going to investigate using Slather instead of lcov to more easily merge coverage between Unit Tests and Calabash tests.

nishabe commented Mar 21, 2016

Thanks, Teresa.
I have updated the env.rb and changed the schema variable. While I do:
cucumber CODE_COVERAGE_BUILD_PARENT_PATH=/Desktop/My-App/
I see and error:
xcodebuild: error: If you specify a workspace then you must also specify a scheme. Use -list to see the schemes in this workspace.

Any thoughts?

nishabe commented Mar 21, 2016

In '01_launch.rb', the below lines are missing.

require 'calabash-cucumber/launcher'

module Calabash::Launcher
@@launcher = nil

def self.launcher
@@launcher ||= Calabash::Cucumber::Launcher.new
end

def self.launcher=(launcher)
@@launcher = launcher
end
end

Before do |scenario|
launcher = Calabash::Launcher.launcher
options = {
# Add launch options here.
}
launcher.relaunch(options)
launcher.calabash_notify(self)
end

My experience show that this will cause a connection failure between client and server.
Is there a reason behind the omission?

Owner

TeresaP commented Mar 24, 2016

@nishabe because this is meant to supplement the code you already have in there :)

With regard to the intermediates_dir line, I suspect you are not using a workspace with projects inside it, but rather you are using only a project. You will need to change that line to one that successfully matches your intermediates directory. I've updated the comments inline and below to indicate this.

Can you try this?

intermediates_dir=xcodebuild -showBuildSettings -project #{xcodeproj_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'

nishabe commented Mar 28, 2016

@TeresaP, thanks for the reply.
After making few more changes, it works now.
This is how my env.rb looks:

require "calabash-cucumber/cucumber"

# This should be set to the parent directory containing the build folders
CODE_COVERAGE_BUILD_PARENT_PATH = ENV['CODE_COVERAGE_BUILD_PARENT_PATH']
if CODE_COVERAGE_BUILD_PARENT_PATH
  # NOTE: The following lines will get you the path to your intermediates dir as of XCode 7.2 if you are using a workspace. 
  # If you are using projects instead, you will want to modify this section.

  schema='Minion-cal'  
  # NOTE: If you are using a workspace
  #xc_workspace_path=Dir["#{CODE_COVERAGE_BUILD_PARENT_PATH}/*.xcworkspace"].first
  #intermediates_dir=`xcodebuild -showBuildSettings -workspace #{xc_workspace_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'`

  # NOTE: If you are using a project
  xcodeproj_path=Dir["#{CODE_COVERAGE_BUILD_PARENT_PATH}/*.xcodeproj"].first
  intermediates_dir=`xcodebuild -showBuildSettings -project #{xcodeproj_path} -scheme #{schema} | grep -m 1 'PROJECT_TEMP_ROOT' | sed -n -e 's/^.*PROJECT_TEMP_ROOT = //p'`


  CODE_COVERAGE_INTERMEDIATES_DIR=intermediates_dir.strip
end

nishabe commented Mar 28, 2016

There are some more observations, that I want to share with you.

  • I have replaced the function log_message with puts.
  • Commented unmount_device
    I also see that, when the number of steps are small (around 13) everything works fine. But when it is around 18, the report generation fails.
    Here is the error that I am getting:
1 scenario (1 passed)
18 steps (18 passed)
1m17.809s
[CODE COVERAGE] Combining 2 info files into a single file at /Users/x08m/Desktop/Minion/CodeCoverage/.
lcov: ERROR: no valid records found in tracefile /Users/me/Desktop/Minion/CodeCoverage//Minion_App_launch__Successful_Applaunch.info
/Users/me/Desktop/Minion/features/support/code_coverage.rb:119:in `generate_lcov_reports_from_info_file': [CODE COVERAGE] generate_lcov_reports expected to find /Users/x08m/Desktop/Minion/CodeCoverage/Minion_combined.info at /Users/me/Desktop/Minion/CodeCoverage/ but it was not there. (RuntimeError)
    from /Users/me/Desktop/Minion/features/support/01_launch.rb:71:in `block in <top (required)>'

There is an extra / that I see in the path, which may be the issue.

nishabe commented Mar 28, 2016

Here is my sample application https://github.com/nishabe/Minion/tree/master in case you want to reproduce the issue.

Owner

TeresaP commented Mar 31, 2016

@nishabe thanks for your feedback! unmount_device and log_message were a couple of internal functions I forgot to take out, so thank you :).

nishabe commented Apr 6, 2016

Teresa, where is calabash_info is defined? I get 'undefined method `calabash_info' for CodeCoverage:Class (NoMethodError)'.
Do you have any suggestions on the error scenario that I mentioned earlier?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment