A script to evenly split the number of tests run between n iOS devices. Useful for splitting UI tests in device farms like Test Lab.

ios_split_device_testing.rb

#!/usr/bin/env ruby

require 'xcodeproj'
require 'open3'

require 'ostruct'
require 'timeout'
require 'benchmark'

class Device
	attr_reader :name, :id, :version

	def initialize(name, id, version)
		@name = name
		@id = id
		@version = version
	end

	def to_s
		"Name: #{@name}, OS: #{@version}, ID: #{@id}"
	end

	class << self
		def connected_device_ids
		  output, error, status = Open3.capture3('system_profiler SPUSBDataType 2>/dev/null')
		  device_id = output.match(/Serial Number: ([\w\d]{40})/)
		  if device_id.nil?
		    raise Exception.new('Could not find a local iOS device to compile the build against.')
		  end

		  device_id[1]
		end

		def connected_devices
			normal_output, output_from_error, status = Open3.capture3('xcrun xctrace list devices')
			output = output_from_error

			# TODO: Use “==” boundary matching so this doesn’t fail on non-English systems?
			devices_header = "== Devices ==\n"
			devices_start = output.index(devices_header)
			header_offset = devices_header.length
			devices_end = output.index("\n\n== Simulators ==")

			if devices_start.nil?
				raise Exception.new('Could not find start of Devices list')
			end

			if devices_end.nil?
				raise Exception.new('Could not find end of Devices list')
			end

			devices = output[devices_start + header_offset, devices_end - header_offset]
			device_info = devices.scan(/(.*)\s\(([\d]{2}.\d+)\)\s\(([\w\d]{40})\)/)

			if device_info.nil?
				raise Exception.new('Could not find a local iOS device')
			end

			device_info.map { |info| Device.new(info[0], info[2], info[1]) }
		end
	end
end

class Simulator
	attr_accessor :name, :type, :version
	attr_reader :uuid

	class << self
		def make_simulators(names, type, version)
			names.map { |name| Simulator.new(name, type, version) }
		end
	end

	def initialize(name, type, version)
		@name = name
		@type = type
		@version = version
	end

	def to_s
		"<Simulator> Name: #{@name}, Type: #{@type}, OS Version: #{@version}, UUID: #{@uuid}"
	end

	def as_destination
		"platform=iOS Simulator,name=#{@name},OS=#{@version}"
	end

	def create
		output, error, status = Open3.capture3("xcrun simctl create '#{@name}' '#{@type}' 'iOS#{@version}'")
		if status.exitstatus != 0
			# TODO: Remove these logs
			puts "output: [#{output}]"
			puts "error: [#{error}]"
			puts "status: [#{status}]"
			raise Exception.new("Could not create iOS Simulator named “#{@name}” of type “#{@type}” using version iOS#{@version}")
		end

		@uuid = output.chomp
	end

	def boot(wait = true)
		raise Exception.new("No UUID defined. Cannot boot Simulator.") if @uuid.nil?

		output, error, status = Open3.capture3("xcrun simctl boot #{@uuid}")
		if status.exitstatus != 0
			raise Exception.new("Could not boot iOS Simulator with UUID #{@uuid}")
		end

		return unless wait

		# NOTE: Yes, Timeout is “bad”. KISS for now.
		timeout = 10 # Make a field with value automatically adjusted higher when in CI, a.k.a. slower systems?
		Timeout.timeout(timeout) do
		  loop do
		    break if is_booted?
		  end
		end
	end

	def is_booted?
		# Would be nice to have models backed by JSON data from `xcrun simctl list -j devices` but let’s keep it simple for now
		devices = `xcrun simctl list devices`
		(devices =~ /#{@uuid}\)\s\(Booted\)\s$/m).nil? == false
	end

	def shutdown
		output, error, status = Open3.capture3("xcrun simctl shutdown #{@uuid}")
		if status.exitstatus != 0
			raise Exception.new("Could not shut down iOS Simulator with UUID #{@uuid}")
		end
	end

	def delete
		output, error, status = Open3.capture3("xcrun simctl delete #{@uuid}")
		if status.exitstatus != 0
			raise Exception.new("Could not delete iOS Simulator with UUID #{@uuid}")
		end
	end
end

class TestSuite
	UNIT_TEST_TYPE = 'com.apple.product-type.bundle.unit-test'.freeze
	UI_TEST_TYPE = 'com.apple.product-type.bundle.ui-testing'.freeze

	attr_reader :type, :class_count
	attr_accessor :configuration, :xcodebuild_command

	def initialize(project_path, target_name, scheme = nil)
		@project = Xcodeproj::Project.open(project_path)
		@target = @project.targets.filter { |target| target.name == target_name }.first
		@type = @target.product_type == UNIT_TEST_TYPE ? 'unit' : 'ui'
		@scheme = scheme ||= Scheme.new("#{project_path}/xcshareddata/xcschemes/#{@target.name}.xcscheme")
		@configuration = @scheme.test_configuration
		@class_count = 0
		@xcodebuild_command = 'env NSUnbufferedIO=YES xcodebuild'
	end

	def test_class_names
		file_names = @target.source_build_phase.files.map(&:display_name).filter { |filename| filename.end_with? 'Tests.swift' }
		file_names.map { |filename| filename[0..filename.index('.swift')-1] }
	end

	def active_classes
		test_class_names - @scheme.skipped_test_identifiers
	end

	def prepare_build_for_simulator(workspace, simulator, derivedDataPath = File.join(Dir.pwd, 'build'))
		# TODO: Coalesce simulator OS versions so this can prepare builds for different types?
		build_command = "#{@xcodebuild_command}  build-for-testing -workspace '#{workspace}' -scheme '#{@target.name}' -destination 'platform=iOS Simulator,name=#{simulator.name},OS=#{simulator.version}' -derivedDataPath #{derivedDataPath}"
		output, error, status = Open3.capture3(build_command)
		if status.exitstatus != 0
			raise Exception.new("Could not prepare build for testing. #{error}")
		end

		File.join(derivedDataPath, 'Build', 'Products', "#{@target.name}_iphonesimulator#{simulator.version}-x86_64.xctestrun")
	end

	def run_on_simulator(workspace, simulator, xctestrun_path, test_classes = nil)
		scheme = @target.name
		cmd = "#{@xcodebuild_command} -workspace #{workspace} -scheme \"#{scheme}\" -configuration #{configuration} -enableCodeCoverage NO -destination '#{simulator.as_destination}' "

		unless test_classes.nil?
			test_classes.each { |test_class| 
				cmd += "'-only-testing:#{scheme}/#{test_class}' "
			}
		end

		cmd += @scheme.skipped_test_identifiers.map { |skipped| 
			"'-skip-testing:#{scheme}/#{skipped}' "
		}.join('')

		cmd += "test-without-building"
		
		cmd
	end

	def split_tests_across_simulators(workspace, simulators, derivedDataPath = nil, test_classes = nil)
		# classes = test_classes ||= active_classes
		classes = test_classes ||= test_class_names
		@class_count = classes.length
		raise Exception.new("Too few classes (#{classes.length}) for number of Simulators provided (#{simulators.length})") unless classes.length > simulators.length

		command = ''
		begin
			simulators.each { |simulator| 
				puts "Creating simulator: #{simulator.name}"
				simulator.create
				puts "Booting…"
				simulator.boot
			}

			puts "Preparing test run file"
			xctestrun = prepare_build_for_simulator(workspace, simulators[0], derivedDataPath)

			puts "Generating test command"
			test_groups = classes.in_groups(simulators.length)
			command = simulators.map.with_index { |simulator, index| run_on_simulator(workspace, simulator, xctestrun, test_groups[index]) }.join(' & ')

			puts "Running tests"
			output, error, status = Open3.capture3(command)
			if status.exitstatus != 0
				# TODO: Should this exit with the `exitstatus` so script consumers can handle faiilure cases appropriately?
				raise Exception.new("Error running tests: #{error}")
			end
		rescue => error
			puts "Error: #{error}"
require 'pry'
binding.pry
		ensure
			simulators.each { |simulator| 
				puts "Shutting down simulator: #{simulator}"
				simulator.shutdown
				puts "Deleting simulator: #{simulator}"
				simulator.delete
			}
		end

		command
	end
end

class Array
	def in_groups(number)
	  group_size = size / number
	  leftovers = size % number

	  groups = []
	  start = 0
	  number.times do |index|
	    length = group_size + (leftovers > 0 && leftovers > index ? 1 : 0)
	    groups << slice(start, length)
	    start += length
	  end

	  groups
	end
end

class Scheme
	attr_reader :test_configuration

	def initialize(scheme_path)
		@scheme = Xcodeproj::XCScheme.new(scheme_path)
		@test_configuration = @scheme.test_action.build_configuration
	end

	def skipped_test_identifiers
		@scheme.test_action.testables.first.skipped_tests.map(&:identifier)
	end
end

project_path = '../mobilesky/SkyGiraffeiOSApp.xcodeproj'

# puts Device.connected_devices
# simulators = [Simulator.new('iPhone 12 Pro Max', '14.2'), Simulator.new('iPhone 11 Pro Max', '13.5'), Simulator.new('iPhone SE (2nd generation)', '13.5')]
# simulators = [Simulator.new('iPhone 12 Pro Max', '14.2'), Simulator.new('iPhone 12 Pro Max', '14.2'), Simulator.new('iPhone 12 Pro Max', '14.2')]
# simulators = [
	# Simulator.new('First iPhone 12 Pro Max', 'iPhone 12 Pro Max', '14.2'),
	# Simulator.new('Second iPhone 12 Pro Max', 'iPhone 12 Pro Max', '14.2'),
	# Simulator.new('Third iPhone 12 Pro Max', 'iPhone 12 Pro Max', '14.2'),
	# Simulator.new('Fourth iPhone 12 Pro Max', 'iPhone 12 Pro Max', '14.2')
# ]

names = [
	'Thirteen one iPhone 11 Pro Max',
	'Thirteen two iPhone 11 Pro Max',
	'Thirteen three iPhone 11 Pro Max',
	'Thirteen four iPhone 11 Pro Max'
	]
simulators = Simulator.make_simulators(names, 'iPhone 11 Pro Max', '13.5')

# integration_tests = TestSuite.new(project_path, 'IntegrationTests')
# puts "💥 IntegrationTests:"
# puts integration_tests.active_classes

unit_tests = TestSuite.new(project_path, 'ApplicationTests')
workspace_path = '../mobilesky/SkyGiraffeiOSApp.xcworkspace'

derivedDataPath = '/tmp/test-builds/14.2'

# TODO: Would be nice to find a way to crawl up from the project to the workspace so the test suite didn't need this specified

# TODO: What happens when one run fails? Gather stderr for all runs? Does `&` fail if one in the chain fails? TEST!
# Maybe answer: An error is generated? Should we exist with the status?

# Set Xcode 11.7 for builds
unit_tests.xcodebuild_command = "time DEVELOPER_DIR=/Applications/Xcode_11.7.app/Contents/Developer #{unit_tests.xcodebuild_command}"

# Track execution time
time = Benchmark.measure {
	# puts unit_tests.split_tests_across_simulators(workspace_path, simulators, derivedDataPath)
	unit_tests.split_tests_across_simulators(workspace_path, simulators, derivedDataPath)
}
puts "#{simulators.length} sims took: #{time.real}"

# puts '💥 ApplicationTests'
# puts unit_tests.active_classes

`tput bel`