Rails System Test Supervisor

README.md

Rails System Test Supervisor

System tests with Rails, Capybara, and Selenium (with the Chrome driver) may go rogue and spike the CPU to 100% on the spawned browser depending on how heavily you've monkey patched these libraries and/or have complex JavaScript that mutates the DOM (morphs etc.).

This supervisor class monitors the driver process and its children and will kill any rogue process that persistently spikes to >=99% CPU utilization for an extended period of time. If and when this occurs, you lose control of the browser and your tests will hang until they eventually time out or error.

The supervisor helps ensure your test suite doesn't hang and trigger unwarranted costs with your CI provider.

Usage

Update your ApplicationSystemTestCase to include the following in your before_setup/after_teardown methods.

class ApplicationSystemTestCase < ActionDispatch::SystemTestCase

  def before_setup
    @supervisor = SystemTestSupervisor.new(Capybara.current_session.driver)
    @supervisor.start
  end
  
  def after_teardown
    @supervisor.stop
  end
end

system_test_supervisor.rb

require "concurrent" # SEE: ruby-concurrency/concurrent-ruby

class SystemTestSupervisor
  def initialize(driver, interval: 10, max_cpu_utilization: 99, max_history: 3, verbose: false)
    @driver = driver
    @interval = interval
    @max_cpu_utilization = max_cpu_utilization
    @max_history = max_history
    @tracker = {}
    @verbose = verbose
    @started = false
  end

  attr_reader :driver, :interval, :max_cpu_utilization, :max_history

  def start
    return false unless chrome?
    return false if started?

    @started = true

    puts "Starting supervisor..." if verbose?
    @promise ||= begin
      promise = Concurrent::Promise.execute(executor: :io) do
        loop do
          break if stopped?
          sleep interval
          puts "Tracking CPU ultilization: #{stopped?} #{tracker.inspect}" if verbose?
          track_cpu_utilization
          kill_rogue_pids!
        end
      end

      promise.then do
        puts "Supervisor stopped!" if verbose?
        :stopped
      end

      promise.rescue do |error|
        puts "Supervisor error! #{error.message} #{error.backtrace[0, 3].join("\n")}"
      end
    end

    true
  end

  def started?
    @started
  end

  def stop
    @started = false
  end

  def stopped?
    !started?
  end

  def verbose?
    @verbose
  end

  private

  attr_reader :promise, :tracker

  def chrome?
    driver.try(:options)&.dig(:browser) == :chrome
  end

  def service_manager
    driver.browser.instance_variable_get(:@service_manager)
  end

  def service_process
    service_manager.instance_variable_get(:@process)
  end

  def service_pid
    service_process.instance_variable_get(:@pid)
  end

  def driver_pids
    return [] unless chrome?
    # `pgrep chromedriver`.strip.split(/\s/).map(&:to_i)
    [service_pid]
  end

  def browser_pids
    pids = driver_pids.map do |pid|
      `pgrep -P #{pid}`.strip.split(/\s/).map(&:to_i)
    end
    pids.flatten
  end

  def helper_pids
    pids = browser_pids.map do |pid|
      `pgrep -P #{pid}`.strip.split(/\s/).map(&:to_i)
    end
    pids.flatten
  end

  def all_pids
    driver_pids + browser_pids + helper_pids
  end

  def known_pid?(pid)
    all_pids.include?(pid)
  end

  def parent_pid(pid)
    ppid = `ps -o ppid= -p #{pid}`.strip.split(/\s/).first.to_i
    ppid = 0 unless known_pid?(ppid)
    ppid
  end

  def current_cpu_utilization(pid)
    return 0.0 unless all_pids.include?(pid)
    `ps -p #{pid} -o %cpu`.strip.split(/\s/).last.to_f
  end

  def track_cpu_utilization
    all_pids.each do |pid|
      history = tracker[pid] || []
      history.shift if history.size >= max_history
      history << current_cpu_utilization(pid)
      tracker[pid] = history
    end
  end

  def rogue?(pid)
    history = tracker[pid] || []
    return false unless history.size >= max_history
    test = history.all? { |cpu| cpu >= max_cpu_utilization }
    puts "Rogue pid? #{pid}=#{test} #{history}" if verbose?
    test
  end

  def kill!(pid, cpid: nil)
    return unless known_pid?(pid)

    ppid = parent_pid(pid)

    if verbose?
      list = ["<pid=#{pid} cpu=#{tracker.dig(pid).inspect}>"]
      list << "<ppid=#{ppid} cpu=#{tracker.dig(ppid).inspect}>" if ppid
      list << "<cpid=#{cpid} cpu=#{tracker.dig(cpid).inspect}>" if cpid
      puts "Killing rogue pid! #{list.join(" ")}"
    end

    Process.kill("TERM", pid)
  rescue
    puts "Failed to kill rogue pid! #{pid}" if verbose?
    kill!(ppid, cpid: pid)
  end

  def kill_rogue_pids!
    all_pids.each { |pid| kill!(pid) if rogue?(pid) }
  end
end