Benchmark custom cops used for linting in Homebrew.

benchmark-custom-homebrew-cops.rb

# Script to manually run each custom cop individually to see if
# there are any very slow cops that can be improved on average.
#
# Note: This is very slow since each cop takes 20-50 seconds to run.

require "pathname"
require "benchmark"

unless system("brew --prefix > /dev/null")
  raise "Missing: Brew is NOT installed locally since `brew --prefix` doesn't work."
end

CASK_TAP = "homebrew/cask"
FORMULA_TAP = "homebrew/core"
INSTALLED_TAPS = `brew tap`.lines.map(&:strip).freeze

[FORMULA_TAP, CASK_TAP].each do |repo|
  next if INSTALLED_TAPS.include?(repo)

  raise "Missing: The #{repo} tap is NOT installed locally."
end

BREW_REPO_DIR = Pathname(`brew --repo`.strip).freeze
BREW_RUBOCOP_DIR = (BREW_REPO_DIR / "Library/Homebrew/rubocops").freeze
BREW_RUBOCOP_CASK_DIR = (BREW_RUBOCOP_DIR / "cask").freeze

def benchmark_rubocop(cop, tap, retries: 3)
  command = "brew style --reset-cache --only-cops=#{cop} #{tap} > /dev/null"
  retries.times.map do
    Benchmark.measure { system(command) }.real
  end.sum / retries # take the average of multiple retries
end

MODULE_REGEX = /^(?<indent>\s*)module (?<module_name>[A-Za-z]+)\s*$/.freeze
CLASS_REGEX = /^(?<indent>\s*)class (?<class_name>[A-Za-z]+) < [A-Za-z]+\s*$/.freeze

def find_cops(dir:, excluded_cops: [])
  all_cops = []
  Dir.children(dir).each do |file|
    file = dir / file
    next unless file.file?
    next unless file.extname == ".rb"

    modules_by_indent = {}
    IO.foreach(file) do |line|
      if (match_data = line.match(MODULE_REGEX))
        captures = match_data.named_captures
        indent_size = captures["indent"].size
        modules_by_indent[indent_size] = captures["module_name"]
      elsif modules_by_indent[0] != "RuboCop" || modules_by_indent[2] != "Cop" || modules_by_indent[4].nil?
        next # We expect to find cops in the following location: "RoboCop::Cop::<namespace>::<class>"
      elsif (match_data = line.match(CLASS_REGEX))
        captures = match_data.named_captures
        next unless captures["indent"].size == 6

        namespace = modules_by_indent[4]
        klass = captures["class_name"]
        all_cops << "#{namespace}/#{klass}"
      end
    end
  end
  all_cops.sort - excluded_cops
end

FORMULA_COPS = find_cops(dir: BREW_RUBOCOP_DIR, excluded_cops: %w[Homebrew/MoveToExtendOS]).freeze
CASK_COPS = find_cops(dir: BREW_RUBOCOP_CASK_DIR).freeze

[
  ["Formula", FORMULA_COPS, FORMULA_TAP],
  ["Cask", CASK_COPS, CASK_TAP],
].each_with_index do |(pkg_type, all_cops, tap), index|
  if index.positive?
    puts
    puts "----------"
    puts
  end

  puts ">>> Benchmarking all #{all_cops.count} Brew #{pkg_type} cops"

  cops_by_run_secs = all_cops.shuffle.map do |cop|
    run_secs = benchmark_rubocop(cop, tap)
    print "." # make it easy to track progress
    [cop, run_secs]
  end.sort_by { |_cop, run_secs| -run_secs }

  puts

  max_width = all_cops.map(&:length).max
  cops_by_run_secs.each do |cop, run_secs|
    printf("%-#{max_width + 1}s  %.3f seconds\n", cop + ":", run_secs)
  end
end

results.txt

>>> Benchmarking all 62 Brew Formula cops
..............................................................
FormulaAudit/Miscellaneous:                     52.209 seconds
FormulaAudit/OnSystemConditionals:              50.800 seconds
FormulaAudit/Text:                              49.500 seconds
FormulaAudit/Lines:                             46.312 seconds
Homebrew/ExecShellMetacharacters:               46.217 seconds
FormulaAudit/LicenseArrays:                     46.117 seconds
FormulaAuditStrict/RustCheck:                   45.330 seconds
FormulaAudit/LivecheckRegexIfPageMatch:         44.366 seconds
FormulaAudit/LivecheckRegexExtension:           43.231 seconds
FormulaAudit/ChecksumCase:                      42.851 seconds
FormulaAudit/LivecheckUrlProvided:              42.753 seconds
FormulaAudit/LivecheckRegexCaseInsensitive:     42.417 seconds
FormulaAuditStrict/MakeCheck:                   42.363 seconds
FormulaAudit/LivecheckSkip:                     42.207 seconds
FormulaAudit/MpiCheck:                          42.052 seconds
FormulaAudit/ComponentsOrder:                   41.972 seconds
FormulaAudit/QuicTLSCheck:                      41.943 seconds
Homebrew/IORead:                                41.941 seconds
FormulaAudit/Licenses:                          41.676 seconds
FormulaAuditStrict/Text:                        41.656 seconds
FormulaAudit/Checksum:                          41.575 seconds
Homebrew/ShellCommands:                         41.486 seconds
FormulaAudit/Urls:                              41.465 seconds
FormulaAuditStrict/GitUrls:                     41.401 seconds
FormulaAudit/PythonVersions:                    41.372 seconds
FormulaAuditStrict/Requirements:                41.145 seconds
FormulaAudit/LivecheckUrlSymbol:                41.070 seconds
FormulaAudit/OptionDeclarations:                41.059 seconds
FormulaAudit/SafePopenCommands:                 40.875 seconds
FormulaAudit/LivecheckRegexParentheses:         40.618 seconds
FormulaAuditStrict/TestPresent:                 40.489 seconds
FormulaAudit/Options:                           40.468 seconds
FormulaAudit/ProvidedByMacos:                   40.456 seconds
FormulaAudit/Patches:                           40.430 seconds
FormulaAudit/DependencyOrder:                   40.375 seconds
FormulaAudit/BottleDigestIndentation:           40.369 seconds
FormulaAudit/PyoxidizerCheck:                   40.359 seconds
FormulaAudit/Test:                              40.318 seconds
FormulaAudit/Comments:                          40.288 seconds
FormulaAudit/Desc:                              40.280 seconds
FormulaAudit/ShellVariables:                    40.274 seconds
FormulaAudit/DeprecateDisableDate:              40.258 seconds
FormulaAudit/Version:                           40.240 seconds
FormulaAudit/ClassInheritance:                  40.236 seconds
FormulaAudit/PyPiUrls:                          40.230 seconds
FormulaAudit/AssertStatements:                  40.216 seconds
FormulaAudit/UsesFromMacos:                     40.209 seconds
FormulaAudit/BottleFormat:                      40.208 seconds
FormulaAudit/ClassName:                         40.119 seconds
FormulaAudit/ComponentsRedundancy:              40.100 seconds
FormulaAudit/BottleTagIndentation:              40.099 seconds
FormulaAudit/Service:                           40.095 seconds
FormulaAudit/BottleOrder:                       40.079 seconds
FormulaAudit/SingleGenerateCompletionsDSLCall:  40.043 seconds
FormulaAudit/GitUrls:                           39.995 seconds
FormulaAudit/KegOnly:                           39.983 seconds
FormulaAudit/GenerateCompletionsDSL:            39.974 seconds
----------

>>> Benchmarking all 9 Brew Cask cops
.........
Cask/OnSystemConditionals:      22.587 seconds
Cask/Variables:                 21.734 seconds
Cask/Desc:                      21.725 seconds
Cask/StanzaGrouping:            21.642 seconds
Cask/HomepageUrlTrailingSlash:  21.463 seconds
Cask/StanzaOrder:               21.405 seconds
Cask/UrlLegacyCommaSeparators:  21.336 seconds
Cask/Url:                       21.335 seconds
Cask/NoOverrides:               21.291 seconds