Benchmarking ActiveJob AsyncAdapter vs. SuckerPunchAdapter

bench_async_vs_sucker_punch.rb

#!/usr/bin/env ruby

#####################################################################
#
# Benchmarking ActiveJob AsyncAdapter vs. SuckerPunchAdapter
#
# Copy this file into the `activejob` directory of the Rails git
# project and, optionally, make it executable. Make sure that
# `bundle install` was successfully run on the Rails repo first. Then
# run this file under a bundler context:
#
# > bundle exec ruby ./bench_async_vs_sucker_punch.rb
#
# The results at the end of this file were captured on my old MacBook
# Air running Ubuntu 15.04.

$: << File.expand_path('../lib', __FILE__)

require 'active_job'
require 'concurrent'
require 'sucker_punch'

require 'benchmark'
require 'benchmark/ips'

IPS_TEST_COUNT = 100
BMBM_TEST_COUNT = 5000

GlobalID.app = 'aj'
ActiveJob::Base.logger.level = 4
SuckerPunch.logger = nil

class BenchmarkJob < ActiveJob::Base
  include ActiveSupport::Configurable
  config_accessor :latch
  queue_as :default
 
  def perform
    self.latch.count_down
  end
end

def benchmark_adapter(tests)
  latch = Concurrent::CountDownLatch.new(tests)
  BenchmarkJob.config.latch = latch
  tests.times { BenchmarkJob.perform_later }
  latch.wait
end

print "\nIPS Benchmark ====================================================\n\n"
 
Benchmark.ips do |bm|
  ActiveJob::Base.queue_adapter = :async

  ActiveJob::QueueAdapters::AsyncAdapter.executor = ActiveJob::QueueAdapters::AsyncAdapter.create_default_executor
  bm.report('async - default pool') { benchmark_adapter(IPS_TEST_COUNT) }

  ActiveJob::QueueAdapters::AsyncAdapter.executor = Concurrent::FixedThreadPool.new(2)
  bm.report('async - fixed pool') { benchmark_adapter(IPS_TEST_COUNT) }

  ActiveJob::Base.queue_adapter = :sucker_punch
  bm.report('sucker_punch') { benchmark_adapter(IPS_TEST_COUNT) }
end

print "\nBMBM Benchmark ===================================================\n\n"

Benchmark.bmbm do |bm|
  ActiveJob::Base.queue_adapter = :async

  ActiveJob::QueueAdapters::AsyncAdapter.executor = ActiveJob::QueueAdapters::AsyncAdapter.create_default_executor
  bm.report('async - default pool') { benchmark_adapter(BMBM_TEST_COUNT) }

  ActiveJob::QueueAdapters::AsyncAdapter.executor = Concurrent::FixedThreadPool.new(2)
  bm.report('async - fixed pool') { benchmark_adapter(BMBM_TEST_COUNT) }

  ActiveJob::Base.queue_adapter = :sucker_punch
  bm.report('sucker_punch') { benchmark_adapter(BMBM_TEST_COUNT) }
end

__END__

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ruby 2.2.2p95 (2015-04-13 revision 50295) [x86_64-linux]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

IPS Benchmark ====================================================

Calculating -------------------------------------
async - default pool     1.000  i/100ms
  async - fixed pool     1.000  i/100ms
        sucker_punch     1.000  i/100ms
-------------------------------------------------
async - default pool      5.844  (±17.1%) i/s -     29.000 
  async - fixed pool      6.492  (±15.4%) i/s -     32.000 
        sucker_punch      6.074  (±16.5%) i/s -     31.000 

BMBM Benchmark ===================================================

Rehearsal --------------------------------------------------------
async - default pool  11.420000   0.290000  11.710000 ( 11.389963)
async - fixed pool    11.260000   0.230000  11.490000 ( 11.201503)
sucker_punch          11.200000   0.340000  11.540000 ( 11.252289)
---------------------------------------------- total: 34.740000sec

                           user     system      total        real
async - default pool  10.890000   0.310000  11.200000 ( 10.869860)
async - fixed pool    10.650000   0.300000  10.950000 ( 10.676605)
sucker_punch          11.380000   0.290000  11.670000 ( 11.325212)

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
jruby 9.0.0.0.pre1 (2.2.0p0) 2015-01-20 d537cab Java HotSpot(TM) 64-Bit Server VM 25.51-b03 on 1.8.0_51-b16 +jit [linux-amd64]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

NOTE: The significant performance difference between the rehearsal and real
benchmarks for AsyncAdapter on JRuby is likely due to JVM warm-up. On JRuby,
`concurrent-ruby` thread pools are thin wrapers around the thread pools in
`java.util.concurrent`. The JVM will optimize them over time.

IPS Benchmark ====================================================

Calculating -------------------------------------
async - default pool     1.000  i/100ms
  async - fixed pool     1.000  i/100ms
        sucker_punch     1.000  i/100ms
-------------------------------------------------
async - default pool      2.615  (±38.2%) i/s -     13.000 
  async - fixed pool      3.175  (± 0.0%) i/s -     16.000 
        sucker_punch      4.112  (±24.3%) i/s -     19.000 

BMBM Benchmark ===================================================

Rehearsal --------------------------------------------------------
async - default pool  39.760000   3.220000  42.980000 ( 17.851281)
async - fixed pool    23.210000   1.910000  25.120000 ( 13.269807)
sucker_punch          15.730000   1.000000  16.730000 ( 11.120381)
---------------------------------------------- total: 84.830000sec

                           user     system      total        real
async - default pool  14.170000   0.720000  14.890000 ( 10.483572)
async - fixed pool    14.820000   0.780000  15.600000 ( 10.971751)
sucker_punch          14.940000   0.880000  15.820000 ( 11.016930)