curb, typhoeus, net:http, em-http quick benchmark

r191.txt

ruby 1.9.1p243 (2009-07-16 revision 24175) [x86_64-linux]

curb (0.6.2.1)
em-http-request (0.2.5)
eventmachine (0.12.10)
typhoeus (0.1.13)

      user     system      total        real   Memory (Kb)
4kb std
  0.000000   0.000000   3.010000 ( 22.408594)  24,484
4kb patron
  0.000000   0.000000   0.490000 ( 11.335008)  23,228
4kb curb
  0.060000   0.030000   3.110000 (189.442267) 154,728
4kb em-http
 -0.010000   0.020000   1.630000 ( 14.981627)  33,736
4kb typhoeus
  0.000000   0.000000   4.780000 ( 21.123887)  35,484

6m std
  0.070000   0.030000   4.030000 (113.118963)  38,844
6m patron
  0.010000   0.040000   0.850000 (109.467519)  39,548
6m curb
  0.000000   0.020000   2.640000 (109.389189) 140,596
6m em-http
  0.030000   0.020000   2.030000 (109.923561) 155,576
6m typhoeus
  0.040000   0.030000   7.880000 (110.425908) 157,104

MIXED std
  0.070000   0.030000   7.060000 (134.863841)  38,968
MIXED patron
  0.010000   0.020000   1.430000 (121.305347)  33,472
MIXED curb
  0.050000   0.030000   3.120000 (189.456940) 154,528
MIXED em-http
 -0.010000   0.010000   1.660000 ( 15.978151)  34,456
MIXED typhoeus
  0.080000   0.050000  26.330000 (186.171735) 172,748

ree.txt

ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux], MBARI 0x6770, Ruby Enterprise Edition 2009.10
Copy On Write Friendly = false

curb (0.6.2.1, 0.6.0.0)
em-http-request (0.2.5)
eventmachine (0.12.10)
typhoeus (0.1.13)

      user     system      total        real  Memory (kb)
4kb std
  0.000000   0.010000   3.210000 ( 23.410652)  28,680
4kb patron
  0.000000   0.000000   0.460000 ( 11.408980)  29,292
4kb curb
  0.000000   0.000000   2.350000 (189.393909) 124,824
4kb em-http
  0.000000   0.000000   1.700000 ( 22.603939)  43,672
4kb typhoeus
  0.000000   0.000000   6.050000 ( 17.929791)  49,428

6m std
  0.000000   0.000000  13.440000 (112.739365)  32,948
6m patron
  0.000000   0.000000   1.020000 (108.827092)  40,748
6m curb
  0.000000   0.000000   2.840000 (109.291002) 184,612
6m em-http
  0.000000   0.000000   0.930000 ( 56.575500) 108,716
6m typhoeus
  0.000000   0.000000   6.630000 (187.015452) 193,316

MIXED std
  0.000000   0.000000  16.770000 (130.540340)  34,780
MIXED patron
  0.000000   0.000000   1.430000 (119.375096)  42,764
MIXED curb
  0.000000   0.000000   2.850000 (189.401198) 125,860
MIXED em-http
  0.000000   0.000000   1.670000 ( 15.745701)  43,432
MIXED typhoeus
  0.000000   0.000000  23.200000 (152.544673) 215,272

ree_cow_friendly.txt

ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux], MBARI 0x6770, Ruby Enterprise Edition 2009.10
Copy On Write Friendly = true

curb (0.6.2.1, 0.6.0.0)
em-http-request (0.2.5)
eventmachine (0.12.10)
typhoeus (0.1.13)

      user     system      total        real  Memory (Kb)
4kb std
  0.000000   0.000000   3.340000 ( 20.742152)  28,640
4kb patron
  0.000000   0.000000   0.450000 ( 11.605030)  29,112
4kb curb
  0.000000   0.000000   2.910000 (189.411345) 124,784
4kb em-http
  0.000000   0.000000   1.170000 ( 15.483167)  41,592
4kb typhoeus
  0.000000   0.000000   4.790000 ( 21.251126)  49,312

6m std
  0.000000   0.000000  13.280000 (112.890136)  32,944
6m patron
  0.000000   0.000000   1.060000 (108.965015)  40,752
6m curb
  0.000000   0.000000   2.660000 (115.235700) 179,608
6m em-http
  0.000000   0.000000   2.330000 (107.030177) 195,452
6m typhoeus
  0.000000   0.000000   7.930000 (109.548357) 195,324

MIXED std
  0.000000   0.000000  16.330000 (127.133653)  34,788
MIXED patron
  0.000000   0.000000   1.550000 (120.289284)  44,368
MIXED curb
  0.000000   0.000000   2.890000 (189.422943) 125,708
MIXED em-http
  0.000000   0.000000   1.610000 ( 24.868122)  42,996
MIXED typhoeus
  0.000000   0.000000  39.390000 (198.821644) 195,316

test.rb

require 'rubygems'
require 'open-uri'
require 'net/http'
require 'curb'
require 'benchmark'
require 'typhoeus'
require 'em-http'
require 'patron'

def std(urls, path)
#  f = File.open(path)
  urls.each do |uri|
    open(URI(uri)).read
    # f << open(URI(uri)).read
  end
#  f.close
end

def curb(urls,save_path = nil)
  downloads = {}
  Curl::Multi.get(urls) {|easy| downloads[easy.url] = easy.body_str }
  # File.open(save_path,"w") do |f|
    urls.each do |url|
      #f << downloads[url]
      downloads[url]
    end
  # end
end

def typo(urls, path = nil)
  hydra = Typhoeus::Hydra.new
  requests = urls.map { |url| t = Typhoeus::Request.new(url); hydra.queue(t); t }
  hydra.disable_memoization
  hydra.run # this is a blocking call
  # f = File.open(path, "w")
  requests.each do |r|
    # f << r.handled_response.body
    r.handled_response.body
  end
  # f.close
end

def em(urls, path = nil)
  EventMachine.run do
    multi = EventMachine::MultiRequest.new
    urls.each do |url| multi.add(EventMachine::HttpRequest.new(url).get) end
    multi.callback do
      # File.open(path, "w") do |f|
        multi.responses[:succeeded].each do |r|
          # f << r.response
          r.response
        end
      # end
      EventMachine.stop
    end
  end
end

def patron(urls, path=nil)
  sess = Patron::Session.new
  sess.timeout = 10
  sess.base_url = "http://#{$server}"
  sess.headers['User-Agent'] = 'Patron'

  urls.each do |u|
    resp = sess.get(u)
    resp.body
  end
end

$server ="192.168.0.102"

i = 0
Small = (["http://#{$server}/4k.html"] * 2000).map { |u| "#{ u }?#{ i += 1 }" }
Big   = (["http://#{$server}/6m.html"] * 20 ).map  { |u| "#{ u }?#{ i += 1 }" }

SmallP = Small.map { |u| u.split(/http:\/\/#{$server}/).last }
BigP   = Big.map   { |u| u.split(/http:\/\/#{$server}/).last }

def check
# GC.copy_on_write_friendly = false

  fork do
  # GC.copy_on_write_friendly = false
    yield
    memory_usage = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes
    puts "\nMemory: #{ memory_usage }\n"
  end

  Process.wait
end

Benchmark.bm do |rep|
  rep.report("4kb std")        do check { std     Small,         "small_curb" } end
  rep.report("4kb patron")     do check { patron  SmallP,        "small_curb" } end
  rep.report("4kb curb")       do check { curb    Small,         "small_curb" } end
  rep.report("4kb em-http")    do check { em      Small,         "small_em"   } end
  rep.report("4kb typhoeus")   do check { typo    Small,         "small_typ"  } end

  rep.report("6m std")         do check { std     Big,           "big_curb"   } end
  rep.report("6m patron")      do check { patron  BigP,          "big_curb"   } end
  rep.report("6m curb")        do check { curb    Big,           "big_curb"   } end
  rep.report("6m em-http")     do check { em      Big,           "big_em"     } end
  rep.report("6m typhoeus")    do check { typo    Big,           "big_typ"    } end

  rep.report("MIXED std")      do check { std     Small  + Big,  "mix_curb"   } end
  rep.report("MIXED patron")   do check { patron  SmallP + BigP, "mix_curb"   } end
  rep.report("MIXED curb")     do check { curb    Small  + Big,  "mix_curb"   } end
  rep.report("MIXED em-http")  do check { em      Small  + Big,  "mix_em"     } end
  rep.report("MIXED typhoeus") do check { typo    Small  + Big,  "mix_typ"    } end
end