http_client_test.rb

require 'rubygems'

# fork to create a detatched server process... (this should be on a different host)
rd,wr = IO.pipe
spid = fork do
  rd.close
  # turn of noise
  STDIN.reopen "/dev/null"
  STDOUT.reopen "/dev/null", "a"
  STDERR.reopen STDOUT

  require 'sinatra'

  class TestApp < Sinatra::Base
    get "/fast" do
      "1"
    end
  end

  wr.write "almost ready"
  wr.flush
  wr.close
  TestApp.run! :host => 'localhost', :port => 9090
end

# wait on the pipe
wr.close
output = rd.read
puts output
rd.close
sleep 1 # test app starts quick, but let's give it 1 extra sec to be certain

require 'open-uri'
require 'thread'
require 'net/http'
require 'curb'
require 'benchmark'
require 'typhoeus'
require 'em-http'


def std(urls, path)
  urls.map{|url| File.open(path,"wb") {|f| f << open(url).read } }
end

def curb(urls,save_path)
  downloads = {}
  Curl::Multi.get(urls) {|easy| downloads[easy.url] = easy.body_str }
  File.open(save_path,"w") {|f| urls.each {|url| f << downloads[url] } }
end

def typo(urls, path)
  hydra = Typhoeus::Hydra.new
  requests = urls.map { |url| t = Typhoeus::Request.new(url); hydra.queue(t); t }
  hydra.run # this is a blocking call
  f = File.open(path, "w")
  requests.each do |r| f << r.handled_response.body end
  f.close
end

def em(urls, path)
  EventMachine.run do
    multi = EventMachine::MultiRequest.new
    urls.each do |url| multi.add(EventMachine::HttpRequest.new(url).get) end
    multi.callback do
      File.open(path, "w") { |f| multi.responses[:succeeded].each { |r| f << r.response } }
      EventMachine.stop
    end
  end
end

FAST = [0..10].map{|i| "http://127.0.0.1:9090/fast" }
N = 100

Benchmark.bmbm do |rep|
  rep.report("curb") do
    N.times do
      curb FAST, "curb_fast.log"
    end
  end

  rep.report("em-http") do
    N.times do
      em(FAST, "em_fast.log")
    end
  end

  rep.report("typhoeus") do
    N.times do
      typo(FAST, "typo_fast.log")
    end
  end

  rep.report("NET::HTTP") do
    N.times do
      std(FAST, "std_fast.log")
    end
  end
end

#Process.kill('INT',spid)
#Process.kill('TERM',spid)
#Process.kill('HUP',spid)
Process.kill(9,spid)

__END__
$ ruby --version
ruby 1.8.7 (2009-06-12 patchlevel 174) [i686-darwin10.0.0]
$ ruby test.rb 
Rehearsal ---------------------------------------------
curb        0.060000   0.750000   0.810000 (  0.914818)
em-http     0.040000   0.030000   0.070000 (  0.124104)
typhoeus    0.100000   7.320000   7.420000 (  8.010210)
NET::HTTP   0.150000   0.170000   0.320000 (  0.725077)
------------------------------------ total: 8.620000sec

                user     system      total        real
curb        0.030000   0.770000   0.800000 (  1.115652)
em-http     0.040000   0.020000   0.060000 (  0.307230)
typhoeus    0.110000   7.290000   7.400000 (  7.777482)
NET::HTTP   0.070000   0.030000   0.100000 (  0.183474)

#####################################################################################################
$ ruby --version
ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux], MBARI 0x6770, Ruby Enterprise Edition 20090928
$ ruby test.rb
almost ready
Rehearsal ---------------------------------------------
curb        0.050000   0.040000   0.090000 (  0.091755)
em-http     0.030000   0.030000   0.060000 (  0.098413)
typhoeus    0.040000   0.050000   0.090000 (  0.134484)
NET::HTTP   0.100000   0.040000   0.140000 (  0.242883)
------------------------------------ total: 0.380000sec

                user     system      total        real
curb        0.030000   0.030000   0.060000 (  0.089375)
em-http     0.040000   0.020000   0.060000 (  0.107818)
typhoeus    0.040000   0.030000   0.070000 (  0.113231)
NET::HTTP   0.040000   0.040000   0.080000 (  0.184679)

#####################################################################################################
$ ruby --version
ruby 1.9.1p376 (2009-12-07 revision 26041) [x86_64-linux]
$ ruby test.rb 
almost ready
Rehearsal ---------------------------------------------
curb        0.110000   0.070000   0.180000 (  0.317914)
em-http     0.040000   0.020000   0.060000 (  0.309513)
typhoeus    0.130000   0.080000   0.210000 (  0.346047)
NET::HTTP   0.120000   0.030000   0.150000 (  0.422395)
------------------------------------ total: 0.600000sec

                user     system      total        real
curb        0.100000   0.050000   0.150000 (  0.292064)
em-http     0.050000   0.030000   0.080000 (  0.304189)
typhoeus    0.110000   0.060000   0.170000 (  0.358280)
NET::HTTP   0.090000   0.020000   0.110000 (  0.359190)