Scatter Gather HTTP with EventMachine (from a non-EM Ruby class, like Rails)

em-http.rb

require 'rubygems'
require 'benchmark'

require 'httparty'

require 'eventmachine'
require 'em-http'

class Interatred
  def self.run(urls)
    bodies = {}
    urls.each do |url|
      response = HTTParty.get(url)
      bodies[url] = response.body.force_encoding(Encoding::UTF_8)
    end

    return bodies
  end
end

class ScatterGather
  def self.run(urls, timeout = 1.0)
    bodies = {}

    EventMachine.run do
      # timeout = Maximum total time in seconds
      EventMachine.add_timer(timeout) do
        puts "Returning #{bodies.length} of #{urls.length} results : #{bodies.keys.sort.inspect}"
        EventMachine.stop
      end

      multi = EventMachine::MultiRequest.new
      urls.each_with_index do |url, idx|
        req = EventMachine::HttpRequest.new(url).get(:redirects => 1)
        req.callback do
          bodies[url] = req.response
        end
        multi.add(idx, req)
      end

      multi.callback do
        # All requests finished in time.
        EventMachine.stop
      end
    end

    return bodies
  end
end

URLS = %w(
  http://twitter.com/404.html
  http://twitter.com/502.html
  http://twitter.com/500.html
  http://google.com
  http://github.com
)

Benchmark.bm do |x|
  x.report(" iter:")  {    Interatred.run(URLS) }
  x.report("em-sg:")  { ScatterGather.run(URLS) }
end