http_async_multi.rb

require 'eventmachine'
require 'em-http'
require 'logger'

class HTTPAsyncMulti
  @@multi            = []
  @@multi_done       = []
  @@concurrency      = 2
  @@logger           = Logger.new($stdout)
  
  attr_reader :url, :request
  
  def self.get_multi(urls, concurrency = nil)
    @@multi      = []
    @@multi_done = []
    @@concurrency = concurrency if concurrency
    EM.run do
      urls.each { |url| HTTPAsyncMulti.new(url) }
      EM::Iterator.new(@@multi, @@concurrency).each do |http, iter|
        @@logger.debug("[#{__method__}] [iterator] [url=#{http.url}]")
        http.perform_http_request do
          @@logger.debug("[#{__method__}] [current_url=#{http.url}] [NEXT iterator]")
          iter.next
        end
      end
      EM.stop
    end
    results_hash = {}
    @@multi.each do |http|
      @@logger.debug("[#{__method__}] [fetching responses] [current_url=#{http.url}]")
      results_hash[http.url] = http.request.response
    end
    results_hash
  end  
    
  def initialize(url)
    @url = url
    @logger = @@logger
    @params = {}
    @connection = ::EM::HttpRequest.new(url)
    @request = nil
    @logger.debug("[#{__method__}] [@url=#{@url}] [@params=#{@params}]")
    @@multi << self
    self
  end

  def perform_http_request(&block)
    @logger.debug("[#{__method__}] [@url=#{@url}] [@params=#{@params}]")
    @request = @connection.get({query: @params})
    @request.stream do |chunk|
      @response ||= ""
      @response << chunk
      @logger.debug("[CHUNK] [@url=#{@url}] [@params=#{@params}] [chunk=#{chunk}]")
    end
    @request.callback do |client|
      @logger.debug("[CALLBACK] [@url=#{@url}] [@params=#{@params}] [response=#{client.inspect}]")
      yield if block_given?
      self
    end
    @request.errback do |client|
      @logger.debug("[ERRBACK] [error=#{client.error}] [@url=#{@url}] [@params=#{@params}] [response=#{client.inspect}]")
      yield if block_given?
      self
    end
    self
  end
  
end

urls = [
  'http://www.google.com',
  'http://www.cnn.com',
  'http://www.yahoo.com',
  'http://www.bing.com'
]
result = HTTPAsyncMulti.get_multi(urls)