Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
require 'eventmachine'
require 'em-http'
require 'logger'
class HTTPAsyncMulti
@@multi = []
@@multi_done = []
@@concurrency = 2
@@logger = Logger.new($stdout)
attr_reader :url, :request
def self.get_multi(urls, concurrency = nil)
@@multi = []
@@multi_done = []
@@concurrency = concurrency if concurrency
EM.run do
urls.each { |url| HTTPAsyncMulti.new(url) }
EM::Iterator.new(@@multi, @@concurrency).each do |http, iter|
@@logger.debug("[#{__method__}] [iterator] [url=#{http.url}]")
http.perform_http_request do
@@logger.debug("[#{__method__}] [current_url=#{http.url}] [NEXT iterator]")
iter.next
end
end
EM.stop
end
results_hash = {}
@@multi.each do |http|
@@logger.debug("[#{__method__}] [fetching responses] [current_url=#{http.url}]")
results_hash[http.url] = http.request.response
end
results_hash
end
def initialize(url)
@url = url
@logger = @@logger
@params = {}
@connection = ::EM::HttpRequest.new(url)
@request = nil
@logger.debug("[#{__method__}] [@url=#{@url}] [@params=#{@params}]")
@@multi << self
self
end
def perform_http_request(&block)
@logger.debug("[#{__method__}] [@url=#{@url}] [@params=#{@params}]")
@request = @connection.get({query: @params})
@request.stream do |chunk|
@response ||= ""
@response << chunk
@logger.debug("[CHUNK] [@url=#{@url}] [@params=#{@params}] [chunk=#{chunk}]")
end
@request.callback do |client|
@logger.debug("[CALLBACK] [@url=#{@url}] [@params=#{@params}] [response=#{client.inspect}]")
yield if block_given?
self
end
@request.errback do |client|
@logger.debug("[ERRBACK] [error=#{client.error}] [@url=#{@url}] [@params=#{@params}] [response=#{client.inspect}]")
yield if block_given?
self
end
self
end
end
urls = [
'http://www.google.com',
'http://www.cnn.com',
'http://www.yahoo.com',
'http://www.bing.com'
]
result = HTTPAsyncMulti.get_multi(urls)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.