Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
em-http-request batch downloader with configurable concurrency level
class Downloader
CONCURRENCY = 50
def initializer(urls)
@urls = urls
end
def download
EventMachine.run do
EM::Iterator.new(@urls, CONCURRENCY).each do |url, iterator|
request = EventMachine::HttpRequest.new(url).get
request.callback { ... }
multi.add url, request
multi.callback { ... } if url == @urls.last
end
end
end
private
def multi
@multi ||= EventMachine::MultiCallback.new
end
end
module EventMachine
# That's a lightweight alternative for EventMachine::MultiRequest (by em-http-request).
# It does not keeps references of all requests and responses thus reducing memory usage when
# tons of requests should be performed.
class MultiCallback
include EventMachine::Deferrable
def initialize
@requests = 0
@responses = 0
end
def add(name, conn)
@requests += 1
conn.callback { finished }
conn.errback { finished }
end
def finished?
@requests == @responses
end
private
def finished
@responses += 1
check_progress
end
def check_progress
succeed(self) if finished?
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment