em-http-request batch downloader with configurable concurrency level

example.rb

class Downloader
  CONCURRENCY = 50
  
  def initializer(urls)
    @urls = urls
  end
  
  def download
    EventMachine.run do
      EM::Iterator.new(@urls, CONCURRENCY).each do |url, iterator|
        request = EventMachine::HttpRequest.new(url).get
        request.callback { ... }
        multi.add url, request
        multi.callback { ... } if url == @urls.last
      end
    end
  end
  
  private
  
  def multi
    @multi ||= EventMachine::MultiCallback.new
  end
end

multi_callback.rb

module EventMachine

  # That's a lightweight alternative for EventMachine::MultiRequest (by em-http-request).
  # It does not keeps references of all requests and responses thus reducing memory usage when
  # tons of requests should be performed.

  class MultiCallback
    include EventMachine::Deferrable

    def initialize
      @requests = 0
      @responses = 0
    end

    def add(name, conn)
      @requests += 1

      conn.callback { finished }
      conn.errback  { finished }
    end

    def finished?
      @requests == @responses
    end

    private

    def finished
      @responses += 1
      check_progress
    end

    def check_progress
      succeed(self) if finished?
    end
  end
end