Created
March 16, 2009 03:19
-
-
Save igrigorik/79685 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "rubygems" | |
require "curb" | |
module Curl | |
class Easy | |
def self.fetch(req, &blk) | |
curl = Curl::Easy.new(req['url'], &blk) | |
curl.follow_location = true | |
curl.max_redirects = 3 | |
curl.connect_timeout = 15 | |
curl.timeout = 30 | |
curl.encoding = "compress, gzip" | |
curl.dns_cache_timeout = 60 * 60 * 12 | |
curl.headers["User-Agent"] = "..." | |
curl | |
end | |
end | |
end | |
class Downloader | |
def initialize | |
@inflight = 0 | |
max = 300 | |
cmulti = Curl::Multi.new | |
cmulti.max_connects = max | |
loop do | |
# initialize the downloader | |
max.times { cmulti.add(fetch) } | |
cmulti.perform do | |
cmulti.add(fetch) if @inflight < max | |
end | |
end | |
end | |
def fetch | |
@inflight += 1 | |
# iterative over some list of urls.... | |
msg = { 'url' => 'http://10.253.207.111/512.dat' } | |
curl = Curl::Easy.fetch(msg) do |page| | |
page.on_complete do |conn| | |
# do something | |
@inflight -= 1 | |
# gotcha: calling fetch from here to replenish the work queue | |
# leads to very, very deep frame stacks (infinite recursion, almost) | |
end | |
end | |
curl | |
end | |
end | |
Downloader.new |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment