Skip to content

Instantly share code, notes, and snippets.

@igrigorik igrigorik/gist:79685
Created Mar 16, 2009

Embed
What would you like to do?
require "rubygems"
require "curb"
module Curl
class Easy
def self.fetch(req, &blk)
curl = Curl::Easy.new(req['url'], &blk)
curl.follow_location = true
curl.max_redirects = 3
curl.connect_timeout = 15
curl.timeout = 30
curl.encoding = "compress, gzip"
curl.dns_cache_timeout = 60 * 60 * 12
curl.headers["User-Agent"] = "..."
curl
end
end
end
class Downloader
def initialize
@inflight = 0
max = 300
cmulti = Curl::Multi.new
cmulti.max_connects = max
loop do
# initialize the downloader
max.times { cmulti.add(fetch) }
cmulti.perform do
cmulti.add(fetch) if @inflight < max
end
end
end
def fetch
@inflight += 1
# iterative over some list of urls....
msg = { 'url' => 'http://10.253.207.111/512.dat' }
curl = Curl::Easy.fetch(msg) do |page|
page.on_complete do |conn|
# do something
@inflight -= 1
# gotcha: calling fetch from here to replenish the work queue
# leads to very, very deep frame stacks (infinite recursion, almost)
end
end
curl
end
end
Downloader.new
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.