gistfile1.txt

$:.unshift File.expand_path('./ext')
$:.unshift File.expand_path('./lib')

require 'curb'

class Curl::Easy
  attr_accessor :retry_timeout
  attr_accessor :hard_error_retry
  attr_accessor :retries

  attr_accessor :multi

  def retry
    self.retries += 1
    if not @multi.nil?
      @multi.add(self)
    else
      perform()
    end
  end

end

def make_request(url)
  Curl::Easy.new(url) do|curl|
    curl.useragent = 'me'#request.user_agent
    # If a zero-length string is set, then an Accept-Encoding header
    # containing all supported encodings is sent.
    curl.encoding = ""
    #curl.verbose = true
    curl.max_redirects = 2
    curl.connect_timeout = 10
    curl.timeout = 300
    # our options
    curl.retry_timeout = 100.0
    curl.retries = 0
    curl.hard_error_retry = 0

    puts "start"

    curl.on_failure do |c,e|
      t = Time.now.to_f.to_s.ljust(14)
      if [404, 400, 500].include?(c.response_code)
        puts "#{t} #{c.object_id}: HTTP response code #{c.response_code}. Retrying in #{curl.retry_timeout}ms (#{curl.retries} retries)..."
        if c.response_code != 503
          if curl.hard_error_retry >= 100
            raise HTTPError, "HTTP response code #{c.response_code} hard error retry: #{curl.hard_error_retry}"
          end
          curl.hard_error_retry += 1
        end
        # sleep a little..
        sleep curl.retry_timeout/1000.0
        curl.retry_timeout += 10
        c.retry
      elsif [Curl::Err::TimeoutError, Curl::Err::GotNothingError].include?(e[0])
        puts "#{t} #{c.object_id}: #{e[1]} (#{e[0]}). Retrying (#{curl.retries} retries)..."
        c.retry
      else
        if not [0, 200].include?(c.response_code)
          if [410, 503].include?(c.response_code)
            yield(c.body_str)
          else
            raise HTTPError, "HTTP response code #{c.response_code}"
          end
        else
          raise *e
        end
      end
    end

    curl.on_success do |c|
      yield(c.body_str)
    end
  end
end

urls = [
  'http://127.0.0.1/',
  'http://127.0.0.1/',
  'http://127.0.0.1/',
  'http://127.0.0.1/',
]
10.times do
  urls += urls
end
urls = urls.flatten
puts "make #{urls.size} requests"

@@curl_multi = Curl::Multi.new
urls.each do |url|
  curl = make_request(url) do|body|
    puts "success"
  end
  curl.multi = @@curl_multi
  @@curl_multi.add(curl)
end

@@curl_multi.perform