get_url_response.rb

require 'net/http'
require 'net/https'

def get_url_response(url, headers = {}, cookies = {}, url_stack = [])
  raise ArgumentError, 'HTTP redirect too deep' if url_stack.length > 20
  uri = URI.parse(url)
  headers.merge!('cookie' => cookies.collect {|k,v| "#{k}=#{v}"}.join('; ')) if not cookies.empty?
  headers.merge!('referer' => url_stack.last) if url_stack.last
  req = Net::HTTP::Get.new(uri.request_uri || '/', headers)
  ses = Net::HTTP.new(uri.host, uri.port)
  ses.use_ssl = true if uri.scheme == "https"
  res = ses.start do |http|
    http.read_timeout = 60
    http.request(req)
  end
  if res['location']
    cookies = res['set-cookie'].split(/;.+?(, |$)/).select {|s| s =~ /\=/ }.inject(cookies) do |sum, pair|
      key, *value = pair.split(/\=/)
      sum.merge(key => value.join('='))
    end if res['set-cookie']
    get_url_response(URI.join(url, res['location']).to_s, headers, cookies, url_stack + [url])
  else
    [url, res]
  end
end