Skip to content

Instantly share code, notes, and snippets.

@bkoski
Created Jul 12, 2010
Embed
What would you like to do?
require 'typhoeus'
require 'nokogiri'
require 'term/ansicolor'
class BaseFetcher
def self.fetch
i = self.new
i.fetch
return i
end
def initialize
@request_queue = Typhoeus::Hydra.new
@xml = {}
end
def fetch
@request_queue.run
end
def [] key
@xml[key.to_sym]
end
def inspect
@xml.keys.inspect
end
private
def queue_request response_key, action, include_match_id=true, params=nil
r = Typhoeus::Request.new(url(action, include_match_id, params))
stub_request(r) if Rails.env.development?
r.on_complete do |response|
puts format_log_entry(r.url, response.time)
xml_response = Nokogiri::XML.parse(response.body)
@xml[response_key] = xml_response
end
@request_queue.queue(r)
return r
end
def queue_cached_request response_key, action, include_match_id=true, params=nil
if include_match_id
cache_key = [response_key, match_id, statset, params]
else
cache_key = [response_key, statset, params]
end
cache_key = cache_key.compact.join('-')
cache_path = File.join(RAILS_ROOT, 'tmp', 'xml_cache', cache_key)
if File.exists?(cache_path)
puts format_log_entry(cache_path, :cache)
@xml[response_key] = Nokogiri::XML.parse(File.read(cache_path))
else
r = queue_request(response_key, action, params)
r.after_complete do
cache_dir = File.dirname(cache_path)
Dir.mkdir(cache_dir) unless File.exists?(cache_dir)
File.open(cache_path, 'w') { |f| f.puts(@xml[response_key]) }
puts format_log_entry(r.url, :cached)
end
end
end
def url action, params
"http://feeds.nytimes.com/#{'?' if params}#{params}"
end
def format_log_entry url, response_time
if self.class == StatsXML
warn, high = 3, 4
else
warn, high = 1.0, 2.0
end
if response_time.is_a?(Symbol)
display_time = response_time.to_s
else
display_time = '%0.2fs' % response_time
if response_time > high
display_time = Color.bold(Color.red(display_time))
elsif response_time > warn
display_time = Color.bold(Color.yellow(display_time))
end
end
"#{self.class.to_s.gsub('XML','')}\t #{display_time} #{url}"
end
def stub_request request
filepath = request.url
response = Typhoeus::Response.new(:code => 200, :body => File.read(File.join(RAILS_ROOT, 'test', 'files', filepath)))
@request_queue.stub(:get, request.url).and_return(response)
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment