Skip to content

Instantly share code, notes, and snippets.

@karmi
Created October 13, 2009 20:20
Show Gist options
  • Save karmi/209521 to your computer and use it in GitHub Desktop.
Save karmi/209521 to your computer and use it in GitHub Desktop.
Cache 200 OK responses in HTTParty models
# = Icebox : Caching for HTTParty
#
# Cache responses in HTTParty models [http://github.com/jnunemaker/httparty]
#
# === Usage
#
# class Foo
# include HTTParty
# include HTTParty::Icebox
# cache :store => 'file', :timeout => 600, :location => MY_APP_ROOT.join('tmp', 'cache')
# end
#
# Modeled after Martyn Loughran's APICache [http://github.com/newbamboo/api_cache]
# and Ruby On Rails's caching [http://api.rubyonrails.org/classes/ActiveSupport/Cache.html]
#
# Author: Karel Minarik [www.karmi.cz]
#
# === Notes
#
# Thanks to Amit Chakradeo for pointing out response objects have to be stored marhalled on FS
# Thanks to Marlin Forbes for pointing out the query parameters have to be included in the cache key
#
#
require 'logger'
require 'ftools'
require 'tmpdir'
require 'pathname'
require 'digest/md5'
module HTTParty #:nodoc:
# == Caching for HTTParty
# See documentation in HTTParty::Icebox::ClassMethods.cache
#
module Icebox
module ClassMethods
# Enable caching and set cache options
# Returns memoized cache object
#
# Following options are available, default values are in []:
#
# +store+:: Storage mechanism for cached data (memory, filesystem, your own) [memory]
# +timeout+:: Cache expiration in seconds [60]
# +logger+:: Path to logfile or logger instance [nil, silent]
#
# Any additional options are passed to the Cache constructor
#
# Usage:
#
# # Enable caching in HTTParty, in memory, for 1 minute
# cache # Use default values
#
# # Enable caching in HTTParty, on filesystem (/tmp), for 10 minutes
# cache :store => 'file', :timeout => 600, :location => '/tmp/'
#
# # Use your own cache store (see +AbstractStore+ class below)
# cache :store => 'memcached', :timeout => 600, :server => '192.168.1.1:1001'
#
def cache(options={})
options[:store] ||= 'memory'
options[:timeout] ||= 60
logger = options[:logger]
@cache ||= Cache.new( options.delete(:store), options )
end
end
# When included, extend class with +cache+ method
# and redefine +get+ method to use cache
#
def self.included(receiver) #:nodoc:
receiver.extend ClassMethods
receiver.class_eval do
# Get reponse from network
#
# TODO: Why alias :new :old is not working here? Returns NoMethodError
#
def self.get_without_caching(path, options={})
perform_request Net::HTTP::Get, path, options
end
# Get response from cache, if available
#
def self.get_with_caching(path, options={})
key = path
key << options[:query].to_s if defined? options[:query]
if cache.exists?(key) and not cache.stale?(key)
Cache.logger.debug "CACHE -- GET #{path}#{options[:query]}"
return cache.get(key)
else
Cache.logger.debug "/!\\ NETWORK -- GET #{path}#{options[:query]}"
response = get_without_caching(path, options)
cache.set(key, response) if response.code == 200
return response
end
end
# Redefine original HTTParty +get+ method to use cache
#
def self.get(path, options={})
self.get_with_caching(path, options={})
end
end
end
# === Cache container
#
# Pass a store name ('memory', etc) to new
#
class Cache
attr_accessor :store
def initialize(store, options={})
self.class.logger = options[:logger]
@store = self.class.lookup_store(store).new(options)
end
def get(key); @store.get encode(key) unless stale?(key); end
def set(key, value); @store.set encode(key), value; end
def exists?(key); @store.exists? encode(key); end
def stale?(key); @store.stale? encode(key); end
def self.logger; @logger || default_logger; end
def self.default_logger; logger = ::Logger.new(STDERR); end
# Pass a filename (String), IO object, Logger instance or +nil+ to silence the logger
def self.logger=(device); @logger = device.kind_of?(::Logger) ? device : ::Logger.new(device); end
private
# Return store class based on passed name
def self.lookup_store(name)
store_name = "#{name.capitalize}Store"
return Store::const_get(store_name)
rescue NameError => e
raise Store::StoreNotFound, "The cache store '#{store_name}' was not found. Did you loaded any such class?"
end
def encode(key); Digest::MD5.hexdigest(key); end
end
# === Cache stores
#
module Store
class StoreNotFound < StandardError; end #:nodoc:
# ==== Abstract Store
# Inherit your store from this class
# *IMPORTANT*: Do not forget to call +super+ in your +initialize+ method!
#
class AbstractStore
def initialize(options={})
raise ArgumentError, "You need to set the :timeout parameter" unless options[:timeout]
@timeout = options[:timeout]
message = "Cache: Using #{self.class.to_s.split('::').last}"
message << " in location: #{options[:location]}" if options[:location]
message << " with timeout #{options[:timeout]} sec"
Cache.logger.info message unless options[:logger].nil?
return self
end
%w{set get exists? stale?}.each do |method_name|
define_method(method_name) { raise NoMethodError, "Please implement method #{method_name} in your store class" }
end
end
# ==== Store objects in memory
# See HTTParty::Icebox::ClassMethods.cache
#
class MemoryStore < AbstractStore
def initialize(options={})
super; @store = {}; self
end
def set(key, value)
Cache.logger.info("Cache: set (#{key})")
@store[key] = [Time.now, value]; true
end
def get(key)
data = @store[key][1]
Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})")
data
end
def exists?(key)
!@store[key].nil?
end
def stale?(key)
return true unless exists?(key)
Time.now - created(key) > @timeout
end
private
def created(key)
@store[key][0]
end
end
# ==== Store objects on the filesystem
# See HTTParty::Icebox::ClassMethods.cache
#
class FileStore < AbstractStore
def initialize(options={})
super
options[:location] ||= Dir::tmpdir
@path = Pathname.new( options[:location] )
FileUtils.mkdir_p( @path )
self
end
def set(key, value)
Cache.logger.info("Cache: set (#{key})")
File.open( @path.join(key), 'w' ) { |file| file << Marshal.dump(value) }
true
end
def get(key)
data = Marshal.load(File.read( @path.join(key)))
Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})")
data
end
def exists?(key)
File.exists?( @path.join(key) )
end
def stale?(key)
return true unless exists?(key)
Time.now - created(key) > @timeout
end
private
def created(key)
File.mtime( @path.join(key) )
end
end
end
end
end
# Major parts of this code are based on architecture of ApiCache.
# Copyright (c) 2008 Martyn Loughran
#
# Other parts are inspired by the ActiveSupport::Cache in Ruby On Rails.
# Copyright (c) 2005-2009 David Heinemeier Hansson
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
require 'rubygems'
require 'fakeweb'
require 'ftools'
require 'test/unit'
require 'shoulda'
require 'httparty'
require 'httparty_icebox'
FakeWeb.register_uri :get, "http://example.com/",
[ {:body => "Hello, World!"}, {:body => "Goodbye, World!"} ]
FakeWeb.register_uri :get, "http://example.com/?name=Joshua",
[ {:body => "Hello, Joshua!"}, {:body => "Goodbye, Joshua!"} ]
FakeWeb.register_uri :get, "http://example.com/teapot",
{ :body => "I'm a teapot", :status => 200, :'x-powered-by'=>"Teapot 1.0" }
FakeWeb.register_uri :get, "http://example.com/bad", {:body => "Not Found", :status => 404}
FakeWeb.register_uri :post, "http://example.com/form", {:body => "Processed", :status => 200}
class HTTPartyIceboxTest < Test::Unit::TestCase
context "When Icebox is included in a class, it" do
setup do
MyResource.class_eval do
@cache = nil
cache :store => 'memory', :timeout => 0.1#, :logger => Logger.new(STDOUT)
# cache :store => 'file', :timeout => 0.1, :location => File.dirname(__FILE__), :logger => Logger.new(STDOUT)
end
end
should "allow setting cache" do
assert_respond_to MyResource, :cache
end
should "set the cache" do
MyResource.cache :store => 'memory', :timeout => 5, :logger => nil
assert_not_nil MyResource.cache
assert_not_nil MyResource.cache.store
assert_instance_of HTTParty::Icebox::Store::MemoryStore, MyResource.cache.store
end
should "get the response from network and cache it" do
MyResource.get('http://example.com')
assert_not_nil MyResource.cache.get('http://example.com')
end
should "get the cached response when the cache still fresh" do
MyResource.get('http://example.com')
assert_equal 'Hello, World!', MyResource.get('http://example.com').body
end
should "get the fresh response when the cache is stale" do
MyResource.get('http://example.com')
sleep 0.3
assert_equal 'Goodbye, World!', MyResource.get('http://example.com').body
end
should "include the query params in key" do
MyResource.get('http://example.com/?name=Joshua')
assert_equal 'Hello, Joshua!', MyResource.get('http://example.com/?name=Joshua').body
end
should "not cache the response when receiving error from network" do
MyResource.get('http://example.com/bad')
assert_nil MyResource.cache.get('http://example.com/bad')
end
should "not cache the response when not a GET request" do
MyResource.post('http://example.com/form')
assert_nil MyResource.cache.get('http://example.com/form')
end
should "store reponse with code, body and headers" do
MyResource.get('http://example.com/teapot')
cached = MyResource.get('http://example.com/teapot')
assert_equal 200, cached.code
assert_equal "I'm a teapot", cached.body
assert_not_nil cached.headers['x-powered-by']
assert_not_nil cached.headers['x-powered-by'].first
assert_equal 'Teapot 1.0', cached.headers['x-powered-by'].first
end
end
# ---------------------------------------------------------------------------
context "A logger for Icebox" do
setup do
HTTParty::Icebox::Cache.class_eval { @logger = nil }
@logpath = Pathname(__FILE__).dirname.expand_path.join('test.log').to_s
end
should "return default logger" do
assert_instance_of Logger, HTTParty::Icebox::Cache.default_logger
end
should "return default logger when none was set" do
assert_instance_of Logger, HTTParty::Icebox::Cache.logger
end
should "set be set to nil when given nil" do
HTTParty::Icebox::Cache.logger=(nil)
assert_instance_of Logger, HTTParty::Icebox::Cache.logger
HTTParty::Icebox::Cache.logger.info "Hi" # Should not see this
assert_equal nil, # UH :/
HTTParty::Icebox::Cache.logger.instance_variable_get(:@logdev).instance_eval{defined?(@filename)}
end
should "should set logger to a Logger instance" do
HTTParty::Icebox::Cache.logger = ::Logger.new(STDOUT)
assert_instance_of Logger, HTTParty::Icebox::Cache.logger
end
should "create a logger to log to file" do
HTTParty::Icebox::Cache.logger = @logpath
assert_equal @logpath, # UH :/
HTTParty::Icebox::Cache.logger.instance_variable_get(:@logdev).instance_variable_get(:@filename)
FileUtils.rm_rf(@logpath)
end
end
# ---------------------------------------------------------------------------
context "When looking up store, it" do
should "use default store" do
# 'memory' is default
assert_equal HTTParty::Icebox::Store::MemoryStore, HTTParty::Icebox::Cache.lookup_store('memory')
end
should "find existing store" do
assert_equal HTTParty::Icebox::Store::FileStore, HTTParty::Icebox::Cache.lookup_store('file')
end
should "raise when passed non-existing store" do
assert_raise(HTTParty::Icebox::Store::StoreNotFound) { HTTParty::Icebox::Cache.lookup_store('ether') }
end
end
# ---------------------------------------------------------------------------
context "AbstractStore class" do
should "raise unless passed :timeout option" do
assert_raise(ArgumentError) { HTTParty::Icebox::Store::AbstractStore.new( :timeout => nil ) }
end
should "raise when abstract methods are called" do
s = HTTParty::Icebox::Store::AbstractStore.new( :timeout => 1, :logger => nil )
assert_raise(NoMethodError) { s.set }
end
end
# ---------------------------------------------------------------------------
context "Memory cache" do
setup do
@cache = HTTParty::Icebox::Cache.new('memory', :timeout => 0.1, :logger => nil)
@key = 'abc'
@value = { :one => [1, 2, 3], :two => 'Hello', :three => 1...5 }
@cache.set @key, @value
end
should "store and retrieve the value" do
assert_equal @value, @cache.get('abc')
end
should "miss when retrieving the value after timeout" do
sleep 0.1
assert_nil @cache.get('abc')
end
end
# ---------------------------------------------------------------------------
context "File store cache" do
setup do
@cache = HTTParty::Icebox::Cache.new('file', :timeout => 1)
@key = 'abc'
@value = { :one => [1, 2, 3], :two => 'Hello', :three => 1...5 }
@cache.set @key, @value
end
should "store and retrieve the value" do
assert_equal @value, @cache.get('abc')
end
should "miss when retrieving the value after timeout" do
sleep 1.1
assert_nil @cache.get('abc')
end
end
# ---------------------------------------------------------------------------
end
# Fake Resource to mixin the functionality into
class MyResource
include HTTParty
include HTTParty::Icebox
cache :store => 'memory', :timeout => 0.2
end
require 'rubygems'
require 'httparty'
require 'httparty_icebox'
class CachedRepresentative
include HTTParty
include HTTParty::Icebox
cache :store => 'memory', :timeout => 0.5, :logger => Logger.new(STDOUT)
end
# -----
response1 = CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544')
response2 = CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544')
puts response1.headers.inspect
puts "\nFresh: " << response1.inspect << " [Code #{response1.code}]"
puts "\nCached: " << response2.inspect << " [Code #{response2.code}]"
puts "\nHeaders work too! => #{response2.headers.inspect}"
# -----
puts "\n\nzzzz...\n\n"
sleep 1.5
# -----
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment