Cache 200 OK responses in HTTParty models

httparty_icebox.rb

# = Icebox : Caching for HTTParty
#
# Cache responses in HTTParty models [http://github.com/jnunemaker/httparty]
#
# === Usage
#
#   class Foo
#     include HTTParty
#     include HTTParty::Icebox
#     cache :store => 'file', :timeout => 600, :location => MY_APP_ROOT.join('tmp', 'cache')
#   end
#
# Modeled after Martyn Loughran's APICache [http://github.com/newbamboo/api_cache]
# and Ruby On Rails's caching [http://api.rubyonrails.org/classes/ActiveSupport/Cache.html]
#
# Author: Karel Minarik [www.karmi.cz]
#
# === Notes
#
# Thanks to Amit Chakradeo for pointing out response objects have to be stored marhalled on FS
# Thanks to Marlin Forbes for pointing out the query parameters have to be included in the cache key
#
#

require 'logger'
require 'ftools'
require 'tmpdir'
require 'pathname'
require 'digest/md5'

module HTTParty #:nodoc:
  # == Caching for HTTParty
  # See documentation in HTTParty::Icebox::ClassMethods.cache
  #
  module Icebox

    module ClassMethods

      # Enable caching and set cache options
      # Returns memoized cache object
      #
      # Following options are available, default values are in []:
      #
      # +store+::       Storage mechanism for cached data (memory, filesystem, your own) [memory]
      # +timeout+::     Cache expiration in seconds [60]
      # +logger+::      Path to logfile or logger instance [nil, silent]
      # 
      # Any additional options are passed to the Cache constructor
      #
      # Usage:
      #
      #   # Enable caching in HTTParty, in memory, for 1 minute
      #   cache # Use default values
      #
      #   # Enable caching in HTTParty, on filesystem (/tmp), for 10 minutes
      #   cache :store => 'file', :timeout => 600, :location => '/tmp/'
      #
      #   # Use your own cache store (see +AbstractStore+ class below)
      #   cache :store => 'memcached', :timeout => 600, :server => '192.168.1.1:1001'
      #
      def cache(options={})
        options[:store]   ||= 'memory'
        options[:timeout] ||= 60
        logger = options[:logger]
        @cache ||= Cache.new( options.delete(:store), options )
      end

    end

    # When included, extend class with +cache+ method
    # and redefine +get+ method to use cache
    #
    def self.included(receiver) #:nodoc:
      receiver.extend ClassMethods
      receiver.class_eval do

        # Get reponse from network
        #
        # TODO: Why alias :new :old is not working here? Returns NoMethodError
        #
        def self.get_without_caching(path, options={})
          perform_request Net::HTTP::Get, path, options
        end

        # Get response from cache, if available
        #
        def self.get_with_caching(path, options={})
          key = path
          key << options[:query].to_s if defined? options[:query]
          if cache.exists?(key) and not cache.stale?(key)
            Cache.logger.debug "CACHE -- GET #{path}#{options[:query]}"
            return cache.get(key)
          else
            Cache.logger.debug "/!\\ NETWORK -- GET #{path}#{options[:query]}"
            response = get_without_caching(path, options)
            cache.set(key, response) if response.code == 200
            return response
          end
        end

        # Redefine original HTTParty +get+ method to use cache
        #
        def self.get(path, options={})
          self.get_with_caching(path, options={})
        end

      end
    end

    # === Cache container
    #
    # Pass a store name ('memory', etc) to new
    #
    class Cache
      attr_accessor :store

      def initialize(store, options={})
        self.class.logger = options[:logger]
        @store = self.class.lookup_store(store).new(options)
      end

      def get(key);            @store.get encode(key) unless stale?(key);        end
      def set(key, value);     @store.set encode(key), value;                    end
      def exists?(key);        @store.exists? encode(key);                       end
      def stale?(key);         @store.stale?  encode(key);                       end

      def self.logger; @logger || default_logger; end
      def self.default_logger; logger = ::Logger.new(STDERR); end

      # Pass a filename (String), IO object, Logger instance or +nil+ to silence the logger
      def self.logger=(device); @logger = device.kind_of?(::Logger) ? device : ::Logger.new(device); end

      private

      # Return store class based on passed name
      def self.lookup_store(name)
        store_name = "#{name.capitalize}Store"
        return Store::const_get(store_name)
      rescue NameError => e
        raise Store::StoreNotFound, "The cache store '#{store_name}' was not found. Did you loaded any such class?"
      end

      def encode(key); Digest::MD5.hexdigest(key); end
    end


    # === Cache stores
    #
    module Store

      class StoreNotFound < StandardError; end #:nodoc:

      # ==== Abstract Store
      # Inherit your store from this class
      # *IMPORTANT*: Do not forget to call +super+ in your +initialize+ method!
      #
      class AbstractStore
        def initialize(options={})
          raise ArgumentError, "You need to set the :timeout parameter" unless options[:timeout]
          @timeout = options[:timeout]
          message =  "Cache: Using #{self.class.to_s.split('::').last}"
          message << " in location: #{options[:location]}" if options[:location]
          message << " with timeout #{options[:timeout]} sec"
          Cache.logger.info message unless options[:logger].nil?
          return self
        end
        %w{set get exists? stale?}.each do |method_name|
          define_method(method_name) { raise NoMethodError, "Please implement method #{method_name} in your store class" }
        end
      end

      # ==== Store objects in memory
      # See HTTParty::Icebox::ClassMethods.cache
      #
      class MemoryStore < AbstractStore
        def initialize(options={})
          super; @store = {}; self
        end
        def set(key, value)
          Cache.logger.info("Cache: set (#{key})")
          @store[key] = [Time.now, value]; true
        end
        def get(key)
          data = @store[key][1]
          Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})")
          data
        end
        def exists?(key)
          !@store[key].nil?
        end
        def stale?(key)
          return true unless exists?(key)
          Time.now - created(key) > @timeout
        end
        private
        def created(key)
          @store[key][0]
        end
      end

      # ==== Store objects on the filesystem
      # See HTTParty::Icebox::ClassMethods.cache
      #
      class FileStore < AbstractStore
        def initialize(options={})
          super
          options[:location] ||= Dir::tmpdir
          @path = Pathname.new( options[:location] )
          FileUtils.mkdir_p( @path )
          self
        end
        def set(key, value)
          Cache.logger.info("Cache: set (#{key})")
          File.open( @path.join(key), 'w' ) { |file| file << Marshal.dump(value)  }
          true
        end
        def get(key)
          data = Marshal.load(File.read( @path.join(key)))
          Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})")
          data
        end
        def exists?(key)
          File.exists?( @path.join(key) )
        end
        def stale?(key)
          return true unless exists?(key)
          Time.now - created(key) > @timeout
        end
        private
        def created(key)
          File.mtime( @path.join(key) )
        end
      end
    end

  end
end


# Major parts of this code are based on architecture of ApiCache.
# Copyright (c) 2008 Martyn Loughran
#
# Other parts are inspired by the ActiveSupport::Cache in Ruby On Rails.
# Copyright (c) 2005-2009 David Heinemeier Hansson
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

httparty_icebox_test.rb

require 'rubygems'
require 'fakeweb'
require 'ftools'
require 'test/unit'
require 'shoulda'

require 'httparty'
require 'httparty_icebox'

FakeWeb.register_uri :get, "http://example.com/",
                     [ {:body => "Hello, World!"},  {:body => "Goodbye, World!"} ]

FakeWeb.register_uri :get, "http://example.com/?name=Joshua",
                     [ {:body => "Hello, Joshua!"}, {:body => "Goodbye, Joshua!"} ]

FakeWeb.register_uri :get, "http://example.com/teapot",
                     { :body => "I'm a teapot", :status => 200, :'x-powered-by'=>"Teapot 1.0" }

FakeWeb.register_uri :get,  "http://example.com/bad",  {:body => "Not Found", :status => 404}
FakeWeb.register_uri :post, "http://example.com/form", {:body => "Processed", :status => 200}

class HTTPartyIceboxTest < Test::Unit::TestCase

  context "When Icebox is included in a class, it" do

    setup do
      MyResource.class_eval do
        @cache = nil
        cache :store => 'memory', :timeout => 0.1#, :logger => Logger.new(STDOUT)
        # cache :store => 'file', :timeout => 0.1, :location => File.dirname(__FILE__), :logger => Logger.new(STDOUT)
      end
    end

    should "allow setting cache" do
      assert_respond_to MyResource, :cache
    end

    should "set the cache" do
      MyResource.cache :store => 'memory', :timeout => 5, :logger => nil
      assert_not_nil MyResource.cache
      assert_not_nil MyResource.cache.store
      assert_instance_of HTTParty::Icebox::Store::MemoryStore, MyResource.cache.store
    end

    should "get the response from network and cache it" do
      MyResource.get('http://example.com')
      assert_not_nil MyResource.cache.get('http://example.com')
    end

    should "get the cached response when the cache still fresh" do
      MyResource.get('http://example.com')
      assert_equal 'Hello, World!', MyResource.get('http://example.com').body
    end

    should "get the fresh response when the cache is stale" do
      MyResource.get('http://example.com')
      sleep 0.3
      assert_equal 'Goodbye, World!', MyResource.get('http://example.com').body
    end

    should "include the query params in key" do
      MyResource.get('http://example.com/?name=Joshua')
      assert_equal 'Hello, Joshua!', MyResource.get('http://example.com/?name=Joshua').body
    end

    should "not cache the response when receiving error from network" do
      MyResource.get('http://example.com/bad')
      assert_nil MyResource.cache.get('http://example.com/bad')
    end

    should "not cache the response when not a GET request" do
      MyResource.post('http://example.com/form')
      assert_nil MyResource.cache.get('http://example.com/form')
    end

    should "store reponse with code, body and headers" do
      MyResource.get('http://example.com/teapot')
      cached = MyResource.get('http://example.com/teapot')
      assert_equal 200, cached.code
      assert_equal "I'm a teapot", cached.body
      assert_not_nil cached.headers['x-powered-by']
      assert_not_nil cached.headers['x-powered-by'].first
      assert_equal 'Teapot 1.0', cached.headers['x-powered-by'].first
    end

  end

  # ---------------------------------------------------------------------------

  context "A logger for Icebox" do

    setup do
      HTTParty::Icebox::Cache.class_eval { @logger = nil }
      @logpath = Pathname(__FILE__).dirname.expand_path.join('test.log').to_s
    end

    should "return default logger" do
      assert_instance_of Logger, HTTParty::Icebox::Cache.default_logger
    end

    should "return default logger when none was set" do
      assert_instance_of Logger, HTTParty::Icebox::Cache.logger
    end

    should "set be set to nil when given nil" do
      HTTParty::Icebox::Cache.logger=(nil)
      assert_instance_of Logger, HTTParty::Icebox::Cache.logger
      HTTParty::Icebox::Cache.logger.info "Hi" # Should not see this
      assert_equal nil, # UH :/
                   HTTParty::Icebox::Cache.logger.instance_variable_get(:@logdev).instance_eval{defined?(@filename)}
    end

    should "should set logger to a Logger instance" do
      HTTParty::Icebox::Cache.logger = ::Logger.new(STDOUT)
      assert_instance_of Logger, HTTParty::Icebox::Cache.logger
    end

    should "create a logger to log to file" do
      HTTParty::Icebox::Cache.logger = @logpath
      assert_equal @logpath, # UH :/
                   HTTParty::Icebox::Cache.logger.instance_variable_get(:@logdev).instance_variable_get(:@filename)
      FileUtils.rm_rf(@logpath)
    end

  end

  # ---------------------------------------------------------------------------

  context "When looking up store, it" do

    should "use default store" do
      # 'memory' is default
      assert_equal HTTParty::Icebox::Store::MemoryStore, HTTParty::Icebox::Cache.lookup_store('memory')
    end

    should "find existing store" do
      assert_equal HTTParty::Icebox::Store::FileStore, HTTParty::Icebox::Cache.lookup_store('file')
    end

    should "raise when passed non-existing store" do
      assert_raise(HTTParty::Icebox::Store::StoreNotFound) { HTTParty::Icebox::Cache.lookup_store('ether') }
    end
  end

  # ---------------------------------------------------------------------------

  context "AbstractStore class" do

    should "raise unless passed :timeout option" do
      assert_raise(ArgumentError) { HTTParty::Icebox::Store::AbstractStore.new( :timeout => nil ) }
    end

    should "raise when abstract methods are called" do
      s = HTTParty::Icebox::Store::AbstractStore.new( :timeout => 1, :logger => nil )
      assert_raise(NoMethodError) { s.set }
    end
  end

  # ---------------------------------------------------------------------------

  context "Memory cache" do
    setup do
      @cache = HTTParty::Icebox::Cache.new('memory', :timeout => 0.1, :logger => nil)
      @key   = 'abc'
      @value = { :one => [1, 2, 3], :two => 'Hello', :three => 1...5 }
      @cache.set @key, @value
    end

    should "store and retrieve the value" do
      assert_equal @value, @cache.get('abc')
    end

    should "miss when retrieving the value after timeout" do
      sleep 0.1
      assert_nil @cache.get('abc')
    end
  end

  # ---------------------------------------------------------------------------

  context "File store cache" do
    setup do
      @cache = HTTParty::Icebox::Cache.new('file', :timeout => 1)
      @key   = 'abc'
      @value = { :one => [1, 2, 3], :two => 'Hello', :three => 1...5 }
      @cache.set @key, @value
    end

    should "store and retrieve the value" do
      assert_equal @value, @cache.get('abc')
    end

    should "miss when retrieving the value after timeout" do
      sleep 1.1
      assert_nil @cache.get('abc')
    end
  end

  # ---------------------------------------------------------------------------


end

# Fake Resource to mixin the functionality into
class MyResource
  include HTTParty
  include HTTParty::Icebox
  cache :store => 'memory', :timeout => 0.2
end

representative.rb

require 'rubygems'
require 'httparty'

require 'httparty_icebox'

class CachedRepresentative
  include HTTParty
  include HTTParty::Icebox
  cache :store => 'memory', :timeout => 0.5, :logger => Logger.new(STDOUT)
end

# -----

response1 = CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544')
response2 = CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544')

puts response1.headers.inspect

puts "\nFresh: "  << response1.inspect << " [Code #{response1.code}]"

puts "\nCached: " << response2.inspect << " [Code #{response2.code}]"

puts "\nHeaders work too! => #{response2.headers.inspect}"

# -----

puts "\n\nzzzz...\n\n"
sleep 1.5

# -----

puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect