gistfile1.txt

# The following code lets you iterate over large collections of Active Record 
# objects without having to load them all at once, thus reducing memory usage. 
# It’s allowed me to run cron jobs which iterate over thousands of records 
# without getting the cron’d process killed for using too much of a system’s 
# resources.
#
# Taken from Flying Machine Studios
# http://www.flyingmachinestudios.com/2008/05/12/chunky-iterator-so-you-dont-have-to-load-all-your-ar-objects-at-once/
# Author: Daniel Higginbotham
# Date: May 12, 2008
#
class ChunkyIterator
  include Enumerable
  
  private
  
  def merged_options(id)
    @options.merge(
      :conditions => merge_conditions("#{@model_class.table_name}.id > #{id}"),
      :limit => @chunk_size
    )
  end

  def merge_conditions(added_condition)
    existing_condition = @options[:conditions]
    new_condition = case existing_condition
    when nil: added_condition
    when String: "(#{existing_condition}) AND (#{added_condition})"
    when Array
      ["(#{existing_condition[0]})" +
       " AND (#{added_condition})"] +
       existing_condition[1..-1]
    end
  end
  public
  
  def initialize(model_class, chunk_size, options)
    @model_class = model_class
    @chunk_size = chunk_size
    @options = options
  end

  def each
    model_objects = @model_class.find(:all, merged_options(0))

    until model_objects.empty?
      model_objects.each{|record| yield record}
      model_objects = @model_class.find(:all, merged_options(model_objects.last.id))
    end
  end

end

# # Example
# Bacon.find_all_in_chunks(500, :conditions => "fresh = TRUE").each do |bacon|
#   bacon.feed_to_cat
# end