gistfile1.txt

def reindex_companies_since(timestamp)
  conditions = ["updated_at >= ?", timestamp]
  total = Company.count(:conditions => conditions)
  indexed = 0

  companies = Queue.new
  indexing = true
  
  Thread.abort_on_exception = true

  producer = Thread.new do
    Company.find_in_batches(:conditions => conditions) do |batch|
      sleep 1 and puts companies.length while companies.length < 4
      companies.push(batch)
      indexed += batch.count
      puts "Indexed #{indexed} of #{total} companies..."
    end
    
    indexing = false
  end

  consumers = Array.new(4) do
    Thread.new do
      while indexing || !companies.empty?
        begin
          Sunspot.index!(*companies.pop)
        rescue ThreadError => e
          sleep 1
        rescue Exception => e
          $stderr.puts "ERROR encountered: #{e.message}"
          $stderr.puts e.backtrace.join("\n")
          sleep 10
          retry
        end
      end
    end
  end
  
  consumers.each(&:join)
end