Ruby Peach

example.rb

  def self.gzip_to_s3
    s3_bucket = Aws::S3::Resource.new(region:'us-east-1')
      .bucket('basf-content-server')
    total = SalesDataDocument.count
    docs = SalesDataDocument.all
    Peach.enum_for(docs).each do |doc, queue|
      progress = total - queue.length
      Rails.logger.info <<-LOG if (progress % 20).zero?
        storing document ##{progress} of #{total}
      LOG
      s3_obj = s3_bucket.object("sales-data/#{Rails.env}/#{doc.name}.gz")
      next if s3_obj.exists? && s3_obj.last_modified >= doc.updated_at
      file = get(doc.name)
      gzipped = ActiveSupport::Gzip.compress(file)
      s3_obj.put body: gzipped
    end
  end

peach.rb

class Peach
  private_class_method :new

  def self.enum_for(*args)
    new(*args)
  end

  def initialize(list, pool_size = 5)
    @queue = list.inject(Queue.new, :<<)
    @pool_size = pool_size
  end

  def each
    Array.new(@pool_size) do
      Thread.new do
        yield(@queue.pop, @queue) until @queue.empty?
      end
    end.each(&:join)
  end
end