Will duplicate contents of one Amazon S3 bucket into another.

s3_bucket_copy.rb

require 'rubygems'
require 'aws/s3'
require 'logger'

AWS::S3::S3Object.class_eval do
  def self.copy_across_buckets(src_bucket, src_key, dest_bucket, dest_key, options = {})
    src_bucket = bucket_name(src_bucket)
    dest_bucket = bucket_name(dest_bucket)

    source_key = path!(src_bucket, src_key)
    target_key = path!(dest_bucket, dest_key)

    default_options = { 'x-amz-copy-source' => source_key }

    returning put(target_key, default_options) do
      acl(dest_key, dest_bucket, acl(src_key, src_bucket)) if options[:copy_acl]
    end
  end
end

class AmazonS3Asset
  include AWS::S3
  
  S3ID       = "xxxx"
  S3KEY      = "xxxx"
  CHUNK_SIZE = 100
  
  def initialize
    @logger = Logger.new('s3_copy.log')
    
    log "connecting..."
    AWS::S3::Base.establish_connection!(
      :access_key_id     => S3ID,
      :secret_access_key => S3KEY
    )
  end
  
  def copy_over_bucket(from_bucket, to_bucket, empty_to_bucket = false)
    bucket_exists?(from_bucket)
    bucket_exists?(to_bucket)
    empty_bucket(to_bucket) if empty_to_bucket
    copy_bucket(from_bucket, to_bucket)
  end
  
  private
  
  def log(message)
    puts message
    @logger.info(message)
  end
  
  def copy_bucket(from_bucket, to_bucket)
    marker = nil
    
    loop do
      find_options = { :max_keys => CHUNK_SIZE }

      if marker
        find_options[:marker] = marker
      end
      
      log "Fetching #{CHUNK_SIZE} keys."
      
      keys_fetched = Bucket.objects(from_bucket, find_options).collect { |o| o.key }
            
      copy_between_buckets(from_bucket, to_bucket, keys_fetched)

      if keys_fetched.size < CHUNK_SIZE
        log 'Done.'
        return
      else
        marker = keys_fetched.last
      end
    end
  end
  
  def bucket_exists?(bucket)
    !!Bucket.find(bucket)
  end
  
  def empty_bucket(bucket)
    log "Emptying #{bucket}..."
    Bucket.find(bucket).delete_all
  end
  
  def copy_between_buckets(from_bucket, to_bucket, keys)
    keys.each do |k|
      begin
        log k
        S3Object.copy_across_buckets(from_bucket, k, to_bucket, k, :copy_acl => true)
      rescue => e
        log e.message
        next        
      end
    end
  end
  
end

a = AmazonS3Asset.new
a.copy_over_bucket(SOURCE_BUCKET, DEST_BUCKET)