sync_aws_buckets

sync_aws_buckets

#!/usr/bin/env ruby
# encoding: utf-8

##
# @example
#   ruby -rubygems `__FILE__` `source_bucket_name` `target_bucket_name`
#
# @author @ipoval
#
# @see https://github.com/ipoval
#

require 'yaml'
require 'right_aws'
require 'benchmark'

fail "Usage:\n ruby -rubygems s3_bucket_copy.rb <source_bucket_name> <target_bucket_name>\n" unless ARGV[0] && ARGV[1]

puts 'Fetching aws config ...'
config = YAML.load(File.read(File.expand_path('.amazonws', ENV['HOME'])))

puts 'Connecting to amazon S3 ...'
s3 = RightAws::S3.new( config['access_key'], config['secret_key'], { :multi_thread => true,
                                                                     :protocol     => 'http',
                                                                     :port         => '80',
                                                                     :server       => 's3.amazonaws.com' } )

source = s3.bucket ARGV[0]
target = s3.bucket ARGV[1]
keys_to_copy = source.keys.collect(&:name) - target.keys.collect(&:name)

if keys_to_copy.empty?
  puts 'These buckets are already in sync.'
  exit 0
else
  puts "There are #{keys_to_copy.size} key(s) to copy ..."

  semaphore = Mutex.new
  elapsed_time = Benchmark.measure do
    i = 0
    threads = []

    100.times do
      threads << Thread.new do
        s3i ||= RightAws::S3Interface.new( config['access_key'], config['secret_key'], { :multi_thread => true,
                                                                                         :protocol     => 'http',
                                                                                         :port         => '80',
                                                                                         :server       => 's3.amazonaws.com' } )

        while(keys_to_copy.size > 0) do
          begin
            key = ''
            semaphore.synchronize { key = keys_to_copy.shift }

            print "Copying #{key} ...\n"
            s3i.copy(source.name, key, target.name, key, :copy, { 'Cache-Control' => 'max-age=315360000', 'Expires' => '315360000' })
            # Do not copy properties
            # acl_prop = s3i.get_acl(source.name, key)
            # s3i.put_acl(target.name, key, acl_prop[:object])

            semaphore.synchronize { print "COPIED: #{i}, LEFT: #{keys_to_copy.size}\n" if (i += 1) % 100 == 0 }
          rescue Exception => e
            STDERR.puts "Error: #{e.message}"
          end
        end
      end
    end

    threads.each(&:join)
  end

  puts elapsed_time
end