copy s3 objects from one bucket to another

copy_s3_objects_from_one_bucket_to_another.rb

# inspired by: https://gist.github.com/dje/763977

def copy(source_bucket:, destination_bucket:, files:, connection:)
  files.each_with_index { |file, index|
    puts "copy #{file['Key']} to #{destination_bucket}" if index % 20 == 0
    connection.copy_object source_bucket,
                           file['Key'],
                           destination_bucket,
                           file['Key']
  }
end

connection = Fog::Storage.new({
  provider: 'AWS',
  path_style: true,
  aws_access_key_id: AWS_KEY,
  aws_secret_access_key: AWS_SECRET
})

source_bucket = 'old'
destination_bucket = 'new'
file_count = 0
files = connection.get_bucket(source_bucket, prefix: 'nested-bucket')
file_count += files.body['Contents'].size
truncated = files.body['IsTruncated']
the_response = files.body['Contents']
copy(
  files: files.body['Contents'],
  source_bucket: source_bucket,
  destination_bucket: destination_bucket,
  connection: connection
)
while truncated
  files = connection.get_bucket(source_bucket,{'max-keys' =>'100000', 'marker' => files.body['Contents'].last["Key"]})
  file_count += files.body['Contents'].size
  truncated = files.body['IsTruncated']
  copy(
    files: files.body['Contents'],
    source_bucket: source_bucket,
    destination_bucket: destination_bucket,
    connection: connection
  )
end

puts "******************** copied #{file_count} files"