Get file size of all the files at a given path on Amazon S3 - recursively, and dump in a CSV

get_s3_file_sizes_recursively.rb

# Using aws-sdk gem

@access_key = "access_key"
@secret_key = "secret_access_key"
@bucket = "my_bucket"

AWS.config(
    :access_key_id => @access_key,
    :secret_access_key => @secret_key
)
@s3 = AWS::S3.new

@path = "path/to/folder"

tree = @s3.buckets[@bucket].as_tree(:prefix => @path)
directories = tree.children.select(&:branch?).collect(&:prefix)
files = tree.children.select(&:leaf?).collect(&:key)
while directories.size > 0
  dir = directories.pop
  files = files + @s3.buckets[@bucket].as_tree(:prefix => dir).children.select(&:leaf?).collect(&:key)
  directories = directories + @s3.buckets[@bucket].as_tree(:prefix => dir).children.select(&:branch?).collect(&:prefix)
end

puts "Total Files: #{files.count}"

f=File.open("/tmp/s3_sizes.csv", "w")
files.each do |file|
  size = @s3.buckets[@bucket].objects[file].content_length
  f.puts size
end
f.close