S3_count_storage_size

count.rb

#!/usr/bin/env ruby

require 'aws-sdk'
require 'optparse'

begin
  require 'aws/profile_parser'
rescue LoadError; end

def conv_unit size
  if size < 1024 # Byte
    "#{size} Byte"
  elsif size < 1024**2 # KiloByte
    "#{sprintf("%5.2f", size/1024)} KB"
  elsif size < (1024**3) # MegaByte
    "#{sprintf("%5.2f", size/(1024**2))} MB"
  elsif size < (1024**4) # GigaByte
    "#{sprintf("%5.2f", size/(1024**3))} GB"
  elsif size < (1024**5) # TeraByte
    "#{sprintf("%5.2f", size/(1024**4))} TB"
  end
end

# count objects size with a specifix prefix
def count_objects_size(s3client, parameters)
  size = 0
  response = s3client.list_objects(parameters)
  size = response.contents.inject(0) {|sum, content| sum += content.size.to_i}
  # if the response is truncated(= has remain objects)
  if response.truncated
    parameters[:next_marker] = response.next_marker
    size += count_objects_size(s3client, parameters)
  end

  if !response.common_prefixes.empty?
    response.common_prefixes.each do |p|
      # initialize next_marker
      parameters[:next_marker] = nil
      # search recursively
      parameters[:prefix] = p[:prefix]
      size += count_objects_size(s3client, parameters)
    end
  end

  size

end #count_objects_size

ARGV.options do |opt|
  begin
    aws_opts = {}
    is_debug = false

    opt.on('-h', '--help')                  { puts opt.help; exit 0 }
    opt.on('-k', '--access-key ACCESS_KEY') { |v| aws_opts[:access_key_id]      = v }
    opt.on('-s', '--secret-key SECRET_KEY') { |v| aws_opts[:secret_access_key]  = v }
    opt.on('-r', '--region REGION')         { |v| aws_opts[:region]             = v }
    opt.on('--debug')                       { is_debug = true }
    opt.on('--profile PROFILE')             { |v| parser = AWS::ProfileParser.new; aws_opts = parser.get(v) }
    opt.parse!

    if aws_opts.empty?
      puts opt.help
      exit 1
    end

    if is_debug
      aws_opts[:logger]    = Logger.new($stdout)
      aws_opts[:log_level] = :debug
    end

    AWS.config(aws_opts)

  rescue => e
    $stderr.puts e
    exit 1
  end
end

s3 = AWS::S3.new
total_size = 0

s3.buckets.each do |bucket|
  print "Bucket[#{bucket.name}] : "
  region = bucket.location_constraint.nil? ?
    "us-east-1" :
    bucket.location_constraint
  s3client = AWS::S3::Client.new(:region => region)
  parameters = {
    :bucket_name => bucket.name,
    :delimiter => "/",
    :prefix => "",
    :next_marker => nil
  }

  bucket_size = count_objects_size(s3client, parameters)
  puts conv_unit(bucket_size.to_f)
  total_size += bucket_size
end

puts "Total : #{conv_unit(total_size.to_f)}"