public
Created

S3 bucket walker

  • Download Gist
gistfile1.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
#!/usr/bin/env ruby
 
require 'rubygems'
require 'aws-sdk'
require 'optparse'
 
def run(options)
# Set up credentials and buckets
s3 = AWS::S3.new({
:access_key_id => options[:key],
:secret_access_key => options[:secret]
})
bucket = s3.buckets[options[:bucket]]
 
# prefixes
prefixes = []
(0..9).each { |x| prefixes << x }
('a'..'z').each { |x| prefixes << x }
('A'..'Z').each { |x| prefixes << x }
 
prefixes.each do |prefix|
Thread.new do
retries = 0
begin
bucket.objects.with_prefix(prefix.to_s).each { |o| puts "#{o.key} #{o.last_modified}" }
rescue OpenSSL::SSL::SSLError
retries += 1
STDERR.puts "Prefix #{prefix} failed #{retries} times."
retry if retries <= 5
end
end
end
 
# Wait for all threads to finish.
Thread.list.each { |t| t.join unless t == Thread.current }
end
 
if __FILE__ == $PROGRAM_NAME
# Hide AWS credentials from process listing
$0 = $PROGRAM_NAME
 
# Parse command line options
options = {}
option_parser = OptionParser.new do |opts|
opts.on('-k', '--key <key>', 'AWS Key') do |k|
options[:key] = k
end
 
opts.on('-s', '--secret <secret>', 'AWS Secret') do |s|
options[:secret] = s
end
 
opts.on('-b', '--bucket <bucket>', 'Bucket to list') do |b|
options[:bucket] = b
end
end
option_parser.parse!(ARGV)
 
# Verify we have all the configuration files available
if [
options[:key],
options[:secret],
options[:bucket]
].include?(nil)
STDERR.puts "Need moar options!\n\n"
exit(1)
end
 
run(options)
end

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.