Created

Embed URL

HTTPS clone URL

SSH clone URL

You can clone with HTTPS or SSH.

Download Gist

S3 bucket walker

View gist:4119222
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
#!/usr/bin/env ruby
 
require 'rubygems'
require 'aws-sdk'
require 'optparse'
 
def run(options)
# Set up credentials and buckets
s3 = AWS::S3.new({
:access_key_id => options[:key],
:secret_access_key => options[:secret]
})
bucket = s3.buckets[options[:bucket]]
 
# prefixes
prefixes = []
(0..9).each { |x| prefixes << x }
('a'..'z').each { |x| prefixes << x }
('A'..'Z').each { |x| prefixes << x }
 
prefixes.each do |prefix|
Thread.new do
retries = 0
begin
bucket.objects.with_prefix(prefix.to_s).each { |o| puts "#{o.key} #{o.last_modified}" }
rescue OpenSSL::SSL::SSLError
retries += 1
STDERR.puts "Prefix #{prefix} failed #{retries} times."
retry if retries <= 5
end
end
end
 
# Wait for all threads to finish.
Thread.list.each { |t| t.join unless t == Thread.current }
end
 
if __FILE__ == $PROGRAM_NAME
# Hide AWS credentials from process listing
$0 = $PROGRAM_NAME
 
# Parse command line options
options = {}
option_parser = OptionParser.new do |opts|
opts.on('-k', '--key <key>', 'AWS Key') do |k|
options[:key] = k
end
 
opts.on('-s', '--secret <secret>', 'AWS Secret') do |s|
options[:secret] = s
end
 
opts.on('-b', '--bucket <bucket>', 'Bucket to list') do |b|
options[:bucket] = b
end
end
option_parser.parse!(ARGV)
 
# Verify we have all the configuration files available
if [
options[:key],
options[:secret],
options[:bucket]
].include?(nil)
STDERR.puts "Need moar options!\n\n"
exit(1)
end
 
run(options)
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.