Skip to content

Instantly share code, notes, and snippets.

@ohookins
Created November 20, 2012 16:54
Show Gist options
  • Save ohookins/4119222 to your computer and use it in GitHub Desktop.
Save ohookins/4119222 to your computer and use it in GitHub Desktop.
S3 bucket walker
#!/usr/bin/env ruby
require 'rubygems'
require 'aws-sdk'
require 'optparse'
def run(options)
# Set up credentials and buckets
s3 = AWS::S3.new({
:access_key_id => options[:key],
:secret_access_key => options[:secret]
})
bucket = s3.buckets[options[:bucket]]
# prefixes
prefixes = []
(0..9).each { |x| prefixes << x }
('a'..'z').each { |x| prefixes << x }
('A'..'Z').each { |x| prefixes << x }
prefixes.each do |prefix|
Thread.new do
retries = 0
begin
bucket.objects.with_prefix(prefix.to_s).each { |o| puts "#{o.key} #{o.last_modified}" }
rescue OpenSSL::SSL::SSLError
retries += 1
STDERR.puts "Prefix #{prefix} failed #{retries} times."
retry if retries <= 5
end
end
end
# Wait for all threads to finish.
Thread.list.each { |t| t.join unless t == Thread.current }
end
if __FILE__ == $PROGRAM_NAME
# Hide AWS credentials from process listing
$0 = $PROGRAM_NAME
# Parse command line options
options = {}
option_parser = OptionParser.new do |opts|
opts.on('-k', '--key <key>', 'AWS Key') do |k|
options[:key] = k
end
opts.on('-s', '--secret <secret>', 'AWS Secret') do |s|
options[:secret] = s
end
opts.on('-b', '--bucket <bucket>', 'Bucket to list') do |b|
options[:bucket] = b
end
end
option_parser.parse!(ARGV)
# Verify we have all the configuration files available
if [
options[:key],
options[:secret],
options[:bucket]
].include?(nil)
STDERR.puts "Need moar options!\n\n"
exit(1)
end
run(options)
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment