S3 bucket walker

gistfile1.rb

#!/usr/bin/env ruby

require 'rubygems'
require 'aws-sdk'
require 'optparse'

  def run(options)
    # Set up credentials and buckets
    s3 = AWS::S3.new({
      :access_key_id     => options[:key],
      :secret_access_key => options[:secret]
    })
    bucket = s3.buckets[options[:bucket]]

    # prefixes
    prefixes = []
    (0..9).each { |x| prefixes << x }
    ('a'..'z').each { |x| prefixes << x }
    ('A'..'Z').each { |x| prefixes << x }

    prefixes.each do |prefix|
      Thread.new do
        retries = 0
        begin
          bucket.objects.with_prefix(prefix.to_s).each { |o| puts "#{o.key} #{o.last_modified}" }
        rescue OpenSSL::SSL::SSLError
          retries += 1
          STDERR.puts "Prefix #{prefix} failed #{retries} times."
          retry if retries <= 5
        end
      end
    end

    # Wait for all threads to finish.
    Thread.list.each { |t| t.join unless t == Thread.current }
  end

if __FILE__ == $PROGRAM_NAME
  # Hide AWS credentials from process listing
  $0 = $PROGRAM_NAME

  # Parse command line options
  options = {}
  option_parser = OptionParser.new do |opts|
    opts.on('-k', '--key <key>', 'AWS Key') do |k|
      options[:key] = k
    end

    opts.on('-s', '--secret <secret>', 'AWS Secret') do |s|
      options[:secret] = s
    end

    opts.on('-b', '--bucket <bucket>', 'Bucket to list') do |b|
      options[:bucket] = b
    end
  end
  option_parser.parse!(ARGV)

  # Verify we have all the configuration files available
  if [
    options[:key],
    options[:secret],
    options[:bucket]
    ].include?(nil)
    STDERR.puts "Need moar options!\n\n"
    exit(1)
  end

  run(options)
end