directory_upload.rb

#!/usr/bin/env ruby

require 'rubygems'
require 'aws-sdk'


class S3FolderUpload
  attr_reader :folder_path, :total_files, :s3_bucket
  attr_accessor :files

  # Initialize the upload class
  #
  # folder_path - path to the folder that you want to upload
  # bucket - The bucket you want to upload to
  # aws_key - Your key generated by AWS defaults to the environemt setting AWS_KEY_ID
  # aws_secret - The secret generated by AWS
  #
  # Examples
  #   => uploader = S3FolderUpload.new("some_route/test_folder", 'your_bucket_name')
  #
  def initialize(folder_path, bucket, aws_key = ENV['AWS_KEY_ID'], aws_secret = ENV['AWS_SECRET'])
    AWS.config(access_key_id: aws_key, secret_access_key: aws_secret)

    @folder_path       = folder_path
    @files             = Dir.glob("#{folder_path}/**/*")
    @total_files       = files.length
    @connection        = AWS::S3.new
    @s3_bucket         = @connection.buckets[bucket]
  end

  # public: Upload files from the folder to S3
  #
  # thread_count - How many threads you want to use (defaults to 5)
  #
  # Examples
  #   => uploader.upload!(20)
  #     true
  #   => uploader.upload!
  #     true
  #
  # Returns true when finished the process
  def upload!(thread_count = 5)
    file_number = 0
    mutex       = Mutex.new
    threads     = []

    thread_count.times do |i|
      threads[i] = Thread.new {
        until files.empty?
          mutex.synchronize do
            file_number += 1
            Thread.current["file_number"] = file_number
          end
          file = files.pop rescue nil
          next unless file

          # I had some more manipulation here figuring out the git sha
          # For the sake of the example, we'll leave it simple
          #
          path = file

          puts "[#{Thread.current["file_number"]}/#{total_files}] uploading..."

          data = File.open(file)

          if File.directory?(data)
            data.close
            next
          else
            obj = s3_bucket.objects[path]
            obj.write(data, { reduced_redundancy: true })
            data.close
          end

        end
      }
    end
    threads.each { |t| t.join }
  end
end


uploader = S3FolderUpload.new('test', 'miles-media-library', AWS_KEY, AWS_SECRET)
uploader.upload!(1)