AWS S3 uploader

s3_uploader.rb

require "aws-sdk-s3"
# Original source: https://gist.github.com/fleveque/816dba802527eada56ab
#
# This version is for the newer aws-s3-sdk v3
#
# You need to require some of the components, at least this one from what I remember:
require 'aws-sdk-s3'

# Upload a folder of files as public files, to S3, using threads. Useful for
# pushing assets compiledi in a rails app on S3, so you can front them with cloudfront.
#
# Original use case was so the assets could be deleted from the Heroku slug after the
# assets were built, to reduce the slug size
class S3Uploader
  attr_reader :folder_path, :total_files, :bucket, :include_folder, :bucket
  attr_accessor :files

  # Initialize the upload class
  #
  # folder_path - path to the folder that you want to upload
  # bucket - The bucket you want to upload to
  # aws_key - Your key generated by AWS defaults to the environemt setting AWS_KEY_ID
  # aws_secret - The secret generated by AWS
  # include_folder - include the root folder on the path? (default: true)
  #
  # Examples
  #   => uploader = S3FolderUpload.new("some_route/test_folder", 'your_bucket_name')
  #
  def initialize(folder_path, bucket, include_folder = true)
    # Use both, AWS is the preferred syntax, AMAZON is the old
    # Could also modify the init to set these so you can pass them in as an argument
    aws_key = ENV["AWS_ACCESS_KEY_ID"] || ENV["AMAZON_ACCESS_KEY_ID"]
    aws_secret = ENV["AWS_SECRET_ACCESS_KEY"] || ENV["AMAZON_SECRET_ACCESS_KEY"]

    # This should/could also be passed in as an argument in the initialize
    # as a potential improvement
    Aws.config.update(
      {
        region: ENV["AWS_REGION"] || "us-east-1",
        credentials: Aws::Credentials.new(aws_key, aws_secret)
      }
    )

    @folder_path       = folder_path
    @files             = Dir.glob("#{folder_path}/**/*")
    @total_files       = files.length
    @resource          = Aws::S3::Resource.new(region: "us-east-1")
    @bucket            = @resource.bucket(bucket)
    @include_folder    = include_folder
  end

  # public: Upload files from the folder to S3
  #
  # thread_count - How many threads you want to use (defaults to 5)
  # simulate - Don't perform upload, just simulate it (default: false)
  # verbose - Verbose info (default: false)
  #
  # Examples
  #   => uploader.upload!(20)
  #     true
  #   => uploader.upload!
  #     true
  #
  # Returns true when finished the process
  def upload!(thread_count = 5, simulate = false, verbose = false)
    file_number = 0
    mutex       = Mutex.new
    threads     = []

    puts "Total files: #{total_files}... uploading (folder #{folder_path} #{include_folder ? '' : 'not '}included)"

    thread_count.times do |i|
      threads[i] = Thread.new {
        until files.empty?
          mutex.synchronize do
            file_number += 1
            Thread.current["file_number"] = file_number
          end
          file = files.pop rescue nil
          next unless file

          # Define destination path
          path =
            if include_folder
              file
            else
              file.sub(/^#{folder_path}\//, '')
            end

          puts "[#{Thread.current["file_number"]}/#{total_files}] uploading... #{file}" if verbose

          data = File.open(file)

          unless File.directory?(data) || simulate
            bucket.put_object(
              key: path,
              body: data,
              acl: "public-read" # FYI: This makes all the files public readable
            )
          end

          data.close
        end
      }
    end
    threads.each { |t| t.join }
  end
end

# Example usage:
uploader = S3FolderUpload.new('my-folder', 'my-bucket')
uploader.upload!(
  10, 
  false, # simulate, rather than upload
  true # verbose logging
)