s3_upload.rb

#!/usr/bin/ruby

#
# This script helps to deploy static content to a S3 bucket.
#
# Send the content from options[:dist_dir] to option[:bucket]
# Also apply the CACHE_CONTROL and EXPIRES header to the files.
# The params can be changed on CACHE_CONTROL_METADATA and EXPIRES_METADATA
# constants previously.
#
# Prereq:
# - gem install aws-sdk
#
# Usage:
# ./s3.rb bucket access_key secret region dist_dir thread_count
#
require 'aws-sdk'

options = { :bucket => "YOUR_BUCKET_HERE",
            :access_key => "YOUR_ACCESS_KEY",
            :secret =>  "YOUR_SECRET",
            :region => "us-east-1",
            :dist_dir => "/app/dist",
            :thread_count => 10,
            :htpasswd =>  "/app/_extras/scripts",
            :metadata => true,
          }


# DEFAULT_METADATA = { metadata: {"mg":"13"} }
DEFAULT_METADATA = { }

CACHE_CONTROL_METADATA = {
    "(html|htm)":"181",
    "(js)":"182",
    "(png|jpe?g|gif|ico|svg)$":"183",
    "(mp4|mov|webm|pdf|zip)$":"184",
    "(ttf|eot|otf|woff(2)?)(\w+)?$":"185",
    "(json)":"186",
    "*":"190"
}

EXPIRES_METADATA = {
  "*":"#{Time.now + 30 * 86400}"
}


puts "[version.start] #{Time.now}"


options[:bucket] = ARGV[0] if ARGV[0]
options[:access_key] = ARGV[1] if ARGV[1]
options[:secret] = ARGV[2] if ARGV[2]
options[:region] = ARGV[3] if ARGV[3]
options[:dist_dir] = ARGV[4] if ARGV[4]
options[:thread_count] = ARGV[5] if ARGV[5]
options[:metadata] = ARGV[6] if ARGV[6]


puts "options: #{options}"

s3 = Aws::S3::Resource.new(access_key_id:options[:access_key], secret_access_key:options[:secret], region:options[:region])
bucket = s3.bucket(options[:bucket])


# 
# 
# 
def clear_bucket(options, bucket)
    puts "-> Cleaning bucket #{options[:bucket]}"
    bucket.clear!
end


# 
# 
# 
def upload_dist(options, bucket)
    puts "-> Uploading files to S3"
    files = Dir.glob("#{options[:dist_dir]}/**/*")
    total_files = files.length

    puts "   Files to upload: #{total_files}"

    file_number = 0
    mutex       = Mutex.new
    threads     = []

    options[:thread_count].times do |i|
      threads[i] = Thread.new {
        until files.empty?
          mutex.synchronize do
            file_number += 1
            Thread.current["file_number"] = file_number
          end
          file = files.pop rescue nil
          next unless file

          # I had some more manipulation here figuring out the git sha
          # For the sake of the example, we'll leave it simple
          #
          path = file

          data = File.open(file)

          if File.directory?(data)
            data.close
            next
          else

            key = file.gsub("#{options[:dist_dir]}/",'')

            puts "[#{Thread.current["file_number"]}/#{total_files}] uploading... #{key}"

            metadata = {}
            if options[:metadata]              
              metadata = _metadata(key, "expires", EXPIRES_METADATA, _metadata(key, "cache_control", CACHE_CONTROL_METADATA, DEFAULT_METADATA))              
              # puts "  META: #{metadata}"
            end

            bucket.object(key).upload_file(file, metadata)
            data.close

          end

        end
      }
    end
    threads.each { |t| t.join }
  
end

# 
# 
# 
def upload_htpasswd (options, bucket)
    puts "-> Uploading .htpasswd to S3"

    # mutex = Mutex.new
    path = "#{options[:htpasswd]}/.htpasswd"
    data = File.open(path)


    key = path.gsub("#{options[:htpasswd]}/",'')
    puts "   From: #{path}   To: #{key}"
    bucket.object(key).upload_file(path)
    data.close

end


# 
# 
def _metadata(key, _entry = "", _values = {}, _ext_meta = {})  
  _meta = _ext_meta

  # puts "-" * 100
  # puts "values: #{_values}"
  # puts "_metadata on #{key}"

  ran_regex = false
  _values.keys.each do |k|  
    if k.to_s != "*" && key =~ Regexp.new(k.to_s)
      # puts "[IF] - key: #{key} / Regexp:#{k}"
      # puts "   - #{(key =~ Regexp.new(k.to_s))}"
      _meta[_entry.to_sym] = _values[k]
      ran_regex = true
    end
  end

  if !ran_regex && _values.key?("*".to_sym)
    # puts "[ELSE] - key: #{key} / Regexp:*"
    # puts "   - running for ALL"
    _meta[_entry.to_sym] = _values["*".to_sym]
  end

  # puts "meta: #{_meta}"
  _meta
end





clear_bucket(options, bucket)
upload_dist(options, bucket)
upload_htpasswd(options, bucket)