An excerpt from the Rakefile I use to deploy my blog. http://blog.alexbrowne.info

aws_deploy_tools.rb

require 'rubygems'
require 'bundler/setup'
require 'openssl'
require 'digest/sha1'
require 'net/https'
require 'base64'
require 'aws-sdk'
require 'digest/md5'
require 'colored'


# A convenient wrapper around aws-sdk
# Also includes the ability to invalidate cloudfront files
# Used for deploying to s3/cloudfront
class AWSDeployTools
	
	def initialize(config = {})

		@access_key_id = config['access_key_id']
		@secret_access_key = config['secret_access_key']

		# for privacy, allow user to store aws credentials in a shell env variable
		@access_key_id ||= ENV['AWS_ACCESS_KEY_ID']
		@secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY']

		@bucket = config['bucket']
		@acl = config['acl']
		@cf_distribution_id = config['cf_distribution_id']
		@dirty_keys = Set.new # a set of keys that are dirty (have been pushed but not invalidated)

		if @bucket.nil?
    	raise "ERROR: Must provide bucket name in constructor. (e.g. :bucket => 'bucket_name')" 
	  end

	  if @cf_distribution_id.nil?
	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n Skipping cf invalidations..."
	  end


		@s3 = AWS::S3.new(config)

	end

	# checks if a local file is in sync with the s3 bucket
	# file can either be a file object or a string with a
	# valid file path
	def synced?(s3_key, file)

		if file.is_a? String
			file = File.open(file, 'r')
		end

		f_content = file.read
		obj_etag = ""

		begin
	    obj = @s3.buckets[@bucket].objects[s3_key]
	    obj_etag = obj.etag # the etag is the md5 of the remote file
	  rescue
	   return false
	  end

		# the etag is surrounded by quotations. chomp removes them
	  obj_etag = obj_etag.gsub('"', '')

		# compare the etag to the md5 hash of the local file
	  obj_etag == md5(f_content)

	end

	# pushes (writes) the file to the s3 bucket at location
	# indicated by s3_key.
	# file can either be a file object or a string with a
	# valid file path
	# options are any options that can be passed to the
	# write method. 
	# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
	def push(s3_key, file, options = {})

		if file.is_a? String
			file = File.open(file, 'r')
		end

		# detect content type		
		require 'mime/types'
		file_path = file.path
		# remove the .gz extension so the base extension will
		# be used to determine content type. E.g. we want the
		# type of index.html.gz to be text/html
		if file_path.include? ".gz"
			file_path.gsub!(".gz", "")
		end
		content_type = MIME::Types.type_for(File.extname(file_path)).first.to_s
		content_type_hash = {:content_type => content_type}
		options.merge! content_type_hash
		
		puts "--> pushing #{file.path} to #{s3_key}...".green
    obj = @s3.buckets[@bucket].objects[s3_key]
    obj.write(file, options)

    @dirty_keys << s3_key
    # Special cases for index files.
    # for /index.html we should also invalidate /
    # for /archive/index.html we should also invalidate /archive/
    if (s3_key == "index.html")
			@dirty_keys << "/"
    elsif File.basename(s3_key) == "index.html"
			@dirty_keys << s3_key.chomp(s3_key.split("/").last)
    end
	  
	end

	# batch pushes (writes) the files to the s3 bucket at locations
	# indicated by s3_keys. (more than one file at a time)
	# files can either be a file object or a string with a
	# valid file path
	# options are any options that can be passed to the
	# write method.
	# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
	def batch_push(s3_keys = [], files = [], options = {})
		
		if (s3_keys.size != files.size) 
			raise "ERROR: There must be a 1-to-1 correspondence of keys to files!"
		end

		files.each_with_index do |file, i|

			s3_key = s3_keys[i]
			push(s3_key, file, options)
		  
		end
	end

	# for each file, first checks if the file is synced.
	# If not, it pushes (writes) the file.
	# options are any options that can be passed to the
	# write method.
	# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html
	def sync(s3_keys = [], files = [], options = {})
		
		if (s3_keys.size != files.size) 
			raise "ERROR: There must be a 1-to-1 correspondence of keys to files!"
		end

		files.each_with_index do |file, i|

			s3_key = s3_keys[i]
	    unless synced?(s3_key, file)
				push(s3_key, file, options)
	    end
		  
		end

	end


	# a convenience method which simply returns the md5 hash of input
	def md5 (input)
	  Digest::MD5.hexdigest(input)
	end

	# invalidates files (accepts an array of keys or a single key)
	# based heavily on https://gist.github.com/601408
	def invalidate(s3_keys)

		if @cf_distribution_id.nil?
	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..."
	    return
	  end

		if s3_keys.nil? || s3_keys.empty?
			puts "nothing to invalidate."
			return
		elsif s3_keys.is_a? String
			puts "--> invalidating #{s3_keys}...".yellow
			# special case for root
			if s3_keys == '/'
				paths = '<Path>/</Path>'
			else
				paths = '<Path>/' + s3_keys + '</Path>'
			end
		elsif s3_keys.length > 0
			puts "--> invalidating #{s3_keys.size} file(s)...".yellow
			paths = '<Path>/' + s3_keys.join('</Path><Path>/') + '</Path>'
			# special case for root
			if s3_keys.include?('/')
				paths.sub!('<Path>//</Path>', '<Path>/</Path>')
			end
		end

		# digest calculation based on http://blog.confabulus.com/2011/05/13/cloudfront-invalidation-from-ruby/
		date = Time.now.strftime("%a, %d %b %Y %H:%M:%S %Z")
		digest = Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new('sha1'), @secret_access_key, date)).strip

		uri = URI.parse('https://cloudfront.amazonaws.com/2010-08-01/distribution/' + @cf_distribution_id + '/invalidation')

		if paths != nil
		  req = Net::HTTP::Post.new(uri.path)
		else
		  req = Net::HTTP::Get.new(uri.path)
		end

		req.initialize_http_header({
		  'x-amz-date' => date,
		  'Content-Type' => 'text/xml',
		  'Authorization' => "AWS %s:%s" % [@access_key_id, digest]
		})
		 
		if paths != nil
		  req.body = "<InvalidationBatch>" + paths + "<CallerReference>ref_#{Time.now.utc.to_i}</CallerReference></InvalidationBatch>"
		end
		 
		http = Net::HTTP.new(uri.host, uri.port)
		http.use_ssl = true
		http.verify_mode = OpenSSL::SSL::VERIFY_NONE
		res = http.request(req)
		
		if res.code == '201'
			puts "Cloudfront Invalidation Success [201]. It may take a few minutes for the new files to propagate.".green
		else
			puts ("Cloudfront Invalidation Error: \n" + res.body).red
		end

		return res.code

	end

	# invalidates all the dirty keys and marks them as clean
	def invalidate_dirty_keys
		if @cf_distribution_id.nil?
	    puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..."
	    return
	  end

		res_code = invalidate(@dirty_keys.to_a)
		# mark the keys as clean iff the invalidation request went through
		@dirty_keys.clear if res_code == '201'
	end


end

Rakefile

# ...

desc "Deploy website to s3/cloudfront via aws-sdk"
task :s3_cloudfront => [:generate, :minify, :gzip, :compress_images] do
  puts "=================================================="
  puts "      Deploying to Amazon S3 & CloudFront"
  puts "=================================================="

  # setup the aws_deploy_tools object
  config = YAML::load( File.open("_config.yml"))
  aws_deploy = AWSDeployTools.new(config)

  # get all files in the public directory
  all_files = Dir.glob("#{$public_dir}/**/*.*")

  # we want the gzipped version of the files, not the regular (non-gzipped) version
  # excluded files contains all the regular versions, which will not be deployed
  excluded_files = []
  $gzip_exts.collect do |ext|
    excluded_files += Dir.glob("#{$public_dir}/**/*.#{ext}")
  end

  # we do gzipped files seperately since they have different metadata (:content_encoding => gzip)
  puts "--> syncing gzipped files...".yellow
  gzipped_files = Dir.glob("#{$public_dir}/**/*.gz")
  gzipped_keys = gzipped_files.collect {|f| (f.split("#{$public_dir}/")[1]).sub(".gz", "")}

  aws_deploy.sync(gzipped_keys, gzipped_files,
          :reduced_redundancy => true,
          :cache_control => "max_age=86400", #24 hours
          :content_encoding => 'gzip',
          :acl => config['acl']
          )

  puts "--> syncing all other files...".yellow
  non_gzipped_files = all_files - gzipped_files - excluded_files
  non_gzipped_keys = non_gzipped_files.collect {|f| f.split("#{$public_dir}/")[1]}

  aws_deploy.sync(non_gzipped_keys, non_gzipped_files,
          :reduced_redundancy => true,
          :cache_control => "max_age=86400", #24 hours
          :acl => config['acl']
          )
  
  # invalidate all the files we just pushed
  aws_deploy.invalidate_dirty_keys  

  puts "DONE."

end

desc "Compress all applicable content in public/ using gzip"
task :gzip do

  unless which('gzip')
    puts "WARNING: gzip is not installed on your system. Skipping gzip..."
    return
  end

  @compressor ||= RedDragonfly.new

  $gzip_exts.each do |ext|
    puts "--> gzipping all #{ext}...".yellow
    files = Dir.glob("#{$gzip_dir}/**/*.#{ext}")
    files.each do |f|
      @compressor.gzip(f)
    end
  end

  puts "DONE."
end

desc "Minify all applicable files in public/ using jitify"
task :minify do
  
  unless which('jitify')
    puts "WARNING: jitify is not installed on your system. Skipping minification..."
    return
  end

  @compressor ||= RedDragonfly.new

  $minify_exts.each do |ext|
    puts "--> minifying all #{ext}...".yellow
    files = Dir.glob("#{$minify_dir}/**/*.#{ext}")
    files.each do |f|
      @compressor.minify(f)
    end
  end

  puts "DONE."

end

desc "Compress all images in public/ using ImageMagick"
task :compress_images do

  unless which('convert')
    puts "WARNING: ImageMagick is not installed on your system. Skipping image compression..."
    return
  end

  @compressor ||= RedDragonfly.new

  $compress_img_exts.each do |ext|
    puts "--> compressing all #{ext}...".yellow
    files = Dir.glob("#{$compress_img_dir}/**/*.#{ext}")
    files.each do |f|
      @compressor.compress_img(f)
    end
  end

  puts "DONE."

end

# ...

##
# invoke system which to check if a command is supported
# from http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
# which('ruby') #=> /usr/bin/ruby
def which(cmd)
  system("which #{ cmd} > /dev/null 2>&1")
end

red_dragonfly.rb

# A set of tools for minifying and compressing content
# 
# Currently supports:
#   	- miniifying js, css, and html
# 		- gzipping any content
#     - compressing and shrinking images
#
# Currently depends on 3 system command line tools:
#			- ImageMagick (http://www.imagemagick.org/script/index.php)
# 		- gzip        (http://www.gzip.org/)
#     - jitify      (http://www.jitify.com/)
#
# These may be swapped out for gem versions in the future,
# but for now you have to manually install command line tools
# above on your system if you don't already have them.
#
# Author: Alex Browne

class RedDragonfly

	def initialize
		# ~~ gzip ~~
		$gzip_options = {
			:output_ext => "gz"
		}

		# ~~ minify (jitify) ~~
		# $minify_options = {
		#
		# }
		

		# ~~ images (ImageMagick) ~~
		# exts          : files extensions which sould be minified during batch operations
		# output_ext    : file extension of the output file ("" means keep the same extension)
		# max_width     : max width for compressed images
		# max_height    : max height for compressed images
		# quality       : image compression quality (1-100, higher is better quality/bigger files)
		# compress_type : type of compression to be used (http://www.imagemagick.org/script/command-line-options.php#compress)
		$img_options = {
			:output_ext => "jpg",
			:max_width => "600",
			:max_height => "1200",
			:quality => "65",
			:compress_type => "JPEG"
		}
	end

	# accepts a single file or an array of files
	# accepts a file object or the path to a file (a string)
	# perserves the original file
	# the output is (e.g.) .html.gz
	def gzip (files = [])

		unless which('gzip')
			puts "WARNING: gzip is not installed on your system. Skipping gzip..."
			return
		end

		unless files.is_a? Array
			files = [files]
		end

		files.each do |file|

			fname = get_filename(file)

		  # invoke system gzip
		  system("gzip -cn9 #{fname} > #{fname + '.' + $gzip_options[:output_ext]}")
		end

	end

	# accepts a single file or an array of files
	# accepts a file object or the path to a file
	# overwrites the original file with the minified version
	# html, css, and js supported only
	def minify (files = [])

		unless which('jitify')
			puts "WARNING: jitify is not installed on your system. Skipping minification..."
			return
		end

		unless files.is_a? Array
			files = [files]
		end

		files.each do |file|

			fname = get_filename(file)

		  # invoke system jitify
		  system("jitify --minify #{fname} > #{fname + '.min'}")
		  # remove the .min extension
		  system("mv #{fname + '.min'} #{fname}")

		end

	end

	# compresses an image file using the options
	# specified at the top
	# accepts either a single file or an array of files
	# accepts either a file object or a path to a file
	def compress_img (files = [])
		
		unless which('convert')
			puts "WARNING: ImageMagick is not installed on your system. Skipping image compression..."
			return
		end
		
		unless files.is_a? Array
			files = [files]
		end

		files.each do |file|

			fname = get_filename(file)

			compress_cmd = "convert #{fname} -resize #{$img_options[:max_width]}x#{$img_options[:max_height]}\\>" + 
		  					" -compress #{$img_options[:compress_type]} -quality #{$img_options[:quality]}" + 
		  					" #{get_raw_filename(fname) + '.' + $img_options[:output_ext]}"

		  # invoke system ImageMagick
		  system(compress_cmd)
		  # remove the old file (if applicable)
		  if (get_ext(fname) != ("." + $img_options[:output_ext]))
		  	system("rm #{fname}")
		  end

		end

	end

	# returns the filename (including path and ext) if the input is a file
	# if the input is a string, returns the same string
	def get_filename (file)
		if file.is_a? File
			file = file.path
		end
		return file
	end

	# returns the extension of a file
	# accepts either a file object or a string path
	def get_ext (file)

		if file.is_a? String
			return File.extname(file)
		elsif file.is_a? File
			return File.extname(file.path)
		end

	end

	# returns the raw filename (minus extension) of a file
	# accepts either a file object or a string path
	def get_raw_filename (file)
	
		# convert to string
		file = get_filename(file)
		# remove extension
		file.sub(get_ext(file), "")
		
	end

	##
	# invoke system which to check if a command is supported
	# from http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
	# which('ruby') #=> /usr/bin/ruby
	def which(cmd)
	  system("which #{ cmd} > /dev/null 2>&1")
	end

end

Thanks for this, but I wanted to point out an issue I came across:

If you don't specify a :content_type option in aws_deploy.sync, then you could end up with a Content-Type of "image/jpeg" for html files. This means that the sync method needs to have logic that detects what kind of file is being uploaded and add the appropriate Content-Type.

Good catch. Turns out S3 is setting the content-type for everything to be "image/jpeg." It doesn't seem to break anything, but I'll probably still fix it at some point for the sanity's sake.

Turns out the content type problem was causing problems on certain versions of IE, plus I can imagine that specifying an incorrect content type might have a performance penalty. I added a few lines to aws_deploy_tools.rb to automatically detect the content type based on the file extension. It uses part of Actionpack, which means you have to add rails to your gemfile.

I've also fixed a problem with cloudfront invalidations. It turns out that they work with paths instead of files. So, e.g., it's necessary to post invalidation requests for both "/index.html" and "/"

By the way– Anyone interested in seeing some of this converted to a gem?

This would make a useful gem.

I'll note that I had to replace "$public_dir" in the Rakefile excerpt with "public_dir"; otherwise the Dir.glob line tried to scan my entire hard drive since $public_dir is unset.

I would really like this into a gem! :)

I think there could be an issue with
def compress_img (files = [])
in that $img_options[:output_ext] is being forced to jpg by
$img_options = {
:output_ext => "jpg",
:max_width => "600",
:max_height => "1200",
:quality => "65",
:compress_type => "JPEG"
}

This means png files are being converted to jpg and deleted.