Last active
December 10, 2015 05:08
-
-
Save albrow/4385707 to your computer and use it in GitHub Desktop.
An excerpt from the Rakefile I use to deploy my blog. http://blog.alexbrowne.info
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'rubygems' | |
require 'bundler/setup' | |
require 'openssl' | |
require 'digest/sha1' | |
require 'net/https' | |
require 'base64' | |
require 'aws-sdk' | |
require 'digest/md5' | |
require 'colored' | |
# A convenient wrapper around aws-sdk | |
# Also includes the ability to invalidate cloudfront files | |
# Used for deploying to s3/cloudfront | |
class AWSDeployTools | |
def initialize(config = {}) | |
@access_key_id = config['access_key_id'] | |
@secret_access_key = config['secret_access_key'] | |
# for privacy, allow user to store aws credentials in a shell env variable | |
@access_key_id ||= ENV['AWS_ACCESS_KEY_ID'] | |
@secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY'] | |
@bucket = config['bucket'] | |
@acl = config['acl'] | |
@cf_distribution_id = config['cf_distribution_id'] | |
@dirty_keys = Set.new # a set of keys that are dirty (have been pushed but not invalidated) | |
if @bucket.nil? | |
raise "ERROR: Must provide bucket name in constructor. (e.g. :bucket => 'bucket_name')" | |
end | |
if @cf_distribution_id.nil? | |
puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n Skipping cf invalidations..." | |
end | |
@s3 = AWS::S3.new(config) | |
end | |
# checks if a local file is in sync with the s3 bucket | |
# file can either be a file object or a string with a | |
# valid file path | |
def synced?(s3_key, file) | |
if file.is_a? String | |
file = File.open(file, 'r') | |
end | |
f_content = file.read | |
obj_etag = "" | |
begin | |
obj = @s3.buckets[@bucket].objects[s3_key] | |
obj_etag = obj.etag # the etag is the md5 of the remote file | |
rescue | |
return false | |
end | |
# the etag is surrounded by quotations. chomp removes them | |
obj_etag = obj_etag.gsub('"', '') | |
# compare the etag to the md5 hash of the local file | |
obj_etag == md5(f_content) | |
end | |
# pushes (writes) the file to the s3 bucket at location | |
# indicated by s3_key. | |
# file can either be a file object or a string with a | |
# valid file path | |
# options are any options that can be passed to the | |
# write method. | |
# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html | |
def push(s3_key, file, options = {}) | |
if file.is_a? String | |
file = File.open(file, 'r') | |
end | |
# detect content type | |
require 'mime/types' | |
file_path = file.path | |
# remove the .gz extension so the base extension will | |
# be used to determine content type. E.g. we want the | |
# type of index.html.gz to be text/html | |
if file_path.include? ".gz" | |
file_path.gsub!(".gz", "") | |
end | |
content_type = MIME::Types.type_for(File.extname(file_path)).first.to_s | |
content_type_hash = {:content_type => content_type} | |
options.merge! content_type_hash | |
puts "--> pushing #{file.path} to #{s3_key}...".green | |
obj = @s3.buckets[@bucket].objects[s3_key] | |
obj.write(file, options) | |
@dirty_keys << s3_key | |
# Special cases for index files. | |
# for /index.html we should also invalidate / | |
# for /archive/index.html we should also invalidate /archive/ | |
if (s3_key == "index.html") | |
@dirty_keys << "/" | |
elsif File.basename(s3_key) == "index.html" | |
@dirty_keys << s3_key.chomp(s3_key.split("/").last) | |
end | |
end | |
# batch pushes (writes) the files to the s3 bucket at locations | |
# indicated by s3_keys. (more than one file at a time) | |
# files can either be a file object or a string with a | |
# valid file path | |
# options are any options that can be passed to the | |
# write method. | |
# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html | |
def batch_push(s3_keys = [], files = [], options = {}) | |
if (s3_keys.size != files.size) | |
raise "ERROR: There must be a 1-to-1 correspondence of keys to files!" | |
end | |
files.each_with_index do |file, i| | |
s3_key = s3_keys[i] | |
push(s3_key, file, options) | |
end | |
end | |
# for each file, first checks if the file is synced. | |
# If not, it pushes (writes) the file. | |
# options are any options that can be passed to the | |
# write method. | |
# See http://docs.aws.amazon.com/AWSRubySDK/latest/frames.html | |
def sync(s3_keys = [], files = [], options = {}) | |
if (s3_keys.size != files.size) | |
raise "ERROR: There must be a 1-to-1 correspondence of keys to files!" | |
end | |
files.each_with_index do |file, i| | |
s3_key = s3_keys[i] | |
unless synced?(s3_key, file) | |
push(s3_key, file, options) | |
end | |
end | |
end | |
# a convenience method which simply returns the md5 hash of input | |
def md5 (input) | |
Digest::MD5.hexdigest(input) | |
end | |
# invalidates files (accepts an array of keys or a single key) | |
# based heavily on https://gist.github.com/601408 | |
def invalidate(s3_keys) | |
if @cf_distribution_id.nil? | |
puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..." | |
return | |
end | |
if s3_keys.nil? || s3_keys.empty? | |
puts "nothing to invalidate." | |
return | |
elsif s3_keys.is_a? String | |
puts "--> invalidating #{s3_keys}...".yellow | |
# special case for root | |
if s3_keys == '/' | |
paths = '<Path>/</Path>' | |
else | |
paths = '<Path>/' + s3_keys + '</Path>' | |
end | |
elsif s3_keys.length > 0 | |
puts "--> invalidating #{s3_keys.size} file(s)...".yellow | |
paths = '<Path>/' + s3_keys.join('</Path><Path>/') + '</Path>' | |
# special case for root | |
if s3_keys.include?('/') | |
paths.sub!('<Path>//</Path>', '<Path>/</Path>') | |
end | |
end | |
# digest calculation based on http://blog.confabulus.com/2011/05/13/cloudfront-invalidation-from-ruby/ | |
date = Time.now.strftime("%a, %d %b %Y %H:%M:%S %Z") | |
digest = Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new('sha1'), @secret_access_key, date)).strip | |
uri = URI.parse('https://cloudfront.amazonaws.com/2010-08-01/distribution/' + @cf_distribution_id + '/invalidation') | |
if paths != nil | |
req = Net::HTTP::Post.new(uri.path) | |
else | |
req = Net::HTTP::Get.new(uri.path) | |
end | |
req.initialize_http_header({ | |
'x-amz-date' => date, | |
'Content-Type' => 'text/xml', | |
'Authorization' => "AWS %s:%s" % [@access_key_id, digest] | |
}) | |
if paths != nil | |
req.body = "<InvalidationBatch>" + paths + "<CallerReference>ref_#{Time.now.utc.to_i}</CallerReference></InvalidationBatch>" | |
end | |
http = Net::HTTP.new(uri.host, uri.port) | |
http.use_ssl = true | |
http.verify_mode = OpenSSL::SSL::VERIFY_NONE | |
res = http.request(req) | |
if res.code == '201' | |
puts "Cloudfront Invalidation Success [201]. It may take a few minutes for the new files to propagate.".green | |
else | |
puts ("Cloudfront Invalidation Error: \n" + res.body).red | |
end | |
return res.code | |
end | |
# invalidates all the dirty keys and marks them as clean | |
def invalidate_dirty_keys | |
if @cf_distribution_id.nil? | |
puts "WARNING: cf_distribution_id is nil. (you can include it with :cf_distribution_id => 'id')\n--> skipping cf invalidations..." | |
return | |
end | |
res_code = invalidate(@dirty_keys.to_a) | |
# mark the keys as clean iff the invalidation request went through | |
@dirty_keys.clear if res_code == '201' | |
end | |
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# ... | |
desc "Deploy website to s3/cloudfront via aws-sdk" | |
task :s3_cloudfront => [:generate, :minify, :gzip, :compress_images] do | |
puts "==================================================" | |
puts " Deploying to Amazon S3 & CloudFront" | |
puts "==================================================" | |
# setup the aws_deploy_tools object | |
config = YAML::load( File.open("_config.yml")) | |
aws_deploy = AWSDeployTools.new(config) | |
# get all files in the public directory | |
all_files = Dir.glob("#{$public_dir}/**/*.*") | |
# we want the gzipped version of the files, not the regular (non-gzipped) version | |
# excluded files contains all the regular versions, which will not be deployed | |
excluded_files = [] | |
$gzip_exts.collect do |ext| | |
excluded_files += Dir.glob("#{$public_dir}/**/*.#{ext}") | |
end | |
# we do gzipped files seperately since they have different metadata (:content_encoding => gzip) | |
puts "--> syncing gzipped files...".yellow | |
gzipped_files = Dir.glob("#{$public_dir}/**/*.gz") | |
gzipped_keys = gzipped_files.collect {|f| (f.split("#{$public_dir}/")[1]).sub(".gz", "")} | |
aws_deploy.sync(gzipped_keys, gzipped_files, | |
:reduced_redundancy => true, | |
:cache_control => "max_age=86400", #24 hours | |
:content_encoding => 'gzip', | |
:acl => config['acl'] | |
) | |
puts "--> syncing all other files...".yellow | |
non_gzipped_files = all_files - gzipped_files - excluded_files | |
non_gzipped_keys = non_gzipped_files.collect {|f| f.split("#{$public_dir}/")[1]} | |
aws_deploy.sync(non_gzipped_keys, non_gzipped_files, | |
:reduced_redundancy => true, | |
:cache_control => "max_age=86400", #24 hours | |
:acl => config['acl'] | |
) | |
# invalidate all the files we just pushed | |
aws_deploy.invalidate_dirty_keys | |
puts "DONE." | |
end | |
desc "Compress all applicable content in public/ using gzip" | |
task :gzip do | |
unless which('gzip') | |
puts "WARNING: gzip is not installed on your system. Skipping gzip..." | |
return | |
end | |
@compressor ||= RedDragonfly.new | |
$gzip_exts.each do |ext| | |
puts "--> gzipping all #{ext}...".yellow | |
files = Dir.glob("#{$gzip_dir}/**/*.#{ext}") | |
files.each do |f| | |
@compressor.gzip(f) | |
end | |
end | |
puts "DONE." | |
end | |
desc "Minify all applicable files in public/ using jitify" | |
task :minify do | |
unless which('jitify') | |
puts "WARNING: jitify is not installed on your system. Skipping minification..." | |
return | |
end | |
@compressor ||= RedDragonfly.new | |
$minify_exts.each do |ext| | |
puts "--> minifying all #{ext}...".yellow | |
files = Dir.glob("#{$minify_dir}/**/*.#{ext}") | |
files.each do |f| | |
@compressor.minify(f) | |
end | |
end | |
puts "DONE." | |
end | |
desc "Compress all images in public/ using ImageMagick" | |
task :compress_images do | |
unless which('convert') | |
puts "WARNING: ImageMagick is not installed on your system. Skipping image compression..." | |
return | |
end | |
@compressor ||= RedDragonfly.new | |
$compress_img_exts.each do |ext| | |
puts "--> compressing all #{ext}...".yellow | |
files = Dir.glob("#{$compress_img_dir}/**/*.#{ext}") | |
files.each do |f| | |
@compressor.compress_img(f) | |
end | |
end | |
puts "DONE." | |
end | |
# ... | |
## | |
# invoke system which to check if a command is supported | |
# from http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby | |
# which('ruby') #=> /usr/bin/ruby | |
def which(cmd) | |
system("which #{ cmd} > /dev/null 2>&1") | |
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# A set of tools for minifying and compressing content | |
# | |
# Currently supports: | |
# - miniifying js, css, and html | |
# - gzipping any content | |
# - compressing and shrinking images | |
# | |
# Currently depends on 3 system command line tools: | |
# - ImageMagick (http://www.imagemagick.org/script/index.php) | |
# - gzip (http://www.gzip.org/) | |
# - jitify (http://www.jitify.com/) | |
# | |
# These may be swapped out for gem versions in the future, | |
# but for now you have to manually install command line tools | |
# above on your system if you don't already have them. | |
# | |
# Author: Alex Browne | |
class RedDragonfly | |
def initialize | |
# ~~ gzip ~~ | |
$gzip_options = { | |
:output_ext => "gz" | |
} | |
# ~~ minify (jitify) ~~ | |
# $minify_options = { | |
# | |
# } | |
# ~~ images (ImageMagick) ~~ | |
# exts : files extensions which sould be minified during batch operations | |
# output_ext : file extension of the output file ("" means keep the same extension) | |
# max_width : max width for compressed images | |
# max_height : max height for compressed images | |
# quality : image compression quality (1-100, higher is better quality/bigger files) | |
# compress_type : type of compression to be used (http://www.imagemagick.org/script/command-line-options.php#compress) | |
$img_options = { | |
:output_ext => "jpg", | |
:max_width => "600", | |
:max_height => "1200", | |
:quality => "65", | |
:compress_type => "JPEG" | |
} | |
end | |
# accepts a single file or an array of files | |
# accepts a file object or the path to a file (a string) | |
# perserves the original file | |
# the output is (e.g.) .html.gz | |
def gzip (files = []) | |
unless which('gzip') | |
puts "WARNING: gzip is not installed on your system. Skipping gzip..." | |
return | |
end | |
unless files.is_a? Array | |
files = [files] | |
end | |
files.each do |file| | |
fname = get_filename(file) | |
# invoke system gzip | |
system("gzip -cn9 #{fname} > #{fname + '.' + $gzip_options[:output_ext]}") | |
end | |
end | |
# accepts a single file or an array of files | |
# accepts a file object or the path to a file | |
# overwrites the original file with the minified version | |
# html, css, and js supported only | |
def minify (files = []) | |
unless which('jitify') | |
puts "WARNING: jitify is not installed on your system. Skipping minification..." | |
return | |
end | |
unless files.is_a? Array | |
files = [files] | |
end | |
files.each do |file| | |
fname = get_filename(file) | |
# invoke system jitify | |
system("jitify --minify #{fname} > #{fname + '.min'}") | |
# remove the .min extension | |
system("mv #{fname + '.min'} #{fname}") | |
end | |
end | |
# compresses an image file using the options | |
# specified at the top | |
# accepts either a single file or an array of files | |
# accepts either a file object or a path to a file | |
def compress_img (files = []) | |
unless which('convert') | |
puts "WARNING: ImageMagick is not installed on your system. Skipping image compression..." | |
return | |
end | |
unless files.is_a? Array | |
files = [files] | |
end | |
files.each do |file| | |
fname = get_filename(file) | |
compress_cmd = "convert #{fname} -resize #{$img_options[:max_width]}x#{$img_options[:max_height]}\\>" + | |
" -compress #{$img_options[:compress_type]} -quality #{$img_options[:quality]}" + | |
" #{get_raw_filename(fname) + '.' + $img_options[:output_ext]}" | |
# invoke system ImageMagick | |
system(compress_cmd) | |
# remove the old file (if applicable) | |
if (get_ext(fname) != ("." + $img_options[:output_ext])) | |
system("rm #{fname}") | |
end | |
end | |
end | |
# returns the filename (including path and ext) if the input is a file | |
# if the input is a string, returns the same string | |
def get_filename (file) | |
if file.is_a? File | |
file = file.path | |
end | |
return file | |
end | |
# returns the extension of a file | |
# accepts either a file object or a string path | |
def get_ext (file) | |
if file.is_a? String | |
return File.extname(file) | |
elsif file.is_a? File | |
return File.extname(file.path) | |
end | |
end | |
# returns the raw filename (minus extension) of a file | |
# accepts either a file object or a string path | |
def get_raw_filename (file) | |
# convert to string | |
file = get_filename(file) | |
# remove extension | |
file.sub(get_ext(file), "") | |
end | |
## | |
# invoke system which to check if a command is supported | |
# from http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby | |
# which('ruby') #=> /usr/bin/ruby | |
def which(cmd) | |
system("which #{ cmd} > /dev/null 2>&1") | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I think there could be an issue with
def compress_img (files = [])
in that $img_options[:output_ext] is being forced to jpg by
$img_options = {
:output_ext => "jpg",
:max_width => "600",
:max_height => "1200",
:quality => "65",
:compress_type => "JPEG"
}
This means png files are being converted to jpg and deleted.