-
-
Save ahoward/f1b7b390fe3e16dd399ae76515272df4 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env ruby | |
# file: ./script/build | |
WORK_IN_PROGRESS_NOTES = | |
<<-____ | |
this script uses rails5/puma + wget to make static build of a rails5 app/site | |
the general idea is | |
- start up puma on an open port | |
- ensure the rails app delivers urls with a trailing slash and will serve | |
assets. this is required to make a clean build. a sample | |
./config/initializers/build.rb is inlined at the end of this script | |
- wget mirror the damn thing | |
- patch up some cruft with ruby | |
shockingly, this works really, really, really well/fast and dojo4 has been | |
using to make rails+static sites for about 3 years. it has just gotten so | |
much easier with rails5/puma that i thought i'd share | |
to make a build just run | |
~> ./script/build | |
and you'll have a preview-able static cache of all reachable pages in | |
public/builds/$uuid | |
check | |
public/builds/$uuid/wget.oe | |
if the build fails for any reasons. examine the detailed wget output for | |
'404' or '500', etc. | |
the build will be suitable for deployment to s3, bitballoon, etc. and will | |
include all images, compiled assets, etc. | |
i like to preview these builds with node-static via | |
~> static ./public/system/builds/$uuid | |
____ | |
# make a build | |
@build = Build.new | |
@build.build! | |
puts @build.directory | |
puts @build.current | |
BEGIN { | |
# | |
require 'fileutils' | |
require 'pathname' | |
require 'thread' | |
require 'socket' | |
require 'timeout' | |
require 'uri' | |
require 'open-uri' | |
require 'securerandom' | |
require 'rubygems' | |
require 'logger' | |
require 'getoptlong' | |
# awesome sauce static build support | |
# | |
class Build | |
# | |
def build! | |
log(:debug, "building #{ @directory }...") | |
log(:debug, "locking...") | |
lock! | |
log(:debug, "locked.") | |
start_puma! unless @url | |
wget_mirror! | |
normalize! | |
current! | |
end | |
# | |
attr_accessor :script_dir | |
attr_accessor :rails_root | |
attr_accessor :lib_dir | |
attr_accessor :directory | |
attr_accessor :current | |
attr_accessor :uuid | |
attr_accessor :env | |
attr_accessor :puma | |
attr_accessor :url | |
# | |
OPTS = GetoptLong.new( | |
[ '--help', '-h', GetoptLong::NO_ARGUMENT ], | |
[ '--url', '-u', GetoptLong::REQUIRED_ARGUMENT ], | |
[ '--base', GetoptLong::REQUIRED_ARGUMENT ], | |
[ '--urls', GetoptLong::REQUIRED_ARGUMENT ], | |
#[ '--repeat', '-n', GetoptLong::REQUIRED_ARGUMENT ], | |
#[ '--name', GetoptLong::OPTIONAL_ARGUMENT ] | |
) | |
# | |
USAGE = <<-__ | |
# TODO | |
__ | |
# | |
def initialize(*args, &block) | |
@opts = parse_opts! | |
if @opts[:help] | |
usage! | |
exit!(42) | |
end | |
@options = args.last.is_a?(Hash) ? args.pop : {} | |
@logger = @options[:logger] || Logger.new(STDERR) | |
@script_dir = File.expand_path(File.dirname(__FILE__)) | |
@rails_root = File.expand_path(File.dirname(@script_dir)) | |
@lib_dir = File.join(@rails_root, 'lib') | |
@uuid = ENV['RAILS_BUILD'] || SecureRandom.uuid | |
@url = ENV['RAILS_BUILD_SERVER'] || ENV['RAILS_URL'] | |
@env = ENV['RAILS_BUILD_ENV'] || ENV['RAILS_ENV'] || 'development' | |
@puma = 'bundle exec puma' | |
@pumactl = 'bundle exec pumactl' | |
@extra_mirror_urls = extra_mirror_urls | |
Dir.chdir(@rails_root) | |
if File.exists?('./Gemfile') | |
require 'bundler/setup' | |
Bundler.setup(:require => false) | |
end | |
$LOAD_PATH.unshift(@lib_dir) | |
if ENV['RAILS_BUILD_DIRECTORY'] | |
@build_directory = File.expand_path(ENV['RAILS_BUILD_DIRECTORY']) | |
else | |
@build_directory = File.join(@rails_root, 'public', 'system', 'builds') | |
end | |
FileUtils.mkdir_p(@build_directory) | |
@directory = File.join(@build_directory, @uuid) | |
ENV['RAILS_BUILD'] ||= @uuid | |
ENV['RAILS_BUILD_ENV'] ||= @env | |
end | |
# | |
def parse_opts! | |
@opts = Hash.new | |
OPTS.each do |opt, arg| | |
case opt | |
when '--help' | |
@opts[:help] = true | |
when '--base' | |
@opts[:base] = arg | |
when '--url' | |
(@opts[:url] ||= []).push(*array_of_strings(arg)) | |
when '--urls' | |
@opts[:urls] = arg | |
end | |
end | |
@opts | |
end | |
# | |
def usage! | |
lines = USAGE.strip.split(/\n/) | |
n = lines[1].to_s.scan(/^\s+/).size | |
indent = ' ' * n | |
re = /^#{ Regexp.escape(indent) }/ | |
usage = lines.map{|line| line.gsub(re, '')}.join("\n") | |
STDERR.puts(usage) | |
end | |
# | |
def lock!(max = 300) | |
started_at = Time.now.to_f | |
loop do | |
if DATA.flock(File::LOCK_EX | File::LOCK_NB) | |
break | |
else | |
warn "could not obtain lock for #{ max } seconds..." | |
if((Time.now.to_f - started_at) > max) | |
warn "ignoring failed lock!" | |
break | |
else | |
sleep(rand) | |
end | |
end | |
end | |
end | |
=begin | |
puma <options> <rackup file> | |
-b, --bind URI URI to bind to (tcp://, unix://, ssl://) | |
-C, --config PATH Load PATH as a config file | |
--control URL The bind url to use for the control server | |
Use 'auto' to use temp unix server | |
--control-token TOKEN The token to use as authentication for the control server | |
-d, --daemon Daemonize the server into the background | |
--debug Log lowlevel debugging information | |
--dir DIR Change to DIR before starting | |
-e, --environment ENVIRONMENT The environment to run the Rack app on (default development) | |
-I, --include PATH Specify $LOAD_PATH directories | |
-p, --port PORT Define the TCP port to bind to | |
Use -b for more advanced options | |
--pidfile PATH Use PATH as a pidfile | |
--preload Preload the app. Cluster mode only | |
--prune-bundler Prune out the bundler env if possible | |
-q, --quiet Do not log requests internally (default true) | |
-v, --log-requests Log requests as they occur | |
-R, --restart-cmd CMD The puma command to run during a hot restart | |
Default: inferred | |
-S, --state PATH Where to store the state details | |
-t, --threads INT min:max threads to use (default 0:16) | |
--tcp-mode Run the app in raw TCP mode instead of HTTP mode | |
-V, --version Print the version information | |
-w, --workers COUNT Activate cluster mode: How many worker processes to create | |
--tag NAME Additional text to display in process listing | |
--redirect-stdout FILE Redirect STDOUT to a specific file | |
--redirect-stderr FILE Redirect STDERR to a specific file | |
--[no-]redirect-append Append to redirected files | |
-h, --help Show help | |
=end | |
# | |
def start_puma! | |
@url = | |
nil | |
puma_port = | |
nil | |
ports = | |
(3001 .. 4001).to_a | |
ports.each do |port| | |
next unless port_open?(port) | |
@port = port | |
@pidfile = './tmp/puma.pid' | |
start_puma = | |
#"#{ @puma } start --daemonize --environment #{ @env } --port #{ port } --max-pool-size 16 --min-instances 16" | |
"puma --port=#{ @port } --environment=#{ @env } --pidfile=#{ @pidfile } --daemon --threads=16:16 --workers=2 --preload config.ru" | |
log(:info, "cmd: #{ start_puma }") | |
puma_output = | |
`#{ start_puma } 2>&1`.strip | |
log(:info, "status: #{ $?.exitstatus }") | |
t = Time.now.to_f | |
timeout = 10 | |
i = 0 | |
loop do | |
i += 1 | |
begin | |
url = "http://0.0.0.0:#{ port }" | |
open(url){|socket| socket.read} | |
@url = url | |
puma_port = port | |
break | |
rescue Object => e | |
if i > 2 | |
log :error, "#{ e.message }(#{ e.class })\n" | |
log :error, "#{ puma_output }\n\n" | |
end | |
if((Time.now.to_f - t) > timeout) | |
abort("could not start puma inside of #{ timeout } seconds ;-/") | |
else | |
sleep(rand(0.42)) | |
end | |
end | |
end | |
break if @url | |
end | |
# barf if puma could not be started | |
# | |
unless @url | |
abort("could not start puma on any of ports #{ ports.first } .. #{ ports.last }") | |
end | |
log(:info, "started puma on #{ @url }") | |
# set assassins to ensure the puma daemon never outlives the build script | |
# no matter how it is killed (even -9) | |
# | |
puma_pid = IO.binread(@pidfile).strip | |
stop_puma = | |
"#{ @pumactl } stop --pidfile=#{ @pidfile }" | |
at_exit{ | |
log(:info, "cmd: #{ stop_puma }") | |
`#{ stop_puma } >/dev/null 2>&1` | |
log(:info, "status: #{ $?.exitstatus }") | |
log(:info, "stopped puma on #{ @url }") | |
} | |
pppid = Process.pid | |
unless fork | |
at_exit{ exit! } | |
Process.setsid | |
if fork | |
exit | |
else | |
loop do | |
begin | |
Process.kill(0, pppid) | |
rescue Object => e | |
if e.is_a?(Errno::ESRCH) | |
`#{ stop_puma } >/dev/null 2>&1` | |
end | |
exit | |
end | |
sleep(1 + rand) | |
end | |
end | |
end | |
end | |
# | |
def port_open?(port, options = {}) | |
seconds = options[:timeout] || 1 | |
ip = options[:ip] || '0.0.0.0' | |
Timeout::timeout(seconds) do | |
begin | |
TCPSocket.new(ip, port).close | |
false | |
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH | |
true | |
rescue Object | |
false | |
end | |
end | |
rescue Timeout::Error | |
false | |
end | |
# | |
def mkdir! | |
FileUtils.rm_rf(@directory) | |
if @opts[:base] | |
log(:info, "base=#{ directory }") | |
directory = File.expand_path(@opts[:base]) | |
FileUtils.cp_r(directory, @directory) | |
else | |
FileUtils.mkdir_p(@directory) | |
end | |
end | |
# this is *tough* to get right but it 'just works' - TM | |
# | |
def wget_mirror! | |
mkdir! | |
Dir.chdir(@directory) do | |
log(:info, "cwd: #{ @directory }") | |
mirrored = [] | |
oe = File.join(@directory, 'wget.log') | |
a = Time.now | |
mirror_urls.each do |url| | |
#--convert-links | |
wget = %W[ | |
wget | |
--mirror | |
--recursive | |
--adjust-extension | |
--no-host-directories | |
--backup-converted | |
--page-requisites | |
--level=4242 | |
--trust-server-names | |
--directory-prefix=. | |
--header 'X-Rails-Build: #{ ENV['RAILS_BUILD'] }' | |
--execute robots=off | |
#{ url.inspect } | |
>> #{ File.basename(oe).inspect } 2>&1 | |
].map(&:strip).join(' ') | |
log(:info, "cmd: #{ wget }") | |
status = system(wget) | |
log(:info, "status: #{ $?.exitstatus }") | |
mirrored.push(status) | |
end | |
b = Time.now | |
if mirrored.size > 0 && mirrored.all? | |
log(:info, "built site in #{ (b.to_f - a.to_f) }s") | |
else | |
log(:error, "failed to build site!") | |
log(:error, "see log at #{ oe }") | |
open(oe){|f| f.each_line{|line| STDERR.puts(line)}} | |
exit(1) | |
end | |
end | |
end | |
# urls to mirror, essentially / plus any extra stuff | |
# | |
def mirror_urls | |
urls = [@url] | |
@extra_mirror_urls.each do |url| | |
u = URI.parse(url) | |
urls.push( | |
if u.absolute? | |
u.to_s | |
else | |
URI.parse(File.join(@url, url)).to_s | |
end | |
) | |
end | |
urls | |
end | |
# for urls that are not discoverable via crawling, pass 'em via the option | |
# and/or a file full of urls | |
# | |
def extra_mirror_urls | |
urls = [] | |
array_of_strings(@opts[:urls]).each do |arg| | |
is_file = arg !~ %r{://} | |
if is_file | |
lines = (arg == '-' ? STDIN.read : IO.binread(arg)).split(/\n/) | |
lines.each do |line| | |
line.strip! | |
next if line.empty? | |
urls.push(line) | |
end | |
else | |
urls.push(arg) | |
end | |
end | |
array_of_strings(@opts[:url]).each do |arg| | |
urls.push(arg) | |
end | |
urls | |
end | |
def array_of_strings(arg) | |
Array(arg).join(',').strip.split(/\s*,\s*/) | |
end | |
# clean up wget shiznit | |
# | |
def normalize! | |
glob = File.join(@directory, '**/**') | |
Dir.glob(glob) do |entry| | |
next unless test(?f, entry) | |
dirname = File.dirname(entry) | |
basename = File.basename(entry) | |
base, query = basename.split('?', 2) | |
if base =~ /\.orig$/ | |
FileUtils.rm(entry) | |
next | |
end | |
if query | |
path = File.join(dirname, base) | |
FileUtils.cp(entry, path) | |
FileUtils.rm(entry) | |
next | |
end | |
end | |
# ensure page.html -> page/index.html cuz static | |
# | |
Dir.glob(glob) do |entry| | |
next unless test(?f, entry) | |
dirname = File.dirname(entry) | |
basename = File.basename(entry) | |
base = basename.split('.', 2).first | |
next if basename =~ /^index.html?$/i | |
if entry =~ /\.html?$/i | |
dir = File.join(dirname, base) | |
src = entry | |
dst = File.join(dir, "index.html") | |
if test(?e, src) && test(?e, dst) | |
same = FileUtils.compare_file(src, dst) | |
unless same | |
raise "#{ src } conflicts with #{ dst }" | |
end | |
end | |
FileUtils.mkdir_p(dir) | |
FileUtils.mv(src, dst) | |
end | |
end | |
end | |
# | |
def current! | |
Dir.chdir(@build_directory) do | |
@current = File.expand_path('current') | |
FileUtils.cp_r(@uuid, 'current') | |
end | |
end | |
# | |
def to_s | |
@directory.to_s | |
end | |
# | |
def log(level, *args, &block) | |
@logger.send(level, *args, &block) | |
end | |
end | |
} | |
__END__ | |
# file ./config/initializers/build.rb | |
if ENV['RAILS_BUILD'] | |
# | |
Rails.configuration.before_initialize do | |
ActionController::Base.module_eval do | |
before_filter :enforce_trailing_slash_when_building | |
protected | |
def enforce_trailing_slash_when_building | |
if request.get? and request.headers['X-Rails-Build'] | |
format = request.fullpath.split('.', 2)[1] | |
if format.nil? and request.format.to_s == 'text/html' | |
url = request.original_url | |
url, query_string = url.split('?') | |
if !url.ends_with?('/') | |
flash.keep | |
url = url + '/' | |
if query_string | |
url = url + '?' + query_string | |
end | |
redirect_to(url) | |
return | |
end | |
end | |
end | |
end | |
end | |
end | |
# | |
Rails.configuration.after_initialize do |app| | |
config = Rails.configuration | |
config.serve_static_assets = true | |
config.assets.compile = true | |
config.assets.compress = true | |
config.assets.digest = true | |
config.assets.initialize_on_precompile = false | |
config.assets.debug = false | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment