Skip to content

Instantly share code, notes, and snippets.

@briandominick
Last active February 8, 2020 01:10
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save briandominick/1e9bc1c4d2c13b642e9314a4f662ab12 to your computer and use it in GitHub Desktop.
An environment- and version-aware command-line wrapper for Jekyll builds

Jekyll Build Scripts

Just a quick release of files written when Codewriting, LLC enhanced the scripts used to build Pepperdata’s Jekyll-based docs site. More documentation may be added after release approval, but for now this code is as-is.

License

Released jointly under the MIT Licence.

#!/usr/bin/env bash
set -e
# Enter sh jklbld.sh --help for full usage options
# Option-parser helpers
function arg_type {
# Every argument must be patterned as an option (flag), a value (string), or empty string
local re_flag="^\-{1,2}[a-z]([a-z\-]+)?$"
if [ "$1" == "" ] ; then
echo "none"
else
if [[ $1 =~ $re_flag ]] ; then
echo "option"
else
echo "value"
fi
fi
}
# True if the given argument is a value rather than an option flag or nil
function is_value {
if [[ $(arg_type $1) == "value" ]] ; then return 0 ; else return 1 ; fi
}
# Parses key/value pairs carried inside a CLI argument
function smuggled_args_parser {
# Accepts a string argument like
# --jekyll 'verbose layouts=_layouts/alternate skip-initial-build incremental'
# Which translates to
# --verbose --layouts _layouts/alternate --skip-initial-build --incremental
args_string=""
args=($1) # Makes an array split on spaces
for arg in $args ; do
IN=$arg
kv=(${IN//=/ })
args_string="$args_string --${kv[0]}"
if [ -n $kv[1] ] ; then
args_string="$args_string ${kv[1]}"
fi
done
echo $args_string
}
# Graceful Ctrl-C
function ctrl_c {
echo "** Process manually stopped with Ctrl+C"
for i in `seq 1 1`; do
sleep 1
echo -n "."
done
}
trap ctrl_c INT
# Default settings
# Note: dev is the default environment, for safety and convenience, but some
# global defaults assume prod and will be overridden by dev mode
env="dev"
action="build" # serve by default for dev
source="src"
bundler_log="--quiet"
search=false # true by default for prod
serve=false # true by default for dev
precheck=false
tests=false # true by default for prod
fail_response="ignore" # for test failures "exit" by default for all others
jekyll_args=""
search_dry=false
verbose=false
reviewnotes=true
serve_after=false
vardump=false
clean_build=true
# Evaluate optional environment arg and port arg up front
re_port_num="^[0-9]{4,5}"
re_version="[4-9]\.[0-9]{1,2}"
while $(is_value $1) ; do
if [[ "$1" =~ ^(dev|stage|prod|search)$ ]] ; then
env=$1
shift
elif [[ "$1" =~ $re_port_num ]] && [ "$1" -gt 1023 -a "$1" -lt 65535 ] ; then
port=$1
shift
elif [ "$1" == "clean" ] ; then
clean_build=true
shift
elif [[ "$1" =~ $re_version(\-$re_version)? ]] ; then
IFS='-' read -r -a vsns <<< "$1"
vsn_start=${vsns[0]}
if [ -n "${vsns[1]}" ] ; then
vsn_end=${vsns[1]}
else
vsn_end=${vsns[0]}
fi
shift
else
echo "Argument $1 not recognized. Aborting."
exit 1
fi
done
# Environment-specific settings
case "$env" in
dev|development)
env_name="development"
clean_build=false
serve=true
config_env="dev"
exclude_env="dev"
dest_dir="_site"
;;
stage|staging|review)
env_name="review"
precheck=true
serve=false
tests=false
fail_response="exit"
search=false
config_env="prod"
exclude_env="dev"
dest_dir="target/www"
;;
prod|production)
env_name="production"
precheck=true
search=dry
tests=true
fail_response="exit"
config_env="prod"
dest_dir="target/www"
reviewnotes=false
exclude_env="prod"
;;
search)
env_name="search"
precheck=false
search=true
tests=false
fail_response="exit"
config_env="prod"
dest_dir="target/www"
reviewnotes=false
exclude_env="prod"
;;
esac
config_arg="--config jekyll/configs/jekyll-global.yml,jekyll/configs/jekyll-$config_env.yml,jekyll/configs/excludes-$exclude_env.yml"
# Versions to build
# If not determined in command arguments
# For establishing environment default ranges
vsn_first=$(ruby src/ruby/version_handler.rb get first id)
vsn_last=$(ruby src/ruby/version_handler.rb get last id)
vsn_2ndto=$(ruby src/ruby/version_handler.rb get 2ndto id)
# Set conditional output vars
if [ $clean_build ] ; then unclean="clean" ; else unclean="unclean" ; fi
if [ -z "$vsn_start" ] ; then
if [ "$env" == "dev" ] ; then
vsn_start=$vsn_2ndto
vsn_end=$vsn_last
else
vsn_start=$vsn_first
vsn_end=$vsn_last
fi
fi
# Generate version-specific configs and return array of versions to build
vsns=($(ruby src/ruby/version_handler.rb gen search $vsn_start-$vsn_end))
# Help menu
if [ "$1" == "--help" ] || [ "$1" == "-h" ] ; then
printf "\n
Usage: $0 [env] [start.version[-end.version]] [clean] [port] [options]
env: Optional build environment, to establish defaults. Must be dev (default), stage, search, or prod.
versioning: Version range can be indicated as 5.5-5.7 or just 5.7.
Note: Versioning only affects which versions are indexed for search, not which are built.
port: Serve port, for dev builds or stage/prod builds with --build-after. (Default: 4000)
clean: Forcibly removes target directory
Options:\n
--build-only | --no-serve | -n Disables serve operation enabled by default in dev mode.
--destination | -d PATH Changes the target directory of the build.
--jekyll 'STRING' | -j 'STRING' Quoted, specially formatted string of option flags and args.
Ex: --jekyll 'incremental layouts=_layouts/new'
--precheck | -k Runs pre-build evaluations.
--search [dry*|true|false] Toggles Algolia search build; dry (*impled when --search is used) performs without push.
--search-key '<KEY>' Where <KEY> is the actual admin API key for Algolia account.
--serve-after Adds a serve operation at the end of the build, even for production operations
--source PATH | -s PATH Path to the content source root (what Jekyll reads).
--tests [exit*|warn|false] Perform tests; abort build (*default for --tests) or warn only on fail, or false to prevent tests.
--verbose Run script in debug mode. (For Jekyll debugging, use: --jekyll 'verbose')
--review-notes-x | -x Fails if review notes are detected. (Default for prod builds.)\n\n\n\n"
exit 0
fi
# Start reporting to the user
echo "Performing $unclean $env_name build."
# Parse options/arguments
while [ "$1" != "" ] ; do
case $1 in
-t|--tests|--test)
tests=true ; fail_response="exit"
shift
if $(is_value $1) ; then
if [[ "$1" =~ ^(ignore|exit|false)$ ]] ; then
if [ "$1" == "false" ] ; then
tests=false
echo "Tests disabled."
else
fail_response=$1
fi
shift
else
echo "Value of --tests unrecognized; must be exit, ignore, or false. Defaulting to exit."
fail_response="exit"
fi
fi
;;
--build-only|--no-serve)
serve=false
shift
;;
--destination|--dest|-d)
shift
if $(is_value $1) ; then
dest_dir=$1
shift
else
echo "--destination/-d flag requires a proper path argument"
fi
;;
--jekyll|-j)
shift
if $(is_value $1) ; then
jekyll_args="$jekyll_args $(smuggled_args_parser $1)"
shift
else
echo "Jekyll arguments missing from --jekyll option. Skipping."
fi
;;
--precheck|-k)
precheck=true
shift
;;
--search)
search=true
shift
if $(is_value $1) ; then
if [ "$1" == true ] || [ "$1" == false ] || [ "$1" == "dry" ] ; then
if [ "$1" == "dry" ] ; then
search_dry=true
else
search=$1
fi
shift
else
search=false
echo "Value of --search unrecognized; must be true, false, or dry (default). Defaulting to false."
fi
else # --search defaults to dry
search_dry=true
fi
;;
--search-key)
shift
if $(is_value $1) ; then
search_key=$1
shift
else
echo "Value of --search-key missing. Skipping search."
search=false ; serve=false
fi
;;
--serve-after)
serve_after=true
serve=false
shift
;;
--source|-s)
shift
if $(is_value $1) ; then
source=$1
shift
else
echo "Option --source (-s) must be followed by a path."
fi
;;
--review-notes-x|-x)
reviewnotes=false
shift
;;
--vardump)
vardump=true
shift
;;
--verbose)
verbose=true
bundler_log=""
shift
;;
-?*)
echo "WARN: Unknown option (ignored): $1"
shift
continue
;;
*)
break
;;
esac
done
if $clean_build ; then
rm -rf $dest_dir
if $verbose ; then
echo "Clean build: clearing $dest_dir"
fi
fi
if $tests ; then
echo "Building for tests."
fi
# Assemble more Jekyll arguments based on options
jekyll_args="$jekyll_args --destination $dest_dir"
jekyll_args="$jekyll_args --source $source"
# Search preparation
if [ $search != false ] ; then
if $search_dry ; then
algolia_api_key='DummyKeyForDryIndexing'
key_from="(Dummy key used.)"
else
if [ -n "$search_key" ] ; then
algolia_api_key=$search_key
key_from="(Key provided on command line.)"
else
if [ -z "${ALGOLIA_API_KEY}" ] ; then
key_from="(Key found in environment variable.)"
else
echo "Search build aborted: Algolia API key missing; either add as --search-key argument or store as \${ALGOLIA_API_KEY} environment variable."
exit 1
fi
fi
fi
if [ "$env" != "prod" ] && ! $search_dry ; then
while true; do
read -p "Are you sure you want to push to Algolia index from a dev build?" yn
case $yn in
[Yy]* ) search=true ; break ;;
[Nn]* ) exit ;;
* ) echo "Please answer yes or no." ;;
esac
done
fi
echo "Building with search index. $key_from"
action="build"
serve=false
fi
# Pre-build prep
if [ "$env" == "dev" ] ; then
if $tests; then
action="build"
serve=false
serve_after=true
fi
fi
# If we haven't suppressed $serve yet...
if $serve ; then
action="serve"
if [ $port ] ; then jekyll_args="$jekyll_args --port $port" ; fi
fi
# Pre-build tests
if $precheck ; then
# precheck ops go here
echo "No prechecks established"
fi
# Fail if content includes review comment includes
if ! $reviewnotes ; then
echo "checking for doc-issue tags"
if [[ -n "$(find $source -type f | xargs grep 'doc-issue.html')" ]]
then
echo "ERROR: doc-issue tag(s) found."
if [ "$fail_response" == "exit" ] ; then
echo "Build command canceled."
exit 1
fi
fi
fi
if $verbose; then
echo "Build vars:"
echo " env:" $env
echo " env_name:" $env_name
echo " action:" $action
echo " config_env:" $config_env
echo " config_arg:" $config_arg
echo " jekyll_args:" $jekyll_args
echo " precheck:" $precheck
echo " reviewnotes:" $reviewnotes
echo " search:" $search
echo " search_dry:" $search_dry
echo " serve:" $serve
echo " serve_after:" $serve_after
echo " tests:" $tests
echo " fail_response:" $fail_response
fi
# Jekyll build command
jekyll_command="bundle exec jekyll $action $config_arg,jekyll/configs/version-base.yml $jekyll_args"
bundle install $bundler_log # ensures dependency gem versions match Gemfile.lock
echo "Running command '$jekyll_command'"
$jekyll_command
if $tests ; then
htmlproofer $dest_dir 2>&1 | tee target/htmlproofer-output.txt
# need to check the output because htmlproofer doesn't return the expected
# nonzero status code on failure
if grep -q "Error: HTML-Proofer found" "target/htmlproofer-output.txt"; then
echo "HTML-Proofer found errors"
if [ "$fail_response" == "exit" ] ; then exit 1 ; fi
fi
fi
if [ "$search" != false ] ; then
if $search_dry ; then jekyll_args="$jekyll_args --dry" ; fi
for vsn in ${vsns[@]} ; do
config_arg_vsn="$config_arg,target/configs/$vsn.yml"
algolia_command="bundle exec jekyll algolia $config_arg_vsn $jekyll_args --destination $dest_dir"
echo "Running search build and index ($vsn)"
ALGOLIA_API_KEY=$algolia_api_key $algolia_command
if [ "$?" -ne 0 ] ; then
echo "Search processing failed"
if [ "$fail_response" == "exit" ] ; then
echo "Canceling search operations."
exit 1
fi
fi
done
fi
if $serve_after ; then
if [ $port ] ; then port_flag="--port $port" ; fi
serve_command="bundle exec jekyll serve $config_arg $port_flag -d $dest_dir --skip-initial-build"
echo "Attempting post-build serve"
echo "Running command: $serve_command"
$serve_command
fi
echo "✔ Script finished."
exit 0
Copyright 2019 Pepperdata, Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# version_handler.rb
# Usage:
# ruby version_handler.rb --help
# This script is executed by `jklbld.sh` to generate version-specific
# Jekyll configuration files based on the docs versions being built and
# indexed by Algolia at that time, and also to return the range of supported
# versions.
#
# The 'gen' action writes the config files to target/configs/ and returns
# a shell-formatted array of version numbers (e.g., 5.5 5.6 5.7).
#
# The 'get' action reads a specified object from the version file and returns
# either the slug or the version ID.
require 'fileutils'
require 'yaml'
@args = ARGV
@base_dir = Dir.pwd
@range = {}
@vsns_file_def = "jekyll/_data/versions.yml"
@vsns_file = @vsns_file_def
@type = "id"
def load_versions_file file
begin
@all_versions = YAML.load_file(file)
rescue
raise "Could not read versions YAML file. Cannot proceed."
end
end
begin
@base_config = YAML.load_file("#{@base_dir}/jekyll/configs/version-base.yml")
rescue
raise "Could not read versions YAML file. Cannot proceed."
end
# Proc for deriving a specific version's data object
def get_version v
# Accepts a position (e.g., -1, 0, 9), returns version object at that position
# Otherwise accepts a float (e.g., 5.7) and returns the version object with that ID
# Otherwise accepts a slug (e.g., v5-7) and returns the version object with that slug
# Returns hash of selected version
if v.include?"."
vsn = @all_versions.select {|vsn| vsn['id'] == v }
return vsn[0]
else
case v
when "first"
v = 0
when "2ndto"
v = -2
when "last"
v = -1
else
raise "UnrecognizedVersionArg"
end
vsn = @all_versions[v]
return vsn
end
end
def build_version_configs startv, endv, target_path="target/configs"
# Builds config files for the range (startv -> endv)
FileUtils::mkdir_p(target_path) unless File.exists?(target_path)
out = []
@all_versions.each do |vsn|
if vsn['float'] >= startv.to_f && vsn['float'] <= endv.to_f
build_config(vsn['id'], target_path)
out << vsn['slug']
end
end
versions = out.join(" ")
versions
end
def build_config v, path
# Builds a single config file for version v
vsn = get_version(v)
config = {}
config.merge!@base_config
config['algolia']['index_name'] = vsn['slug']
config['algolia']['files_to_exclude'] = other_versions(v, true, true)
File.open("#{@base_dir}/#{path}/#{vsn['slug']}.yml", 'w') { |file| file.write(config.to_yaml) }
end
def other_versions v, underscore=false, wildcards=false
# returns an array of version IDs other than passed v
u=""
w=""
oth = []
u = "_" if underscore
w = "/**/**" if wildcards
@all_versions.each do |vsn|
oth << u + vsn['slug'] + w if vsn['id'] != v
end
return oth
end
def show_help action=nil
gen_desc = "Generates configuration files of <kind>: 'search' or 'build', which generate their respective kind of version-specific config files. Returns a Bash-formatted 'array' of versions generated."
get_desc = "Returns the proper version ID or slug for the version at this position or ID in versions.yml"
if action == "gen"
print "
#{gen_desc}
Usage: #{$0} gen [<version1>[-<version2>]] [path/to/yaml]
Ex: #{$0} gen search 5.7 jekyll/_data/versions.yml
Ex: #{$0} gen search 5.5-5.7
Ex: #{$0} gen search"
elsif action == "get"
print "
#{get_desc}
Usage: #{$0} get [first|last|2ndto|<version>] [slug|id] [path/to/yaml]
Ex: #{$0} get 5.7 jekyll/_data/versions.yml
Ex: #{$0} get 2ndto jekyll/_data/versions.yml
Ex: #{$0} get last"
else
print "
Usage: #{$0} [get|gen] [version options] [path/to/yaml]
get: #{get_desc}
gen: #{gen_desc}
Use #{$0} [get|gen] --help to display version options.
"
end
puts "\n\n"
exit 0
end
# Begin command-line parsing
while @args[0] do
arg = @args[0]
case arg
when "get"
@action = "get" ; @args.shift
when "gen"
@action = "gen" ; @gen_kind = "search" ; @args.shift
when "--help"
show_help(@action)
else
raise "MissingAction" unless @action
if arg.match(/^[4-9]\.[0-9]{1,2}$|first|last|2ndto/)
@version = arg
@range['start'] = arg
@range['end'] = arg
@args.shift
elsif arg.match(/^[4-9]\.[0-9]{1,2}\-[4-9]\.[0-9]{1,2}$/)
raise "WrongAction" unless @action == "gen"
range = arg.split("-")
@range['start'] = range[0]
@range['end'] = range[1]
@args.shift
elsif arg.match(/^(.*)\/.*\.yml$/)
@vsns_file = arg ; @args.shift
elsif arg.match(/build|search/) # NOTE build gen not yet implemented
@gen_kind = arg ; @args.shift
elsif arg.match(/slug|id/)
@type = arg ; @args.shift
else
raise "Argument #{arg} not recognized"
end
end
end
load_versions_file(@vsns_file)
if @action == "get"
raise "MissingVersion" unless @version
vsn = get_version(@version)
puts vsn[@type]
end
if @action == "gen"
unless (@range['start'] && @range['end'])
@range['start'] = @all_versions[0]['id']
@range['end'] = @all_versions[-1]['id']
end
vsns = build_version_configs(@range['start'], @range['end'])
puts vsns
end
@charlespepper
Copy link

Pepperdata, Inc approves this matter for release as-is, without warranty, under The MIT License.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment