An environment- and version-aware command-line wrapper for Jekyll builds

README.adoc

Jekyll Build Scripts

Just a quick release of files written when Codewriting, LLC enhanced the scripts used to build Pepperdata’s Jekyll-based docs site. More documentation may be added after release approval, but for now this code is as-is.

License

Released jointly under the MIT Licence.

Jekyll Build Scripts

# See README.adoc

jklbld.sh

#!/usr/bin/env bash
set -e

# Enter sh jklbld.sh --help for full usage options

# Option-parser helpers
function arg_type {
  # Every argument must be patterned as an option (flag), a value (string), or empty string
  local re_flag="^\-{1,2}[a-z]([a-z\-]+)?$"
  if [ "$1" == "" ] ; then
    echo "none"
  else
    if [[ $1 =~ $re_flag ]] ; then
      echo "option"
    else
      echo "value"
    fi
  fi
}

# True if the given argument is a value rather than an option flag or nil
function is_value {
  if [[ $(arg_type $1) == "value" ]] ; then return 0 ; else return 1 ; fi
}

# Parses key/value pairs carried inside a CLI argument
function smuggled_args_parser {
  # Accepts a string argument like
  #  --jekyll 'verbose layouts=_layouts/alternate skip-initial-build incremental'
  # Which translates to
  #  --verbose --layouts _layouts/alternate --skip-initial-build --incremental
  args_string=""
  args=($1) # Makes an array split on spaces
  for arg in $args ; do
    IN=$arg
    kv=(${IN//=/ })
    args_string="$args_string --${kv[0]}"
    if [ -n $kv[1] ] ; then
      args_string="$args_string ${kv[1]}"
    fi
  done
  echo $args_string
}

# Graceful Ctrl-C
function ctrl_c {
  echo "** Process manually stopped with Ctrl+C"
  for i in `seq 1 1`; do
    sleep 1
    echo -n "."
  done
}
trap ctrl_c INT

# Default settings
# Note: dev is the default environment, for safety and convenience, but some
# global defaults assume prod and will be overridden by dev mode
env="dev"
action="build" # serve by default for dev
source="src"
bundler_log="--quiet"
search=false # true by default for prod
serve=false # true by default for dev
precheck=false
tests=false # true by default for prod
fail_response="ignore" # for test failures "exit" by default for all others
jekyll_args=""
search_dry=false
verbose=false
reviewnotes=true
serve_after=false
vardump=false
clean_build=true

# Evaluate optional environment arg and port arg up front
re_port_num="^[0-9]{4,5}"
re_version="[4-9]\.[0-9]{1,2}"
while $(is_value $1) ; do
  if [[ "$1" =~ ^(dev|stage|prod|search)$ ]] ; then
    env=$1
    shift
  elif [[ "$1" =~ $re_port_num ]] && [ "$1" -gt 1023 -a "$1" -lt 65535 ] ; then
    port=$1
    shift
  elif [ "$1" == "clean" ] ; then
    clean_build=true
    shift
  elif [[ "$1" =~ $re_version(\-$re_version)? ]] ; then
    IFS='-' read -r -a vsns <<< "$1"
    vsn_start=${vsns[0]}
    if [ -n "${vsns[1]}" ] ; then
      vsn_end=${vsns[1]}
    else
      vsn_end=${vsns[0]}
    fi
    shift
  else
    echo "Argument $1 not recognized. Aborting."
    exit 1
  fi
done
# Environment-specific settings
case "$env" in
  dev|development)
  env_name="development"
  clean_build=false
  serve=true
  config_env="dev"
  exclude_env="dev"
  dest_dir="_site"
  ;;

  stage|staging|review)
  env_name="review"
  precheck=true
  serve=false
  tests=false
  fail_response="exit"
  search=false
  config_env="prod"
  exclude_env="dev"
  dest_dir="target/www"
  ;;

  prod|production)
  env_name="production"
  precheck=true
  search=dry
  tests=true
  fail_response="exit"
  config_env="prod"
  dest_dir="target/www"
  reviewnotes=false
  exclude_env="prod"
  ;;

  search)
  env_name="search"
  precheck=false
  search=true
  tests=false
  fail_response="exit"
  config_env="prod"
  dest_dir="target/www"
  reviewnotes=false
  exclude_env="prod"
  ;;
esac
config_arg="--config jekyll/configs/jekyll-global.yml,jekyll/configs/jekyll-$config_env.yml,jekyll/configs/excludes-$exclude_env.yml"

# Versions to build
# If not determined in command arguments
# For establishing environment default ranges
vsn_first=$(ruby src/ruby/version_handler.rb get first id)
vsn_last=$(ruby src/ruby/version_handler.rb get last id)
vsn_2ndto=$(ruby src/ruby/version_handler.rb get 2ndto id)

# Set conditional output vars
if [ $clean_build ] ; then unclean="clean" ; else unclean="unclean" ; fi

if [ -z "$vsn_start" ] ; then
  if [ "$env" == "dev" ] ; then
    vsn_start=$vsn_2ndto
    vsn_end=$vsn_last
  else
    vsn_start=$vsn_first
    vsn_end=$vsn_last
  fi
fi
# Generate version-specific configs and return array of versions to build
vsns=($(ruby src/ruby/version_handler.rb gen search $vsn_start-$vsn_end))

# Help menu
if [ "$1" == "--help" ] || [ "$1" == "-h" ] ; then
  printf "\n
Usage: $0 [env] [start.version[-end.version]] [clean] [port] [options]

env: Optional build environment, to establish defaults. Must be dev (default), stage, search, or prod.
versioning: Version range can be indicated as 5.5-5.7 or just 5.7.
            Note: Versioning only affects which versions are indexed for search, not which are built.
port: Serve port, for dev builds or stage/prod builds with --build-after. (Default: 4000)
clean: Forcibly removes target directory

Options:\n
  --build-only | --no-serve | -n   Disables serve operation enabled by default in dev mode.
  --destination | -d PATH          Changes the target directory of the build.
  --jekyll 'STRING' | -j 'STRING'  Quoted, specially formatted string of option flags and args.
                                     Ex: --jekyll 'incremental layouts=_layouts/new'
  --precheck | -k                  Runs pre-build evaluations.
  --search [dry*|true|false]       Toggles Algolia search build; dry (*impled when --search is used) performs without push.
  --search-key '<KEY>'             Where <KEY> is the actual admin API key for Algolia account.
  --serve-after                    Adds a serve operation at the end of the build, even for production operations
  --source PATH | -s PATH          Path to the content source root (what Jekyll reads).
  --tests [exit*|warn|false]       Perform tests; abort build (*default for --tests) or warn only on fail, or false to prevent tests.
  --verbose                        Run script in debug mode. (For Jekyll debugging, use: --jekyll 'verbose')
  --review-notes-x | -x            Fails if review notes are detected. (Default for prod builds.)\n\n\n\n"
  exit 0
fi

# Start reporting to the user
echo "Performing $unclean $env_name build."

# Parse options/arguments
while [ "$1" != "" ] ; do
  case $1 in

    -t|--tests|--test)
    tests=true ; fail_response="exit"
    shift
    if $(is_value $1) ; then
      if [[ "$1" =~ ^(ignore|exit|false)$ ]] ; then
        if [ "$1" == "false" ] ; then
          tests=false
          echo "Tests disabled."
        else
          fail_response=$1
        fi
        shift
      else
        echo "Value of --tests unrecognized; must be exit, ignore, or false. Defaulting to exit."
        fail_response="exit"
      fi
    fi
    ;;

    --build-only|--no-serve)
    serve=false
    shift
    ;;

    --destination|--dest|-d)
    shift
    if $(is_value $1) ; then
      dest_dir=$1
      shift
    else
      echo "--destination/-d flag requires a proper path argument"
    fi
    ;;

    --jekyll|-j)
    shift
    if $(is_value $1) ; then
      jekyll_args="$jekyll_args $(smuggled_args_parser $1)"
      shift
    else
      echo "Jekyll arguments missing from --jekyll option. Skipping."
    fi
    ;;

    --precheck|-k)
    precheck=true
    shift
    ;;

    --search)
    search=true
    shift
    if $(is_value $1) ; then
      if [ "$1" == true ] || [ "$1" == false ] || [ "$1" == "dry" ] ; then
        if [ "$1" == "dry" ] ; then
          search_dry=true
        else
          search=$1
        fi
        shift
      else
        search=false
        echo "Value of --search unrecognized; must be true, false, or dry (default). Defaulting to false."
      fi
    else # --search defaults to dry
      search_dry=true
    fi
    ;;

    --search-key)
    shift
    if $(is_value $1) ; then
      search_key=$1
      shift
    else
      echo "Value of --search-key missing. Skipping search."
      search=false ; serve=false
    fi
    ;;

    --serve-after)
    serve_after=true
    serve=false
    shift
    ;;

    --source|-s)
    shift
    if $(is_value $1) ; then
      source=$1
      shift
    else
      echo "Option --source (-s) must be followed by a path."
    fi
    ;;

    --review-notes-x|-x)
    reviewnotes=false
    shift
    ;;

    --vardump)
    vardump=true
    shift
    ;;

    --verbose)
    verbose=true
    bundler_log=""
    shift
    ;;

    -?*)
    echo "WARN: Unknown option (ignored): $1"
    shift
    continue
    ;;

    *)
    break
    ;;
  esac
done

if $clean_build ; then
  rm -rf $dest_dir
  if $verbose ; then
    echo "Clean build: clearing $dest_dir"
  fi
fi

if $tests ; then
  echo "Building for tests."
fi

# Assemble more Jekyll arguments based on options
jekyll_args="$jekyll_args --destination $dest_dir"
jekyll_args="$jekyll_args --source $source"

# Search preparation
if [ $search != false ] ; then
  if $search_dry ; then
    algolia_api_key='DummyKeyForDryIndexing'
    key_from="(Dummy key used.)"
  else
    if [ -n "$search_key" ] ; then
      algolia_api_key=$search_key
      key_from="(Key provided on command line.)"
    else
      if [ -z "${ALGOLIA_API_KEY}" ] ; then
        key_from="(Key found in environment variable.)"
      else
        echo "Search build aborted: Algolia API key missing; either add as --search-key argument or store as \${ALGOLIA_API_KEY} environment variable."
        exit 1
      fi
    fi
  fi

  if [ "$env" != "prod" ] && ! $search_dry ; then
    while true; do
      read -p "Are you sure you want to push to Algolia index from a dev build?" yn
      case $yn in
        [Yy]* ) search=true ; break ;;
        [Nn]* ) exit ;;
        * ) echo "Please answer yes or no." ;;
      esac
    done
  fi

  echo "Building with search index. $key_from"
  action="build"
  serve=false
fi

# Pre-build prep
if [ "$env" == "dev" ] ; then
  if $tests; then
    action="build"
    serve=false
    serve_after=true
  fi
fi

# If we haven't suppressed $serve yet...
if $serve ; then
  action="serve"
  if [ $port ] ; then jekyll_args="$jekyll_args --port $port" ; fi
fi

# Pre-build tests
if $precheck ; then
  # precheck ops go here
  echo "No prechecks established"
fi
# Fail if content includes review comment includes
if ! $reviewnotes ; then
  echo "checking for doc-issue tags"
  if [[ -n "$(find $source -type f | xargs grep 'doc-issue.html')" ]]
  then
    echo "ERROR: doc-issue tag(s) found."
    if [ "$fail_response" == "exit" ] ; then
      echo "Build command canceled."
      exit 1
    fi
  fi
fi

if $verbose; then
  echo "Build vars:"
  echo "           env:" $env
  echo "      env_name:" $env_name
  echo "        action:" $action
  echo "    config_env:" $config_env
  echo "    config_arg:" $config_arg
  echo "   jekyll_args:" $jekyll_args
  echo "      precheck:" $precheck
  echo "   reviewnotes:" $reviewnotes
  echo "        search:" $search
  echo "    search_dry:" $search_dry
  echo "         serve:" $serve
  echo "   serve_after:" $serve_after
  echo "         tests:" $tests
  echo " fail_response:" $fail_response
fi

# Jekyll build command
jekyll_command="bundle exec jekyll $action $config_arg,jekyll/configs/version-base.yml $jekyll_args"
bundle install $bundler_log # ensures dependency gem versions match Gemfile.lock
echo "Running command '$jekyll_command'"
$jekyll_command



if $tests ; then
  htmlproofer $dest_dir 2>&1 | tee target/htmlproofer-output.txt
  # need to check the output because htmlproofer doesn't return the expected
  # nonzero status code on failure
  if grep -q "Error:  HTML-Proofer found" "target/htmlproofer-output.txt"; then
    echo "HTML-Proofer found errors"
    if [ "$fail_response" == "exit" ] ; then exit 1 ; fi
  fi
fi

if [ "$search" != false ] ; then
  if $search_dry ; then jekyll_args="$jekyll_args --dry" ; fi
  for vsn in ${vsns[@]} ; do
    config_arg_vsn="$config_arg,target/configs/$vsn.yml"
    algolia_command="bundle exec jekyll algolia $config_arg_vsn $jekyll_args --destination $dest_dir"
    echo "Running search build and index ($vsn)"
    ALGOLIA_API_KEY=$algolia_api_key $algolia_command
    if [ "$?" -ne 0 ] ; then
      echo "Search processing failed"
      if [ "$fail_response" == "exit" ] ; then
        echo "Canceling search operations."
        exit 1
      fi
    fi
  done
fi

if $serve_after ; then
  if [ $port ] ; then port_flag="--port $port" ; fi
  serve_command="bundle exec jekyll serve $config_arg $port_flag -d $dest_dir --skip-initial-build"
  echo "Attempting post-build serve"
  echo "Running command: $serve_command"
  $serve_command
fi

echo "✔ Script finished."
exit 0

LICENSE

Copyright 2019 Pepperdata, Inc

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation 
files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, 
modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software 
is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

version_handler.rb

# version_handler.rb
# Usage:
#        ruby version_handler.rb --help
# This script is executed by `jklbld.sh` to generate version-specific
# Jekyll configuration files based on the docs versions being built and
# indexed by Algolia at that time, and also to return the range of supported
# versions.
#
# The 'gen' action writes the config files to target/configs/ and returns
# a shell-formatted array of version numbers (e.g., 5.5 5.6 5.7).
#
# The 'get' action reads a specified object from the version file and returns
# either the slug or the version ID.

require 'fileutils'
require 'yaml'

@args = ARGV
@base_dir = Dir.pwd
@range = {}
@vsns_file_def = "jekyll/_data/versions.yml"
@vsns_file = @vsns_file_def
@type = "id"

def load_versions_file file
  begin
    @all_versions = YAML.load_file(file)
  rescue
    raise "Could not read versions YAML file. Cannot proceed."
  end
end

begin
  @base_config = YAML.load_file("#{@base_dir}/jekyll/configs/version-base.yml")
rescue
  raise "Could not read versions YAML file. Cannot proceed."
end

# Proc for deriving a specific version's data object
def get_version v
  # Accepts a position (e.g., -1, 0, 9), returns version object at that position
  # Otherwise accepts a float (e.g., 5.7) and returns the version object with that ID
  # Otherwise accepts a slug (e.g., v5-7) and returns the version object with that slug
  # Returns hash of selected version
  if v.include?"."
    vsn = @all_versions.select {|vsn| vsn['id'] == v }
    return vsn[0]
  else
    case v
    when "first"
      v = 0
    when "2ndto"
      v = -2
    when "last"
      v = -1
    else
      raise "UnrecognizedVersionArg"
    end
    vsn = @all_versions[v]
    return vsn
  end
end

def build_version_configs startv, endv, target_path="target/configs"
  # Builds config files for the range (startv -> endv)
  FileUtils::mkdir_p(target_path) unless File.exists?(target_path)
  out = []
  @all_versions.each do |vsn|
    if vsn['float'] >= startv.to_f && vsn['float'] <= endv.to_f
      build_config(vsn['id'], target_path)
      out << vsn['slug']
    end
  end
  versions = out.join(" ")
  versions
end

def build_config v, path
  # Builds a single config file for version v
  vsn = get_version(v)
  config = {}
  config.merge!@base_config
  config['algolia']['index_name'] = vsn['slug']
  config['algolia']['files_to_exclude'] = other_versions(v, true, true)
  File.open("#{@base_dir}/#{path}/#{vsn['slug']}.yml", 'w') { |file| file.write(config.to_yaml) }
end

def other_versions v, underscore=false, wildcards=false
  # returns an array of version IDs other than passed v
  u=""
  w=""
  oth = []
  u = "_" if underscore
  w = "/**/**" if wildcards
  @all_versions.each do |vsn|
    oth << u + vsn['slug'] + w if vsn['id'] != v
  end
  return oth
end

def show_help action=nil
  gen_desc = "Generates configuration files of <kind>: 'search' or 'build', which generate their respective kind of version-specific config files. Returns a Bash-formatted 'array' of versions generated."
  get_desc = "Returns the proper version ID or slug for the version at this position or ID in versions.yml"
  if action == "gen"
    print "
    #{gen_desc}

    Usage: #{$0} gen [<version1>[-<version2>]] [path/to/yaml]

    Ex: #{$0} gen search 5.7 jekyll/_data/versions.yml
    Ex: #{$0} gen search 5.5-5.7
    Ex: #{$0} gen search"
  elsif action == "get"
    print "
    #{get_desc}

    Usage: #{$0} get [first|last|2ndto|<version>] [slug|id] [path/to/yaml]

    Ex: #{$0} get 5.7 jekyll/_data/versions.yml
    Ex: #{$0} get 2ndto jekyll/_data/versions.yml
    Ex: #{$0} get last"
  else
    print "
    Usage: #{$0} [get|gen] [version options] [path/to/yaml]
    get: #{get_desc}
    gen: #{gen_desc}

    Use #{$0} [get|gen] --help to display version options.
    "
  end
  puts "\n\n"
  exit 0
end

# Begin command-line parsing
while @args[0] do
  arg = @args[0]
  case arg
  when "get"
    @action = "get" ; @args.shift
  when "gen"
    @action = "gen" ; @gen_kind = "search" ; @args.shift
  when "--help"
    show_help(@action)
  else
    raise "MissingAction" unless @action
    if arg.match(/^[4-9]\.[0-9]{1,2}$|first|last|2ndto/)
      @version = arg
      @range['start'] = arg
      @range['end'] = arg
      @args.shift
    elsif arg.match(/^[4-9]\.[0-9]{1,2}\-[4-9]\.[0-9]{1,2}$/)
      raise "WrongAction" unless @action == "gen"
      range = arg.split("-")
      @range['start'] = range[0]
      @range['end'] = range[1]
      @args.shift
    elsif arg.match(/^(.*)\/.*\.yml$/)
      @vsns_file = arg ; @args.shift
    elsif arg.match(/build|search/) # NOTE build gen not yet implemented
      @gen_kind = arg ; @args.shift
    elsif arg.match(/slug|id/)
      @type = arg ; @args.shift
    else
      raise "Argument #{arg} not recognized"
    end
  end
end

load_versions_file(@vsns_file)

if @action == "get"
  raise "MissingVersion" unless @version
  vsn = get_version(@version)
  puts vsn[@type]
end

if @action == "gen"
  unless (@range['start'] && @range['end'])
    @range['start'] = @all_versions[0]['id']
    @range['end'] = @all_versions[-1]['id']
  end
  vsns = build_version_configs(@range['start'], @range['end'])
  puts vsns
end

Pepperdata, Inc approves this matter for release as-is, without warranty, under The MIT License.