CI release script, uses Github API and internal release manager API to validate and release projects

release_pipeline.sh

#!/usr/bin/env bash
#
# Author: dataday
# created: 23/02/2016
#
# Description:
# Runs release tasks on CI (Delivery Pipeline)
# For input variables see $SCRIPT_ROOT/environment-ci
# debug: $0 2>&1 | tee $SCRIPT_ROOT/release.log
#
# fail on error
set -e

export LOCAL_ROOT=$(pwd -L)
export SCRIPT_NAME=${BASH_SOURCE##*/}
export SCRIPT_ROOT=${BASH_SOURCE[0]%/*}

source $SCRIPT_ROOT/sandbox/environment
source $SCRIPT_ROOT/sandbox/environment-ci

# http://www.bagill.com/ascii-sig.php
cat "$SCRIPT_ROOT/sandbox/etc/banner"

# https://manager.api.domain.co.uk/docs
RELEASE_MANAGER_URL_API=https://manager.api.domain.co.uk
RELEASE_MANAGER_URL_CREATE=$RELEASE_MANAGER_URL_API/releases/create
RELEASE_MANAGER_URL_RELEASES=$RELEASE_MANAGER_URL_API/component/$APP_ID/releases
RELEASE_MANAGER_URL_COMPONENT=$RELEASE_MANAGER_URL_API/env/%s/component/%s
RELEASE_MANAGER_URL_REPOSITORIES=$RELEASE_MANAGER_URL_API/component/$APP_ID/repositories
RELEASE_MANAGER_URL_DEPLOYMENT=$RELEASE_MANAGER_URL_COMPONENT/deploy_release
RELEASE_MANAGER_URL_DEPLOYMENT_ADMIN=https://admin.live.domain.co.uk/manager/env/int/deployment/%s
RELEASE_MANAGER_URL_DEPLOYMENT_STATUS=$RELEASE_MANAGER_URL_API/%s

# https://repository.api.domain.co.uk/docs
# https://developer.github.com/v3/repos/statuses
REPO_URL_HOME=git@github.com:domain/$APP_ID.git
REPO_SHA_HEAD=$(git rev-parse --verify HEAD)
REPO_URL_STATUSES=https://api.github.com/repos/domain/$APP_ID/statuses/$REPO_SHA_HEAD
REPO_URL_HEAD=https://repository.api.domain.co.uk/manager-$APP_ID/revisions/head
REPO_RPM_PATH=$LOCAL_ROOT/RPMS/$APP_RPM_NAME

CURL_SSL_PREFIX="curl --insecure -v --cert $APP_CLIENT_CERT --key $APP_CLIENT_KEY --cacert $APP_CA"

DEPLOY=""
PROMOTE=""
PACKAGE=""
STATUS=""

REBUILD=false
BUILD=false
TEST=false
SNIFF=false
REGISTER=false

#
# display usage information
function usage () {
  printf "Usage: $SCRIPT_NAME --build --rebuild [--test --sniff --package=rpm|translations --deploy=assets|repositories|rpm --promote=int|test|live]

  Triggers sequence specific release tasks from options
  For input variables see $SCRIPT_ROOT/sandbox/environment-ci

  requires either options
  \t--build - Builds containers and machine
  \t--rebuild - Hard rebuilds the containers and machine

  optional options
  \t--test - Tests container code (PHP)
  \t--sniff - Sniffs container code (PHP)
  \n--package=rpm|translations - Builds an RPM using mbt (magic build tool) or translations
  \n--deploy=assets|repositories|rpm - Deploys assets, YUM repositories or the packaged up RPM
  \n--promote=int|test|live - Releases to a specified environment
  \n" 1>&2
  exit $?
}

#
# check for options
if (($# == 0)); then
  usage
fi

#
# provide feedback from status
function status () {
  local type="$1"
  local message="$2"
  local data='{"state":"%s", "target_url":"%sconsole", "description":"Job: %s No.: %s Date: %s. %s", "context":"continuous-integration/jenkins"}'
  local submission=$(printf "$data" "$type" "$BUILD_URL" "$APP_ID" "$APP_VERSION" "$(date)" "$message")

  response=$(curl -X POST --data '$submission' --header 'Authorization: token $GITHUB_TOKEN' --header 'Content-Type:application/json' $REPO_URL_STATUSES)

  log "status: $APP_VERSION $type $REPO_URL_STATUSES"
}

#
# provide log feedback to console
function log () {
  local message="$1"
  local stop=${2:-false}

  if [ $? -ne 0 -o "$stop" = true ]; then
    status "failure" $message
    echo "${CMD_PROMPT/$SCRIPT_NAME/failed} (line:$LINENO) $message" >&2
    $SCRIPT_ROOT/sandbox/run --status=stop || true
    [ "$stop" != true ] || { exit $?; }
    exit 1
  else
    echo "$CMD_PROMPT $message" >&2
  fi
}

#
# provide http response feedback
function parse_http_response () {
  local success_codes=( 204 201 200 )
  local response=$(eval "$1 -w '\n%{http_code}'")
  local status=$(echo "$response" | tail -n1)
  local body=$(echo "$response" | sed \$d)

  if [[ ! " ${success_codes[@]} " =~ " $status " ]]; then
    log "http: $body ${1/$CURL_SSL_PREFIX\ /}" true
  fi

  echo "$body"
}

#
# provide text response feedback
function parse_text_response () {
  local key="$1"
  local message="$2"
  local match=${3:-true}

  # error if empty
  [ ! -z "$message" ] || error=true

  # fail if key is found
  if [ "$match" = true ]; then
    [[ ! "$message" =~ "$key" ]] || error=true
  else
    # fail if key is not found
    [[ "$message" =~ "$key" ]] || error=true
  fi

  log "text: '$key' $message" ${error:-false}
}

#
# provide json response feedback
function parse_json_response () {
  local json="$1"
  local path="$2"
  local cmd="import sys,json;print json.load(sys.stdin)$path"
  local response=$(echo $json | python -c "$cmd")
  log "json: $response $path"
  echo "$response"
}

#
# provide deployment status feedback (GET)
# [pending_bake, pending_stack_update, pending_stack_update_resolution, ...]
function parse_status_response () {
  log "status: $RELEASE_MANAGER_URL_DEPLOYMENT_STATUS ($RELEASE_MANAGER_POLL_INTERVAL)"

  # provide release status
  data=$(parse_http_response "$CURL_SSL_PREFIX $RELEASE_MANAGER_URL_DEPLOYMENT_STATUS")
  status=$(parse_json_response "$data" "['status']")
  id=$(parse_json_response "$data" "['id']")
  job=$(printf "$RELEASE_MANAGER_URL_DEPLOYMENT_ADMIN" $id)

  # update release status
  while [ "$status" != "done" -a "$status" != "failed" ]; do
    sleep $RELEASE_MANAGER_POLL_INTERVAL
    log "status: $status $job ($id)"
    parse_status_response
  done

  # release failed
  if [ $status != "done" ]; then
    log "status: $status $job ($id)" true
  elif [ $status = "done" ]; then
   STATUS="status: $status $job ($id)"
  fi
}

#
# provide unit test feedback
function tests () {
  if [ "$TEST" = true ]; then
    parse_text_response "FAILURES!" "$($SCRIPT_ROOT/sandbox/run --ci --test)"
  fi
}

#
# provide code sniff feedback
function sniff () {
  if [ "$SNIFF" = true ]; then
    parse_text_response " ERROR " "$($SCRIPT_ROOT/sandbox/run --ci --sniff)"
  fi
}

#
# provide RPM package
function package_rpm () {
  log "package: RPM $APP_VERSION ($REPO_RPM_PATH)"

  # check mbt exists
  type mbt >/dev/null 2>&1 || error=true

  if [ "${error:-false}" != true ]; then
    # use mbt directly instead of calling
    # ../make VERSION=$APP_VERSION
    mbt -v $APP_VERSION
  fi

  log "package: RPM $(cat $LOCAL_ROOT/SPECS/$APP_ID.spec)"
}

#
# package translations
function package_translations () {
  log "translations: $APP_ID"
  #parse_text_response "Created lang directory" "$($SCRIPT_ROOT/sandbox/run --ci --translations)" false
  echo "$($SCRIPT_ROOT/sandbox/run --ci --translations)"
  # restart the container as translations exits on success and failure
  $SCRIPT_ROOT/sandbox/run --status=restart || true
}

#
# filter package type
function package () {
  if [ ! -z "$PACKAGE" ]; then
    case $PACKAGE in
      "rpm") package_rpm;;
      "translations") package_translations;;
    esac
  fi
}

#
# update YUM repositories
function deploy_repositories () {
  log "deploy: repositories $APP_REPOS_FILE"
  response=$(parse_http_response "$CURL_SSL_PREFIX -X PUT --data @$WORKSPACE/$APP_REPOS_FILE -H 'Content-Type:application/json' $RELEASE_MANAGER_URL_REPOSITORIES")
  log "deploy: repositories ($response)"
}

#
# upload assets
function deploy_assets () {
  # deploy environment specific assets
  # path should match /$APP_ID/$APP_VERSION, see ./app/bootstrap.php
  log "deploy: assets $APP_ASSET_ROOT -> $APP_BUCKET"
  parse_text_response "Error " "$($LOCAL_ROOT/vendor/bin/rmp-assets assets:upload $APP_ASSETS_ROOT $APP_BUCKET)"
}

#
# deploy RPM
function deploy_rpm () {
  log "deploy: RPM $APP_RPM_NAME ($APP_TARGET_ENV:$GIT_COMMIT)"

  # deploy RPM to repository (POST)
  release=$(parse_http_response "$CURL_SSL_PREFIX -X POST -H 'Content-Type:application/octet-stream' -T $REPO_RPM_PATH $REPO_URL_HEAD")

  # get deployment reference
  reference=$(parse_json_response "$release" "['ref']")
  log "deploy: RPM reference $reference"

  # create associated meta data for deployment
  submission="{\"components\": [\"$APP_ID\"],\"version\": \"$APP_VERSION\",\"release\": {\"source\": {\"type\": \"git\",\"url\": \"$REPO_URL_HOME\",\"revision\": \"$GIT_COMMIT\"},\"repositories\": {\"manager-$APP_ID\": {\"type\": \"direct\", \"url\": \"$reference\"}},\"packages\": [{\"name\": \"$APP_ID\",\"version\": \"1.domain.e16\",\"release\": \"$APP_VERSION\",\"arch\": \"noarch\"}]}}"
  log "deploy: RPM submission $submission"

  # create release (POST)
  response=$(parse_http_response "$CURL_SSL_PREFIX --data '$submission' -H 'Content-Type:application/json' $RELEASE_MANAGER_URL_CREATE")
}

#
# filter deploy type
function deploy () {
  if [ ! -z "$DEPLOY" ]; then
    case $DEPLOY in
      "rpm") deploy_rpm;;
      "assets") deploy_assets;;
      "repositories") deploy_repositories;;
    esac
  fi
}

#
# promote the release to a selected environment
function promote () {
  if [ ! -z "$PROMOTE" ]; then
    log "promote: ($RELEASE_MANAGER_URL_RELEASES)"

    # provide list of releases to date (GET)
    releases=$(parse_http_response "$CURL_SSL_PREFIX $RELEASE_MANAGER_URL_RELEASES")

    # provide latest release version
    version=$(parse_json_response "$releases" "['releases'][0]['version']")
    log "promote: version ($version)"

    # provide deployment response (POST)
    release=$(parse_http_response "$CURL_SSL_PREFIX --data '{\"release_version\":\"$version\"}' -H 'Content-Type:application/json' $RELEASE_MANAGER_URL_DEPLOYMENT")
    log "promote: release $release"

    # provide deployment status url
    RELEASE_MANAGER_URL_DEPLOYMENT_STATUS=$(printf "$RELEASE_MANAGER_URL_DEPLOYMENT_STATUS" $(parse_json_response "$release" "['url']"))
    log "promote: sleep ($BUILD_SLEEP)"

    sleep $BUILD_SLEEP

    parse_status_response

    # avoid duplicate messages appearing
    # when polling status
    log "$STATUS"

    # stop the container
    $SCRIPT_ROOT/sandbox/run --status=stop || true
  fi
}

#
# provide registry updates
function register () {
  if [ "$REGISTER" = true ]; then
    local file=$SCRIPT_ROOT/registry.json
    local data="[{\"repository\":\"git@github.com:domain/$APP_ID.git\",\"containers\":[{\"name\":\"base\",\"dockerfile\":\"Dockerfile\"}]}]"

    log "register: data $data"

    # create registry meta data
    rm -f $file && echo "$data" > $file

    # ensure rotterdam is available
    type rotterdam >/dev/null 2>&1 || error=true

    # asset CI and not in error
    if [ "$CI" = true -a "${error:-false}" != true ]; then
      rotterdam $file $APP_REGISTRY rmp
    fi

    log "register: update (success)"
  fi
}

#
# create the container
function setup () {
  if [ "$BUILD" = true -o "$REBUILD" = true ]; then

    # invoke build
    if [ "$BUILD" = true ]; then
      $SCRIPT_ROOT/sandbox/run --ci --build

    # invoke rebuild
    elif [ "$REBUILD" = true ]; then
      $SCRIPT_ROOT/sandbox/run --ci --rebuild
    fi
  fi

  # decide target environment
  if [ ! -z "$PROMOTE" ]; then
    if [[ ! "$PROMOTE" =~ ^(int|test|live)$ ]]; then
      PROMOTE=int
    fi

    # set target environment and deployment endpoints (@note)
    export APP_TARGET_ENV=$PROMOTE
    export APP_STATUS_URL=$(printf "$APP_STATUS_URL" $APP_TARGET_ENV)
    export RELEASE_MANAGER_URL_COMPONENT=$(printf "$RELEASE_MANAGER_URL_COMPONENT" $APP_TARGET_ENV $APP_ID)
    export RELEASE_MANAGER_URL_DEPLOYMENT=$(printf "$RELEASE_MANAGER_URL_DEPLOYMENT" $APP_TARGET_ENV $APP_ID)
    export RELEASE_MANAGER_URL_DEPLOYMENT_ADMIN=$(printf "$RELEASE_MANAGER_URL_DEPLOYMENT_ADMIN" $APP_TARGET_ENV)
  fi
}

#
# assign options
while getopts ":-:" opt; do
  case $opt in
    -)
    case $OPTARG in
      test) TEST=true;;
      sniff) SNIFF=true;;
      build) BUILD=true;;
      rebuild) REBUILD=true;;
      register) REGISTER=true;;
      package=*) PACKAGE="${OPTARG#*=}";;
      deploy=*) DEPLOY="${OPTARG#*=}";;
      promote=*) PROMOTE="${OPTARG#*=}";;
    esac
    ;;
  \?) log "invalid: -$OPTARG" && usage;;
  *) usage;;
  esac
done

#
# init
function init () {
  status "pending"
  tests
  sniff
  setup
  package
  deploy
  promote
  register
  status "success"
}

init

exit 0