Updated gitlab-runner supporting docker in docker

gitlab-local.sh

#!/usr/bin/env bash
set -euo pipefail

usage () {
  echo "Usage: $(basename $0) {up|down|[job stage]}"
  echo "[job stage] - defaults to "build" and supports multiple comma delimited jobs"
  echo "$(basename $0) test,build"
}


gitlab_down () {
    set +e
    docker stop $PROJECT_RUNNER_NAME
    docker rm $PROJECT_RUNNER_NAME
    #for container in $(docker ps -q -f 'name=runner*'); do docker stop $container; docker rm $container; done
    #docker volume rm gitlab-runner-config
    set -e
}

PWD_RESOLVED="$(pwd -P)"
# .gitlab-ci.yml has to live in the root of a project/git repo, so if we aren't in that folder, cd to it
GIT_ROOT=$(git rev-parse --show-toplevel)
if [ "$PWD_RESOLVED" != "$GIT_ROOT" ]; then cd $GIT_ROOT; PWD_RESOLVED="$(pwd -P)"; fi
# Need short name for project to allow concurrent runs of different projects
PROJECT_RUNNER_NAME="gitlab-runner-$(basename $PWD_RESOLVED)"
# If we are in a Git(lab) repo the .gitlab-ci.yml should be in the root of it
[ -f "./.gitlab-ci.yml" ] || { echo "No .gitlab-ci.yml found"; exit 3; }
DEFAULT_STAGE="build"


declare -a ENVVARS
if [ -f $HOME/.aws/credentials ]; then
  # Need this to get secrets from the host, but don't pass in AWS_PROFILE itself to runner because it shouldn't have ~/.aws/credentials where it looks up profiles
  awsProfile=${AWS_PROFILE:-default}
  # echo "AWS_PROFILE=$awsProfile"
  set -a
  AWS_ACCESS_KEY_ID=$(aws --profile $awsProfile configure get aws_access_key_id)
  AWS_SECRET_ACCESS_KEY=$(aws --profile $awsProfile configure get aws_secret_access_key)
  AWS_SESSION_TOKEN=$(aws --profile $awsProfile configure get aws_session_token)
  set +a
  # Exit early if the AWS credentials we are trying to use are expired by actually hitting AWS
  aws sts get-caller-identity || { OOPS=$?; gitlab_down; echo "AWS profile creds missing or session was invalid, refresh your session and try again"; exit $OOPS; }
  ENVVARS=("AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" "CI_PIPELINE_IID=$(date -u +"%H%M%s")" "CI=1")
  CREDVARS=$(printf " --env %s" "${ENVVARS[@]}")
  add_ecr_auth () {
    # The * isn't actually valid, insert your account ID (if more than one, copy the line, replacing * with each account)
    cat >/tmp/docker-ecr-config.json <<-'EOD'
{
    "credHelpers":{
       "*.dkr.ecr.us-east-1.amazonaws.com": "ecr-login"
    }
}
EOD
}

  # Not sure if there is a way to get "latest" automatically, with newer versions of Docker it might be good to pull a newer release
  ECR_HELPER_URL="https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.4.0/linux-amd64/docker-credential-ecr-login"
  # need set -a / +a before after the above variable if wanting to use it in the { subshell } below
  [ -f /tmp/docker-credential-ecr-login ] || { echo "Grabbing ecr-helper"; curl -L -o /tmp/docker-credential-ecr-login https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.4.0/linux-amd64/docker-credential-ecr-login && chmod +x /tmp/docker-credential-ecr-login; }
  [ -f /tmp/docker-ecr-config.json ] || { echo "Creating docker-ecr-config to mount"; add_ecr_auth ; }

fi

committedCodeCheck() {
    # Gitlab-runner already checks this but we wanted to give a message about how to abort and rectify
    set +e
    git update-index --refresh
    set -e
    if ! git diff-index --quiet HEAD -- ; then
        echo "WARNING: Git has some uncommitted changes which will not run!"
        git status
        echo
        echo "gitlab-runner local builds require you commit changes before they take effect (anything outside the .gitlab-ci.yml)"
        echo "press Ctrl+C now to abort in the next 10 seconds and commit your changes, unless you want to run old code"
        sleep 10
    fi
}

gitlabLoop() {
    stages=($(echo "$1" | tr ',' '\n'))
    if [ -z "$stages" ]; then
        echo "Running $DEFAULT_STAGE stage only"
        gitlabExec "$DEFAULT_STAGE"
    else
        echo "Running stage(s): ${stages[*]}"
        for stage in ${stages[*]}; do
            gitlabExec $stage
        done
    fi
}

gitlabExec() {
    docker exec -w $PWD_RESOLVED -it $PROJECT_RUNNER_NAME \
        gitlab-runner exec docker $1 \
        --docker-privileged \
        --custom_build_dir-enabled \
        --docker-volumes="/certs" \
        --docker-disable-cache=false \
        --timeout 3600 \
        --env ROOT_PWD=$PWD_RESOLVED \
        ${CREDVARS:-}

        #--docker-volumes="/root/.docker/config.json" \
        # Want at least 1 hour timeout for debugging
  #      --cache-dir=/tmp/gitlabrunner \
  #      --docker-pull-policy="if-not-present" \
  # The custom_build_dir requires a workaround in local scripts because not all CI_ variables present on a Gitlab instance show up in the local runner
  # https://docs.gitlab.com/ee/ci/variables/where_variables_can_be_used.html

  # The /tmp/gitlabrunner is inside the outer gitlab-runner container so isn't the volume from host_runner
  # https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnersdocker-section
  # Don't disable cache as it behaves poorly with our /certs volume
  # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4596
      # why disable cache?
      #--docker-disable-cache \
}

host_runner () {
  committedCodeCheck
  # Checks if gitlab-runner is already running 
  # Need to start with --name gitlab-runner-$(basename $PWD_RESOLVED) and only kill instances matching that so this script can be used concurrently from multiple directories
  # ensures it is using the current repo AND
  # has a currently valid AWS token otherwise restarts the host_runner
  { [ -n "$(docker ps -q -f "name=$PROJECT_RUNNER_NAME")" ] && \
     docker inspect $PROJECT_RUNNER_NAME | jq -r .[].Mounts[].Source | grep -q "$PWD_RESOLVED" && aws sts get-caller-identity >/dev/null 2>&1 ; } || \
     {
     gitlab_down
     #aws sts get-caller-identity || { OOPS=$?; echo "AWS profile creds missing or session was invalid, refresh your session and try again"; exit $OOPS; }
     # Use --mount type=bind to avoid creating root owned paths on host
     # Use --rm to cleanup 'anonymous volumes'
     docker run -d --rm --name $PROJECT_RUNNER_NAME \
        ${CREDVARS:-} \
       --env DOCKER_DRIVER=overlay2 \
       --mount type=bind,source=$PWD_RESOLVED,target=$PWD_RESOLVED \
       --mount type=volume,target=/certs \
       --mount type=bind,source=/tmp/docker-ecr-config.json,target=/root/.docker/config.json \
       --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
       --mount type=bind,source=/tmp/docker-credential-ecr-login,target=/usr/bin/docker-credential-ecr-login \
       --workdir $PWD_RESOLVED \
       --mount type=volume,target=/etc/gitlab-runner \
       gitlab/gitlab-runner:latest

       # We need $HOME/.docker/docker.json for ECR configs
       # We want to try and share images with the host for faster repeated builds
       # We want to make this available in the container without forking from upstream
  }
}
case "${1:-}" in

  up) ## ONLY CREATE/START DOCKER CONTAINER FOR GITLAB
    host_runner
    ;;

  job) ## TO RUN RUNNER AFTER GITLAB CONTAINER HAS STARTED
    host_runner
    gitlabLoop ${2}
    ;;

  down) ## TO STOP AND CLEANUP GITLAB CONTAINER
    gitlab_down
    ;;

  -h) usage; exit 0
    ;;

  '')
     usage
     echo "User didn't pass job/stage to run, defaulting to 'build'"
     sleep 2
     host_runner
     gitlabLoop ${1:-build}
    ;;

  *)
    usage
    host_runner
    gitlabLoop ${1:-build}
   ;;

esac