Skip to content

Instantly share code, notes, and snippets.

@dmjio

dmjio/.gitlab-ci.yml

Created April 11, 2022 21:31
Show Gist options
  • Save dmjio/6abcd272f653094584af2c6f80605073 to your computer and use it in GitHub Desktop.
Save dmjio/6abcd272f653094584af2c6f80605073 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
.gitlab-ci.yml
################################################################################
# GitLab docs on types of pipelines:
# https://docs.gitlab.com/ee/ci/pipelines/pipeline_architectures.html
#
# This one is set to be a basic pipeline of the following steps:
#
# build binaries -> test binaries -> build and push image -> deploy image
#
# we do fan out during deploy and push to all environments at once since our
# deployes are driven by infra changes
#
# of important note is this
# > Jobs on the same stage run in parallel
# we exploit this when we deploy
################################################################################
### CHEDR GitLab is currently broken into 3 stages.
stages:
- analysis_build_and_tests
- publish_docs
- build_image # Run on merges to master
- push_and_scan_image # Run on merges to master
- deploy # Run only on commits containing 'DEPLOY' or 'FAIL'
- report # Run only on commits containing 'DEPLOY' or 'FAIL'
- replay # Run only on commits containing 'REPLAY'
# Required for the stack build tool
variables:
STACK_ROOT: "${CI_PROJECT_DIR}/.stack"
BACKSTAGE_URL: https://4crpg59pwd.execute-api.us-east-1.amazonaws.com/stage/deployments
# Disable double pipeline creation
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
# Default CHEDR docker image
default:
image: us.gcr.io/heb-cx-nonprod/chedr-dev-docker-cx:40a02ac7ab8c6f2659d8dd0a74bac09ffe663dc7
tags:
- gcp_large
# always run analysis as a way to get parallelism and a hack so deploy commits get a pipeline
shellcheck:
tags:
- gcp_small
stage: analysis_build_and_tests
script:
- sh scripts/shellcheck.sh
interruptible:
true
hlint:
tags:
- gcp_small
stage: analysis_build_and_tests
rules:
- if: $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
cache:
key:
files:
- stack.yaml
prefix: hlint-stack-v1
paths:
- .stack/
- .stack-work
script:
- sh scripts/hlint.sh
interruptible:
true
format-haskell:
tags:
- gcp_small
stage: analysis_build_and_tests
rules:
- if: $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
script:
- sh scripts/format-haskell.sh
interruptible:
true
modulint:
tags:
- gcp_small
stage: analysis_build_and_tests
rules:
- if: $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
cache:
key:
files:
- stack.yaml
prefix: modulint-stack-v1
paths:
- .stack/
- .stack-work
script:
- sh scripts/modulint.sh
interruptible:
true
# Run on all commits to any branch, except for commits containing 'DEPLOY' or 'REPLAY'.
.build_executable_run_tests: &build_executable_run_tests_core
tags:
- gcp_large
stage: analysis_build_and_tests
before_script:
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud auth configure-docker -q
cache:
key:
files:
- stack.yaml
- package.yaml
prefix: stack-v5
paths:
- .stack/
script:
- ./scripts/build.sh
- ./scripts/charcuterie-box.sh
artifacts:
paths:
- .stack-work/dist/charcuterie-exe
- .stack-work/dist/chedr-dev-tools
build_executable_run_tests_non_master:
<<: *build_executable_run_tests_core
rules:
- if: $CI_COMMIT_BRANCH !~ $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
interruptible:
true
build_executable_run_tests_master:
<<: *build_executable_run_tests_core
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
after_script:
- .stack-work/dist/chedr-dev-tools publish-confluence-docs
interruptible:
false
build_image:
tags:
- gcp_small
stage: build_image
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
before_script:
- export IMAGE_NAME=charcuterie
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_NONPROD
- gcloud auth configure-docker -q
script:
- cp .stack-work/dist/charcuterie-exe ./containers/deploy/charcuterie/
- cp ./scripts/wait-for-postgres.sh ./containers/deploy/charcuterie
- docker build --pull ./containers/deploy/charcuterie -f ./containers/deploy/charcuterie/Dockerfile -t "${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- docker save "${IMAGE_NAME}:${CI_COMMIT_SHA}" -o charcuterie-image.tar.gz
artifacts:
paths:
- ./charcuterie-image.tar.gz
build_base_image:
stage: build_image
tags:
- gcp_small
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
before_script:
- export IMAGE_NAME=base-deploy-image-buster-slim
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_NONPROD
- gcloud auth configure-docker -q
script:
- cp ./containers/deploy/certs/* ./containers/deploy/
- docker build --pull ./containers/deploy -f ./containers/deploy/Dockerfile.base -t "${IMAGE_NAME}:${CI_COMMIT_SHA}"
- docker save "${IMAGE_NAME}:${CI_COMMIT_SHA}" -o "${IMAGE_NAME}.tar.gz"
artifacts:
paths:
- ./base-deploy-image-buster-slim.tar.gz
push_base_image:
stage: push_and_scan_image
dependencies:
- build_base_image
tags:
- gcp_small
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
before_script:
- export IMAGE_NAME=base-deploy-image-buster-slim
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_NONPROD
- gcloud auth configure-docker -q
script:
- docker load < "${IMAGE_NAME}.tar.gz"
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_NONPROD
- docker tag "${IMAGE_NAME}":"${CI_COMMIT_SHA}" us.gcr.io/heb-cx-nonprod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- docker push us.gcr.io/heb-cx-nonprod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_PROD
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_PROD
- docker tag "${IMAGE_NAME}":"${CI_COMMIT_SHA}" us.gcr.io/heb-cx-prod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- docker push us.gcr.io/heb-cx-prod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
push_image:
tags:
- gcp_small
stage: push_and_scan_image
dependencies:
- build_image
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
before_script:
- export IMAGE_NAME=charcuterie
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_NONPROD
- gcloud auth configure-docker -q
script:
- docker load < charcuterie-image.tar.gz
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_NONPROD
- docker tag "${IMAGE_NAME}":"${CI_COMMIT_SHA}" us.gcr.io/heb-cx-nonprod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- docker push us.gcr.io/heb-cx-nonprod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_PROD
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_PROD
- docker tag "${IMAGE_NAME}":"${CI_COMMIT_SHA}" us.gcr.io/heb-cx-prod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
- docker push us.gcr.io/heb-cx-prod/"${IMAGE_NAME}":"${CI_COMMIT_SHA}"
scan_image:
tags:
- gcp_small
stage: push_and_scan_image
dependencies:
- build_image
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
allow_failure: true
before_script:
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_NONPROD
- gcloud auth configure-docker -q
script:
- docker load < charcuterie-image.tar.gz
- scripts/gitlab/scan-image.sh charcuterie
scan_base_image:
tags:
- gcp_small
stage: push_and_scan_image
dependencies:
- build_base_image
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
allow_failure: true
before_script:
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_NONPROD
- gcloud auth configure-docker -q
script:
- docker load < base-deploy-image-buster-slim.tar.gz
- scripts/gitlab/scan-image.sh base-deploy-image-buster-slim
auto_deploy_dev:
tags:
- gcp_small
stage: deploy
environment:
name: cx-dev
deployment_tier: development
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-dev --region us-central1 --project heb-cx-nonprod
script:
- ./infrastructure/scripts/auto-deploy.sh cx-dev
auto_deploy_cert:
tags:
- gcp_small
stage: deploy
environment:
name: cx-cert
deployment_tier: development
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /DEPLOY/ && $CI_COMMIT_MESSAGE !~ /REPLAY/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-cert --region us-central1 --project heb-cx-nonprod
script:
- ./infrastructure/scripts/auto-deploy.sh cx-cert
deploy_dev:
tags:
- gcp_small
stage: deploy
environment:
name: cx-dev
deployment_tier: development
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /DEPLOY-DEV/ && $CI_COMMIT_MESSAGE !~ /DEPLOY-(CERT|PREPROD|PROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-dev --region us-central1 --project heb-cx-nonprod
script:
- ./infrastructure/scripts/deploy.sh -c cx-dev
deploy_cert:
tags:
- gcp_small
stage: deploy
environment:
name: cx-cert
deployment_tier: development
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /DEPLOY-CERT/ && $CI_COMMIT_MESSAGE !~ /DEPLOY-(DEV|PREPROD|PROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-cert --region us-central1 --project heb-cx-nonprod
script:
- ./infrastructure/scripts/deploy.sh -c cx-cert
deploy_preprod:
tags:
- gcp_small
stage: deploy
environment:
name: cx-preprod
deployment_tier: staging
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /DEPLOY-PREPROD/ && $CI_COMMIT_MESSAGE !~ /DEPLOY-(DEV|CERT|PROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_PROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-preprod --region us-central1 --project heb-cx-prod
script:
- ./infrastructure/scripts/deploy.sh -c cx-preprod
deploy_prod:
tags:
- gcp_small
stage: deploy
environment:
name: cx-prod
deployment_tier: production
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /DEPLOY-PROD/ && $CI_COMMIT_MESSAGE !~ /DEPLOY-(DEV|CERT|PREPROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_PROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-prod --region us-central1 --project heb-cx-prod
script:
- ./infrastructure/scripts/deploy.sh -c cx-prod
report_successful_deploy:
tags:
- gcp_small
stage: report
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /DEPLOY-PROD/ && $CI_COMMIT_MESSAGE !~ /FAIL/ && $CI_COMMIT_MESSAGE !~ /DEPLOY-(DEV|CERT|PREPROD)/
before_script:
- export BACKSTAGE_URL=$BACKSTAGE_URL
- export BACKSTAGE_API_KEY=$BACKSTAGE_API_KEY
script:
- ./infrastructure/scripts/report-deploy.sh
report_fail_deploy:
tags:
- gcp_small
stage: report
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /DEPLOY-PROD/ && $CI_COMMIT_MESSAGE =~ /FAIL/ && $CI_COMMIT_MESSAGE !~ /DEPLOY-(DEV|CERT|PREPROD)/
before_script:
- export BACKSTAGE_URL=$BACKSTAGE_URL
- export BACKSTAGE_API_KEY=$BACKSTAGE_API_KEY
script:
- ./infrastructure/scripts/report-deploy.sh -f
replay_dev:
tags:
- gcp_small
stage: replay
environment:
name: cx-dev
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /REPLAY-DEV/ && $CI_COMMIT_MESSAGE !~ /REPLAY-(CERT|PREPROD|PROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-dev --region us-central1 --project heb-cx-nonprod
script:
- ./infrastructure/scripts/deploy.sh -c cx-dev --replay
replay_cert:
tags:
- gcp_small
stage: replay
environment:
name: cx-cert
deployment_tier: development
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /REPLAY-CERT/ && $CI_COMMIT_MESSAGE !~ /REPLAY-(DEV|PREPROD|PROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_NONPROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-cert --region us-central1 --project heb-cx-nonprod
script:
- ./infrastructure/scripts/deploy.sh -c cx-cert --replay
replay_preprod:
tags:
- gcp_small
stage: replay
environment:
name: cx-preprod
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /REPLAY-PREPROD/ && $CI_COMMIT_MESSAGE !~ /REPLAY-(DEV|CERT|PROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_PROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-preprod --region us-central1 --project heb-cx-prod
script:
- ./infrastructure/scripts/deploy.sh -c cx-preprod --replay
replay_prod:
tags:
- gcp_small
stage: replay
environment:
name: cx-prod
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /REPLAY-PROD/ && $CI_COMMIT_MESSAGE !~ /REPLAY-(DEV|CERT|PREPROD)/
before_script:
- export PROJECT_DIR=$CI_PROJECT_DIR
- export GOOGLE_KEY_CX=$GOOGLE_KEY_PROD
- export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_KEY_CX
- gcloud auth activate-service-account --key-file=$GOOGLE_KEY_CX
- gcloud container clusters get-credentials cx-prod --region us-central1 --project heb-cx-prod
script:
- ./infrastructure/scripts/deploy.sh -c cx-prod --replay
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment