CI+CD pipeline for k8s nodejs deployment with helm

.gitlab-ci.yaml

stages:
  - test
  - docker-build
  - version-push
  - deploy
  - cleanup

variables:
  # The prefix to add to commit messages that only include upgrades of versions
  # Due to the limitations of the workflow expressions. You must also update the workflow rules below if you change this
  # This text is matched with regular expression below. Regex characters such as `[` must be escaped!
  # For more info on the problem see: https://stackoverflow.com/questions/2933474/escape-characters-contained-by-bash-variable-in-regex-pattern
  BUMP_VERSION_MESSAGE: "[bump version]"

  # When a new commit is pushed or merged into master, does that commit get automatically deployed to the development.1 environment
  CD_TO_DEVEOPMENT_1_FROM_MASTER: "true"

  # The prefix to add to each container registry image tag for images that are created for commits. Does not apply to registry tags created for git tags.
  CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX: "commit-"

  # How long container tag that are created for commits will survive. Container registry tags created for git tags will survive forever.
  # Can be in format: `1d` `1h` `1month`
  # See: https://docs.gitlab.com/ee/api/container_registry.html#delete-registry-repository-tags-in-bulk
  CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_LIFETIME: "3month"

# Only run when there is [bump version] in the commit message and rev type is a tag or there is not [bump version] in the commit message
workflow:
  rules:
    - if: $CI_COMMIT_MESSAGE =~ /\[bump version\]/ && $CI_COMMIT_TAG
      when: always
    - if: $CI_COMMIT_MESSAGE =~ /\[bump version\]/ && $CI_COMMIT_BRANCH
      when: never
    - if: $CI_COMMIT_MESSAGE !~ /\[bump version\]/
      when: always

.version-validate: &version-validate
  - >
      echo $CI_PROJECT_ID;
      latest_tag_json=$(curl -X GET --header "Private-Token: $SHARED_CI_ACCESS_TOKEN" "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/tags?per_page=1");
      echo "$latest_tag_json";
      latest_tag_length=$(echo "$latest_tag_json" | jq '. | length');
      latest_tag_name=;
      if [[ $latest_tag_length > 0 ]]; then
        latest_tag_name=$(echo "$latest_tag_json" | jq -r '.[0].name');
        package_version=$(jq -r '.version' app/package.json);
        if [[ "$package_version" != "$latest_tag_name" ]]; then
          echo -e "\e[31mVersion does not match latest tag. Please change version in package.json to '$latest_tag_name'\e[39m";
          exit 1;
        fi;
      fi;

.git-configure: &git-configure
  - echo "$GIT_SSH_PK" | base64 -d > id_rsa
  - chmod 400 id_rsa
  - echo 'ssh -i ./id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $*' > ssh
  - chmod +x ssh
  - git config --global user.name "$GITLAB_USER_NAME"
  - git config --global user.email "$GITLAB_USER_EMAIL"
  - git remote set-url origin git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git

test:
  image: cryptexlabs/ts-node-ci:1.2.0
  stage: test
  allow_failure: false
  except:
    refs:
      - tags
      - schedules
  script:
    - *version-validate
    - export NODE_ENV=test
    - cd app
    - yarn install
    - yarn lint
    - yarn test

.docker-variables: &docker-variables
  DOCKER_TLS_CERTDIR: ""
  DOCKER_DRIVER: overlay2
  DOCKER_HOST: tcp://docker:2375

docker-build:
  image: cryptexlabs/ts-node-ci:1.2.0
  stage: docker-build
  allow_failure: false
  needs:
    - test
  services:
    - docker:dind
  variables:
    <<: *docker-variables
  except:
    refs:
      - tags
      - schedules
  script:
    - docker build --cache-from $CI_REGISTRY_IMAGE:latest . --tag $CI_REGISTRY_IMAGE:$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
    - docker push $CI_REGISTRY_IMAGE:$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA

.version-push: &version-push
  image: cryptexlabs/ts-node-ci:1.2.0
  stage: version-push
  needs:
    - test
    - docker-build
  when: manual
  allow_failure: false
  services:
    - docker:dind
  only:
    refs:
      - master
  except:
    refs:
      - schedules
  script:
    - *version-validate
    - *git-configure
    - git checkout master
    - BASE_DIR=$PWD
    - cd app
    - yarn version --$INCREMENT_TYPE --no-git-tag-version --no-commit-hooks
    - version=$(jq -r '.version' package.json)
    - cd $BASE_DIR
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
    - docker pull $CI_REGISTRY_IMAGE:$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA
    - docker tag $CI_REGISTRY_IMAGE:$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA $CI_REGISTRY_IMAGE:$version
    - docker tag $CI_REGISTRY_IMAGE:$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA $CI_REGISTRY_IMAGE:latest
    - yq w -i k8s/$CI_PROJECT_NAME/Chart.yaml appVersion "$version"
    - yq w -i k8s/$CI_PROJECT_NAME/values.yaml nodejs.image.tag "$version"
    - git add k8s/$CI_PROJECT_NAME/Chart.yaml k8s/$CI_PROJECT_NAME/values.yaml app/package.json
    - git commit -m "$BUMP_VERSION_MESSAGE $version"
    - git tag $version
    - GIT_SSH='./ssh' git push origin $version
    - GIT_SSH='./ssh' git push origin master
    - docker push $CI_REGISTRY_IMAGE:$version
    - docker push $CI_REGISTRY_IMAGE:latest

patch:
  <<: *version-push
  variables:
    INCREMENT_TYPE: patch
    <<: *docker-variables

minor:
  <<: *version-push
  variables:
    INCREMENT_TYPE: minor
    <<: *docker-variables

major:
  <<: *version-push
  variables:
    INCREMENT_TYPE: major
    <<: *docker-variables

.deploy-script: &deploy-script
  - >
      if [[ -z "$(kubectl get namespace $KUBE_NAMESPACE)" ]]; then
        kubectl create namespace $KUBE_NAMESPACE;
      fi;
  - >
      if [[ -z "$(kubectl get secret gitlab-registry --namespace $KUBE_NAMESPACE)" ]]; then
        kubectl create secret docker-registry gitlab-registry \
        --docker-server=https://registry.gitlab.com \
        --docker-username=$CI_REGISTRY_USER \
        --docker-password=$SHARED_CI_ACCESS_TOKEN \
        --namespace $KUBE_NAMESPACE;
      fi;
  -  helm dep up $CI_PROJECT_NAME
  - >
      helm upgrade $CI_PROJECT_NAME k8s/$CI_PROJECT_NAME
      --install
      --namespace $KUBE_NAMESPACE
      --set nodejs.annotations.'app\.gitlab\.com/env'=$CI_ENVIRONMENT_SLUG
      --set nodejs.annotations.'app\.gitlab\.com/app'=$CI_PROJECT_PATH_SLUG
      --set nodejs.namespace=$KUBE_NAMESPACE
      --set nodejs.image.tag=$APP_VERSION

.deploy: &deploy
  image:
    name: cryptexlabs/helm-yq:3.2.4
    entrypoint: [""]
  stage: deploy
  allow_failure: false

.rev-deploy: &rev-deploy
  <<: *deploy
  script:
    - yq w -i k8s/$CI_PROJECT_NAME/Chart.yaml appVersion "$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA"
    - APP_VERSION=$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX$CI_COMMIT_SHORT_SHA
    - *deploy-script

.tag-deploy: &tag-deploy
  <<: *deploy
  script:
    - APP_VERSION=$CI_COMMIT_TAG
    - *deploy-script

deploy-development-1-ref:
  <<: *rev-deploy
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - if: $CI_COMMIT_BRANCH == "master" && $CD_TO_DEVEOPMENT_1_FROM_MASTER == "true"
      # Makes continuous deployment to dev environment from master
      when: on_success
    - if: $CI_COMMIT_BRANCH != "master" && $CI_COMMIT_BRANCH
      when: manual
  needs:
    - test
    - docker-build
  environment:
    name: development.1

deploy-development-1:
  <<: *tag-deploy
  # Makes continuous deployment to dev environment from all new tags
  when: on_success
  only:
    refs:
      - tags
  except:
    refs:
      - schedules
  environment:
    name: development.1

deploy-production-1:
  <<: *tag-deploy
  when: manual
  only:
    refs:
      - tags
  except:
    refs:
      - schedules
  environment:
    name: production.1

cleanup-image-tags:
  stage: cleanup
  image: cryptexlabs/ts-node-ci:1.2.0
  rules:
    - if: $SCHEDULE_NAME == 'cleanup-image-tags' && $CI_PIPELINE_SOURCE == "schedule"
      when: always
  # Issues regarding "regex" in name_regex_delete https://gitlab.com/gitlab-org/gitlab/-/issues/27072#note_239340436
  # Throttling of bulk delete API: https://gitlab.com/gitlab-org/gitlab/-/issues/32900
  script:
    - >
        registries_json=$(curl -X GET --header "Private-Token: $SHARED_CI_ACCESS_TOKEN" "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/registry/repositories?per_page=1");

        registries_length=$(echo "$registries_json" | jq '. | length');

        if [[ $registries_length > 0 ]]; then
          registry_id=$(echo "$registries_json" | jq -r '.[0].id');
          curl -X DELETE \
          --fail \
          --header "Private-Token: $SHARED_CI_ACCESS_TOKEN" \
          --data "name_regex_delete=^$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_PREFIX.*" \
          --data "older_than=$CONTAINER_REGISTRY_IMAGE_TAG_FOR_COMMIT_LIFETIME" \
          "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/registry/repositories/$registry_id/tags";
        fi;

README.md

Some important notes:

1. You need to create SHARED_CI_ACCESS_TOKEN which is a PERSONAL yes I said PERSONAL access token with read and write access. At some point gitlab will probably have project level access tokens but for now the only way to get a lot of features to work is to use your own personal access token. Maybe you will want to use a service account but service accounts can be expensive. If you leave the organization, all the things that used your access token will suddenly break after you leave! Sounds funny unless you're not the person leaving... :(.
2. You need to create a UNMANAGED kubernetes cluster in gitlab. That means that the checkbox that enables gitlab to manage it must be UNCHECKED. That kubernetes cluster must be matched to the environment development.1
3. You should have a separate kubernetes cluster for each environment but this pipeline would still work for multiple environments on a single cluster provided you have enough resources. Using same k8s cluster for multiple environments is definitely not recommended.
4. You need to create GIT_SSH_PK which is a base64 encoded private key for a deploy key. Not a deploy token. A deploy key that has WRITE access to the repository.
5. You need to install the gitlab runner on your kubernetes cluster. You can do this through the gilab ui under the Applications tab of your settings for your cluster.
6. You need to have a helm chart which sets the values of .Values.nodejs.annotations to the meta.annotations of the deployment specification for the helm chart. I haven't tested whether or not daemonsets or statefulsets work.
7. This of course could be modified for non nodejs apps. Just switch out the cryptexlabs/ts-node-ci:1.2.0 with an image that has all the tools you need installed
8. WARNING: Automatic deletion of tags is enabled by default so that tags more than 90 days old will be automatically deleted. You need to configure expiration policy using one of the two methods:
   1. Use pipeline:
      1. You need to create a schedule to delete old jobs with a variable name SCHEDULE_NAME and value cleanup-image-tags otherwise the job to cleanup old image tags will not run.
      2. Disable the setting in Settings -> CI / CD -> Cleanup policy for tags
   2. Use built in expiration policy:
      1. Go to Settings -> CI / CD -> Cleanup policy for tags
      2. Add an expression ^commit-.* for the setting Tags with names matching this regex pattern will expire: so that only your commit tags will be deleted
9. The pipeline is broken basically into these steps:
   • test: run unit tests and make sure app version in package.json hasn't been modified manually
   • docker-build: builds a docker image for the revision number and pushes it to gitlab ci.
   • version-push: This creates a new semantic version based on whether you chose to run the patch, minor or major job.
     • Having different jobs for each of these things is basically a hack workaround for gitlab not having a dropdown to select these things. For a potential solution to this problem see https://gitlab.com/gitlab-org/gitlab/-/issues/22629
     • Update the version in the npm package.json using yarn version
     • Pull docker image previously built add a tag with the same version as the version in the package.json
     • Update appVersion in the helm Chart.yaml
     • Update nodejs.image.tag in the helm values.yaml
     • Create a commit with the changes in package.json, Chart.yaml, and values.yaml
     • Create a git tag with the same version
     • Push the tag
     • Push the new commit to master
   • deploy:
     • After the docker-build step this pipeline will also automatically create a new deployment to the development.1 environment. You can disable this by changing the rule from when: on_success to when: manual
     • After the version-push step pushes a new tag with the commit message containing [bump version] a new pipeline will be created only for the tag. A pipeline will not be created when pushing a commit containing [bump version] to any branch. So you shouldn't use that in your commit messages.
     • When the deployment pipeline is created after receiving a new tag it will automatically deploy the tag (which is the same image that was previously deployed) to the development.1 environment. You can disable auto deploy by changing when: on_success to when: manual
10. You can deploy manually to the development.1. environment from any pipeline created on any branch without having to create a new tag. The images for these deployments will be eventually deleted automatically so make sure to eventually create a tag.
11. You CANNOT modify the the namespace that assets are deployed to by overriding $KUBE_NAMESPACE variable if you want Deploy Boards to work. For more information on the issue see https://gitlab.com/gitlab-org/gitlab/-/issues/228717
12. Deploy Boards only work on Silver and above plans. Personally $20/month with a 12 month commitment isn't worth it for me considering all the problems.

Pipeline:

Deploy Board:

helm ls:

environment.png

helm-ls.png

pipeline.png