K8.Testing.CloudBuild

cloudbuild-docker.yaml

# A Cloud Build file for the Docker image. We run it manually only on when we update the container.
steps:

- name: 'gcr.io/cloud-builders/docker'
  id: 'build_image'
  args:
    - 'build'
    - '--tag=gcr.io/$PROJECT_ID/cloud-builders-python:latest'
    - '--tag=gcr.io/$PROJECT_ID/cloud-builders-python:$SHORT_SHA'
    - '.'
  dir: 'external/cloud-builders-python'

images:
- 'gcr.io/$PROJECT_ID/cloud-builders-python:latest'
- 'gcr.io/$PROJECT_ID/cloud-builders-python:$SHORT_SHA'

cloudbuild-main.yaml

steps:

# ......

 # Create namespace first, otherwise K8 will complain 
  - name: 'gcr.io/cloud-builders/kubectl'
    args: ['apply', '-f', 'deployment/kubernetes/namespace.yaml']
    env:
      - 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE'
      - 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER'

 # Create configmap as Tests need it
  - name: 'gcr.io/cloud-builders/kubectl'
    args: ['apply', '-f', 'deployment/kubernetes/deployments/0.configmap.yaml']
    env:
      - 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE'
      - 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER'
      
# Remove any old tests and run new one
  - name: gcr.io/cloud-builders/gcloud
    entrypoint: '/bin/bash'
    args:
      - '-c'
      - |
        echo "=========================================================="
        [[ "$_ENVR" == "prod" ]] && exit 0 || echo "Running Unit Tests"
        echo "[Deleting old job]"
        kubectl delete -f deployment/kubernetes/jobs/tests.yaml
        echo "[Applying new job]"
        kubectl apply -f deployment/kubernetes/jobs/tests.yaml 
    env:
      - 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE'
      - 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER'

  # Check if Test Job has completed
  - name: 'gcr.io/$PROJECT_ID/cloud-builders-python:latest'
    args: ['python3', 'deployment/scripts/k8_watch_job.py']
    env:
      - 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE'
      - 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER'
      - '_ENVR=$_ENVR'
      - 'JOB_NAME=api-test'

  # Apply K8 Deployment & Services
  - name: 'gcr.io/cloud-builders/kubectl'
    args: ['apply', '-f', 'deployment/kubernetes/deployments',]
    env:
      - 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE'
      - 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER'

Dockerfile

FROM gcr.io/cloud-builders/gcloud

RUN apt-get -y update && \
    apt-get -y install python3 python3-dev python3-setuptools python3-pip

RUN pip3 install PyYAML Jinja2 simplejson kubernetes \
    google-cloud-storage google-api-python-client

RUN apt-get -y remove python-dev python-setuptools wget && \
    rm -rf /var/lib/apt/lists/* && \
    rm -rf ~/.config/gcloud && \
    chmod +x /entrypoint.bash
    
COPY entrypoint.bash /entrypoint.bash
ENTRYPOINT ["/entrypoint.bash"]

entrypoint.bash

#!/bin/bash

set -e
#######################################################################################
# A Docker ENTRYPOINT script that authenticates with GCP Kubernetes and then runs any CMD
# Requires CLOUDSDK_CONTAINER_CLUSTER, CLOUDSDK_COMPUTE_REGION and CLOUDSDK_COMPUTE_ZONE
#  env variables set 
# https://success.docker.com/article/use-a-script-to-initialize-stateful-container-data
# https://github.com/kubernetes-client/python/blob/master/kubernetes/client/api/batch_v1_api.py
#
#######################################################################################

# If there is no current context, get one.
if [[ $(kubectl config current-context 2> /dev/null) == "" ]]; then
    # This tries to read environment variables. If not set, it grabs from gcloud
    cluster=${CLOUDSDK_CONTAINER_CLUSTER:-$(gcloud config get-value container/cluster 2> /dev/null)}
    region=${CLOUDSDK_COMPUTE_REGION:-$(gcloud config get-value compute/region 2> /dev/null)}
    zone=${CLOUDSDK_COMPUTE_ZONE:-$(gcloud config get-value compute/zone 2> /dev/null)}
    project=${GCLOUD_PROJECT:-$(gcloud config get-value core/project 2> /dev/null)}

    function var_usage() {
        cat <<EOF
No cluster is set. To set the cluster (and the region/zone where it is found), set the environment variables
  CLOUDSDK_COMPUTE_REGION={region} (regional clusters)
  CLOUDSDK_COMPUTE_ZONE=${zone} (zonal clusters)
  CLOUDSDK_CONTAINER_CLUSTER=${cluster}
EOF
        exit 1
    }

    [[ -z "$cluster" ]] && var_usage
    [ ! "$zone" -o "$region" ] && var_usage

    if [ -n "$region" ]; then
      echo "Running: gcloud config set container/use_v1_api_client false"
      gcloud config set container/use_v1_api_client false
      echo "Running: gcloud beta container clusters get-credentials --project=\"$project\" --region=\"$region\" \"$cluster\""
      gcloud beta container clusters get-credentials --project="$project" --region="$region" "$cluster" || exit
    else
      echo "Running: gcloud container clusters get-credentials --project=\"$project\" --zone=\"$zone\" \"$cluster\""
      gcloud container clusters get-credentials --project="$project" --zone="$zone" "$cluster" || exit
    fi
fi


eval "$@"
exitCode=$?
exit $exitCode

job_status_output.py

# Bellow are the 3 example outputs of read_namespaced_job_status(),
# I collected during the development of this:

SUCCESS = {'status': {'active': None,
            'completion_time': datetime.datetime(2020, 2, 4, 15, 32, 52, tzinfo=tzlocal()),
            'conditions': [{'last_probe_time': datetime.datetime(2020, 2, 4, 15, 32, 52, tzinfo=tzlocal()),
                            'last_transition_time': datetime.datetime(2020, 2, 4, 15, 32, 52, tzinfo=tzlocal()),
                            'message': None,
                            'reason': None,
                            'status': 'True',
                            'type': 'Complete'}],
            'failed': None,
            'start_time': datetime.datetime(2020, 2, 4, 15, 32, 35, tzinfo=tzlocal()),
            'succeeded': 1}}

ONGOING = {'status': {'active': 1,
             'completion_time': None,
             'conditions': None,
             'failed': None,
             'start_time': datetime.datetime(2020, 2, 5, 16, 4, 20, tzinfo=tzlocal()),
             'succeeded': None}}


FAILED ='status': {'active': None,
            'completion_time': None,
            'conditions': [{'last_probe_time': datetime.datetime(2020, 2, 5, 16, 6, 18, tzinfo=tzlocal()),
                            'last_transition_time': datetime.datetime(2020, 2, 5, 16, 6, 18, tzinfo=tzlocal()),
                            'message': 'Job has reached the specified backoff '
                                       'limit',
                            'reason': 'BackoffLimitExceeded',
                            'status': 'True',
                            'type': 'Failed'}],
            'failed': None,
            'start_time': datetime.datetime(2020, 2, 5, 16, 4, 20, tzinfo=tzlocal()),
            'succeeded': None}}

k8_watch_job.py

"""
A tool that uses the kubernetes API to "watch" a K8 job till completion.
If the job failed for any reason an exception is raised with the failure as a message.

The script expects the folling ENV vars:
- JOB_NAME: the name of the Kubernetes Job
- _ENVR: the namespace in Kubernetes

In addition the kubernetes.config expects that there is a `~/.kube/config` file that holds the K8 credentials
"""

import os
import time
import sys

from kubernetes import client, config

config.load_kube_config()
kclient = client.BatchV1Api()


def check_google_auth():
    from google.auth import compute_engine
    credentials = compute_engine.Credentials()
    print(credentials)


def get_timeout():
    timeout = os.environ.get('TIMEOUT')
    if timeout is not None:
        return int(timeout)
    else:
        return 300


def watch_job(timeout=300, **kwargs):
    """
    :param timeout: max seconds/cycles to run the script for
    :param kwargs: arguments to be passed to `read_namespaced_job_status()`
    :return: True if success, Exception otherwise
    """
    active = True
    this_job = None
    counter = timeout

    while active:
        if counter == 0:
            raise Exception('Timed out after: {}s'.format(timeout))
        this_job = kclient.read_namespaced_job_status(**kwargs)
        active = bool(this_job._status.active)
        time.sleep(1)
        counter -= 1

    if bool(this_job._status.succeeded):
        print("Job completed successfully")
        return True
    elif this_job._status.succeeded is None:
        print("Job failed")
        raise Exception(this_job._status.conditions[0].reason)
    else:
        msg = "Job failed for unknown reason"
        print(msg)
        print(this_job._status)
        raise Exception(msg)


if __name__ == '__main__':
    if os.environ['_ENVR'] == 'prod':
        # The tests don't run in prod, so this is a 'temporary' workaround
        sys.exit(0)
    watch_job(
        timeout=get_timeout(),
        name=os.environ['JOB_NAME'],
        namespace=os.environ['_ENVR']
    )