Google Kubernetes Engine as an alternative to Cloud Run

cloudbuild_CloudRun.yaml

 steps:
 # Build the container image
 - name: 'gcr.io/cloud-builders/docker'
   args: ['build', '-t', 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:${_VERSION}', '-f', './${_DIRECTORY_PROJECT}/Dockerfile', '.', '--build-arg',  'directory=${_DIRECTORY_PROJECT}','--build-arg', 'number_workers=${_NB_WORKERS}']
 # Push the container image to Container Registry
 - name: 'gcr.io/cloud-builders/docker'
   args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:${_VERSION}']
 # Deploy container image to Cloud Run
 - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
   entrypoint: gcloud
   args:
   - 'run'
   - 'deploy'
   - '${_SERVICE_NAME}'
   - '--image'
   - 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:${_VERSION}'
   - '--platform'
   - 'managed'
   - '--concurrency'
   - '${_NB_WORKERS}'
   - '--max-instances'
   - '50'
   - '--port'
   - '5000'
   - '--memory'
   - '512Mi'
   - '--cpu'
   - '1'
   - '--timeout'
   - '120'
   - "--set-env-vars"
   - "VARIABLE_1=${_VARIABLE_1},VARIABLE_2=${_VARIABLE_2},PROJECT_ID=${_PROJECT_ID},TOPIC_ID=${_TOPIC_ID}"
 images:
 - 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:${_VERSION}'

cloudbuild_K8s.yaml

steps:
# Build the container image
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:${_VERSION}', '-f', './${_DIRECTORY_PROJECT}/Dockerfile', '.',  '--build-arg', 'directory=${_DIRECTORY_PROJECT}']
# Push the container image to Container Registry
- name: 'gcr.io/cloud-builders/docker'
  args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:${_VERSION}']

deploy_K8s_pod._useful_commands.txt

# To check whether something is happening:
kubectl get pods -n  extraction-containers
# To delete everything:
kubectl delete $( kubectl get pods -n  extraction-containers -o name) -n extraction-containers

deploy_K8s_pod.txt

#bin/bash

# 1
gcloud config set project {PROJECT_ID}
# 2
sudo apt-get install
# 3
gcloud container clusters get-credentials {CLUSTER_NAME} --zone {ZONE_NAME}
# 4
kubectl config use-context gke_{PROJECT_ID}_{ZONE_NAME}_{CLUSTER_NAME}
# 5
kubectl create ns extraction-containers

# 6
cat <<EOF | kubectl apply -n extraction-containers -f  -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: extractor-article
  labels:
    app: extractor-article
spec:
  replicas: 180 # (1)
  selector:
    matchLabels:
      app: extractor-article
  template:
    metadata:
      labels:
        app:  extractor-article
    spec:
      containers:
      - name: extractor-article-container
        image: gcr.io/my_project-3051257/article-extractor@sha256:75bbebdfgb451e3a301fe9dd2cf1facsdfdsf445164cdd2fd0462b78781 # (2)
        imagePullPolicy: Always
        ports:
        - containerPort: 8000
        resources:
          requests: # (3)
            memory: "256Mi"
            cpu: "150m"
          limits: # (4)
            memory: "512Mi"
            cpu: "250m"
        env: # (5)
          - name: VARIABLE_1
            value: "abc"
          - name: VARIABLE_2
            value: "60"
          - name: PROJECT_ID
            value: "my_project-3051257"
          - name: TOPIC_ID
            value: "postextraction-queue"
          - name: SUBSRIPTION_ID
            value: "preextraction-sub"
          - name: TIMEOUT
            value: "60"
          - name: MAX_SENT
            value: "10"
EOF

deploy_K8s_pod_1_4.txt

# 1
gcloud config set project {PROJECT_ID}
# 2
sudo apt-get install
# 3
gcloud container clusters get-credentials {CLUSTER_NAME} --zone {ZONE_NAME}
# 4
kubectl config use-context gke_{PROJECT_ID}_{ZONE_NAME}_{CLUSTER_NAME}

deploy_K8s_pod_5.txt

# 5
kubectl create ns extraction-containers

deploy_K8s_pod_6.txt

# 6
cat <<EOF | kubectl apply -n extraction-containers -f  -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: extractor-article
  labels:
    app: extractor-article
spec:
  replicas: 180 # (1)
  selector:
    matchLabels:
      app: extractor-article
  template:
    metadata:
      labels:
        app:  extractor-article
    spec:
      containers:
      - name: extractor-article-container
        image: gcr.io/my_project-3051257/article-extractor@sha256:75bbebdfgb451e3a301fe9dd2cf1facsdfdsf445164cdd2fd0462b78781 # (2)
        imagePullPolicy: Always
        ports:
        - containerPort: 8000
        resources:
          requests: # (3)
            memory: "256Mi"
            cpu: "150m"
          limits: # (4)
            memory: "512Mi"
            cpu: "250m"
        env: # (5)
          - name: VARIABLE_1
            value: "abc"
          - name: VARIABLE_2
            value: "60"
          - name: PROJECT_ID
            value: "my_project-3051257"
          - name: TOPIC_ID
            value: "postextraction-queue"
          - name: SUBSRIPTION_ID
            value: "preextraction-sub"
          - name: TIMEOUT
            value: "60"
          - name: MAX_SENT
            value: "10"
EOF

deploy_K8s_pod_example.txt

gcloud config set project my_project-3051257
sudo apt-get install
gcloud container clusters get-credentials cluster-article --zone us-central1-a
kubectl config use-context gke_my_project-3051257_us-central1-a_cluster-article

deployment_K8s.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: extractor-article
  labels:
    app: extractor-article
spec:
  replicas: 180 # (1)
  selector:
    matchLabels:
      app: extractor-article
  template:
    metadata:
      labels:
        app:  extractor-article
    spec:
      containers:
      - name: extractor-article-container
        image: gcr.io/my_project-3051257/article-extractor@sha256:75bbebdfgb451e3a301fe9dd2cf1facsdfdsf445164cdd2fd0462b78781 # (2)
        imagePullPolicy: Always
        ports:
        - containerPort: 8000
        resources:
          requests: # (3)
            memory: "256Mi"
            cpu: "150m"
          limits: # (4)
            memory: "512Mi"
            cpu: "250m"
        env: # (5)
          - name: VARIABLE_1
            value: "abc"
          - name: VARIABLE_2
            value: "60"
          - name: PROJECT_ID
            value: "my_project-3051257"
          - name: TOPIC_ID
            value: "postextraction-queue"
          - name: SUBSRIPTION_ID
            value: "preextraction-sub"
          - name: TIMEOUT
            value: "60"
          - name: MAX_SENT
            value: "10"

Dockerfile_CloudRun

# Use Python36
FROM python:3.6

# Arguments
ARG directory
ARG number_workers # number of threads

WORKDIR /app

# Copy requirements.txt to the docker image and install packages
COPY ./${directory}/requirements.txt ./
RUN pip install -r requirements.txt

# Copy all files in the folder app
COPY ./${directory}/ /app/

# Expose port 5000
EXPOSE 5000
ENV PORT 5000
ENV NB ${number_workers}

# Use gunicorn as the entrypoint
CMD exec gunicorn --bind :$PORT main:app --workers 1 --threads $NB --timeout 0

Dockerfile_K8s

# Use Python36
FROM python:3.6

# Arguments to use in docker run
ARG directory

# Copy requirements.txt to the docker image and install packages
COPY ./${directory}/. /app
WORKDIR /app
RUN pip install -r requirements.txt

# Use gunicorn as the entrypoint
CMD [ "python", "./main.py" ]

main_CloudRun.py

"""import default packages"""
import os
import logging
import json
import base64


# Import Global Env Variables
Variable_1 = os.environ(['VARIABLE_1'])
Variable_2 = os.environ(['VARIABLE_2'])
project_id = os.environ(['PROJECT_ID'])
topic_id = os.environ(['TOPIC_ID'])

"""Import downloaded packages"""


from flask import Flask, request
from flask_cors import CORS
from flask_sslify import SSLify


"""Everything related to Google Pub/Sub"""
from google.cloud import pubsub_v1

publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)


# Initialise flask app
app = Flask(__name__)
CORS(app, supports_credentials=True)
sslify = SSLify(app)


def transform_message(input_message):
    try:
        output_message = json.loads(base64.b64decode(input_message).decode('utf-8'))
        return output_message
    except:
        return None


@app.route("/", methods=["POST"])
def extract_article():
    logging.basicConfig(
        level='INFO',
        format='[%(levelname)-5s] %(asctime)s\t-\t%(message)s',
        datefmt='%d/%m/%Y %I:%M:%S %p'
    )

    envelope = request.get_json()
    if not envelope:
        msg = "no Pub/Sub message received"
        logging.info(f"ERROR: {msg}")
        return {'message': f"Bad Request: {msg}",
                'status': 'error'}, 400

    if not isinstance(envelope, dict) or "message" not in envelope:
        msg = "invalid Pub/Sub message format"
        logging.info(f"ERROR: {msg}")
        return {'message': {msg},
                'status': 'error'}, 400

    pubsub_message = envelope["message"]

    if isinstance(pubsub_message, dict) and "data" in pubsub_message:
        message = transform_message(pubsub_message["data"])
        if not message:
            logging.info(f'Impossible to consume the message {str(pubsub_message["data"])}')
            msg = "Impossible to transform the message from Pub/Sub"
            return {"message": f"Error: {msg}",
                    'status': 'error'}, 400

    """Function continues"""



    message_to_send = json.dumps({'title': 'Toyota reveals a hybrid pickup at last—but it\'s probably not what you think',
                                  'summary': '	After more than 20 years of hybrid sales in the U.S.,'
                                             ' it\'s hard to believe that Toyota hasn\'t made a single hybrid pickup.'
                                             ' The wait is at last over. But no, the hybrid pickup Toyota\'s delivering isn\'t'
                                             ' going to land anywhere close to the 40-mpg Ford Maverick hybrid pickup revealed'
                                             ' earlier this year.Toyota\'s jumping in the hybrid truck arena with the arrival'
                                             ' of a completely redesigned 2022 Toyota Tundra—its full-size truck'
                                             ' that has been one of the thirstiest models in its peer set'
                                             ' (14 mpg combined, for the 2021 4WD models).'}).encode('utf-8')

    try:
        publish_future = publisher.publish(topic_path, data=message_to_send)
        publish_future.result()
    except Exception as e:
        logging.info(f"ERROR: while sending a message")

if __name__ == "__main__":
    app.run(ssl_context="adhoc", host="0.0.0.0", port=5000)

main_K8s.py

"""Import packages"""
import os
import json
import logging


# Import Global Env Variables
Variable_1 = os.environ(['VARIABLE_1'])
Variable_2 = os.environ(['VARIABLE_2'])
project_id = os.environ(['PROJECT_ID'])
topic_id = os.environ['TOPIC_ID']
subscription_id = os.environ(['SUBSRIPTION_ID'])
timeout = os.environ(['TIMEOUT'])
max_messages_sent = os.environ(['MAX_SENT'])

"""Everything related to Google Pub/Sub"""
from google.cloud import pubsub_v1

# Subscriber which extract news article message from the pull
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id, subscription_id)
flow_control = pubsub_v1.types.FlowControl(max_messages=max_messages_sent)

# Topic where the extracted article is sent to
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)



def consume_message(message_input):
    message = json.loads(message_input.data.decode('utf-8'))

    """Function continue and extract data from an article"""

    message_to_send = json.dumps(
        {'title': 'Toyota reveals a hybrid pickup at last—but it\'s probably not what you think',
         'summary': '	After more than 20 years of hybrid sales in the U.S.,'
                    ' it\'s hard to believe that Toyota hasn\'t made a single hybrid pickup.'
                    ' The wait is at last over. But no, the hybrid pickup Toyota\'s delivering isn\'t'
                    ' going to land anywhere close to the 40-mpg Ford Maverick hybrid pickup revealed'
                    ' earlier this year.Toyota\'s jumping in the hybrid truck arena with the arrival'
                    ' of a completely redesigned 2022 Toyota Tundra—its full-size truck'
                    ' that has been one of the thirstiest models in its peer set'
                    ' (14 mpg combined, for the 2021 4WD models).'}).encode('utf-8')

    try:
        publish_future = publisher.publish(topic_path, data=message_to_send)
        publish_future.result()
    except Exception as e:
        logging.error(e)

    message_input.ack()


if __name__ == "__main__":
    while True:
        streaming_pull_future = subscriber.subscribe(subscription_path, callback=consume_message, flow_control=flow_control)
        try:
            streaming_pull_future.result(timeout=timeout)
        except Exception as e:
            logging.error(e)
            streaming_pull_future.cancel()
            subscriber.close()
            subscriber = pubsub_v1.SubscriberClient()
            subscription_path = subscriber.subscription_path(project_id, subscription_id)
            flow_control = pubsub_v1.types.FlowControl(max_messages=max_messages_sent)

requirements_CloudRun.txt

flask==1.1.1
Flask-Cors==3.0.10
Flask-SSLify==0.1.5
google-cloud-pubsub==2.4.0

requirements_K8s.txt

google-cloud-pubsub==2.4.0