Zuna tutorial code

.gcloudignore

# This file specifies files that are *not* uploaded to Google Cloud Platform
# using gcloud. It follows the same syntax as .gitignore, with the addition of
# "#!include" directives (which insert the entries of the given .gitignore-style
# file at that point).
#
# For more information, run:
#   $ gcloud topic gcloudignore
#
.gcloudignore
# If you would like to upload your .git directory, .gitignore file or files
# from your .gitignore file, remove the corresponding line
# below:
.git
.gitignore

node_modules
#!include:.gitignore

.gitignore

*-key.json
sa-zuna

001_create_resources.sh

#!/usr/bin/env bash

GCP_PROJECT_ID=$(gcloud config get-value project)
LOCATION=EU

# Create Pub/Sub topics
gcloud pubsub topics create \
    test-zuna-topic \
    --labels=department=engineering

# Create Pub/Sub subscription that should be removed
gcloud pubsub subscriptions create \
    test-zuna-subscription \
    --topic=test-zuna-topic \
    --labels=department=engineering,autodelete=true

# Create Pub/Sub subscription that should NOT be removed
gcloud pubsub subscriptions create \
    test-zuna-subscription-dontremove \
    --topic=test-zuna-topic \
    --labels=department=engineering

001_example_output.txt

$ gcloud pubsub subscriptions describe test-zuna-subscription
ackDeadlineSeconds: 10
expirationPolicy:
  ttl: 2678400s
labels:
  autodelete: 'true'
  department: engineering
messageRetentionDuration: 604800s
name: projects/jonnys-project-304716/subscriptions/test-zuna-subscription
pushConfig: {}
topic: projects/jonnys-project-304716/topics/test-zuna-topic

002_create_sa.sh

#!/usr/bin/env bash

GCP_PROJECT_ID=$(gcloud config get-value project)

gcloud iam service-accounts create sa-zuna  
gcloud iam service-accounts keys create sa-key.json --iam-account=sa-zuna@${GCP_PROJECT_ID}.iam.gserviceaccount.com

003_set_permissions.sh

#!/usr/bin/env bash

GCP_PROJECT_ID=$(gcloud config get-value project)

# Create a role specifically for ZUNA inside the project
gcloud iam roles create zuna --project=${GCP_PROJECT_ID} \
  --file=zuna-role-definition.yaml

# Bind the role to the ZUNA SA on a project level
gcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \
    --member="serviceAccount:sa-zuna@${GCP_PROJECT_ID}.iam.gserviceaccount.com" \
    --role="projects/${GCP_PROJECT_ID}/roles/zuna"

004_prepare_env.sh

#!/usr/bin/env bash

virtualenv -p python3 venv
source venv/bin/activate
pip install google-cloud-pubsub==2.4.0

005_example_output.txt

Skipping subscription projects/jonnys-project-304716/subscriptions/test-zuna-subscription-dontremove (not tagged for removal)
Subscription projects/jonnys-project-304716/subscriptions/test-zuna-subscription is up for removal
Skipping removal of 1 subscriptions

005_run_script_locally.sh

#!/usr/bin/env bash

source venv/bin/activate

export GCP_PROJECT_ID=$(gcloud config get-value project)
export GOOGLE_APPLICATION_CREDENTIALS=sa-key.json

python clean_subscriptions.py

006_deploy_cloud_function.sh

#!/usr/bin/env bash

GCP_PROJECT_ID=$(gcloud config get-value project)

source venv/bin/activate

# Deploy the function
gcloud functions deploy \
    app-zuna \
    --entry-point=app_zuna \
    --region=europe-west1 \
    --runtime python38 \
    --service-account=sa-zuna@${GCP_PROJECT_ID}.iam.gserviceaccount.com \
    --trigger-topic app-zuna-cloudscheduler \
    --set-env-vars GCP_PROJECT_ID=${GCP_PROJECT_ID} \
    --timeout=540s \
    --quiet

007_test_cloud_function.sh

#!/usr/bin/env bash

gcloud pubsub topics publish app-zuna-cloudscheduler \
    --message="Hello ZUNA!"

008_example_output.txt

$gcloud functions logs read app-zuna --region=europe-west1
D      app-zuna  3brcusf1xgve  2021-02-28 20:37:12.444  Function execution started
       app-zuna  3brcusf1xgve  2021-02-28 20:37:12.458  This Function was triggered by messageId 2041279672282325 published at 2021-02-28T20:37:10.501Z
       app-zuna  3brcusf1xgve  2021-02-28 20:37:12.458
       app-zuna  3brcusf1xgve  2021-02-28 20:37:12.458  ZUNA started with payload "Hello ZUNA"!
       app-zuna  3brcusf1xgve  2021-02-28 20:37:12.458  ZUNA project: jonnys-project-304716
       app-zuna  3brcusf1xgve  2021-02-28 20:37:12.458  Cleaning PubSub Subscriptions...
       app-zuna  3brcusf1xgve  2021-02-28 20:37:13.139  Skipping subscription projects/jonnys-project-304716/subscriptions/test-zuna-subscription-dontremove (not tagged for removal)
       app-zuna  3brcusf1xgve  2021-02-28 20:37:13.139  Subscription projects/jonnys-project-304716/subscriptions/test-zuna-subscription is up for removal
       app-zuna  3brcusf1xgve  2021-02-28 20:37:13.139  Skipping subscription projects/jonnys-project-304716/subscriptions/gcf-app-zuna-europe-west1-app-zuna-cloudscheduler (not tagged for removal)
       app-zuna  3brcusf1xgve  2021-02-28 20:37:13.139  Removing projects/jonnys-project-304716/subscriptions/test-zuna-subscription...
       app-zuna  3brcusf1xgve  2021-02-28 20:37:17.869  Removed 1 subscriptions
       app-zuna  3brcusf1xgve  2021-02-28 20:37:17.945  PubSub Subscriptions cleaned!
D      app-zuna  3brcusf1xgve  2021-02-28 20:37:17.946  Function execution took 5505 ms, finished with status: 'ok'
D      app-zuna  9cbqexeyaie8  2021-02-28 20:38:49.068  Function execution started
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.147  [2021-02-28 20:38:50 +0000] [1] [INFO] Starting gunicorn 20.0.4
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.148  [2021-02-28 20:38:50 +0000] [1] [INFO] Listening at: http://0.0.0.0:8080 (1)
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.148  [2021-02-28 20:38:50 +0000] [1] [INFO] Using worker: threads
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.202  [2021-02-28 20:38:50 +0000] [6] [INFO] Booting worker with pid: 6
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.232  This Function was triggered by messageId 2041279609298300 published at 2021-02-28T20:38:48.348Z
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.233
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.233  ZUNA started with payload "Test ZUNA using Cloud Console"!
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.233  ZUNA project: jonnys-project-304716
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.233  Cleaning PubSub Subscriptions...
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.851  Skipping subscription projects/jonnys-project-304716/subscriptions/test-zuna-subscription-dontremove (not tagged for removal)
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.851  Skipping subscription projects/jonnys-project-304716/subscriptions/gcf-app-zuna-europe-west1-app-zuna-cloudscheduler (not tagged for removal)
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.851  Removed 0 subscriptions
       app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.852  PubSub Subscriptions cleaned!
D      app-zuna  9cbqexeyaie8  2021-02-28 20:38:50.854  Function execution took 1787 ms, finished with status: 'ok'

009_create_scheduler.sh

#!/usr/bin/env bash

gcloud scheduler jobs create pubsub \
    zuna-weekly \
    --time-zone="Europe/Brussels" \
    --schedule="0 22 * * 5" \
    --topic=app-zuna-cloudscheduler \
    --message-body "Go Zuna! (source: Cloud Scheduler job: zuna-weekly)"

00Y_create_resources.sh

#!/usr/bin/env bash

GCP_PROJECT_ID=$(gcloud config get-value project)
LOCATION=EU

# Create Pub/Sub topics
gcloud pubsub topics create \
    test-zuna-topic \
    --labels=department=engineering,autodelete=true

gcloud pubsub topics create \
    test-zuna-topic-dontremove \
    --labels=department=engineering

# Create Pub/Sub subscriptions
gcloud pubsub subscriptions create \
    test-zuna-subscription \
    --topic=test-zuna-topic \
    --labels=department=engineering,autodelete=true

gcloud pubsub subscriptions create \
    test-zuna-subscription-dontremove \
    --topic=test-zuna-topic-dontremove \
    --labels=department=engineering

# BigQuery dataset
bq mk --dataset \
--location=${LOCATION} \
--description="This is a dataset that should not be removed" \
--label=department:engineering --label=autodelete:true \
$GCP_PROJECT_ID:zuna_dataset

# --default_table_expiration integer1 \
# --default_partition_expiration integer2 \

# BigQuery table
bq mk --table \
--description="This is an example table" \
--label=department:engineering \
$GCP_PROJECT_ID:zuna_dataset.zuna_table \
name:STRING,temperature:FLOAT,error_message:STRING

010_example_output.txt

name: projects/jonnys-project-304716/locations/europe-west1/jobs/zuna-weekly
pubsubTarget:
  data: R28gWnVuYSEgKHNvdXJjZTogQ2xvdWQgU2NoZWR1bGVyIGpvYjogenVuYS13ZWVrbHkp
  topicName: projects/jonnys-project-304716/topics/app-zuna-cloudscheduler
retryConfig:
  maxBackoffDuration: 3600s
  maxDoublings: 16
  maxRetryDuration: 0s
  minBackoffDuration: 5s
schedule: 0 22 * * 5
state: ENABLED
timeZone: Europe/Brussels
userUpdateTime: '2021-03-07T12:46:38Z'

011_test_scheduler.sh

#!/usr/bin/env bash

gcloud scheduler jobs run zuna-weekly

012_example_output.txt

D      app-zuna  yboj7rwqd6ji  2021-03-07 12:52:51.069  Function execution started
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.419  [2021-03-07 12:52:52 +0000] [1] [INFO] Starting gunicorn 20.0.4
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.420  [2021-03-07 12:52:52 +0000] [1] [INFO] Listening at: http://0.0.0.0:8080 (1)
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.420  [2021-03-07 12:52:52 +0000] [1] [INFO] Using worker: threads
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.485  [2021-03-07 12:52:52 +0000] [6] [INFO] Booting worker with pid: 6
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.523  This Function was triggered by messageId 2087983919547225 published at 2021-03-07T12:52:48.910Z
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.523
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.523  ZUNA started with payload "Go Zuna! (source: Cloud Scheduler job: zuna-weekly)"!
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.523  ZUNA project: jonnys-project-304716
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:52.523  Cleaning PubSub Subscriptions...
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:53.185  Skipping subscription projects/jonnys-project-304716/subscriptions/test-zuna-subscription-dontremove (not tagged for removal)
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:53.185  Skipping subscription projects/jonnys-project-304716/subscriptions/gcf-app-zuna-europe-west1-app-zuna-cloudscheduler (not tagged for removal)
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:53.185  Removed 0 subscriptions
       app-zuna  yboj7rwqd6ji  2021-03-07 12:52:53.239  PubSub Subscriptionscleaned!
D      app-zuna  yboj7rwqd6ji  2021-03-07 12:52:53.243  Function execution took 2175 ms, finished with status: 'ok'

013_cleanup.sh

#!/usr/bin/env bash

GCP_PROJECT_ID=$(gcloud config get-value project)

# Remove virtual environment
rm -rf venv

# Remove Pub/Sub subscriptions
gcloud pubsub subscriptions delete test-zuna-subscription
gcloud pubsub subscriptions delete test-zuna-subscription-dontremove

# Remove Pub/Sub topics
gcloud pubsub topics delete test-zuna-topic

# Cloud Scheduler
gcloud scheduler jobs delete zuna-weekly --quiet

# Cloud Function
gcloud functions delete app-zuna --region=europe-west1 --quiet

# Roles
gcloud iam roles delete zuna --project=${GCP_PROJECT_ID}

# Service Account
gcloud iam service-accounts delete sa-zuna@${GCP_PROJECT_ID}.iam.gserviceaccount.com

clean_subscriptions.py

import os
from google.cloud import pubsub_v1

def clean_pubsub_subscriptions(project_id, delete=False):

    # Subscriptions
    # see: https://github.com/GoogleCloudPlatform/python-docs-samples/blob/efe5e78451c59415a7dcaaf72db77b13085cfa51/pubsub/cloud-client/subscriber.py#L43
    client = pubsub_v1.SubscriberClient()
    project_path = f"projects/{project_id}"

    to_delete = []

    # Go over ALL subscriptions in the project
    for subscription in client.list_subscriptions(request={"project": project_path}):
        # Collect those with the correct label
        if subscription.labels['autodelete'] == 'true':
            print(f'Subscription {subscription.name} is up for removal')
            to_delete.append(subscription.name)
        else:
            print(f'Skipping subscription {subscription.name} (not tagged for removal)')

    # Remove subscriptions if needed
    if delete:
        for subscription_name in to_delete:
            print(f'Removing {subscription_name}...')
            client.delete_subscription(subscription=subscription_name)
        print(f'Removed {len(to_delete)} subscriptions')
    else:
        print(f'Skipping removal of {len(to_delete)} subscriptions')
   
    client.close()

if __name__ == "__main__":
    project_id = os.environ['GCP_PROJECT_ID']
    clean_pubsub_subscriptions(project_id, False)

custom_role.yaml

title: resource-cleaner
description: Permissions to list and remove Pub/Sub and BigQuery resources
stage: alpha
includedPermissions:
- pubsub.topics.list
- pubsub.topics.delete
- pubsub.subscriptions.list
- pubsub.subscriptions.delete

main.py

import base64
import os

from clean_subscriptions import clean_pubsub_subscriptions
    
# see: https://cloud.google.com/functions/docs/tutorials/pubsub#functions-prepare-environment-python
def app_zuna(event, context):
    """
    Background Cloud Function to be triggered by Pub/Sub.
    Args:
         event (dict):  The dictionary with data specific to this type of
         event. The `data` field contains the PubsubMessage message. The
         `attributes` field will contain custom attributes if there are any.
         context (google.cloud.functions.Context): The Cloud Functions event
         metadata. The `event_id` field contains the Pub/Sub message ID. The
         `timestamp` field contains the publish time.
    """

    print("""This Function was triggered by messageId {} published at {}
    """.format(context.event_id, context.timestamp))

    if 'data' in event:
        payload = base64.b64decode(event['data']).decode('utf-8')
    else:
        payload = 'N/A'
    print('ZUNA started with payload "{}"!'.format(payload))

    run_cleanup_steps()

def run_cleanup_steps():
    project_id = os.environ['GCP_PROJECT_ID']
    print("ZUNA project:", project_id)

    print("Cleaning Pub/Sub Subscriptions...")
    clean_pubsub_subscriptions(project_id=project_id, delete=True)
    print("Pub/Sub Subscriptions cleaned!")

    # TODO Clean-up Pub/Sub Topics
    # TODO Clean-up BigQuery Datasets

if __name__ == "__main__":
    run_cleanup_steps()

requirements.txt

# google-cloud-bigquery
google-cloud-pubsub==2.4.0
# google-api-python-client

zuna-role-definition.yaml

title: "ZUNA"
description: "Permissions for ZUNA."
stage: "ALPHA"
includedPermissions:
- pubsub.subscriptions.list
- pubsub.subscriptions.delete