Experiment with GCP Asset Inventory feeds publishing to Pub/Sub and importing into BigQuery using subscriptions and Avro schema

gcp-asset-inv-bq-feed.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/asset-inventory/docs/monitoring-asset-changes#organizations
# - https://cloud.google.com/asset-inventory/docs/reference/rest/v1/TopLevel/batchGetAssetsHistory#temporalasset
# - https://cloud.google.com/iam/docs/creating-custom-roles#creating
# - https://cloud.google.com/sdk/gcloud/reference/organizations/add-iam-policy-binding
# - https://gist.github.com/mikesparr/38260f9d457fc2f9452f1910213b4a71
# - https://aravind-ramaiahk.medium.com/streamlining-cloud-asset-management-automating-the-export-of-google-cloud-organization-asset-46cc8b428038
#####################################################################

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain
export ORG_ID=$(gcloud projects get-ancestors $PROJECT_ID --format="value(id)" | tail -n 1)

export GCP_REGION="us-central1" # CHANGEME (OPT)
export GCP_ZONE="us-central1-a" # CHANGEME (OPT)
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    cloudresourcemanager.googleapis.com \
    cloudasset.googleapis.com \
    storage.googleapis.com \
    pubsub.googleapis.com \
    bigquery.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE


##########################################################
# Permissions
##########################################################
export ROLE_CONFIG=role-config.yaml
export ROLE_ID="testAssetInvRole"
export ROLE_TITLE="Test-Asset-Inv-Role"
export ROLE_DESCRIPTION="Test-Asset-Inv-Role"
export LAUNCH_STAGE="ALPHA"
export SA_NAME="asset-inv-sa"
export SA_EMAIL="$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com"

# create config file
cat > $ROLE_CONFIG << EOF
title: $ROLE_TITLE
description: $ROLE_DESCRIPTION
stage: $LAUNCH_STAGE
includedPermissions:
- cloudasset.feeds.create
- cloudasset.assets.exportResource
- cloudasset.feeds.update
- cloudasset.assets.exportResource
- cloudasset.feeds.delete
- cloudasset.feeds.get
- cloudasset.feeds.list
EOF

# create the role
gcloud iam roles create $ROLE_ID --organization=$ORG_ID \
    --file=$ROLE_CONFIG

# create service account
gcloud iam service-accounts create $SA_NAME \
    --description="$SA_NAME" \
    --display-name="$SA_NAME"

# bind custom role to service account
gcloud organizations add-iam-policy-binding $ORG_ID \
    --member="serviceAccount:$SA_EMAIL" \
    --role="organizations/$ORG_ID/roles/$ROLE_ID"



#############################################################
# BigQuery setup
# - TODO: change from example to actual schema for asset inventory feed
# - SEE: https://cloud.google.com/asset-inventory/docs/reference/rest/v1/TopLevel/batchGetAssetsHistory#temporalasset
#############################################################
export DATASET_NAME="cmdb"
export TABLE_NAME="asset_inventory_changes"
export BQ_SCHEMA_FILE="$TABLE_NAME-schema.json"

# bigquery dataset
bq --location $GCP_REGION mk -d \
    --description "Config Mgmt Database" \
    $DATASET_NAME

# schema definition
cat > $BQ_SCHEMA_FILE << EOF
[

    {
        "name": "asset",
        "type": "JSON",
        "mode": "NULLABLE"
    },
    {
        "name": "priorAsset",
        "type": "JSON",
        "mode": "NULLABLE"
    },
    {
        "name": "priorAssetState",
        "type": "STRING",
        "mode": "NULLABLE"
    },
    {
        "name": "window",
        "type": "JSON",
        "mode": "NULLABLE"
    },
    {
        "name": "deleted",
        "type": "BOOLEAN",
        "mode": "NULLABLE"
    }
]
EOF

# bigquery table
bq --location $GCP_REGION mk -t \
    --description "Example illustrating map Avro type to BQ schema" \
    --label purpose:demo \
    $DATASET_NAME.$TABLE_NAME \
    $(PWD)/$BQ_SCHEMA_FILE



#############################################################
# Pub/Sub setup
# - TODO: change example to asset inventory schema
# - TODO: use service account created with custom role
# - SEE: https://cloud.google.com/asset-inventory/docs/reference/rest/v1/TopLevel/batchGetAssetsHistory#temporalasset
#############################################################
export SCHEMA_ID="asset-inv-changes"
export SCHEMA_FILE="$SCHEMA_ID.avsc"
export MSG_FILE="example-message.json"
export TOPIC_ID="asset-inv-feed"
export SUBSCRIPTION_ID="asset-inv-change-bq-import"
export PUBSUB_SA_ID="service-$PROJECT_NUMBER@gcp-sa-pubsub.iam.gserviceaccount.com"

# create avro schema file
cat > $SCHEMA_FILE << EOF
{
  "type" : "record",
  "name" : "Avro",
  "fields" : [
    {
      "name" : "priorAssetState",
      "type" : "string"
    },
    {
      "name" : "deleted",
      "type" : "boolean"
    },
    {
      "name" : "asset",
      "type" : {
        "type": "record",
        "name": "ResourceRecord",
        "fields": [
          {"name": "type", "type": "string"},
          {"name": "ARN", "type": "string"},
          {"name": "accountId", "type": "string"},
          {"name": "ARNPrefix", "type": "string"}
        ]
      }
    },
    {
      "name" : "priorAsset",
      "type" : {
        "type": "record",
        "name": "ResourceRecord",
        "fields": [
          {"name": "type", "type": "string"},
          {"name": "ARN", "type": "string"},
          {"name": "accountId", "type": "string"},
          {"name": "ARNPrefix", "type": "string"}
        ]
      }
    },
    {
      "name" : "window",
      "type" : {
        "type": "record",
        "name": "ResourceRecord",
        "fields": [
          {"name": "type", "type": "string"},
          {"name": "ARN", "type": "string"},
          {"name": "accountId", "type": "string"},
          {"name": "ARNPrefix", "type": "string"}
        ]
      }
    }
  ]
}
EOF

# create pubsub schema
# gcloud pubsub schemas create $SCHEMA_ID \
#     --type AVRO \
#     --definition="$(cat $SCHEMA_FILE)"

# test schema
cat > $MSG_FILE << EOF
{
  "name": "AWS",
  "content": {
    "foo": "bar", 
    "boo": "baz"
  }, 
  "resources": { 
    "type": "blah", 
    "ARN": "blah", 
    "accountId": "blah", 
    "ARNPrefix": "blah" 
  } 
}
EOF

# gcloud pubsub schemas validate-message \
#     --type AVRO \
#     --definition="$(cat $SCHEMA_FILE)" \
#     --message-encoding JSON \
#     --message="$(cat $MSG_FILE)"

# pubsub topics
# gcloud pubsub topics create $TOPIC_ID \
#     --schema $SCHEMA_ID \
#     --message-encoding JSON

gcloud pubsub topics create $TOPIC_ID

# add BQ service account permissions to subscribe to topic
gcloud projects add-iam-policy-binding $PROJECT_ID \
    --member serviceAccount:$PUBSUB_SA_ID \
    --role roles/bigquery.dataEditor \
    --condition=None
gcloud projects add-iam-policy-binding $PROJECT_ID \
    --member serviceAccount:$PUBSUB_SA_ID \
    --role roles/bigquery.metadataViewer \
    --condition=None

# add BQ subscription to clean text topic
# gcloud pubsub subscriptions create $SUBSCRIPTION_ID \
#     --topic=$TOPIC_ID \
#     --bigquery-table=$PROJECT_ID:$DATASET_NAME.$TABLE_NAME \
#     --use-topic-schema



##########################################################
# Asset Inventory Feed
# - TODO: export to Pub/Sub topic
# - TODO: research asset types to include in feed (e.g. compute, ...)
# - TODO: add vars for other params in command as needed
# - TODO: test service account (impersonate) to create feed
##########################################################
export FEED_ID="asset-inv-feed"

gcloud asset feeds create $FEED_ID \
    --organization=$ORG_ID \
    --pubsub-topic=projects/$PROJECT_ID/topics/$TOPIC_ID \
    --asset-types="compute.googleapis.com.*" \
    --content-type=resource