create_stackdriver_sinks.sh

# Filter out a specific message type and store it to gcs as schemaless .jsonl and bq (which requires that schema does not change).
# Different stackdriver labels creates different partitions of the output.
# I recommend adding a version field to each log so that you can easily filter them to different sinks.

filter="jsonPayload.message_type=\"$1\""
PROJECT_NAME=my_proj

gsutil mb gs://$PROJECT_NAME-logs-$1
gcloud beta logging sinks create $1-to-gcs storage.googleapis.com/$PROJECT_NAME-logs-$1 --log-filter="$filter" --project=$PROJECT_NAME

bq mk --dataset $PROJECT_NAME:logs_$1
gcloud beta logging sinks create $1-to-bq bigquery.googleapis.com/projects/depict-ai-production/datasets/logs_$1 --log-filter="$filter" --project=depict-ai-production