Terraform template creating resources to create a centralized processing pipeline for GCP audit logs

gcp_audit_ingest.tf

provider "google" {
  project     = var.google-cloud-project
}
provider "google-beta" {
  project     = var.google-cloud-project
}
# ---------------------------------------------------------
#          Google Cloud
# ---------------------------------------------------------
variable "google-cloud-project" { type = string }
variable "audit_topic_publishers" { type = list }
variable "elastic_url" {
    type = "string"
    default = "https://es.example.com:9200"
}
variable "api_key" { type = string }
# Service Account for the dataflow ingest of pupsub audit logs
resource "google_service_account" "service_account" {
  account_id   = "gcp-audit"
  display_name = "gcp-audit"
  description = "Service account to be used by DataFlow to ingest pupsub logs"
}

resource "google_project_iam_member" "svc_permissions" {
  for_each = toset(["roles/pubsub.publisher", "roles/viewer", "roles/storage.objectAdmin", "roles/dataflow.worker", "roles/storage.objectViewer"])
  project = var.google-cloud-project
  role = each.key
  member  = "serviceAccount:${google_service_account.service_account.email}"
}

resource "google_dataflow_flex_template_job" "gcp_ingest" {
    provider = google-beta
    name = "gcp-audit-ingest"
    container_spec_gcs_path = "gs://dataflow-templates-europe-west1/latest/flex/PubSub_to_Elasticsearch"
    parameters = {
      "inputSubscription": google_pubsub_subscription.gcp_audit.id,
      "connectionUrl": var.elastic_url,
      "errorOutputTopic": google_pubsub_topic.failure_topic.id,
      "apiKey": var.api_key,
      "dataset": "audit",
      "tempLocation": "${google_storage_bucket.data_flow.url}/tmp",
      "serviceAccount": google_service_account.service_account.email,
      "stagingLocation": "${google_storage_bucket.data_flow.url}/staging"
    }
}

resource "google_pubsub_topic" "audit_topic" {
  name = "topic-gcp-audit"
  message_retention_duration = "86600s"
}

resource "google_pubsub_subscription" "gcp_audit" {
  name  = "subscription-gcp-audit"
  topic = google_pubsub_topic.audit_topic.name
  # 20 minutes
  message_retention_duration = "1200s"
  enable_message_ordering    = false
}

resource "google_pubsub_topic" "failure_topic" {
  name = "topic-ingest-failures"
  message_retention_duration = "86600s"
}

resource "google_storage_bucket" "data_flow" {
  name          = "storage-gcp-audit-dataflow"
  location      = "EU"
  force_destroy = true
}

resource "google_storage_bucket_iam_member" "member" {
  for_each = toset(["serviceAccount:${google_service_account.service_account.email}"])
  bucket = google_storage_bucket.data_flow.name
  role = "roles/storage.admin"
  member = each.key
}

resource "google_pubsub_topic_iam_member" "member" {
  for_each = toset(var.audit_topic_publishers)
  project = var.google-cloud-project
  topic = google_pubsub_topic.audit_topic.name
  role = "roles/pubsub.publisher"
  member = each.key
}