This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import google.auth.transport.requests | |
from google.oauth2 import id_token | |
from google.oauth2 import service_account | |
from google.auth.transport.requests import AuthorizedSession | |
import google.auth | |
import requests | |
import json | |
import os | |
##set variables |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"artifact": { | |
"name": "cdap-data-pipeline", | |
"version": "6.7.1", | |
"scope": "SYSTEM" | |
}, | |
"description": "Data Pipeline Application", | |
"name": "getRunIDMetrics_v4", | |
"config": { | |
"resources": { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import datetime | |
import re | |
from airflow import models | |
from airflow.providers.google.cloud.operators.datafusion import CloudDataFusionStartPipelineOperator | |
from airflow.providers.google.cloud.sensors.datafusion import CloudDataFusionPipelineStateSensor | |
from airflow.providers.google.cloud.hooks.datafusion import PipelineStates | |
from airflow.utils.dates import days_ago | |
PROJECT_ID="" | |
REGION="" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
variable "project_id" { | |
description = "The project id of the cdf deployment" | |
type = string | |
} | |
variable "instance_name" { | |
description = "The instance name." | |
type = string | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
terraform { | |
required_providers { | |
google = { | |
source = "hashicorp/google" | |
version = "4.22.0" | |
} | |
cdap = { | |
source = "GoogleCloudPlatform/cdap" | |
# Pin to a specific version as 0.x releases are not guaranteed to be backwards compatible. | |
version = "0.9.0" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
resource "google_data_fusion_instance" "create_instance" { | |
name = var.instance_name | |
description = var.description | |
region = var.region | |
type = var.cdf_version | |
enable_stackdriver_logging = true | |
enable_stackdriver_monitoring = true | |
labels = { | |
instance_owner = var.instance_owner | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
instance_name = "my-cdf" | |
cdf_version = "ENTERPRISE" | |
cdf_release = "6.6.0" | |
cdf_network = "default" | |
cdf_ip_range = "172.28.76.0/22" | |
description = "Enterprise CDF deployed by Terraform" | |
default_service_account = "my-service-account@developer.gserviceaccount.com" | |
instance_owner = "Bob" | |
region = "us-central1" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
terraform { | |
required_providers { | |
google = { | |
source = "hashicorp/google" | |
version = "4.22.0" | |
} | |
} | |
} | |
provider "google" { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
variable "instance_name" { | |
description = "The instance name." | |
type = string | |
} | |
variable "cdf_version" { | |
description = "The version of CDF to deploy: BASIC, DEVELOPER, ENTERPRISE" | |
type = string | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
resource "google_data_fusion_instance" "create_instance" { | |
name = var.instance_name | |
description = var.description | |
region = var.region | |
type = var.cdf_version | |
enable_stackdriver_logging = true | |
enable_stackdriver_monitoring = true | |
labels = { | |
instance_owner = var.instance_owner | |
} |
NewerOlder