jster1357

from google.cloud import pubsub_v1
import subprocess
import requests
import json
import os
import time

# get the access token for the API call
def get_access_token():

jster1357

resource "google_data_fusion_instance" "create_instance" {
  name = var.instance_name
  description = var.description
  region = var.region
  type = var.cdf_version
  enable_stackdriver_logging = true
  enable_stackdriver_monitoring = true
  labels = {
    instance_owner = var.instance_owner
  }

jster1357

variable "instance_name" {
  description = "The instance name."
  type        = string
}

variable "cdf_version" {
  description = "The version of CDF to deploy: BASIC, DEVELOPER, ENTERPRISE"
  type        = string
}

jster1357

terraform {
  required_providers {
    google = {
      source = "hashicorp/google"
      version = "4.22.0"
    }
  }
}

provider "google" {

jster1357

instance_name = "my-cdf"
cdf_version = "ENTERPRISE"
cdf_release = "6.6.0"
cdf_network = "default"
cdf_ip_range = "172.28.76.0/22"
description = "Enterprise CDF deployed by Terraform"
default_service_account = "my-service-account@developer.gserviceaccount.com"
instance_owner = "Bob"
region = "us-central1"

jster1357

resource "google_data_fusion_instance" "create_instance" {
  name = var.instance_name
  description = var.description
  region = var.region
  type = var.cdf_version
  enable_stackdriver_logging = true
  enable_stackdriver_monitoring = true
  labels = {
    instance_owner = var.instance_owner
  }

jster1357

terraform {
  required_providers {
    google = {
      source = "hashicorp/google"
      version = "4.22.0"
    }
    cdap = {
      source = "GoogleCloudPlatform/cdap"
      # Pin to a specific version as 0.x releases are not guaranteed to be backwards compatible.
      version = "0.9.0"

jster1357

variable "project_id" {
  description = "The project id of the cdf deployment"
  type        = string
}

variable "instance_name" {
  description = "The instance name."
  type        = string
}

jster1357

import datetime
import re
from airflow import models
from airflow.providers.google.cloud.operators.datafusion import CloudDataFusionStartPipelineOperator
from airflow.providers.google.cloud.sensors.datafusion  import CloudDataFusionPipelineStateSensor
from airflow.providers.google.cloud.hooks.datafusion import PipelineStates
from airflow.utils.dates import days_ago

PROJECT_ID=""
REGION=""

jster1357

{
    "artifact": {
        "name": "cdap-data-pipeline",
        "version": "6.7.1",
        "scope": "SYSTEM"
    },
    "description": "Data Pipeline Application",
    "name": "getRunIDMetrics_v4",
    "config": {
        "resources": {