Skip to content

Instantly share code, notes, and snippets.

@pascalwhoop
Created April 30, 2020 16:46
Show Gist options
  • Save pascalwhoop/b17e5d5f18905d76478b1dae4ff035f3 to your computer and use it in GitHub Desktop.
Save pascalwhoop/b17e5d5f18905d76478b1dae4ff035f3 to your computer and use it in GitHub Desktop.
locals {
foo_directory = "../../../../services/foo/target"
jobName = "company-${var.environment}-datalayer-foo-app-v1"
region = "europe-west1"
//parameters for the java jar
dataflow_parameters = {
runner = "DataflowRunner"
jobName = local.jobName
update = data.external.running_job.result.name == local.jobName ? true : false
project = var.project
kafkaInputEnabled = "true"
serviceAccount = module.dataflow_job.service_account.email
region = local.region
zone = "${local.region}-b"
workerMachineType = "n1-standard-1"
diskSizeGb = "30"
maxWorkers = "1"
streaming = "true"
gcpTempLocation = "gs://${google_storage_bucket.default.name}/${google_storage_bucket_object.temp_subfolder.name}"
stagingLocation = "gs://${google_storage_bucket.default.name}/${google_storage_bucket_object.staging_subfolder.name}"
pubSubFailureTopic = google_pubsub_topic.failure_topic.name
deadLetterDatasetName = google_bigquery_dataset.dead_letter_dataset.id
env = var.environment
jobOwner = "DeLorean"
}
// turns the map above into a list of parameters to the java process
parameters_string = join(" ", formatlist("--%s=%s", keys(local.dataflow_parameters), values(local.dataflow_parameters)))
}
data "external" "running_job" {
working_dir = local.foo_directory
//check for existing job
program = ["bash", "-c", "gcloud dataflow jobs list --status active --filter='name=${local.jobName}' --format=json | jq '.[0]'"]
}
resource "null_resource" "deploy_job" {
triggers = {
jar_hash = filesha256("${local.foo_directory}/foo-bundled-0.0.2.jar")
}
provisioner "local-exec" {
command = "java -cp foo-bundled-0.0.2.jar com.company.datalayer.pipelines.MetlGrid ${local.parameters_string}"
working_dir = local.foo_directory
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment