run.sh

#!/usr/bin/env bash

# Pipeline vars
PROJECT=my-project
BUCKET_NAME=my-bucket
TABLE={$PROJECT}:my-dataset.my-table
NODE_ADDRESSES=comma-separated-list-nodes
INDEX=my-index
DOCUMENT_TYPE=my-type

# Containerization vars
IMAGE_NAME=my-image-name
TARGET_GCR_IMAGE=gcr.io/${PROJECT}/${IMAGE_NAME}
BASE_CONTAINER_IMAGE=my-base-container-image
BASE_CONTAINER_IMAGE_VERSION=my-base-container-image-version
APP_ROOT=/path/to/app-root
COMMAND_SPEC=/path/to/command-spec

# Build and upload image
mvn clean package \
-Dimage=${TARGET_GCR_IMAGE} \
-Dbase-container-image=${BASE_CONTAINER_IMAGE} \
-Dbase-container-image.version=${BASE_CONTAINER_IMAGE_VERSION} \
-Dapp-root=${APP_ROOT} \
-Dcommand-spec=${COMMAND_SPEC}


# Execute template:
API_ROOT_URL="https://dataflow.googleapis.com"
TEMPLATES_LAUNCH_API="${API_ROOT_URL}/v1b3/projects/${PROJECT}/templates:launch"
JOB_NAME="bigquery-to-elasticsearch-`date +%Y%m%d-%H%M%S-%N`"

curl -X POST -H "Content-Type: application/json" \
    -H "Authorization: Bearer $(gcloud auth print-access-token)" \
    "${TEMPLATES_LAUNCH_API}"`
    `"?validateOnly=false"`
    `"&dynamicTemplate.gcsPath=${BUCKET_NAME}/path/to/image-spec"`
    `"&dynamicTemplate.stagingLocation=${BUCKET_NAME}/staging" \
    -d '
     {
      "jobName":"'$JOB_NAME'",
      "parameters": {
           "inputTableSpec":"'$TABLE'",
           "nodeAddresses":"'$NODE_ADDRESSES'",
           "index":"'$INDEX'",
           "documentType":"'$DOCUMENT_TYPE'"
       }
      }
     '

# Create and stage template https://cloud.google.com/dataflow/docs/guides/templates/creating-templates#creating-and-staging-templates
TEMPLATE_NAME=bigquery-to-elasticsearch-template #TODO: was this previously defined?

mvn compile exec:java \
     -Dexec.mainClass=com.euromonitor.via.dataflow.BigQueryToElasticsearch \
     -Dexec.args="--runner=DataflowRunner \
                  --project=$PROJECT \
                  --stagingLocation=gs://$BUCKET_NAME/staging \
                  --templateLocation=gs://$BUCKET_NAME/templates/$TEMPLATE_NAME"