Instantly share code, notes, and snippets.

Embed
What would you like to do?
jobs:
- sparkJob:
jarFileUris:
- gs://dataproc-demo-bucket/dataprocJavaDemo-1.0-SNAPSHOT.jar
mainClass: org.example.dataproc.InternationalLoansAppDataprocSmall
stepId: ibrd-small-spark
- sparkJob:
jarFileUris:
- gs://dataproc-demo-bucket/dataprocJavaDemo-1.0-SNAPSHOT.jar
mainClass: org.example.dataproc.InternationalLoansAppDataprocLarge
stepId: ibrd-large-spark
- pysparkJob:
args:
- gs://dataproc-demo-bucket
- ibrd-statement-of-loans-historical-data.csv
- ibrd-summary-large-python
mainPythonFileUri: gs://dataproc-demo-bucket/international_loans_dataproc.py
stepId: ibrd-large-pyspark
placement:
managedCluster:
clusterName: three-node-cluster
config:
gceClusterConfig:
zoneUri: us-east1-b
masterConfig:
diskConfig:
bootDiskSizeGb: 500
machineTypeUri: n1-standard-4
softwareConfig:
imageVersion: 1.3-deb9
workerConfig:
diskConfig:
bootDiskSizeGb: 500
machineTypeUri: n1-standard-4
numInstances: 2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment