Instantly share code, notes, and snippets.

Embed
What would you like to do?
projects/$PROJECT_ID/regions/$REGION/operations/896b7922-da8e-49a9-bd80-b1ac3fda5105
done: true
metadata:
'@type': type.googleapis.com/google.cloud.dataproc.v1beta2.ClusterOperationMetadata
clusterName: three-node-cluster-j6q2al2mkkqck
clusterUuid: 10656c6e-ef49-4264-805b-463e1e819626
description: Create cluster with 2 workers
operationType: CREATE
status:
innerState: DONE
state: DONE
stateStartTime: '2018-12-15T17:10:12.722Z'
statusHistory:
- state: PENDING
stateStartTime: '2018-12-15T17:08:38.322Z'
- state: RUNNING
stateStartTime: '2018-12-15T17:08:38.380Z'
name: projects/dataproc-demo-224523/regions/us-east1/operations/896b7922-da8e-49a9-bd80-b1ac3fda5105
response:
'@type': type.googleapis.com/google.cloud.dataproc.v1beta2.Cluster
clusterName: three-node-cluster-j6q2al2mkkqck
clusterUuid: 10656c6e-ef49-4264-805b-463e1e819626
config:
configBucket: dataproc-5214e13c-d3ea-400b-9c70-11ee08fac5ab-us-east1
gceClusterConfig:
networkUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/global/networks/default
serviceAccountScopes:
- https://www.googleapis.com/auth/bigquery
- https://www.googleapis.com/auth/bigtable.admin.table
- https://www.googleapis.com/auth/bigtable.data
- https://www.googleapis.com/auth/cloud.useraccounts.readonly
- https://www.googleapis.com/auth/devstorage.full_control
- https://www.googleapis.com/auth/devstorage.read_write
- https://www.googleapis.com/auth/logging.write
zoneUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/zones/us-east1-b
masterConfig:
diskConfig:
bootDiskSizeGb: 500
bootDiskType: pd-standard
imageUri: https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-1-3-deb9-20181206-000000-rc01
machineTypeUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/zones/us-east1-b/machineTypes/n1-standard-4
minCpuPlatform: AUTOMATIC
numInstances: 1
softwareConfig:
imageVersion: 1.3.19-deb9
properties:
capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy: fair
core:fs.gs.block.size: '134217728'
core:fs.gs.metadata.cache.enable: 'false'
distcp:mapreduce.map.java.opts: -Xmx768m
distcp:mapreduce.map.memory.mb: '1024'
distcp:mapreduce.reduce.java.opts: -Xmx768m
distcp:mapreduce.reduce.memory.mb: '1024'
hdfs:dfs.datanode.address: 0.0.0.0:9866
hdfs:dfs.datanode.http.address: 0.0.0.0:9864
hdfs:dfs.datanode.https.address: 0.0.0.0:9865
hdfs:dfs.datanode.ipc.address: 0.0.0.0:9867
hdfs:dfs.namenode.handler.count: '20'
hdfs:dfs.namenode.http-address: 0.0.0.0:9870
hdfs:dfs.namenode.https-address: 0.0.0.0:9871
hdfs:dfs.namenode.lifeline.rpc-address: three-node-cluster-j6q2al2mkkqck-m:8050
hdfs:dfs.namenode.secondary.http-address: 0.0.0.0:9868
hdfs:dfs.namenode.secondary.https-address: 0.0.0.0:9869
hdfs:dfs.namenode.service.handler.count: '10'
hdfs:dfs.namenode.servicerpc-address: three-node-cluster-j6q2al2mkkqck-m:8051
mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE: '3840'
mapred:mapreduce.job.maps: '21'
mapred:mapreduce.job.reduce.slowstart.completedmaps: '0.95'
mapred:mapreduce.job.reduces: '7'
mapred:mapreduce.map.cpu.vcores: '1'
mapred:mapreduce.map.java.opts: -Xmx2457m
mapred:mapreduce.map.memory.mb: '3072'
mapred:mapreduce.reduce.cpu.vcores: '1'
mapred:mapreduce.reduce.java.opts: -Xmx2457m
mapred:mapreduce.reduce.memory.mb: '3072'
mapred:mapreduce.task.io.sort.mb: '256'
mapred:yarn.app.mapreduce.am.command-opts: -Xmx2457m
mapred:yarn.app.mapreduce.am.resource.cpu-vcores: '1'
mapred:yarn.app.mapreduce.am.resource.mb: '3072'
presto-jvm:MaxHeapSize: 12288m
presto:query.max-memory-per-node: 7372MB
presto:query.max-total-memory-per-node: 7372MB
spark-env:SPARK_DAEMON_MEMORY: 3840m
spark:spark.driver.maxResultSize: 1920m
spark:spark.driver.memory: 3840m
spark:spark.executor.cores: '2'
spark:spark.executor.instances: '2'
spark:spark.executor.memory: 5586m
spark:spark.executorEnv.OPENBLAS_NUM_THREADS: '1'
spark:spark.scheduler.mode: FAIR
spark:spark.sql.cbo.enabled: 'true'
spark:spark.yarn.am.memory: 640m
yarn-env:YARN_TIMELINESERVER_HEAPSIZE: '3840'
yarn:yarn.nodemanager.resource.memory-mb: '12288'
yarn:yarn.scheduler.maximum-allocation-mb: '12288'
yarn:yarn.scheduler.minimum-allocation-mb: '1024'
workerConfig:
diskConfig:
bootDiskSizeGb: 500
bootDiskType: pd-standard
imageUri: https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-1-3-deb9-20181206-000000-rc01
machineTypeUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/zones/us-east1-b/machineTypes/n1-standard-4
minCpuPlatform: AUTOMATIC
numInstances: 2
labels:
goog-dataproc-cluster-name: three-node-cluster-j6q2al2mkkqck
goog-dataproc-cluster-uuid: 10656c6e-ef49-4264-805b-463e1e819626
goog-dataproc-location: us-east1
goog-dataproc-workflow-instance-id: b3c5063f-e3cf-3833-b613-83db12b82f32
goog-dataproc-workflow-template-id: template-demo-3
projectId: dataproc-demo-224523
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment