dataproc-operations-describe.yaml

  projects/$PROJECT_ID/regions/$REGION/operations/896b7922-da8e-49a9-bd80-b1ac3fda5105
done: true
metadata:
  '@type': type.googleapis.com/google.cloud.dataproc.v1beta2.ClusterOperationMetadata
  clusterName: three-node-cluster-j6q2al2mkkqck
  clusterUuid: 10656c6e-ef49-4264-805b-463e1e819626
  description: Create cluster with 2 workers
  operationType: CREATE
  status:
    innerState: DONE
    state: DONE
    stateStartTime: '2018-12-15T17:10:12.722Z'
  statusHistory:
  - state: PENDING
    stateStartTime: '2018-12-15T17:08:38.322Z'
  - state: RUNNING
    stateStartTime: '2018-12-15T17:08:38.380Z'
name: projects/dataproc-demo-224523/regions/us-east1/operations/896b7922-da8e-49a9-bd80-b1ac3fda5105
response:
  '@type': type.googleapis.com/google.cloud.dataproc.v1beta2.Cluster
  clusterName: three-node-cluster-j6q2al2mkkqck
  clusterUuid: 10656c6e-ef49-4264-805b-463e1e819626
  config:
    configBucket: dataproc-5214e13c-d3ea-400b-9c70-11ee08fac5ab-us-east1
    gceClusterConfig:
      networkUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/global/networks/default
      serviceAccountScopes:
      - https://www.googleapis.com/auth/bigquery
      - https://www.googleapis.com/auth/bigtable.admin.table
      - https://www.googleapis.com/auth/bigtable.data
      - https://www.googleapis.com/auth/cloud.useraccounts.readonly
      - https://www.googleapis.com/auth/devstorage.full_control
      - https://www.googleapis.com/auth/devstorage.read_write
      - https://www.googleapis.com/auth/logging.write
      zoneUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/zones/us-east1-b
    masterConfig:
      diskConfig:
        bootDiskSizeGb: 500
        bootDiskType: pd-standard
      imageUri: https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-1-3-deb9-20181206-000000-rc01
      machineTypeUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/zones/us-east1-b/machineTypes/n1-standard-4
      minCpuPlatform: AUTOMATIC
      numInstances: 1
    softwareConfig:
      imageVersion: 1.3.19-deb9
      properties:
        capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy: fair
        core:fs.gs.block.size: '134217728'
        core:fs.gs.metadata.cache.enable: 'false'
        distcp:mapreduce.map.java.opts: -Xmx768m
        distcp:mapreduce.map.memory.mb: '1024'
        distcp:mapreduce.reduce.java.opts: -Xmx768m
        distcp:mapreduce.reduce.memory.mb: '1024'
        hdfs:dfs.datanode.address: 0.0.0.0:9866
        hdfs:dfs.datanode.http.address: 0.0.0.0:9864
        hdfs:dfs.datanode.https.address: 0.0.0.0:9865
        hdfs:dfs.datanode.ipc.address: 0.0.0.0:9867
        hdfs:dfs.namenode.handler.count: '20'
        hdfs:dfs.namenode.http-address: 0.0.0.0:9870
        hdfs:dfs.namenode.https-address: 0.0.0.0:9871
        hdfs:dfs.namenode.lifeline.rpc-address: three-node-cluster-j6q2al2mkkqck-m:8050
        hdfs:dfs.namenode.secondary.http-address: 0.0.0.0:9868
        hdfs:dfs.namenode.secondary.https-address: 0.0.0.0:9869
        hdfs:dfs.namenode.service.handler.count: '10'
        hdfs:dfs.namenode.servicerpc-address: three-node-cluster-j6q2al2mkkqck-m:8051
        mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE: '3840'
        mapred:mapreduce.job.maps: '21'
        mapred:mapreduce.job.reduce.slowstart.completedmaps: '0.95'
        mapred:mapreduce.job.reduces: '7'
        mapred:mapreduce.map.cpu.vcores: '1'
        mapred:mapreduce.map.java.opts: -Xmx2457m
        mapred:mapreduce.map.memory.mb: '3072'
        mapred:mapreduce.reduce.cpu.vcores: '1'
        mapred:mapreduce.reduce.java.opts: -Xmx2457m
        mapred:mapreduce.reduce.memory.mb: '3072'
        mapred:mapreduce.task.io.sort.mb: '256'
        mapred:yarn.app.mapreduce.am.command-opts: -Xmx2457m
        mapred:yarn.app.mapreduce.am.resource.cpu-vcores: '1'
        mapred:yarn.app.mapreduce.am.resource.mb: '3072'
        presto-jvm:MaxHeapSize: 12288m
        presto:query.max-memory-per-node: 7372MB
        presto:query.max-total-memory-per-node: 7372MB
        spark-env:SPARK_DAEMON_MEMORY: 3840m
        spark:spark.driver.maxResultSize: 1920m
        spark:spark.driver.memory: 3840m
        spark:spark.executor.cores: '2'
        spark:spark.executor.instances: '2'
        spark:spark.executor.memory: 5586m
        spark:spark.executorEnv.OPENBLAS_NUM_THREADS: '1'
        spark:spark.scheduler.mode: FAIR
        spark:spark.sql.cbo.enabled: 'true'
        spark:spark.yarn.am.memory: 640m
        yarn-env:YARN_TIMELINESERVER_HEAPSIZE: '3840'
        yarn:yarn.nodemanager.resource.memory-mb: '12288'
        yarn:yarn.scheduler.maximum-allocation-mb: '12288'
        yarn:yarn.scheduler.minimum-allocation-mb: '1024'
    workerConfig:
      diskConfig:
        bootDiskSizeGb: 500
        bootDiskType: pd-standard
      imageUri: https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-1-3-deb9-20181206-000000-rc01
      machineTypeUri: https://www.googleapis.com/compute/v1/projects/dataproc-demo-224523/zones/us-east1-b/machineTypes/n1-standard-4
      minCpuPlatform: AUTOMATIC
      numInstances: 2
  labels:
    goog-dataproc-cluster-name: three-node-cluster-j6q2al2mkkqck
    goog-dataproc-cluster-uuid: 10656c6e-ef49-4264-805b-463e1e819626
    goog-dataproc-location: us-east1
    goog-dataproc-workflow-instance-id: b3c5063f-e3cf-3833-b613-83db12b82f32
    goog-dataproc-workflow-template-id: template-demo-3
  projectId: dataproc-demo-224523