spark-operator.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: spark-operator
---
apiVersion: v1
kind: Namespace
metadata:
  name: spark-apps
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark
  namespace: spark-apps
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: spark-operator-role
  namespace: spark-apps
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: edit
subjects:
  - kind: ServiceAccount
    name: spark
    namespace: spark-apps

spark-pi.sh

$ kubectl describe SparkApplication spark-pi -n spark-apps
Name:         spark-pi
Namespace:    spark-apps
Labels:       <none>
Annotations:  API Version:  sparkoperator.k8s.io/v1beta2
Kind:         SparkApplication
Metadata:
  Creation Timestamp:  2020-07-15T02:30:58Z
  Generation:          1
  Managed Fields:
    API Version:  sparkoperator.k8s.io/v1beta2
    Fields Type:  FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          .:
          f:kubectl.kubernetes.io/last-applied-configuration:
      f:spec:
        .:
        f:driver:
          .:
          f:coreLimit:
          f:cores:
          f:labels:
            .:
            f:version:
          f:memory:
          f:serviceAccount:
          f:volumeMounts:
        f:executor:
          .:
          f:cores:
          f:instances:
          f:labels:
            .:
            f:version:
          f:memory:
          f:volumeMounts:
        f:image:
        f:imagePullPolicy:
        f:mainApplicationFile:
        f:mainClass:
        f:mode:
        f:restartPolicy:
          .:
          f:type:
        f:sparkVersion:
        f:type:
        f:volumes:
    Manager:      kubectl
    Operation:    Update
    Time:         2020-07-15T02:30:58Z
    API Version:  sparkoperator.k8s.io/v1beta2
    Fields Type:  FieldsV1
    fieldsV1:
      f:spec:
        f:deps:
      f:status:
        .:
        f:applicationState:
          .:
          f:state:
        f:driverInfo:
          .:
          f:podName:
          f:webUIAddress:
          f:webUIPort:
          f:webUIServiceName:
        f:executionAttempts:
        f:executorState:
          .:
          f:spark-pi-1594780259870-exec-1:
        f:lastSubmissionAttemptTime:
        f:sparkApplicationId:
        f:submissionAttempts:
        f:submissionID:
        f:terminationTime:
    Manager:         spark-operator
    Operation:       Update
    Time:            2020-07-15T02:31:16Z
  Resource Version:  2431
  Self Link:         /apis/sparkoperator.k8s.io/v1beta2/namespaces/spark-apps/sparkapplications/spark-pi
  UID:               042607cd-a6be-4878-b1c4-1c1007f9b967
Spec:
  Driver:
    Core Limit:  1200m
    Cores:       1
    Labels:
      Version:        2.4.5
    Memory:           512m
    Service Account:  spark
    Volume Mounts:
      Mount Path:  /tmp
      Name:        test-volume
  Executor:
    Cores:      1
    Instances:  1
    Labels:
      Version:  2.4.5
    Memory:     512m
    Volume Mounts:
      Mount Path:         /tmp
      Name:               test-volume
  Image:                  gcr.io/spark-operator/spark:v2.4.5
  Image Pull Policy:      Always
  Main Application File:  local:///opt/spark/examples/jars/spark-examples_2.11-2.4.5.jar
  Main Class:             org.apache.spark.examples.SparkPi
  Mode:                   cluster
  Restart Policy:
    Type:         Never
  Spark Version:  2.4.5
  Type:           Scala
  Volumes:
    Host Path:
      Path:  /tmp
      Type:  Directory
    Name:    test-volume
Status:
  Application State:
    State:  COMPLETED
  Driver Info:
    Pod Name:             spark-pi-driver
    Web UI Address:       10.111.23.203:4040
    Web UI Port:          4040
    Web UI Service Name:  spark-pi-ui-svc
  Execution Attempts:     1
  Executor State:
    spark-pi-1594780259870-exec-1:  COMPLETED
  Last Submission Attempt Time:     2020-07-15T02:31:01Z
  Spark Application Id:             spark-c06415bc59fe4c828cadcc1f59c546d5
  Submission Attempts:              1
  Submission ID:                    9abd2c52-eccc-4cc2-bc60-2d24b09ff128
  Termination Time:                 2020-07-15T02:31:16Z
Events:
  Type    Reason                     Age   From            Message
  ----    ------                     ----  ----            -------
  Normal  SparkApplicationAdded      17s   spark-operator  SparkApplication spark-pi was added, enqueuing it for submission
  Normal  SparkApplicationSubmitted  14s   spark-operator  SparkApplication spark-pi was submitted successfully
  Normal  SparkDriverRunning         12s   spark-operator  Driver spark-pi-driver is running
  Normal  SparkExecutorPending       6s    spark-operator  Executor spark-pi-1594780259870-exec-1 is pending
  Normal  SparkExecutorRunning       4s    spark-operator  Executor spark-pi-1594780259870-exec-1 is running
  Normal  SparkDriverCompleted       0s    spark-operator  Driver spark-pi-driver completed
  Normal  SparkApplicationCompleted  0s    spark-operator  SparkApplication spark-pi completed

spark-pi.yaml

apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
  name: spark-pi
  namespace: spark-apps
spec:
  type: Scala
  mode: cluster
  image: "gcr.io/spark-operator/spark:v2.4.5"
  imagePullPolicy: Always
  mainClass: org.apache.spark.examples.SparkPi
  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.5.jar"
  sparkVersion: "2.4.5"
  restartPolicy:
    type: Never
  volumes:
    - name: "test-volume"
      hostPath:
        path: "/tmp"
        type: Directory
  driver:
    cores: 1
    coreLimit: "1200m"
    memory: "512m"
    labels:
      version: 2.4.5
    serviceAccount: spark
    volumeMounts:
      - name: "test-volume"
        mountPath: "/tmp"
  executor:
    cores: 1
    instances: 1
    memory: "512m"
    labels:
      version: 2.4.5
    volumeMounts:
      - name: "test-volume"
        mountPath: "/tmp"