spark-on-k8s-running-from-pod.sh

spark-on-k8s-running-from-pod.sh

#!/bin/bash
IMAGE_REPO=$(kubectl get svc registry-docker-registry -o=jsonpath='{.spec.clusterIP}'):5000

cat > clustermode-podspec.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
  name: spark-submit-example
spec:
  serviceAccountName: spark
  containers:
  - name: spark-submit-example
    args:
    - /opt/spark/bin/spark-submit
    - --master
    - k8s://https://\$(KUBERNETES_PORT_443_TCP_ADDR):\$(KUBERNETES_PORT_443_TCP_PORT)
    - --deploy-mode
    - cluster
    - --conf
    - spark.kubernetes.container.image=$IMAGE_REPO/spark
    - --conf
    - spark.kubernetes.authenticate.driver.serviceAccountName=spark
    - --class
    - org.apache.spark.examples.SparkPi
    - local:///opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar
    env:
    - name: SPARK_HOME
      value: /opt/spark
    resources: {}
    image: $IMAGE_REPO/spark:latest
    imagePullPolicy: Always
EOF

# this will start a pod that runs spark-submit
# see: https://spark.apache.org/docs/latest/running-on-kubernetes.html#submitting-applications-to-kubernetes
# for differences in running in client or cluster mode
kubectl apply -f clustermode-podspec.yaml