k8s Spark external shuffler

pod-spark-shuffle.yaml

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    app: spark-shuffle-service
    spark-version: 2.2.0
  name: shuffle
spec:
  template:
    metadata:
      labels:
        app: spark-shuffle-service
        spark-version: 2.2.0
    spec:
      volumes:
        - name: temp-volume
          hostPath:
            path: '/tmp/spark-local' # change this path according to your cluster configuration.
      containers:
        - name: shuffle
          # This is an official image that is built
          # from the dockerfiles/shuffle directory
          # in the spark distribution.
          image: kubespark/spark-shuffle:v2.2.0-kubernetes-0.5.0
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - mountPath: '/tmp/spark-local'
              name: temp-volume
              # more volumes can be mounted here.
              # The spark job must be configured to use these
              # mounts using the configuration:
              #   spark.kubernetes.shuffle.dir=<mount-1>,<mount-2>,...
          resources:
             requests:
               cpu: "1"
             limits:
               cpu: "1"