setup Spark HA with docker swarm mode

spark_swarm.sh

#!/usr/bin/env bash

docker network create -d overlay spark 2>/dev/null || true

# ignore service remove error
docker service rm spark-master 2>/dev/null || true
sleep 3s

echo "create spark-master service..."

docker service create \
        --name spark-master \
        --replicas 3 \
        --env SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=10.0.10.114:2181,10.0.10.115:2181,10.0.10.116:2181" \
        --network spark \
        --limit-cpu 0.4 \
        --limit-memory 2g \
        --with-registry-auth \
        singularities/spark bash -c "\${SPARK_HOME}/sbin/start-master.sh && tail -f \${SPARK_HOME}/logs/*"

sleep 3s

# ignore service remove error
docker service rm spark-slave 2>/dev/null || true
sleep 3s

echo "create spark-slave service..."

docker service create \
        --name spark-slave \
        --replicas 3 \
        --env SPARK_DAEMON_JAVA_OPTS="-Dspark.shuffle.service.enabled=true" \
        --env SPARK_WORKER_CORES=4 \
        --env SPARK_WORKER_MEMORY=12g \
        --network spark \
        --with-registry-auth \
        singularities/spark bash -c "SPARK_MASTER_URL=\$(getent hosts tasks.spark-master|awk '{print \$1}'|sed 's/\$/:7077/g'|paste -d, -s -|tr -d '[:space:]'); echo \${SPARK_MASTER_URL} && \${SPARK_HOME}/sbin/start-slave.sh spark://\${SPARK_MASTER_URL} && tail -f \${SPARK_HOME}/logs/*"