Dockerized mulit-worker Spark

docker-compose.yaml

version: "3.8"

volumes:
  local-workspace-fs:
    name: "local-workspace-fs"
    external: true
  local-data-fs:
    name: "local-data-fs"
    external: true

networks:
  sparknet:
    driver: bridge

services:
  jupyterlab:
    image: jupyter/pyspark-notebook:python-3.10
    container_name: jupyterlab
    ports:
      - "8888:8888"
      - "4040:4040"
    volumes:
      - local-workspace-fs:/home/jovyan/work
      - local-data-fs:/opt/data
    networks:
      - sparknet

  spark-master:
    image: apache/spark-py:v3.4.0
    container_name: spark-master
    hostname: spark-master
    user: root
    ports:
      - "8080:8080"
      - "7077:7077"
    command:
      - "bash"
      - "-c"
      - "/opt/spark/sbin/start-master.sh && tail -f /opt/spark/logs/spark*"
    volumes:
      - local-workspace-fs:/opt/workspace
      - local-data-fs:/opt/data
    networks:
      - sparknet

  spark-worker-1:
    image: apache/spark-py:v3.4.0
    container_name: spark-worker-1
    hostname: spark-worker-1
    user: root
    environment:
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=512m
    ports:
      - "8081:8081"
    command:
      - "bash"
      - "-c"
      - "pip install pyarrow pandas && /opt/spark/sbin/start-worker.sh spark://spark-master:7077 && tail -f /opt/spark/logs/spark*"
    volumes:
      - local-workspace-fs:/opt/workspace
      - local-data-fs:/opt/data
    networks:
      - sparknet
    depends_on:
      - spark-master

  spark-worker-2:
    image: apache/spark-py:v3.4.0
    container_name: spark-worker-2
    hostname: spark-worker-2
    user: root
    environment:
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=512m
    ports:
      - "8082:8081"
    command:
      - "bash"
      - "-c"
      - "pip install pyarrow pandas && /opt/spark/sbin/start-worker.sh spark://spark-master:7077 && tail -f /opt/spark/logs/spark*"
    volumes:
      - local-workspace-fs:/opt/workspace
      - local-data-fs:/opt/data
    networks:
      - sparknet
    depends_on:
      - spark-master