Native Spark Cluster on Docker with Jupyter Notebook and PySpark

docker-compose.yml

master:
  image: jupyter/pyspark-notebook
  command: bash -c "/usr/local/spark/bin/spark-class org.apache.spark.deploy.master.Master -h master && \
           start-notebook.sh"
  hostname: master
  environment:
    MASTER: spark://master:7077
    SPARK_CONF_DIR: /conf
    SPARK_PUBLIC_DNS: localhost
  expose:
    - 7001
    - 7002
    - 7003
    - 7004
    - 7005
    - 7006
    - 7077
    - 6066
  ports:
    - 4040:4040
    - 6066:6066
    - 7077:7077
    - 8080:8080
    - 8888:8888
    - 10000:10000
  volumes:
    - ./conf/master:/conf
    - ./data:/home/jovyan/work

worker:
  image: jupyter/pyspark-notebook
  command: /usr/local/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077 -d /home/jovyan/work
  hostname: worker
  environment:
    SPARK_CONF_DIR: /conf
    SPARK_WORKER_CORES: 2
    SPARK_WORKER_MEMORY: 1g
    SPARK_WORKER_PORT: 8881
    SPARK_WORKER_WEBUI_PORT: 8081
    SPARK_PUBLIC_DNS: localhost
  links:
    - master
  expose:
    - 7012
    - 7013
    - 7014
    - 7015
    - 7016
    - 8881
  ports:
    - 8081:8081
  volumes:
    - ./conf/worker:/conf
    - ./data:/home/jovyan/work