Skip to content

Instantly share code, notes, and snippets.

@kjmkznr
Created March 30, 2017 15:43
Show Gist options
  • Save kjmkznr/1171e2a663c8a803cc117922c724724e to your computer and use it in GitHub Desktop.
Save kjmkznr/1171e2a663c8a803cc117922c724724e to your computer and use it in GitHub Desktop.
spark-master:
image: spark-2
command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
hostname: spark-master
environment:
MASTER: spark://spark-master:7077
SPARK_CONF_DIR: /conf
SPARK_PUBLIC_DNS: 127.0.0.1
expose:
- 7001
- 7002
- 7003
- 7004
- 7005
- 7006
- 7077
- 6066
ports:
- 4040:4040
- 6066:6066
- 7077:7077
- 8080:8080
volumes:
- ./conf/spark-master:/conf
- ./data:/tmp/data
spark-worker-1:
image: spark-2
command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
hostname: spark-worker-1
environment:
SPARK_CONF_DIR: /conf
SPARK_PUBLIC_DNS: 127.0.0.1
SPARK_WORKER_CORES: 2
SPARK_WORKER_MEMORY: 2g
SPARK_WORKER_PORT: 8881
SPARK_WORKER_WEBUI_PORT: 8081
links:
- spark-master
expose:
- 7012
- 7013
- 7014
- 7015
- 7016
- 8881
ports:
- 8081:8081
volumes:
- ./conf/spark-worker-1:/conf
- ./data:/tmp/data
FROM java:8
# SPARK
ARG SPARK_ARCHIVE=http://ftp.jaist.ac.jp/pub/apache/spark/spark-2.1.0/spark-2.1.0-bin-hadoop2.7.tgz
ENV SPARK_HOME /usr/local/spark-2.1.0-bin-hadoop2.7
ENV PATH $PATH:${SPARK_HOME}/bin
RUN curl -s ${SPARK_ARCHIVE} | tar -xz -C /usr/local/
WORKDIR $SPARK_HOME
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment