spark-kafka-consumer_docker-compose.yml

version: '2'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"
  kafka:
    image: wurstmeister/kafka:2.11-1.1.1
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 111.222.3.4 #your IP
      KAFKA_CREATE_TOPICS: "wiki-changes:1:1"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
  spark:
    image: jupyter/all-spark-notebook
    ports:
      - "8888:8888"
      - "4040-4080:4040-4080"
    links:
      - "kafka:kafka-server" #allows spark notebook to discover kafka service by name "kafka-server"
    volumes:
      - ../notebooks:/home/jovyan/work/notebooks/