scrapy-cluster-docker-cloud-stack

docker-compose.yml

crawler:
  environment:
    - REDIS_PORT=6379
  image: 'istresearch/scrapy-cluster:crawler-dev-alpine'
  links:
    - kafka
    - redis
    - zookeeper
  restart: always
  target_num_containers: 3
kafka:
  environment:
    - KAFKA_ADVERTISED_HOST_NAME=$DOCKERCLOUD_CONTAINER_HOSTNAME
    - KAFKA_ADVERTISED_PORT=9092
    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    - 'KAFKA_ZOOKEEPER_CONNECT=zookeeper-1:2181'
  expose:
    - '9092'
  image: 'wurstmeister/kafka:latest'
  links:
    - zookeeper
  restart: always
  volumes:
    - '/var/run/docker.sock:/var/run/docker.sock'
kafka-monitor:
  environment:
    - REDIS_PORT=6379
  image: 'istresearch/scrapy-cluster:kafka-monitor-dev-alpine'
  links:
    - kafka
    - redis
  restart: always
redis:
  expose:
    - '6379'
  image: 'redis:latest'
  restart: always
redis-monitor:
  environment:
    - REDIS_PORT=6379
  image: 'istresearch/scrapy-cluster:redis-monitor-dev-alpine'
  links:
    - kafka
    - redis
    - zookeeper
  restart: always
zookeeper:
  expose:
    - '2181'
  image: 'wurstmeister/zookeeper:latest'
  restart: always