Confluent Platform Docker

README.md

Complete Confluent Platform docker-compose.yml file. Includes an nginx configuration to load-balance between the rest-proxy and schema-registry components.

Running - make sure both docker-compose.yml and nginx_kafka.conf are in the same directory:

$ docker-compose up

docker-compose.yml

version: '2'
networks:
  zk:
    driver: bridge
services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: INFO
      ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
    networks:
    - default
    - zk
    ports:
    - 22181:2181
    - 22888:2888
    - 23888:3888

  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: INFO
      ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
    networks:
    - default
    - zk
    ports:
    - 32181:2181
    - 32888:2888
    - 33888:3888

  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: INFO
      ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
    networks:
    - default
    - zk
    ports:
    - 42181:2181
    - 42888:2888
    - 43888:3888

  kafka-1:
    image: confluentinc/cp-kafka:latest
    networks:
    - default
    - zk
    links:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 19092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:19092
      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
  kafka-2:
    image: confluentinc/cp-kafka:latest
    networks:
    - default
    - zk
    links:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 29092:9092
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:29092
      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO

  kafka-3:
    image: confluentinc/cp-kafka:latest
    networks:
    - default
    - zk
    links:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 39092:9092
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:39092
      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO

  control-center:
    image: confluentinc/cp-control-center:latest
    networks:
      - zk
      - default
    links:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181"
      BOOTSTRAP_SERVERS: "kafka-1:19092,kafka-2:29092,kafka-3:39092"
      CONTROL_CENTER_REPLICATION_FACTOR: 3
      MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 6
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 6
      CONTROL_CENTER_STREAM_THREADS: 2
      CONTROL_CENTER_LOG4J_ROOT_LOGLEVEL: INFO

    ports:
      - 9021:9021

  schema-registry-1:
    image: confluentinc/cp-schema-registry:latest
    networks:
      - zk
      - default
    links:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      SCHEMA_REGISTRY_HOST_NAME: schema-registry-1
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO

    ports:
      - 18081:8081

  schema-registry-2:
    image: confluentinc/cp-schema-registry:latest
    networks:
      - zk
      - default
    links:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      SCHEMA_REGISTRY_HOST_NAME: schema-registry-2
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
    ports:
    - 28081:8081

  schema-registry-3:
    image: confluentinc/cp-schema-registry:latest
    networks:
    - zk
    - default
    links:
        - zookeeper-1
        - zookeeper-2
        - zookeeper-3
        - kafka-1
        - kafka-2
        - kafka-3
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      SCHEMA_REGISTRY_HOST_NAME: schema-registry-3
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
    ports:
    - 38081:8081

  restproxy-1:
      image: confluentinc/cp-kafka-rest:latest
      links:
          - zookeeper-1
          - zookeeper-2
          - zookeeper-3
          - schema-registry-1
          - schema-registry-2
          - schema-registry-3
      ports:
        - 18082:8082
      environment:
        KAFKA_REST_HOST_NAME: restproxy-1
        KAFKA_REST_ID_CONFIG: 1
        KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
        KAFKA_REST_LOG4J_ROOT_LOGLEVEL: INFO

  restproxy-2:
      image: confluentinc/cp-kafka-rest:latest
      links:
          - zookeeper-1
          - zookeeper-2
          - zookeeper-3
          - schema-registry-1
          - schema-registry-2
          - schema-registry-3
      ports:
        - 28082:8082
      environment:
        KAFKA_REST_HOST_NAME: restproxy-2
        KAFKA_REST_ID_CONFIG: 2
        KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
        KAFKA_REST_LOG4J_ROOT_LOGLEVEL: INFO

  restproxy-3:
      image: confluentinc/cp-kafka-rest:latest
      links:
          - zookeeper-1
          - zookeeper-2
          - zookeeper-3
          - schema-registry-1
          - schema-registry-2
          - schema-registry-3
      ports:
        - 38082:8082
      environment:
        KAFKA_REST_HOST_NAME: restproxy-3
        KAFKA_REST_ID_CONFIG: 3
        KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
        KAFKA_REST_LOG4J_ROOT_LOGLEVEL: INFO

  connect-1:
    image: confluentinc/cp-kafka-connect:latest
    restart: on-failure:3
    links:
        - zookeeper-1
        - zookeeper-2
        - zookeeper-3
        - kafka-1
        - kafka-2
        - kafka-3
    # include Kafka Connectors
    # these volumes should have the files from the `target/kafka-connect-twitter-1.0-SNAPSHOT-pacage/share/java` dir
    # after running mvn package
    # volumes:
    #   - ./connectors/kafka-connect-twitter:/usr/share/java/kafka-connect-twitter
    #   - ./connectors/kafka-connect-spooldir:/usr/share/java/kafka-connect-spooldir
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "default"
      CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
      CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "default.status"
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "connect-1"
      CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181

  connect-2:
    image: confluentinc/cp-kafka-connect:latest
    restart: on-failure:3
    links:
        - zookeeper-1
        - zookeeper-2
        - zookeeper-3
        - kafka-1
        - kafka-2
        - kafka-3
    # include Kafka Connectors
    # these volumes should have the files from the `target/kafka-connect-twitter-1.0-SNAPSHOT-pacage/share/java` dir
    # after running mvn package
    # volumes:
    #   - ./connectors/kafka-connect-twitter:/usr/share/java/kafka-connect-twitter
    #   - ./connectors/kafka-connect-spooldir:/usr/share/java/kafka-connect-spooldir
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "default"
      CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
      CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "default.status"
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "connect-2"
      CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181

  nginx:
    image: nginx:latest
    ports:
      - "8080:8080"
    volumes:
      - ./nginx_kafka.conf:/etc/nginx/conf.d/kafka_proxy.template
    links:
      - restproxy-1
      - restproxy-2
      - restproxy-3
      - schema-registry-1
      - schema-registry-2
      - schema-registry-3
      - control-center
    environment:
      - NGINX_HOST=confluent-platform
      - NGINX_PORT=8080
    command: /bin/bash -c "envsubst < /etc/nginx/conf.d/kafka_proxy.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"

nginx_kafka.conf

upstream restproxy {
  ip_hash;
  server restproxy-1:8082;
  server restproxy-2:8082;
  server restproxy-3:8082;
}

upstream schemaregistry {
  ip_hash;
  server schema-registry-1:8081;
  server schema-registry-2:8081;
  server schema-registry-3:8081;
}

server {
  server_name rest-proxy;
  listen 8082;
  location / {
    proxy_pass http://restproxy;
  }
}

server {
  server_name schema-registry;
  listen 8081;
  location / {
    proxy_pass http://schemaregistry;
  }
}

server {
  server_name restproxy-1;
  location / {
    proxy_pass http://restproxy-1:8082;
  }
}

server {
  server_name restproxy-2;
  location / {
    proxy_pass http://restproxy-2:8082;
  }
}

server {
  server_name restproxy-3;
  location / {
    proxy_pass http://restproxy-3:8082;
  }
}