Running Zookeeper, 5 Kafka brokers, Kafka UI, and kafka-init using docker compose on Windows

README.md

Running Kafka Brokers on Docker Containers

How to Run

Docker Compose A

This config uses Docker images created by wurstmeister.

1. Retrieve the IPv4 address of the host machine and assign it to the environment variable HOST_IP.

$env:HOST_IP = (Get-NetIPConfiguration | Where-Object { $_.IPv4DefaultGateway -ne $null }).IPv4Address.IPAddress

2. Run the containers.

docker compose -f ./docker-compose-A.yaml up -d

Docker Compose B

This config uses Docker images created by Confluent.

1. Run the containers.

docker compose -f ./docker-compose-B.yaml up -d

Listeners

Containers	Host:Port
Kafka UI	localhost:9000
Kafka Broker 1	localhost:9001
Kafka Broker 2	localhost:9002
Kafka Broker 3	localhost:9003
Kafka Broker 4	localhost:9004
Kafka Broker 5	localhost:9005

docker-compose-A.yaml

services:
  kafdrop:
    image: obsidiandynamics/kafdrop:3.31.0
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: kafka:9092
      JVM_OPTS: -Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify
    depends_on:
      - kafka

  kafka-init:
    image: monasca/kafka-init:0.0.4
    environment:
      KAFKA_CREATE_TOPICS: topic.1:3:3,topic.2:5:5
    depends_on:
      - kafka

  kafka:
    image: wurstmeister/kafka:2.13-2.8.1
    deploy:
      replicas: 5
    ports:
      - 9001-9005:9092
    environment:
      KAFKA_ADVERTISED_HOST_NAME: ${HOST_IP}
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      COMPOSE_CONVERT_WINDOWS_PATH: true
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - zookeeper

  zookeeper:
    image: wurstmeister/zookeeper:3.4.6

docker-compose-B.yaml

---
version: '3.5'

networks:
  kafka:
    name: kafka

services:
  kafdrop:
    image: obsidiandynamics/kafdrop:3.31.0
    ports:
      - 9000:9000
    networks:
      - kafka
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
      - kafka-4
      - kafka-5
    environment:
      KAFKA_BROKERCONNECT: kafka-1:9092
      JVM_OPTS: -Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify

  kafka-init:
    image: monasca/kafka-init:0.0.4
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
      - kafka-4
      - kafka-5
    networks:
      - kafka
    environment:
      KAFKA_HOST: kafka-1:9092
      KAFKA_CREATE_TOPICS: topic.1:3:3,topic.2:5:5

  kafka-1:
    image: confluentinc/cp-kafka:5.5.0
    ports:
      - "9001:9001"
    networks:
      - kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9092,EXTERNAL://localhost:9001
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

  kafka-2:
    image: confluentinc/cp-kafka:5.5.0
    ports:
      - "9002:9002"
    networks:
      - kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:9092,EXTERNAL://localhost:9002
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

  kafka-3:
    image: confluentinc/cp-kafka:5.5.0
    ports:
      - "9003:9003"
    networks:
      - kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:9092,EXTERNAL://localhost:9003
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

  kafka-4:
    image: confluentinc/cp-kafka:5.5.0
    ports:
      - "9004:9004"
    networks:
      - kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-4:9092,EXTERNAL://localhost:9004
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

  kafka-5:
    image: confluentinc/cp-kafka:5.5.0
    ports:
      - "9005:9005"
    networks:
      - kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-5:9092,EXTERNAL://localhost:9005
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL

  zookeeper:
    image: confluentinc/cp-zookeeper:5.5.0
    networks:
      - kafka
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181