docker multinode

docker multinode.md

download all the files and spread across the node adapt and change according to your existing hostnames replace vm0[1-3]

if running on a single node create entries in /etc/hosts.

put the kafka_config.yml in a subdirectory "volumes" download https://github.com/sknop/kafka-docker-composer/blob/master/volumes/jmx_prometheus_javaagent-0.20.0.jar and put it to volumes as well.

kafka_config.yml

---
startDelaySeconds: 120
lowercaseOutputName: true
lowercaseOutputLabelNames: true
blacklistObjectNames:
  - "kafka.consumer:type=*,id=*"
  - "kafka.consumer:type=*,client-id=*"
  - "kafka.consumer:type=*,client-id=*,node-id=*"
  - "kafka.producer:type=*,id=*"
  - "kafka.producer:type=*,client-id=*"
  - "kafka.producer:type=*,client-id=*,node-id=*"
  - "kafka.*:type=kafka-metrics-count,*"
  # This will ignore the admin client metrics from Kafka Brokers and will blacklist certain metrics
  # that do not make sense for ingestion.
  # "kafka.admin.client:type=*, node-id=*, client-id=*"
  # "kafka.admin.client:type=*, client-id=*"
  # "kafka.admin.client:type=*, id=*"
  - "kafka.admin.client:*"
  - "kafka.server:type=*,cipher=*,protocol=*,listener=*,networkProcessor=*"
  #- "kafka.server:type=*"
  - "kafka.server:type=app-info,id=*"
  - "kafka.rest:*"
  - "rest.utils:*"
  - "io.confluent.common.security.jetty:*"
  - "io.confluent.rest:*"
  - "confluent.metadata.service:type=app-info,id=*"
  - "confluent.metadata.service:type=app-info,client-id=*"
  - "confluent.metadata:type=kafkaauthstore,*"
rules:
  # This rule is more specific than the next rule; it has to come before it otherwise it will never be hit
  # "kafka.server:type=*,name=*, client-id=*, topic=*, partition=*"
  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
    name: kafka_server_$1_$2
    type: GAUGE
    cache: true
    labels:
      clientId: "$3"
      topic: "$4"
      partition: "$5"
  # This is by far the biggest contributor to the number of sheer metrics being produced.
  # Always keep it near the top for the case of probability when so many metrics will hit the first condition and exit.
  # "kafka.cluster:type=*, name=*, topic=*, partition=*"
  # "kafka.log:type=*,name=*, topic=*, partition=*"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
    name: kafka_$1_$2_$3
    type: GAUGE
    cache: true
    labels:
      topic: "$4"
      partition: "$5"
  # Next two rules are similar; Value version is a GAUGE; Count version is not
  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
    name: kafka_server_$1_$2
    type: GAUGE
    cache: true
    labels:
      clientId: "$3"
      broker: "$4:$5"
  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Count
    name: kafka_server_$1_$2
    cache: true
    labels:
      clientId: "$3"
      broker: "$4:$5"
  # Needed for Cluster Linking metrics
  # "kafka.server:type=*, name=*, *=*, *=*, *=*, *=*"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(Count|Value)
    name: kafka_$1_$2_$3
    cache: true
    labels:
      "$4": "$5"
      "$6": "$7"
      "$8": "$9"
      "$10": "$11"
  # "kafka.server:type=*, name=*, *=*, *=*, *=*"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(Count|Value)
    name: kafka_$1_$2_$3
    cache: true
    labels:
      "$4": "$5"
      "$6": "$7"
      "$8": "$9"
  # "kafka.network:type=*, name=*, request=*, error=*"
  # "kafka.network:type=*, name=*, request=*, version=*"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>(Count|Value)
    name: kafka_$1_$2_$3
    cache: true
    labels:
      "$4": "$5"
      "$6": "$7"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    cache: true
    labels:
      "$4": "$5"
      "$6": "$7"
      quantile: "0.$8"
  # "kafka.rest:type=*, topic=*, partition=*, client-id=*"
  # "kafka.rest:type=*, cipher=*, protocol=*, client-id=*"
  - pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>Value
    name: kafka_$1_$2
    cache: true
    labels:
      "$3": "$4"
      "$5": "$6"
      "$7": "$8"
  # Count and Value
  # "kafka.server:type=*, name=*, topic=*"
  # "kafka.server:type=*, name=*, clientId=*"
  # "kafka.server:type=*, name=*, delayedOperation=*"
  # "kafka.server:type=*, name=*, fetcherType=*"
  # "kafka.network:type=*, name=*, networkProcessor=*"
  # "kafka.network:type=*, name=*, processor=*"
  # "kafka.network:type=*, name=*, request=*"
  # "kafka.network:type=*, name=*, listener=*"
  # "kafka.log:type=*, name=*, logDirectory=*"
  # "kafka.log:type=*, name=*, op=*"
  # "kafka.rest:type=*, node-id=*, client-id=*"
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>(Count|Value)
    name: kafka_$1_$2_$3
    cache: true
    labels:
      "$4": "$5"
  # "kafka.consumer:type=*, topic=*, client-id=*"
  # "kafka.producer:type=*, topic=*, client-id=*"
  # "kafka.rest:type=*, topic=*, client-id=*"
  # "kafka.server:type=*, broker-id=*, fetcher-id=*"
  # "kafka.server:type=*, listener=*, networkProcessor=*"
  - pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(Count|Value)
    name: kafka_$1_$2
    cache: true
    labels:
      "$3": "$4"
      "$5": "$6"
  # - pattern: "kafka.(.+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(.+):"
  #   name: kafka_$1_$2
  #   cache: true
  #   labels:
  #     "$3": "$4"
  #     "$5": "$6"
  #     attribute_name: "$7"
  # "kafka.network:type=*, name=*"
  # "kafka.server:type=*, name=*"
  # "kafka.controller:type=*, name=*"
  # "kafka.databalancer:type=*, name=*"
  # "kafka.log:type=*, name=*"
  # "kafka.utils:type=*, name=*"
  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(Count|Value)
    name: kafka_$1_$2_$3
  # "kafka.producer:type=*, client-id=*"
  # "kafka.producer:type=*, id=*"
  # "kafka.rest:type=*, client-id=*"
  # "kafka.rest:type=*, http-status-code=*"
  # "kafka.server:type=*, BrokerId=*"
  # "kafka.server:type=*, listener=*"
  # "kafka.server:type=*, id=*"
  - pattern: kafka.(\w+)<type=(.+), (.+)=(.+)><>Value
    name: kafka_$1_$2
    cache: true
    labels:
      "$3": "$4"
  # - pattern: "kafka.(.+)<type=(.+), (.+)=(.+)><>(.+):"
  #   name: kafka_$1_$2
  #   cache: true
  #   labels:
  #     "$3": "$4"
  #     attribute_name: "$5"
  - pattern: kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
    name: kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total
    type: GAUGE
  # "kafka.server:type=*, listener=*, networkProcessor=*, clientSoftwareName=*, clientSoftwareVersion=*"
  - pattern: kafka.server<type=socket-server-metrics, clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
    name: kafka_server_socketservermetrics_connections
    type: GAUGE
    cache: true
    labels:
      client_software_name: "$1"
      client_software_version: "$2"
      listener: "$3"
      network_processor: "$4"
  - pattern: "kafka.server<type=socket-server-metrics, listener=(.+), networkProcessor=(.+)><>(.+):"
    name: kafka_server_socketservermetrics_$3
    type: GAUGE
    cache: true
    labels:
      listener: "$1"
      network_processor: "$2"
  # - pattern: "kafka.server<type=socket-server-metrics, listener=(.+)><>(.+):"
  #   name: kafka_server_socketservermetrics
  #   type: GAUGE
  #   cache: true
  #   labels:
  #     listener: "$1"
  #     attribute_name: "$2"
  # "kafka.coordinator.group:type=*, name=*"
  # "kafka.coordinator.transaction:type=*, name=*"
  - pattern: kafka.coordinator.(\w+)<type=(.+), name=(.+)><>(Count|Value)
    name: kafka_coordinator_$1_$2_$3
  # Percentile
  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    cache: true
    labels:
      "$4": "$5"
      quantile: "0.$6"
  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
    name: kafka_$1_$2_$3
    type: GAUGE
    cache: true
    labels:
      quantile: "0.$4"
  # Additional Rules for Confluent Server Metrics
  # 'kafka.server:type=confluent-auth-store-metrics'
  - pattern: kafka.server<type=confluent-auth-store-metrics><>(.+):(.*)
    name: confluent_auth_store_metrics_$1
    type: GAUGE
    cache: true
  # 'confluent.metadata:type=*, name=*, topic=*, partition=*'
  - pattern: confluent.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(Value|Count)
    name: confluent_$1_$2
    type: GAUGE
    cache: true
    labels:
      "$3": "$4"
      "$5": "$6"
      "$7": "$8"
  # 'confluent.metadata.service:type=*, node-id=*, client-id=*'
  - pattern: confluent.(.+)<type=(.+), (.+)=(.+), (.+)=(.+)><>Value
    name: confluent_$1_$2
    type: GAUGE
    cache: true
    labels:
      "$3": "$4"
      "$5": "$6"
  # 'confluent.metadata.service:type=*, node-id=*, client-id=*'
  - pattern: 'confluent.metadata.service<type=(.+), (.+)=(.+), (.+)=(.+)><>(.+):'
    name: $1
    type: GAUGE
    cache: true
    labels:
      "$2": "$3"
      "$4": "$5"
      attribute_name: "$6"
  # 'confluent.metadata.service:type=*, client-id=*'
  # 'confluent.metadata.service:type=*, id=*'
  # 'confluent.metadata:type=*, name=*'
  # 'confluent.license:type=*, name=*'
  - pattern: confluent.(.+)<type=(.+), (.+)=(.+)><>Value
    name: confluent_$1_$2
    type: GAUGE
    cache: true
    labels:
      "$3": "$4"
  - pattern: 'confluent.(.+)<type=(.+), (.+)=(.+)><>(.+):'
    name: confluent_$1_$2
    type: GAUGE
    cache: true
    labels:
      "$3": "$4"
      attribute_name: "$5"
  # Quotas
  - pattern : 'kafka.server<type=(Produce|Fetch|Request), user=(.+), client-id=(.+)><>(.+):'
    name: kafka_server_$1_$4
    type: GAUGE
    cache: true
    labels:
      user: "$2"
      client-id: "$3"

  - pattern : 'kafka.server<type=(Produce|Fetch|Request), user=(.+)><>(.+):'
    name: kafka_server_$1_$3
    type: GAUGE
    cache: true
    labels:
      user: "$2"

  - pattern : 'kafka.server<type=(Produce|Fetch|Request), client-id=(.+)><>(.+):'
    name: kafka_server_$1_$3
    type: GAUGE
    cache: true
    labels:
      client-id: "$2"

# Broker  Metrics
  - pattern : 'kafka.server<type=BrokerTopicMetrics, name=(MessagesInPerSec|BytesInPerSec|BytesOutPerSec|TotalProduceRequestsPerSec|TotalFetchRequestsPerSec), topic=(.+)><>(Count|OneMinuteRate|FiveMinuteRate|FifteenMinuteRate)'
    name: kafka_server_brokertopicmetrics_$1_$3
    type: GAUGE
    cache: true
    labels:
      topic: "$2"

  - pattern : 'kafka.server<type=BrokerTopicMetrics, name=(MessagesInPerSec|BytesInPerSec|BytesOutPerSec)><>(Count|OneMinuteRate|FiveMinuteRate|FifteenMinuteRate)'
    name: kafka_server_brokertopicmetrics_$1_$2_alltopics
    type: GAUGE

# Network Request  Metrics
  - pattern : 'kafka.network<type=RequestMetrics, name=RequestsPerSec, request=(.+), version=([0-9]+)><>(Count|OneMinuteRate|FiveMinuteRate|FifteenMinuteRate)'
    name: kafka_network_requestmetrics_requestspersec_$3
    type: GAUGE
    cache: true
    labels:
      request: "$1"

# "kafka.server:type=raft-metrics"
  - pattern: kafka.server<type=raft-metrics><>(.+):(.*)
    name: kafka_server_raft_metrics_$1
    type: GAUGE
    cache: true

# kafka.server:type=transaction-coordinator-metrics
  - pattern: kafka.server<type=transaction-coordinator-metrics><>(.+):(.*)
    name: kafka_server_transaction_coordinator_metrics_$1
    type: GAUGE
    cache: true

  # Needed for Cluster Linking metrics
  # kafka.server.link:type=ClusterLinkFetcherManager,name=*,clientId=&,link-name=*
  - pattern: kafka.server.link<type=ClusterLinkFetcherManager, name=(.+), (.+)=(.+), (.+)=(.+)><>Value
    name: kafka_server_link_clusterlinkfetchermanager_$1
    type: GAUGE
    cache: true
    labels:
      "$2": "$3"
      "$4": "$5"
  # kafka.server:type=cluster-link-fetcher-metrics,link-name=*,broker-id=*,fetcher-id=*, mechanism=*
  - pattern: 'kafka.server<type=cluster-link-fetcher-metrics, (.+)=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(.+):'
    name: kafka_server_cluster_link_fetcher_metrics_$9
    type: GAUGE
    labels:
      "$1": "$2"
      "$3": "$4"
      "$5": "$6"
      "$7": "$8"
  # kafka.server:type=cluster-link-fetcher-metrics,link-name=*,broker-id=*,fetcher-id=*
  - pattern: 'kafka.server<type=cluster-link-fetcher-metrics, (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(.+):'
    name: kafka_server_cluster_link_fetcher_metrics_$7
    type: GAUGE
    cache: true
    labels:
      "$1": "$2"
      "$3": "$4"
      "$5": "$6"
  # kafka.server:type=cluster-link-metrics, mode=*, state=*, link-name=*, name=*
  - pattern: 'kafka.server<type=cluster-link-metrics, (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(.+):'
    name: kafka_server_cluster_link_metrics_$7
    type: GAUGE
    labels:
      "$1": "$2"
      "$3": "$4"
      "$5": "$6"
  # kafka.server:type=cluster-link-metrics,state=*,link-name=*,name=*
  - pattern: 'kafka.server<type=cluster-link-metrics, (.+)=(.+), (.+)=(.+)><>(.+):'
    name: kafka_server_cluster_link_metrics_$5
    type: GAUGE
    cache: true
    labels:
      "$1": "$2"
      "$3": "$4"
  # kafka.server:type=cluster-link-metrics,name=*,link-name=*
  - pattern: 'kafka.server<type=cluster-link-metrics, (.+)=(.+)><>(.+):'
    name: kafka_server_cluster_link_metrics_$3
    type: GAUGE
    labels:
      "$1": "$2"
  # kafka.server:type=cluster-link-source-metrics,request=*,link-id=*
  - pattern: 'kafka.server<type=cluster-link-source-metrics, request=(.+), link-id=(.+)><>(.+):(.+)'
    name: kafka_server_cluster_link_source_metrics_$1_$3
    value: $4
    type: GAUGE
    labels:
      request: "$1"
      link-id: "$2"

  ## Needed for Tiered Storage Metrics
  # kafka.tier.tasks.archive:type=TierArchiver,name=*
  - pattern: 'kafka.tier.tasks.archive<type=TierArchiver, name=(.+)><>(.+):(.+)'
    name: kafka_tier_tasks_archive_tierarchiver_$1
    type: GAUGE
  # kafka.tier.tasks:type=TierTasks,name=*
  - pattern: 'kafka.tier.tasks<type=TierTasks, name=(.+)><>(.+):(.+)'
    name: kafka_tier_tasks_tiertasks_$1
    type: GAUGE

  ## Consumer Lag Offsets
  # "kafka.server:type=tenant-metrics,name=*,consumer-group=*,client-id=*,topic=*,partition=*"
  - pattern: 'kafka.server<type=tenant-metrics, member=(.+), topic=(.+), consumer-group=(.+), partition=(.+), client-id=(.+)><>(.+):(.+)'
    name: kafka_server_tenant_metrics_$6
    type: GAUGE
    labels:
      consumerGroup: "$3"
      client-id: "$5"
      topic: "$2"
      partition: "$4"

vm01.yml

services:
    controller-1:
        image: confluentinc/cp-kafka:7.7.0
        hostname: vm01
        container_name: controller-1
        environment:
            KAFKA_NODE_ID: 1
            CLUSTER_ID: Nk018hRAQFytWskYqtQduw
            KAFKA_PROCESS_ROLES: controller
            KAFKA_LISTENERS: CONTROLLER://controller-1:19091
            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
            KAFKA_INTER_BROKER_LISTENER_NAME: CONTROLLER
            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@vm01:19091,2@vm02:19092,3@vm03:19093
            KAFKA_JMX_PORT: 9999
            KAFKA_JMX_HOSTNAME: controller-1
            KAFKA_BROKER_RACK: rack-0
            KAFKA_DEFAULT_REPLICATION_FACTOR: 3
            KAFKA_OFFSET_REPLICATION_FACTOR: 3
            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_METADATA_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 3            
            KAFKA_OPTS: -javaagent:/tmp/jmx_prometheus_javaagent-0.20.0.jar=8091:/tmp/kafka_config.yml
        cap_add:
            - NET_ADMIN
        ports:
            - 19091:19091
        volumes:
            - $PWD/volumes/jmx_prometheus_javaagent-0.20.0.jar:/tmp/jmx_prometheus_javaagent-0.20.0.jar
            - $PWD/volumes/kafka_config.yml:/tmp/kafka_config.yml

    kafka-1:
        image: confluentinc/cp-kafka:7.7.0
        hostname: vm01
        container_name: kafka-1

        healthcheck:
            test: curl -fail --silent http://kafka-1:8090/kafka/v3/clusters/ --output /dev/null || exit 1
            interval: 10s
            retries: 10
            start_period: 20s
        depends_on:
            - controller-1
        environment:
            KAFKA_LISTENERS: PLAINTEXT://vm01:19094, EXTERNAL://vm01:9091
            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://vm01:19094, EXTERNAL://vm01:9091
            KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
            KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
            KAFKA_JMX_PORT: 10001
            KAFKA_JMX_HOSTNAME: localhost
            KAFKA_BROKER_RACK: rack-0
            KAFKA_OPTS: -javaagent:/tmp/jmx_prometheus_javaagent-0.20.0.jar=8091:/tmp/kafka_config.yml
            KAFKA_MIN_INSYNC_REPLICAS: 2
            KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: True
            KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: True
            KAFKA_NODE_ID: 4
            CLUSTER_ID: Nk018hRAQFytWskYqtQduw
            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@vm01:19091,2@vm02:19092,3@vm03:19093
            KAFKA_PROCESS_ROLES: broker
            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
        cap_add:
            - NET_ADMIN
        ports:
            - 9091:9091
            - 10001:10001
            - 10101:8091
            - 10201:8090
            - 19094:19094

        volumes:
            - $PWD/volumes/jmx_prometheus_javaagent-0.20.0.jar:/tmp/jmx_prometheus_javaagent-0.20.0.jar
            - $PWD/volumes/kafka_config.yml:/tmp/kafka_config.yml            

vm02.yml

services:
    controller-2:
        image: confluentinc/cp-kafka:7.7.0
        hostname: vm02
        container_name: controller-2

        environment:
            KAFKA_NODE_ID: 2
            CLUSTER_ID: Nk018hRAQFytWskYqtQduw
            KAFKA_PROCESS_ROLES: controller
            KAFKA_LISTENERS: CONTROLLER://controller-2:19092
            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
            KAFKA_INTER_BROKER_LISTENER_NAME: CONTROLLER
            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@vm01:19091,2@vm02:19092,3@vm03:19093
            KAFKA_JMX_PORT: 9999
            KAFKA_JMX_HOSTNAME: controller-2
            KAFKA_BROKER_RACK: rack-0
            KAFKA_DEFAULT_REPLICATION_FACTOR: 3
            KAFKA_OFFSET_REPLICATION_FACTOR: 3
            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_METADATA_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_OPTS: -javaagent:/tmp/jmx_prometheus_javaagent-0.20.0.jar=8091:/tmp/kafka_config.yml
        cap_add:
            - NET_ADMIN
        ports:
            - 19092:19092
        volumes:
            - $PWD/volumes/jmx_prometheus_javaagent-0.20.0.jar:/tmp/jmx_prometheus_javaagent-0.20.0.jar
            - $PWD/volumes/kafka_config.yml:/tmp/kafka_config.yml

    kafka-2:
        image: confluentinc/cp-kafka:7.7.0
        hostname: vm02
        container_name: kafka-2
        healthcheck:
            test: curl -fail --silent http://kafka-2:8090/kafka/v3/clusters/ --output /dev/null || exit 1
            interval: 10s
            retries: 10
            start_period: 20s
        depends_on:
            - controller-2
        environment:
            KAFKA_LISTENERS: PLAINTEXT://vm02:19095, EXTERNAL://vm02:9092
            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://vm02:19095, EXTERNAL://vm02:9092
            KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
            KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
            KAFKA_JMX_PORT: 10002
            KAFKA_JMX_HOSTNAME: localhost
            KAFKA_BROKER_RACK: rack-0
            KAFKA_OPTS: -javaagent:/tmp/jmx_prometheus_javaagent-0.20.0.jar=8091:/tmp/kafka_config.yml
            KAFKA_MIN_INSYNC_REPLICAS: 2
            KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: True
            KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: True
            KAFKA_NODE_ID: 5
            CLUSTER_ID: Nk018hRAQFytWskYqtQduw
            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@vm01:19091,2@vm02:19092,3@vm03:19093
            KAFKA_PROCESS_ROLES: broker
            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
        cap_add:
            - NET_ADMIN
        ports:
            - 9092:9092
            - 10002:10002
            - 10102:8091
            - 10202:8090
            - 19095:19095
        volumes:
            - $PWD/volumes/jmx_prometheus_javaagent-0.20.0.jar:/tmp/jmx_prometheus_javaagent-0.20.0.jar
            - $PWD/volumes/kafka_config.yml:/tmp/kafka_config.yml            

vm03.yml

services:
    controller-3:
        image: confluentinc/cp-kafka:7.7.0
        hostname: vm03
        container_name: controller-3
        environment:
            KAFKA_NODE_ID: 3
            CLUSTER_ID: Nk018hRAQFytWskYqtQduw
            KAFKA_PROCESS_ROLES: controller
            KAFKA_LISTENERS: CONTROLLER://controller-3:19093
            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
            KAFKA_INTER_BROKER_LISTENER_NAME: CONTROLLER
            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@vm01:19091,2@vm02:19092,3@vm03:19093
            KAFKA_JMX_PORT: 9999
            KAFKA_JMX_HOSTNAME: controller-3
            KAFKA_BROKER_RACK: rack-0
            KAFKA_DEFAULT_REPLICATION_FACTOR: 3
            KAFKA_OFFSET_REPLICATION_FACTOR: 3
            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_METADATA_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_OPTS: -javaagent:/tmp/jmx_prometheus_javaagent-0.20.0.jar=8091:/tmp/kafka_config.yml
        cap_add:
            - NET_ADMIN
        ports:
            - 19093:19093
        volumes:
            - $PWD/volumes/jmx_prometheus_javaagent-0.20.0.jar:/tmp/jmx_prometheus_javaagent-0.20.0.jar
            - $PWD/volumes/kafka_config.yml:/tmp/kafka_config.yml

    kafka-3:
        image: confluentinc/cp-kafka:7.7.0
        hostname: vm03
        container_name: kafka-3

        healthcheck:
            test: curl -fail --silent http://kafka-2:8090/kafka/v3/clusters/ --output /dev/null || exit 1
            interval: 10s
            retries: 10
            start_period: 20s
        depends_on:
            - controller-3
        environment:
            KAFKA_LISTENERS: PLAINTEXT://vm03:19096, EXTERNAL://vm03:9093
            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://vm03:19096, EXTERNAL://vm03:9093
            KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
            KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
            KAFKA_JMX_PORT: 10003
            KAFKA_JMX_HOSTNAME: localhost
            KAFKA_BROKER_RACK: rack-0
            KAFKA_OPTS: -javaagent:/tmp/jmx_prometheus_javaagent-0.20.0.jar=8091:/tmp/kafka_config.yml
            KAFKA_MIN_INSYNC_REPLICAS: 2
            KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
            KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: True
            KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: True
            KAFKA_NODE_ID: 6
            CLUSTER_ID: Nk018hRAQFytWskYqtQduw
            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@vm01:19091,2@vm02:19092,3@vm03:19093
            KAFKA_PROCESS_ROLES: broker
            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
        cap_add:
            - NET_ADMIN
        ports:
            - 9093:9093
            - 10003:10003
            - 10103:8091
            - 10203:8090
            - 19096:19096
        volumes:
            - $PWD/volumes/jmx_prometheus_javaagent-0.20.0.jar:/tmp/jmx_prometheus_javaagent-0.20.0.jar
            - $PWD/volumes/kafka_config.yml:/tmp/kafka_config.yml