Created
December 27, 2023 16:28
-
-
Save Elanza-48/c14ffe890f0da996c3872372da67f37e to your computer and use it in GitHub Desktop.
Kafka 3 Node Cluster with Conduktor UI
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
services: | |
zoo1: | |
image: confluentinc/cp-zookeeper:7.3.2 | |
hostname: zoo1 | |
container_name: zoo1 | |
ports: | |
- "2181:2181" | |
environment: | |
ZOOKEEPER_CLIENT_PORT: 2181 | |
ZOOKEEPER_SERVER_ID: 1 | |
ZOOKEEPER_SERVERS: zoo1:2888:3888 | |
kafka1: | |
image: confluentinc/cp-kafka:7.3.2 | |
hostname: kafka1 | |
container_name: kafka1 | |
ports: | |
- "9092:9092" | |
- "29092:29092" | |
environment: | |
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 | |
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT | |
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL | |
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" | |
KAFKA_BROKER_ID: 1 | |
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" | |
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer | |
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" | |
depends_on: | |
- zoo1 | |
kafka2: | |
image: confluentinc/cp-kafka:7.3.2 | |
hostname: kafka2 | |
container_name: kafka2 | |
ports: | |
- "9093:9093" | |
- "29093:29093" | |
environment: | |
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 | |
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT | |
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL | |
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" | |
KAFKA_BROKER_ID: 2 | |
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" | |
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer | |
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" | |
depends_on: | |
- zoo1 | |
kafka3: | |
image: confluentinc/cp-kafka:7.3.2 | |
hostname: kafka3 | |
container_name: kafka3 | |
ports: | |
- "9094:9094" | |
- "29094:29094" | |
environment: | |
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 | |
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT | |
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL | |
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" | |
KAFKA_BROKER_ID: 3 | |
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" | |
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer | |
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" | |
depends_on: | |
- zoo1 | |
kafka-schema-registry: | |
image: confluentinc/cp-schema-registry:7.3.2 | |
hostname: kafka-schema-registry | |
container_name: kafka-schema-registry | |
depends_on: | |
- zoo1 | |
- kafka1 | |
- kafka2 | |
- kafka3 | |
ports: | |
- "8081:8081" | |
environment: | |
SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry | |
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094' | |
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 | |
kafka-rest-proxy: | |
image: confluentinc/cp-kafka-rest:7.3.2 | |
hostname: kafka-rest-proxy | |
container_name: kafka-rest-proxy | |
ports: | |
- "8082:8082" | |
environment: | |
# KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181 | |
KAFKA_REST_LISTENERS: http://0.0.0.0:8082/ | |
KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/ | |
KAFKA_REST_HOST_NAME: kafka-rest-proxy | |
KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092 | |
depends_on: | |
- zoo1 | |
- kafka1 | |
- kafka-schema-registry | |
ksqldb-server: | |
image: confluentinc/cp-ksqldb-server:7.3.2 | |
hostname: ksqldb-server | |
container_name: ksqldb-server | |
ports: | |
- "8088:8088" | |
environment: | |
KSQL_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094' | |
KSQL_LISTENERS: http://0.0.0.0:8088/ | |
KSQL_KSQL_SERVICE_ID: ksqldb-server_ | |
depends_on: | |
- zoo1 | |
- kafka1 | |
kafka-connect: | |
image: confluentinc/cp-kafka-connect:7.3.2 | |
hostname: kafka-connect | |
container_name: kafka-connect | |
ports: | |
- "8083:8083" | |
environment: | |
CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092,kafka2:19093,kafka3:19094" | |
CONNECT_REST_PORT: 8083 | |
CONNECT_GROUP_ID: compose-connect-group | |
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs | |
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets | |
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status | |
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter | |
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081' | |
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter | |
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081' | |
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect" | |
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" | |
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" | |
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" | |
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" | |
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" | |
CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars,/usr/share/confluent-hub-components' | |
volumes: | |
- ./connectors:/etc/kafka-connect/jars/ | |
depends_on: | |
- zoo1 | |
- kafka1 | |
- kafka2 | |
- kafka3 | |
- kafka-schema-registry | |
- kafka-rest-proxy | |
command: | |
- bash | |
- -c | |
- | | |
confluent-hub install --no-prompt debezium/debezium-connector-mysql:latest | |
confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.4.0 | |
/etc/confluent/docker/run | |
postgresql: | |
image: postgres:14 | |
hostname: postgresql | |
volumes: | |
- pg_data:/var/lib/postgresql/data | |
environment: | |
POSTGRES_DB: "conduktor-platform" | |
POSTGRES_USER: "conduktor" | |
POSTGRES_PASSWORD: "some_password" | |
POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" | |
conduktor-platform-cortex: | |
image: conduktor/conduktor-platform-cortex:latest | |
ports: | |
- "9009:9009" | |
- "9010:9010" | |
- "9090:9090" | |
environment: | |
CDK_CONSOLE-URL: 'http://conduktor-platform:8080' | |
conduktor-platform: | |
image: conduktor/conduktor-platform:latest | |
ports: | |
- "8080:8080" | |
volumes: | |
- conduktor_data:/var/conduktor | |
environment: | |
CDK_ORGANIZATION_NAME: "demo" | |
CDK_ADMIN_EMAIL: "admin@admin.io" | |
CDK_ADMIN_PASSWORD: "admin" | |
CDK_DATABASE_URL: "postgresql://conduktor:some_password@postgresql:5432/conduktor-platform" | |
CDK_CLUSTERS_0_ID: "default" | |
CDK_CLUSTERS_0_NAME: "My Local Kafka Cluster" | |
CDK_CLUSTERS_0_COLOR: "#0013E7" | |
CDK_CLUSTERS_0_BOOTSTRAPSERVERS: 'PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094' | |
CDK_CLUSTERS_0_ZOOKEEPERSERVER: zoo1:2181" | |
CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081" | |
CDK_CLUSTERS_0_KAFKACONNECTS_0_NAME: "full stack kafka connect" | |
CDK_MONITORING_CORTEX-URL: 'http://conduktor-platform-cortex:9009' | |
CDK_MONITORING_ALERT-MANAGER-URL: 'http://conduktor-platform-cortex:9010' | |
CDK_MONITORING_CALLBACK-URL: 'http://conduktor-platform:8080/monitoring/api/' | |
CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: 'http://0.0.0.0:8080' | |
CDK_CLUSTERS_0_KAFKACONNECTS_0_URL: "http://kafka-connect:8083" | |
volumes: | |
pg_data: {} | |
conduktor_data: {} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment