Created
May 26, 2021 20:14
-
-
Save crockpotveggies/02d3eced4b212d3863b2ceb8da867fc2 to your computer and use it in GitHub Desktop.
Run Kafka in one command using docker-compose
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| version: '2' | |
| services: | |
| zookeeper: | |
| image: confluentinc/cp-zookeeper:latest | |
| hostname: zookeeper | |
| container_name: zookeeper | |
| ports: | |
| - '2181:2181' | |
| environment: | |
| ZOOKEEPER_CLIENT_PORT: '2181' | |
| ZOOKEEPER_TICK_TIME: '2000' | |
| KAFKA_OPTS: '-Djava.security.auth.login.config=/etc/kafka/server-jaas.conf -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider' | |
| volumes: | |
| - ./testHelpers/kafka/server-jaas.conf:/etc/kafka/server-jaas.conf:ro,z | |
| kafka1: | |
| image: confluentinc/cp-kafka:6.1.1 | |
| hostname: kafka1 | |
| container_name: kafka1 | |
| labels: | |
| - 'custom.project=kafkajs' | |
| - 'custom.service=kafka1' | |
| depends_on: | |
| - zookeeper | |
| ports: | |
| - '29092:29092' | |
| - '9092:9092' | |
| - '29093:29093' | |
| - '9093:9093' | |
| - '29094:29094' | |
| - '9094:9094' | |
| environment: | |
| KAFKA_BROKER_ID: '0' | |
| KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092,SSL://kafka1:29093,SSL_HOST://localhost:9093,SASL_SSL://kafka1:29094,SASL_SSL_HOST://localhost:9094 | |
| KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' | |
| KAFKA_DELETE_TOPIC_ENABLE: 'true' | |
| KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0' | |
| KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks' | |
| KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds' | |
| KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds' | |
| KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks' | |
| KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds' | |
| KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN' | |
| KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512' | |
| KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf' | |
| KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" | |
| KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' | |
| # suppress verbosity | |
| # https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template | |
| KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO' | |
| CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' | |
| volumes: | |
| - ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z | |
| - ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z | |
| - ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z | |
| - ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z | |
| - ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z | |
| - ./testHelpers/kafka/server-jaas.conf:/opt/kafka/config/server-jaas.conf:ro,z | |
| kafka2: | |
| image: confluentinc/cp-kafka:5.4.2 | |
| hostname: kafka2 | |
| container_name: kafka2 | |
| labels: | |
| - 'custom.project=kafkajs' | |
| - 'custom.service=kafka2' | |
| depends_on: | |
| - zookeeper | |
| ports: | |
| - '29095:29095' | |
| - '9095:9095' | |
| - '29096:29096' | |
| - '9096:9096' | |
| - '29097:29097' | |
| - '9097:9097' | |
| environment: | |
| KAFKA_BROKER_ID: '1' | |
| KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:29095,PLAINTEXT_HOST://localhost:9095,SSL://kafka2:29096,SSL_HOST://localhost:9096,SASL_SSL://kafka2:29097,SASL_SSL_HOST://localhost:9097 | |
| KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' | |
| KAFKA_DELETE_TOPIC_ENABLE: 'true' | |
| KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0' | |
| KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks' | |
| KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds' | |
| KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds' | |
| KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks' | |
| KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds' | |
| KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN' | |
| KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512' | |
| KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf' | |
| KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" | |
| KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' | |
| # suppress verbosity | |
| # https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template | |
| KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO' | |
| CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' | |
| volumes: | |
| - ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z | |
| - ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z | |
| - ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z | |
| - ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z | |
| - ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z | |
| - ./testHelpers/kafka/server-jaas.conf:/opt/kafka/config/server-jaas.conf:ro,z | |
| kafka3: | |
| image: confluentinc/cp-kafka:5.4.2 | |
| hostname: kafka3 | |
| container_name: kafka3 | |
| labels: | |
| - 'custom.project=kafkajs' | |
| - 'custom.service=kafka3' | |
| depends_on: | |
| - zookeeper | |
| ports: | |
| - '29098:29098' | |
| - '9098:9098' | |
| - '29099:29099' | |
| - '9099:9099' | |
| - '29100:29100' | |
| - '9100:9100' | |
| environment: | |
| KAFKA_BROKER_ID: '2' | |
| KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
| KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:29098,PLAINTEXT_HOST://localhost:9098,SSL://kafka3:29099,SSL_HOST://localhost:9099,SASL_SSL://kafka3:29100,SASL_SSL_HOST://localhost:9100 | |
| KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' | |
| KAFKA_DELETE_TOPIC_ENABLE: 'true' | |
| KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0' | |
| KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks' | |
| KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds' | |
| KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds' | |
| KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks' | |
| KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds' | |
| KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN' | |
| KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512' | |
| KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf' | |
| KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" | |
| KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' | |
| # suppress verbosity | |
| # https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template | |
| KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO' | |
| CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' | |
| volumes: | |
| - ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z | |
| - ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z | |
| - ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z | |
| - ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z | |
| - ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z | |
| - ./testHelpers/kafka/server-jaas.conf:/opt/kafka/config/server-jaas.conf:ro,z |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash -e | |
| COMPOSE_FILE=${COMPOSE_FILE:="docker-compose.kafka.yml"} | |
| echo "Running compose file: ${COMPOSE_FILE}:" | |
| docker-compose -f "${COMPOSE_FILE}" up --force-recreate -d | |
| if [ -z ${NO_LOGS} ]; then | |
| docker-compose -f "${COMPOSE_FILE}" logs -f | |
| fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment