Skip to content

Instantly share code, notes, and snippets.

@acdcjunior
Last active December 9, 2020 20:23
Show Gist options
  • Save acdcjunior/9508c05e8ea5d60ff173a663e5ebc679 to your computer and use it in GitHub Desktop.
Save acdcjunior/9508c05e8ea5d60ff173a663e5ebc679 to your computer and use it in GitHub Desktop.
Basic Kafka + Kafka GitOps
version: '2.1'
services:
kafka-zookeeper-1:
image: zookeeper:3.4.9
hostname: kafka-zookeeper-1
ports:
- "2181:2181"
environment:
ZOO_MY_ID: 1
ZOO_PORT: 2181
ZOO_SERVERS: server.1=kafka-zookeeper-1:2888:3888
volumes:
- v-kafka-zookeeper:/data
- v-kafka-zookeeper-log:/datalog
kafka-broker:
image: confluentinc/cp-kafka:5.3.1
hostname: kafka-broker
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-broker:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
KAFKA_ZOOKEEPER_CONNECT: kafka-zookeeper-1:2181
KAFKA_BROKER_ID: 1
KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/kafka_broker_jaas.conf"
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
ZOOKEEPER_SASL_ENABLED: "false"
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_SUPER_USERS: User:kafka_user
KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
volumes:
- v-kafka-broker-1:/var/lib/kafka/data
- v-kafka-broker-1-log:/var/lib/kafka/log
- ./kafka_broker_jaas.conf:/etc/kafka/secrets/kafka_broker_jaas.conf
depends_on:
- kafka-zookeeper-1
kafka-gitops:
image: devshawn/kafka-gitops:0.2.12
environment:
KAFKA_BOOTSTRAP_SERVERS: 'kafka-broker:19092'
KAFKA_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka_user" password="kafka-pwd";'
KAFKA_SASL_MECHANISM: 'PLAIN'
KAFKA_SECURITY_PROTOCOL: 'SASL_PLAINTEXT'
volumes:
- ./state.yml:/state.yml
command: kafka-gitops -v -f /state.yml apply
depends_on:
- kafka-broker
volumes:
v-kafka-zookeeper:
v-kafka-zookeeper-log:
v-kafka-broker-1:
v-kafka-broker-1-log:
KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="kafka_user"
password="kafka-pwd"
user_kafka_user="kafka-pwd";
};
KafkaClient {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="kafka_user"
password="kafka-pwd";
};
topics:
topic-one:
partitions: 1
replication: 1
topic-two:
partitions: 1
replication: 1
services:
cg-aaaaaa:
type: application
principal: User:kafka_user
consumes:
- topic-one
- topic-two
produces:
- topic-one
- topic-two
cg-bbbbb:
type: application
principal: User:kafka_user
consumes:
- topic-two
produces:
- topic-one
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment