Basic Kafka + Kafka GitOps

docker-compose.yml

version: '2.1'

services:
  kafka-zookeeper-1:
    image: zookeeper:3.4.9
    hostname: kafka-zookeeper-1
    ports:
      - "2181:2181"
    environment:
      ZOO_MY_ID: 1
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=kafka-zookeeper-1:2888:3888
    volumes:
      - v-kafka-zookeeper:/data
      - v-kafka-zookeeper-log:/datalog

  kafka-broker:
    image: confluentinc/cp-kafka:5.3.1
    hostname: kafka-broker
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-broker:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: kafka-zookeeper-1:2181
      KAFKA_BROKER_ID: 1
      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/kafka_broker_jaas.conf"

      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
      ZOOKEEPER_SASL_ENABLED: "false"

      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
      KAFKA_SUPER_USERS: User:kafka_user

      KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
    volumes:
      - v-kafka-broker-1:/var/lib/kafka/data
      - v-kafka-broker-1-log:/var/lib/kafka/log
      - ./kafka_broker_jaas.conf:/etc/kafka/secrets/kafka_broker_jaas.conf
    depends_on:
      - kafka-zookeeper-1

  kafka-gitops:
    image: devshawn/kafka-gitops:0.2.12
    environment:
      KAFKA_BOOTSTRAP_SERVERS: 'kafka-broker:19092'
      KAFKA_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka_user" password="kafka-pwd";'
      KAFKA_SASL_MECHANISM: 'PLAIN'
      KAFKA_SECURITY_PROTOCOL: 'SASL_PLAINTEXT'
    volumes:
      - ./state.yml:/state.yml
    command: kafka-gitops -v -f /state.yml apply
    depends_on:
      - kafka-broker

volumes:

  v-kafka-zookeeper:
  v-kafka-zookeeper-log:

  v-kafka-broker-1:
  v-kafka-broker-1-log:

kafka_broker_jaas.conf

KafkaServer {

  org.apache.kafka.common.security.plain.PlainLoginModule required

  username="kafka_user"

  password="kafka-pwd"

  user_kafka_user="kafka-pwd";



};

KafkaClient {

  org.apache.kafka.common.security.plain.PlainLoginModule required

  username="kafka_user"

  password="kafka-pwd";

};

state.yml

topics:

  topic-one:

    partitions: 1

    replication: 1

  topic-two:

    partitions: 1

    replication: 1

services:

  cg-aaaaaa:

    type: application

    principal: User:kafka_user

    consumes:

      - topic-one

      - topic-two

    produces:

      - topic-one

      - topic-two

  cg-bbbbb:

    type: application

    principal: User:kafka_user

    consumes:

      - topic-two

    produces:

      - topic-one