Debezium Postgres Kafka CDC

docker-compose.yml

version: "3"

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      KAFKA_OPTS:
        -Djava.security.auth.login.config=/etc/kafka/zookeeper_server_jaas.conf
        -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
        -Dzookeeper.allowSaslFailedClients=false
        -Dzookeeper.requireClientAuthScheme=sasl
    volumes:
      - ./kafka/zookeeper.sasl.jaas.config:/etc/kafka/zookeeper_server_jaas.conf
    networks:
      - kafka-net

  kafka:
    image: confluentinc/cp-kafka:latest
    hostname: kafka
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
      KAFKA_INTER_BROKER_LISTENER_NAME: SASL_PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_SASL_MECHANISM: PLAIN
      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
    volumes:
      - ./kafka/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
      - ./kafka/client.properties:/tmp/client.properties
    networks:
      - kafka-net

  schema-registry:
    image: confluentinc/cp-schema-registry:latest
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      - kafka
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
      SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
      SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username=${KAFKA_USERNAME} password=${KAFKA_PASSWORD};
    networks:
      - kafka-net

  postgres_db:
    container_name: product_dev_db
    image: postgres:13-alpine
    environment:
      - POSTGRES_USER=${MASTER_DB_USER}
      - POSTGRES_PASSWORD=${MASTER_DB_PASSWORD}
      - POSTGRES_DB=${MASTER_DB_NAME}
    volumes:
      - dev_postgres_data:/var/lib/postgresql/data/
    ports:
      - 5432:${MASTER_DB_PORT}
    command:
      - "postgres"
      - "-c"
      - "wal_level=logical"
    networks:
      - db-net
    restart: always

  kowl:
    image: quay.io/cloudhut/kowl
    restart: on-failure
    container_name: kowl
    ports:
      - "8080:8080"
    depends_on:
      - kafka
    environment:
      KAFKA_BROKERS: kafka:9092
      KAFKA_SCHEMAREGISTRY_ENABLED: "true"
      KAFKA_SCHEMAREGISTRY_URLS: "http://schema-registry:8081"
      KAFKA_SASL_ENABLED: "true"
      KAFKA_SASL_MECHANISM: "PLAIN"
      KAFKA_SASL_USERNAME: ${KAFKA_USERNAME}
      KAFKA_SASL_PASSWORD: ${KAFKA_PASSWORD}
    networks:
      - kafka-net

  kafka-connect-ui:
    image: landoop/kafka-connect-ui
    hostname: kafka-connect-ui
    container_name: kafka-connect-ui
    ports:
      - "8000:8000"
    environment:
      CONNECT_URL: "http://kafka-connect:8083;dev"
    depends_on:
      - kafka-connect
    networks:
      - kafka-net

  kafka-connect:
    image: debezium/connect:latest
    hostname: kafka-connect
    container_name: kafka-connect
    depends_on:
      - postgres_db
      - zookeeper
      - kafka
      - schema-registry
    volumes:
      - ./connectors/confluentinc-kafka-connect-jdbc-10.7.4:/usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc
    networks:
      - kafka-net
      - db-net
    ports:
      - "8083:8083"
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      REST_ADVERTISED_HOST_NAME: "kafka-connect"
      REST_PORT: 8083
      GROUP_ID: "kafka-connect-group1"
      CONFIG_STORAGE_TOPIC: "_kafka_connect_group1-configs"
      OFFSET_STORAGE_TOPIC: "_kafka_connect_group1-offsets"
      STATUS_STORAGE_TOPIC: "_kafka_connect_group1-status"
      CONFIG_STORAGE_REPLICATION_FACTOR: 1
      OFFSET_FLUSH_INTERVAL_MS: 10000
      OFFSET_STORAGE_REPLICATION_FACTOR: 1
      STATUS_STORAGE_REPLICATION_FACTOR: 1
      KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
      LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
      CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
      CONNECT_SASL_MECHANISM: PLAIN
      CONNECT_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username=${KAFKA_USERNAME} password=${KAFKA_PASSWORD};
      CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
      CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
      CONNECT_CONSUMER_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username=${KAFKA_USERNAME} password=${KAFKA_PASSWORD};
      CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
      CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
      CONNECT_PRODUCER_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username=${KAFKA_USERNAME} password=${KAFKA_PASSWORD};

volumes:
  dev_postgres_data:

networks:
  kafka-net:
  db-net: