sink connector(basic components without monitoring)

docker-compose.yaml

version: "3.4"

# Ubuntu , set this for redpanda to start
# https://sort.veritas.com/public/documents/HSO/2.0/linux/productguides/html/hfo_admin_ubuntu/ch04s03.htm

# Clickhouse Table Schema
# create table test(id int, message String) ENGINE=MergeTree() PRIMARY KEY id;

services:
  mysql-master:
    container_name: mysql-master
    image: docker.io/bitnami/mysql:latest
    restart: "no"
    ports:
      - "3306:3306"
    environment:
      - MYSQL_ROOT_PASSWORD=root
      - MYSQL_DATABASE=test
      - MYSQL_REPLICATION_MODE=master
      - MYSQL_REPLICATION_USER=repl_user
      - ALLOW_EMPTY_PASSWORD=yes
    volumes:
     - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
    # - ../sql/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql
    healthcheck:
      test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
      interval: 15s
      timeout: 5s
      retries: 6

  schemaregistry:
    container_name: schemaregistry
    #image: apicurio/apicurio-registry-mem:latest-release
    image: confluentinc/cp-schema-registry:latest
    restart: "no"
    ports:
      - "8081:8081"
    environment:
        - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
        - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
        - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
        - SCHEMA_REGISTRY_DEBUG=true

    depends_on:
      - kafka

  debezium:
    container_name: debezium
    #image: debezium/connect:1.9.5.Final
    build:
      context: ../../docker/debezium_jmx
      args:
        DEBEZIUM_VERSION: 2.1.0.Alpha1
        JMX_AGENT_VERSION: 0.15.0
    restart: "no"
    ports:
      - "8083:8083"
      - "1976:1976"
    extra_hosts:
      - "host.docker.internal:host-gateway"

    environment:
      # Where to find Kafka broker
      - BOOTSTRAP_SERVERS=kafka:9092
      - GROUP_ID=1
      - CONFIG_STORAGE_TOPIC=config-storage-topic-debezium
      - OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium
      - STATUS_STORAGE_TOPIC=status-storage-topic-debezium
      - LOG_LEVEL=INFO
      - KEY_CONVERTER=io.confluent.connect.avro.AvroConverter
      - VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter
      - KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml
      - JMXHOST=localhost
      - JMXPORT=1976
      #- LOG_LEVEL=DEBUG
    depends_on:
      - kafka

  kafka:
    container_name: kafka
    image: vectorized/redpanda
    restart: "no"
    ports:
      # Expose LOCALHOST_NETWORK listener on localhost
      - "19092:19092"
    command:
      - redpanda
      - start
      - --overprovisioned
      # Setup listeners for docker network and localhost network
      - --kafka-addr
      - DOCKER_NETWORK://0.0.0.0:9092,LOCALHOST_NETWORK://0.0.0.0:19092
      - --advertise-kafka-addr
      - DOCKER_NETWORK://kafka:9092,LOCALHOST_NETWORK://127.0.0.1:19092

  sink:
    container_name: sink
    image: altinity/clickhouse-sink-connector:${SINK_VERSION}
    restart: "no"
    ports:
      - "18083:8083"
      - "5005:5005"
      - "39999:39999"
    environment:
      # Where to find Kafka broker
      - BOOTSTRAP_SERVERS=kafka:9092
      - GROUP_ID=2
      - CONFIG_STORAGE_TOPIC=config-storage-topic-sink
      - OFFSET_STORAGE_TOPIC=offset-storage-topic-sink
      - STATUS_STORAGE_TOPIC=status-storage-topic-sink
      - LOG_LEVEL=INFO
      - JAVA_DEBUG_PORT=*:5005
      - DEFAULT_JAVA_DEBUG_PORT=*:5005
      - KAFKA_DEBUG=true
      - JMX_PORT=39999
      #- LOG_LEVEL=DEBUG
    depends_on:
      - kafka

  clickhouse:
    # clickhouse-client --host=127.0.0.1 --port=9000 --user=root --password=root --database=test
    container_name: clickhouse
    image: clickhouse/clickhouse-server:latest
    restart: "no"
    ports:
      - "8123:8123"
      - "9000:9000"
    environment:
      - CLICKHOUSE_USER=root
      - CLICKHOUSE_PASSWORD=root
      - CLICKHOUSE_DB=test
    ulimits:
      nofile:
        soft: "262144"
        hard: "262144"
    volumes:
      - ../sql/init_clickhouse.sql:/docker-entrypoint-initdb.d/init_clickhouse.sql
      -  ./clickhouse/users.xml:/etc/clickhouse-server/users.xml

  ##### MONITORING #####
  #https://stackoverflow.com/questions/55473562/configuring-a-jmx-prometheus-exporter-docker-container-to-read-jmx-local-applica
  # Bitnami bitnami/jmx-exporter vs sscaling images

  ### KAFKA UI #####
#  console:
#    image: docker.redpanda.com/vectorized/console:master-217260f
#    restart: on-failure
#    entrypoint: /bin/sh
#    command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
#    environment:
#      CONFIG_FILEPATH: /tmp/config.yml
#      CONSOLE_CONFIG_FILE: |
#        kafka:
#          brokers: ["kafka:9092"]
#        connect:
#          enabled: true
#          clusters:
#            - name: datagen
#              url: http://debezium:8083
#    ports:
#      - "8089:8080"
#    depends_on:
#      - kafka
#
#  jmx_exporter:
#    container_name: jmx_exporter
#    image: sscaling/jmx-prometheus-exporter
#    restart: "no"
#    ports:
#      - "9072:9072"
#    environment:
#      SERVICE_PORT: 9072
#    volumes:
#      - ../config/jmx-config.yml:/opt/jmx_exporter/config.yml
#
#  prometheus:
#    container_name: prometheus
#    image: bitnami/prometheus:2.36.0
#    restart: "no"
#    ports:
#      - "9090:9090"
#    volumes:
#      - ../config/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
#
#  grafana:
#    build:
#      context: ../../docker/grafana
#      args:
#        GRAFANA_VERSION: 9.1.1
#    #container_name: grafana
#    #image: grafana/grafana
#    restart: "no"
#    #volumes:
#    #  - ../config/grafana/dashboards:/etc/grafana/provisioning/dashboards
#    ports:
#      - "3000:3000"
#    environment:
#      - DS_PROMETHEUS=prometheus
#      - GF_USERS_DEFAULT_THEME=light
#    depends_on:
#      - prometheus
  #### END OF MONITORING ###