Skip to content

Instantly share code, notes, and snippets.

@gschmutz
Last active February 18, 2024 03:53
Show Gist options
  • Star 16 You must be signed in to star a gist
  • Fork 6 You must be signed in to fork a gist
  • Save gschmutz/db582679c07c11f645b8cb9718e31209 to your computer and use it in GitHub Desktop.
Save gschmutz/db582679c07c11f645b8cb9718e31209 to your computer and use it in GitHub Desktop.
Docker Compose with Kafka Single Broker, Connect, Schema-Registry, REST Proxy, Kafka Manager
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:3.3.0
hostname: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker-1:
image: confluentinc/cp-enterprise-kafka:3.3.0
hostname: broker-1
depends_on:
- zookeeper
ports:
- "9092:9092"
environment:
KAFKA_BROKER_ID: 1
KAFKA_BROKER_RACK: rack-a
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://${DOCKER_HOST_IP}:9092'
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_DELETE_TOPIC_ENABLE: "true"
KAFKA_JMX_PORT: 9999
KAFKA_JMX_HOSTNAME: 'broker-1'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-1:9092
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
schema_registry:
image: confluentinc/cp-schema-registry:3.3.0
hostname: schema_registry
container_name: schema_registry
depends_on:
- zookeeper
- broker-1
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema_registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_ORIGIN: '*'
SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_METHODS: 'GET,POST,PUT,OPTIONS'
connect:
image: confluentinc/cp-kafka-connect:3.3.0
hostname: connect
container_name: connect
depends_on:
- zookeeper
- broker-1
- schema_registry
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker-1:9092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
volumes:
- ./kafka-connect:/etc/kafka-connect/jars
rest-proxy:
image: confluentinc/cp-kafka-rest
hostname: rest-proxy
depends_on:
- broker-1
- schema_registry
ports:
- "8084:8084"
environment:
KAFKA_REST_ZOOKEEPER_CONNECT: '${DOCKER_HOST_IP}:2181'
KAFKA_REST_LISTENERS: 'http://0.0.0.0:8084'
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
KAFKA_REST_HOST_NAME: 'rest-proxy'
adminer:
image: adminer
ports:
- 8080:8080
db:
image: mujz/pagila
environment:
- POSTGRES_PASSWORD=sample
- POSTGRES_USER=sample
- POSTGRES_DB=sample
kafka-manager:
image: trivadisbds/kafka-manager
hostname: kafka-manager
depends_on:
- zookeeper
ports:
- "9000:9000"
environment:
ZK_HOSTS: 'zookeeper:2181'
APPLICATION_SECRET: 'letmein'
connect-ui:
image: landoop/kafka-connect-ui
container_name: connect-ui
depends_on:
- connect
ports:
- "8001:8000"
environment:
- "CONNECT_URL=http://connect:8083"
schema-registry-ui:
image: landoop/schema-registry-ui
hostname: schema-registry-ui
depends_on:
- broker-1
- schema_registry
ports:
- "8002:8000"
environment:
SCHEMAREGISTRY_URL: 'http://${DOCKER_HOST_IP}:8081'
## Request
curl -X "POST" "http://192.168.69.154:8083/connectors" \
-H "Content-Type: application/json" \
-d $'{
"name": "jdbc-source",
"config": {
"connector.class": "JdbcSourceConnector",
"tasks.max": "1",
"connection.url":"jdbc:postgresql://db/sample?user=sample&password=sample",
"mode": "timestamp",
"timestamp.column.name":"last_update",
"table.whitelist":"actor",
"validate.non.null":"false",
"topic.prefix":"postgres-jdbc-",
"name": "jdbc-source"
}
}'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment