Created
June 8, 2023 05:19
-
-
Save abhishekmishragithub/fadc65b5fd13f2b3d28efac7bd838b06 to your computer and use it in GitHub Desktop.
kafka_dozer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
version: '3' | |
services: | |
zookeeper: | |
image: confluentinc/cp-zookeeper:7.3.0 | |
hostname: zookeeper | |
container_name: zookeeper | |
environment: | |
ZOOKEEPER_CLIENT_PORT: 2181 | |
ZOOKEEPER_TICK_TIME: 2000 | |
broker: | |
image: confluentinc/cp-kafka:7.3.0 | |
container_name: broker | |
ports: | |
- "9092:9092" | |
depends_on: | |
- zookeeper | |
environment: | |
KAFKA_BROKER_ID: 1 | |
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT | |
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092 | |
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 | |
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 | |
schema-registry: | |
image: confluentinc/cp-schema-registry:7.3.0 | |
container_name: schema-registry | |
ports: | |
- "8081:8081" | |
depends_on: | |
- zookeeper | |
- broker | |
environment: | |
# SCHEMA_REGISTRY_HOST_NAME: schema-registry | |
# SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' | |
# SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' | |
# SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 | |
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://broker:29092 | |
SCHEMA_REGISTRY_HOST_NAME: schema-registry | |
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 |
Dozer config:
connections:
- config : !Kafka
broker: localhost:9092
schema_registry_url:
name: kafka_store
sources:
- name: purchases
table_name: testable
connection: !Ref kafka_store
columns:
endpoints:
- name: movies_with_bookings
path: /movies_with_bookings
table_name: purchases
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
the
getting_started.ini
to store the configRun project:
1 . docker-compose up
2. Create a topic:
docker compose exec broker \ kafka-topics --create \ --topic purchases \ --bootstrap-server localhost:9092 \ --replication-factor 1 \ --partitions 1