Last active
February 22, 2023 12:29
-
-
Save moonraker595/8c401ef7d0e1e425446d06ae353dd089 to your computer and use it in GitHub Desktop.
Docker Compose file for the Kafka Connect Websocket Data Sink
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
version: '3' | |
services: | |
minio1: | |
# viewable at http://127.0.0.1:9001/ | |
image: minio/minio:RELEASE.2020-03-14T02-21-58Z | |
container_name: minio1 | |
restart: always | |
volumes: | |
- data1:/data | |
ports: | |
- "9001:9000" | |
environment: | |
MINIO_ACCESS_KEY: minio | |
MINIO_SECRET_KEY: minio123 | |
command: server /export | |
# Create a bucket for the kafka data to go in | |
createbuckets: | |
image: minio/mc | |
depends_on: | |
- minio1 | |
entrypoint: > | |
/bin/sh -c " | |
echo Waiting for minio service to start...; | |
while ! nc -z minio1 9000; | |
do | |
sleep 3; | |
done; | |
/usr/bin/mc config host add minio http://minio1:9000 minio minio123; | |
/usr/bin/mc mb minio/test-bucket; | |
" | |
streaming: | |
restart: always | |
build: ../streaming | |
image: streaming | |
depends_on: | |
- zookeeper | |
- broker | |
container_name: Streaming-Service | |
ports: | |
- "5000:5000" | |
zookeeper: | |
image: confluentinc/cp-zookeeper:5.4.1 | |
hostname: zookeeper | |
container_name: zookeeper | |
ports: | |
- "2181:2181" | |
environment: | |
ZOOKEEPER_CLIENT_PORT: 2181 | |
ZOOKEEPER_TICK_TIME: 2000 | |
broker: | |
image: confluentinc/cp-server:5.4.1 | |
hostname: broker | |
container_name: broker | |
depends_on: | |
- zookeeper | |
ports: | |
- "9092:9092" | |
environment: | |
KAFKA_BROKER_ID: 1 | |
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT | |
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 | |
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter | |
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 | |
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 | |
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092 | |
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181 | |
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 | |
CONFLUENT_METRICS_ENABLE: 'true' | |
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' | |
schema-registry: | |
image: confluentinc/cp-schema-registry:5.4.1 | |
hostname: schema-registry | |
container_name: schema-registry | |
depends_on: | |
- zookeeper | |
- broker | |
ports: | |
- "8081:8081" | |
environment: | |
SCHEMA_REGISTRY_HOST_NAME: schema-registry | |
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' | |
connect: | |
# viewable at http://localhost:9021/ | |
image: confluentinc/cp-kafka-connect-base:5.4.1 | |
hostname: connect | |
container_name: connect | |
depends_on: | |
- zookeeper | |
- broker | |
- schema-registry | |
ports: | |
- "8083:8083" | |
environment: | |
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092' | |
CONNECT_REST_ADVERTISED_HOST_NAME: connect | |
CONNECT_REST_PORT: 8083 | |
CONNECT_GROUP_ID: compose-connect-group | |
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs | |
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets | |
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status | |
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter | |
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' | |
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter | |
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 | |
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" | |
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" | |
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR | |
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 | |
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 | |
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 | |
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" | |
AWS_ACCESS_KEY_ID: "minio" | |
AWS_SECRET_ACCESS_KEY: "minio123" | |
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
command: | |
- bash | |
- -c | |
- | | |
# | |
echo "Installing connector plugins" | |
confluent-hub install --no-prompt confluentinc/kafka-connect-s3:5.4.1 | |
confluent-hub install --no-prompt mdrogalis/voluble:0.1.0 | |
# | |
echo "Launching Kafka Connect worker" | |
/etc/confluent/docker/run & | |
# | |
echo "Waiting for Kafka Connect to start listening on localhost ⏳" | |
while : ; do | |
curl_status=$$(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors) | |
echo -e $$(date) " Kafka Connect listener HTTP state: " $$curl_status " (waiting for 200)" | |
if [ $$curl_status -eq 200 ] ; then | |
break | |
fi | |
sleep 5 | |
done | |
echo -e "\n--\n+> Creating S3 Sink source" | |
curl -s -X PUT -H "Content-Type:application/json" http://localhost:8083/connectors/test_connector/config \ | |
-d '{ | |
"connector.class": "io.confluent.connect.s3.S3SinkConnector", | |
"tasks.max": "1", | |
"key.converter": "org.apache.kafka.connect.storage.StringConverter", | |
"value.converter": "org.apache.kafka.connect.storage.StringConverter", | |
"topics": "test_topic", | |
"format.class": "io.confluent.connect.s3.format.json.JsonFormat", | |
"flush.size": "100", | |
"schema.compatibility": "NONE", | |
"s3.bucket.name": "test-bucket", | |
"s3.region": "us-east-1", | |
"storage.class": "io.confluent.connect.s3.storage.S3Storage", | |
"store.url": "http://minio1:9000/", | |
"partitioner.class": "io.confluent.connect.storage.partitioner.DefaultPartitioner" | |
}' | |
sleep infinity | |
control-center: | |
image: confluentinc/cp-enterprise-control-center:5.4.1 | |
hostname: control-center | |
container_name: control-center | |
depends_on: | |
- zookeeper | |
- broker | |
- schema-registry | |
- connect | |
- ksql-server | |
ports: | |
- "9021:9021" | |
environment: | |
AWS_ACCESS_KEY_ID: 'minio' | |
AWS_SECRET_KEY_ID: 'minio123' | |
CONTROL_AWS_ACCESS_KEY_ID: 'minio' | |
CONTROL_AWS_SECRET_KEY_ID: 'minio123' | |
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092' | |
CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181' | |
CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083' | |
CONTROL_CENTER_KSQL_URL: "http://ksql-server:8088" | |
CONTROL_CENTER_KSQL_ADVERTISED_URL: "http://localhost:8088" | |
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" | |
CONTROL_CENTER_REPLICATION_FACTOR: 1 | |
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 | |
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 | |
CONFLUENT_METRICS_TOPIC_REPLICATION: 1 | |
PORT: 9021 | |
ksql-server: | |
image: confluentinc/cp-ksql-server:5.4.1 | |
hostname: ksql-server | |
container_name: ksql-server | |
depends_on: | |
- broker | |
- connect | |
ports: | |
- "8088:8088" | |
environment: | |
KSQL_CONFIG_DIR: "/etc/ksql" | |
KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties" | |
KSQL_BOOTSTRAP_SERVERS: "broker:29092" | |
KSQL_HOST_NAME: ksql-server | |
KSQL_LISTENERS: "http://0.0.0.0:8088" | |
KSQL_CACHE_MAX_BYTES_BUFFERING: 0 | |
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" | |
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" | |
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" | |
KSQL_KSQL_CONNECT_URL: "http://connect:8083" | |
ksql-cli: | |
image: confluentinc/cp-ksql-cli:5.4.1 | |
container_name: ksql-cli | |
depends_on: | |
- broker | |
- connect | |
- ksql-server | |
entrypoint: /bin/sh | |
tty: true | |
ksql-datagen: | |
# Downrev ksql-examples to 5.1.2 due to DEVX-798 (work around issues in 5.2.0) | |
image: confluentinc/ksql-examples:5.4.1 | |
hostname: ksql-datagen | |
container_name: ksql-datagen | |
depends_on: | |
- ksql-server | |
- broker | |
- schema-registry | |
- connect | |
command: "bash -c 'echo Waiting for Kafka to be ready... && \ | |
cub kafka-ready -b broker:29092 1 40 && \ | |
echo Waiting for Confluent Schema Registry to be ready... && \ | |
cub sr-ready schema-registry 8081 40 && \ | |
echo Waiting a few seconds for topic creation to finish... && \ | |
sleep 11 && \ | |
tail -f /dev/null'" | |
environment: | |
KSQL_CONFIG_DIR: "/etc/ksql" | |
KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties" | |
STREAMS_BOOTSTRAP_SERVERS: broker:29092 | |
STREAMS_SCHEMA_REGISTRY_HOST: schema-registry | |
STREAMS_SCHEMA_REGISTRY_PORT: 8081 | |
rest-proxy: | |
image: confluentinc/cp-kafka-rest:5.4.1 | |
depends_on: | |
- zookeeper | |
- broker | |
- schema-registry | |
ports: | |
- 8082:8082 | |
hostname: rest-proxy | |
container_name: rest-proxy | |
environment: | |
KAFKA_REST_HOST_NAME: rest-proxy | |
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092' | |
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" | |
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' | |
volumes: | |
data1: # used for minio |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment