kafka local dev environment

demo.sh

docker-compose up

# optional: open local control center
open http://localhost:9021

# demo 1
# ------

# create topic
docker exec -it broker kafka-topics --bootstrap-server localhost:9092 --create --topic demo1 --partitions 3 --replication-factor 1

# start console producer
docker exec -it schema-registry kafka-avro-console-producer --broker-list broker:29092 --topic demo1 --property value.schema="{\"type\":\"record\",\"name\":\"demo1\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"username\",\"type\":\"string\"}]}"

# start console consumer
docker exec -it schema-registry kafka-avro-console-consumer --bootstrap-server broker:29092 --topic demo1 --property print.key=true

# demo 2
# ------

# create topic
docker exec -it broker kafka-topics --bootstrap-server localhost:9092 --create --topic demo3 --partitions 3 --replication-factor 1

# start console consumer
docker exec -it schema-registry kafka-avro-console-consumer --bootstrap-server broker:29092 --topic demo3

# generate data
docker cp demo2.avsc connect:/demo2.avsc
curl -X POST -H "Content-Type: application/json" http://localhost:8083/connectors -d @demo2.json

demo2.avsc

{
  "type": "record",
  "name": "demo2",
  "namespace": "ua.rabota.topics",
  "fields": [
    {
      "name": "userId",
      "type": {
        "type": "int",
        "arg.properties": {
          "range": {
            "min": 1,
            "max": 100
          }
        }
      }
    },
    {
      "name": "vacancyId",
      "type": {
        "type": "long",
        "arg.properties": {
          "range": {
            "min": 7710732,
            "max": 7711732
          }
        }
      }
    },
    {
      "name": "platform",
      "type": ["null", {
        "type": "string",
        "arg.properties": {
          "options": ["desktop", "mobile", "ios", "android"]
        }
      }],
      "default": null
    }
  ]
}

demo2.json

{
    "name": "demo2",
    "config": {
      "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
      "kafka.topic": "demo2",
      "value.converter": "io.confluent.connect.avro.AvroConverter",
      "value.converter.schema.registry.url": "http://schema-registry:8081",
      "max.interval": 1000,
      "iterations": 1000,
      "tasks.max": "1",
      "schema.filename": "/demo2.avsc"
    }
}