Cheat sheet for Standalone Kafka Connect with Avro serialisation

kafak-connect-avro.sh

# 0. Delete the existing topics: app-users, app-movies
make kafka-shell

kafka-topics --delete --topic app-users --zookeeper $ZK
kafka-topics --delete --topic app-movies --zookeeper $ZK

kafka-topics --create --topic app-users --partitions 2 --zookeeper $ZK --replication-factor 2
kafka-topics --create --topic app-movies --partitions 2 --zookeeper $ZK --replication-factor 2

exit

# 1. Kafka Connect with Avro serialization
# Enter Kafka Connect Shell Client
make connect-shell

# Kafka Connect with avro-standalone setting, connecting via postgres
connect-standalone dev/connect-avro-standalone.properties dev/source-postgres-incremental.properties

# 2. Console consumer with Avro deserialization
# Enter Kafka Avro Shell Client
make avro-shell

# Console Consumer with Avro deserialization
TOPIC=app-users

kafka-avro-console-consumer --bootstrap-server $BROKERS \
 --topic $TOPIC \
 --from-beginning \
 --property print.key=true \
 --property print.value=true \
 --property schema.registry.url=$SCHEMA_REGISTRY \
 --max-messages 20

# Expected console-consumer output from topic: app-users
# null {"id":101,"first_name":"Cass","last_name":"Truss","email":"ctruss0@salon.com"}
# null {"id":102,"first_name":"Joete","last_name":"Scowcroft","email":"jscowcroft1@gizmodo.com"}
# null {"id":103,"first_name":"Celine","last_name":"Wortley","email":"cwortley2@bizjournals.com"}
# null {"id":104,"first_name":"Othilia","last_name":"Gumme","email":"ogumme3@issuu.com"}
# null {"id":105,"first_name":"Maynord","last_name":"Buttrick","email":"mbuttrick4@mashable.com"}
# null {"id":106,"first_name":"Kendall","last_name":"Yitzovitz","email":"kyitzovitz5@storify.com"}
# null {"id":107,"first_name":"Tedi","last_name":"Danne","email":"tdanne6@earthlink.net"}
# null {"id":108,"first_name":"Diena","last_name":"Knight","email":"dknight7@cbsnews.com"}
# null {"id":109,"first_name":"Court","last_name":"Hensmans","email":"chensmans8@over-blog.com"}
# null {"id":110,"first_name":"Lissi","last_name":"Andrea","email":"landrea9@gizmodo.com"}
# ...

# Exit Docker when Kafka Connect is done
(Ctrl-C)
exit