kafka.drawio

<mxfile host="app.diagrams.net" modified="2020-07-30T13:24:04.078Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" etag="za19b1gBxxY2w0qTRGfJ" version="13.5.5" type="device"><diagram id="n5W6CcNsMwQ7RIXeHPj_" name="Page-1">7V1bc9pIFv4t++CqmQendEEXHm2TZDOxK3acVDL71pIa0FioWUnEJr9+u0EC0edgZCO1hLyemgo0QsD39bmf7j4zr2ZPHxMyn96wgEZnhhY8nZmjM8PQHd3l/4iR5XpkMMwHJkkY5BdtB+7D3zQf1PLRRRjQdOfCjLEoC+e7gz6LY+pnO2MkSdjj7mVjFu1+6pxMKBi490kER3+EQTZdj7qWth3/Nw0n0+KTdS1/ZUaKi/OBdEoC9lgaMt+fmVcJY9n60ezpikYCvAKX9fs+7Hl188USGmeV3vD4T/bhzg8DNrok3+5/2oH149zN2fhFokX+i/Nvmy0LCBK2iAMq7qKfmZeP0zCj93Pii1cfOel8bJrNovxl+K3yL/qLJhl9Kg3l3/IjZTOaJUt+ydMu9fmM0c0C7MctAeYwH5uWwDedfJDkpE82997iwh/k0LwApqEFYKIeYw/pmWFH/NMvvYQ/mohHf2RsHvp/HofiOIyiKxaxZPVec2yJ//h4miXsgZZesVd/4h0szkrj6z8+LmAP+XS+iMJJzF/j364elgYSSwOMpQHC0kYgjmHJv7AuRp+uvUvi/shI9vdfc//zuW4DlvhPzMIsZDEf1o7jBEdSYorqgUUdjKmh7ZjE3mAPgEbo2Iu9bUnYF0q1hL1RzNky9nZj0DvPQq8rgH48prbvY9AHztDj7NcDvfFOAt9y4cRvDHxUO+kA+/tb/vyc/89nvQuw578+2wV4F7KYxVTCNx8iOQE+R4smCDOzMAjEx6CMbjnX6tFBjqSDIBG6i2mgpogwnyPCgPqnr0QgpkAtEQNAxFfx4wztQmil2epxX7nYoJ+TYUIyULPcFBfQdeq3UjJl97Vl/KFTlNGIzqcCPOC9xouZR5MUsMODlzldQZ7tA7LEnEf8h8kK2C+LLAoFS6vxgCQPX/i7wkyAo73TrHogl2a8jlhkTP1sYrX64wUMcwAqDXiomT9lSTZlExaT6P12VJqc22uumfCAVpj+Q7NsmcfNZJGxXSboU5j9XGFt5c/+3nk2esrvvXqyLD25pUnIwRACtR6LOTA/y0/WdzKlv+LV7Z1Xz5blZ/K9ZU/O9SnuyXmuJeKMZyZNyhaJT5+hJndUM5JMaPbMdbmGEhw9OwUTGpEs/LWbLsAm1OqtF0lClqUL5iyMs7R051sxUFImcjBsS0H+y67nD9bfYDu1Nz/l9bMd+v4zuleDLGbRhZ+xsp6+Jh6Nblm6jhbMkceyjM32+v2l6c3W6uVqk/ypSYfrw8NK3EQ0SmMuPkzTJJSId9GUx2BUfOQE82rUoV6jDL/A1g7lSLhlngrxKxH1SMIq3mZ1LyahafibeKtbCRxzHcLva12eWSNxL24H0rVJqCsnJwsEEmsp9Wp06OL3O8SS8m0m4uOoJQD69b3zKyXMB4gVUOtXGjCR1gsn0jrgNb7e3Stywgf9vUJQ6nP4jhMvGLadvFPlHlZhSo21MWhVmnZkCQujXida+q5oGQ3KVtVYqmuyBQOWjnvTAaHuGPWmbd+l3rgmcycFj5t6amsCar0BAXUaFNDhiQoozJ91XEAr132P9Eclh7R1AYXh7o8pyVIynwNuVLeRyEWpDXgHvPdBY2AZAKy9E3ieMJ+m6etDolYMyGaiPWNAlAapBqzIKoNcjUqQax9YG45ayGHgcr+M/WnCYraACYD+5Gekut+wbR6gk3uRvgki5Pp32wVYAybvQ4HUjAYhSULWXyrkVk6r7ayxCdW/woCieFwpe6Y8U2ZUDRbMbgULBgwWbhdeFKZTLgFtO6FSBtlGUl66odJjN6HH/l50MsOU/Uca04SIIOooEFupC8qBko04oopxh02aqzIBzHCcWmlwIHUj24gHailV8tDp/8SBm/DJHMYTONHvl2lGZ1U8oo4TscuDVdTGSzw4CA9mYzzAEu3LIoFuw6270ry34bzHVqk0N+9hRRZ6O3FwIVZFCVQjkqahX9HkHTT2B6S9GHtZuxTobzrfZ1CLW6zdnPxdW0DBjawD91l7QeA+dbVMmTBK7h9XUjbIHr6SK/PAfZrmCkbS/eNKkgdHbmGoytXgwH2a5qrCesaT50rGWK9HB4L7NM0VDNz6w1WBqWztq3Ij9duB+zTMTTHFeskNvs7YkR3hqlTp2oEbNc0VDC17x5UhQSwX6SpzJZMu36hprgzA1feUJrCt76Q6u84lbWW33S4/qFD3O3WRkLxix3qtSMiyJd+oaZGAaQIhEnwEcnZaUuHs8a9akwqYIGDj8aqCLQOtNBnTSorY0qX0DZIiVpq2LD6s+7WpvegfrDENqnZjD4YVda2aGtMApmvyGhOaUb5iszmJl4BP1dUnyUS4SGZebRXEQsxwWw2djc3xql3RHZvjdrvd8Hmjek6P/iw5eCvt5kntpfFTpXQAM5fNFnvr0Flu15TWAOYUcwf11MM2GWqkgqXWQYUZwZuVV/nIf2brM1NOGLU/My0k8+Nc7Z2UR3e41gCivJDYNSCIarengeEni99qTCRlMl1kOb3amAgpaXd8ghvmblzZ/gRHSs0dx1DupW4fw6ObqbvdywLy60j3utJeFrvdlX+7C90bChTtPEl9alGFDZPr72ckjJDOOsr5aD+kMLRdlTxEZrdax82GOvmGK12xFzUA8VIYfAhibxYJyH10w7aVvY0o+xn5vdrI9v7uvr9MSOGgriEpcbVWAEbeX2kQcnuriRtzOsKA/jd8eDuU6MgiVrWUwBD9K/G8MLu5e0MsIPVtpSwUgllurf725ea2vxSAUwH0ts2EA5MvN3ffvr0lCpD0gFoKDGipb+7elBQgKVu1FMBg4DZhGfNZ1IOw2JK2l8S2PcTW1DQHN8xW/nXTY5fUlLf4b1/tw5Tkxe2n/jIgbwqFyoBaBmDwfOGLtIdwQfPA4IL/vFnY410OgEva9uoz1wZgd7R9Sf3SeiePYw9vOp6r945k+hwYf38m4wcCmD5mRXhgUTcYAEnkr7iGZ9q1bZ21Ky/Y5pNqc4AuNOX/YewzpfNjU6atAAx8pfYRHjr/V0l7Z1/V3T6KVeQdUUkuzD9dioP4MnZsh0grUnMOTztsfauKIUxsXF332MM9lxfb6Ubb+9YNoW2YJyxY+JUKQd0Oqs8NTZ7ySDrVVgo3VCrvP8E00mnvFOjIB1Nhu3Gh3T1WQ7BvihnlTXEMsW2IXAMdjUY9Y0NefqJbGrJdC3KKpFkDG/gRnsM2vaWX9OZvvaV3w6Gz6zENDnhMFDt/CvVwXrnlctEUXHasnjmtti0/avM929oITI1/JXc06tguwGrdK71Y11hC3mdxupj1wLgP5AOZTCQvpTRjrmswYXulzpqMx2Njz6nEtmdbtU1zCXbbhtZEsW2HWdqITSpM8BONIuRChd32BsC6BvuYbht0amvA8Fw+Ika3kNBA8TyGOVDyi4QR8cJInDfVzQk9ZnFWeG77vZmXuKtyiNz6+WMaDNr4GwxtLBwVLZuKUyJeYFdPVO3YMjEDB2p+tcToSGNM3+ytLWv71u2tDr3K2wZXebQTNBtgsrdtHnSk/+X9iRlZ7MyIplDEQ2F4HHxHUw9nNWYKiiNMD2cKHJxQRZkC5KjVt5ApsNvPFCCHQCY04EiS2K/ieXY7V2DLIRPmumAnNNXhuqCCBk3oZo5vMtGLZAWTod0vvNRPQk94l+uL+GdurusCO4AKhLD98mDKmRxMHrDtARqjByZyLiNuJ/nQhyQMSCck4ijMwY7JSLOjUoFAXHkVG4gcBaIp7zCAZCAb0+M4ilCvdB5F+QQNV2VbAo4idENSLribPEze+cl9opRL9EmlZo5jynHeyaoaW5+E1BKb0xvYUQSSJRWbpNH1iRzaVZ6nSftpSkFrHLJwRm9qYw3c04GV92769McBL/v0DiIYarWYAS1qKGbmyc9xOTXjIu6iUm/RgFbXC2Oy+tLzfIkSf/hHOqd+OA79syJ1vGpx/vPkGTmX+32wlP2mF0sNJdCEr5TIecI4NaePOOhsw7b8VRvRVjDE12zySrPbXs3kSJ5A+l7hSlacJ5jpcfhTHtSKzQ488eUDOiaLCJ553h9aQGIai3/V0oId/fMhm82F4fBWFUb+88aL6F9V7MXJ8gJSowqXAOK8YMf8gPiiqP1qHzkm89fGFqfKGlRyCkv2OGuwYg9YuyUctNUOqW+cLadt3YecFHuzvL+7BiQUxU1/GYVivy/jcDTorfG79g7VOwsHzapLlUlnXukuosqwtQfy+YD1wQwDwiwhcUr8shRs4/GIIRvO/8HdZ/4CNEIFNzHL9s3lClXngCQPX/i7RFJN1Ei1uuiQDT62pS7mL+vysXL10WECDDtaisY3QdfeDYwzZS3wOIR54H2wrm2qWkmIf3rL+92/dIvJxrjB11m6rXIDczffykpRW8Ws2g1XUQmiDUfUo7/DxQzqQ9VlFKDgsMyv0gQkekqw5IR9pss3734hlkit+wUzAimZiZDzga6qXXxIjGr56HzrOPeXJZBfw7bgb2p5CU4TjET3LGFkMyJynNoo4epTRKUjmgoU97lsrbVoHsWRLa9EsTQTUoRlneVj5+qjCIad379/GvVXThy5dWfQdiyJHCfLxuOU9jmbCXZ2wE6Yq4kF/jRhLCu99pH/rOkNC6i44n8=</diagram></mxfile>

kafka.sh

#https://kafka.apache.org/downloads 
#Scala 2.13 - kafka_2.13-2.5.0.tgz (asc, sha512) 

bin/zookeeper-server-start.sh config/zookeeper.properties 

bin/kafka-server-start.sh config/server.properties 

bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic ebooks

bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic ebooks

bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic ebooks --from-beginning

bin/kafka-topics.sh --bootstrap-server localhost:9092 --list
bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic emails
bin/kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic emails

bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic ebooks --from-beginning --group ebook-generators

bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list
bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ebook-generators

bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic ebooks --property "parse.key=true" --property "key.separator=;"

bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic ebooks --partitions 2

links.md

Links

Day 1

Indirection quote (via Kazuki): https://en.wikipedia.org/wiki/Indirection

"Enterprise Integration Patterns", an old book on Messaging: https://www.enterpriseintegrationpatterns.com/patterns/messaging/

Several podcasts on Kafka (in pt_BR): https://hipsters.tech/?s=kafka

Kafka uses: https://kafka.apache.org/uses

Kafka without Zookeeper (only in the future): https://www.confluent.io/blog/removing-zookeeper-dependency-in-kafka/ Kafka without Zookeeper (the Kafka Improvement Proposal): https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum

Sequential disk access can be faster than random memory access: https://kafka.apache.org/documentation/#design_filesystem

-ilities: https://en.wikipedia.org/wiki/List_of_system_quality_attributes

Day 2

Talk on Distributed Systems: https://research.cs.cornell.edu/ladis2009/talks/dean-keynote-ladis2009.pdf

Durable Subscriber Pattern: https://www.enterpriseintegrationpatterns.com/patterns/messaging/DurableSubscription.html

Competing Consumers Pattern: https://www.enterpriseintegrationpatterns.com/patterns/messaging/CompetingConsumers.html

Debezium: https://debezium.io/documentation/reference/1.2/architecture.html Kafka Connect: https://kafka.apache.org/documentation/#connect

Confluent Kafka certifications: https://www.confluent.io/certification/

Alternative CLI: https://github.com/edenhill/kafkacat

GUIs: https://www.conduktor.io/ https://www.kafkatool.com/

Kafka Clients: https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-ClientLibrariesPreviouslySupported

Producer Configs: https://kafka.apache.org/documentation/#producerconfigs

Consumer Configs: https://kafka.apache.org/documentation/#consumerconfigs

Books on Kafka by Confluent: https://www.confluent.io/resources/kafka-the-definitive-guide/ https://www.confluent.io/designing-event-driven-systems/ https://www.confluent.io/blog/kafka-streams-action/