[kafka: data manipulations] Data manipulations using kafka docker image #docker #shell #kafka

kafka_data_manipulation.sh

# Run and connect to the container
docker run --net host -it wurstmeister/kafka:0.9.0.1 bin/bash
# NOTE: if `kafka-docker` is used and container is already sterted, connecting can be done directly to started container with `docker exec -it <containerId> bash`
# navigate to kafka dir
cd $KAFKA_HOME

# listing topics
bin/kafka-topics.sh --zookeeper <zookeeper_ip>:2181 --list
# creating a kafka topic
bin/kafka-topics.sh --create --zookeeper <zookeeper_ip>:2181 --replication-factor 1 --partitions 1 --topic <topic_name>
# describing a topic
bin/kafka-topics.sh --describe --zookeeper <zookeeper_ip>:2181 --topic <topic_name>
# run producer
bin/kafka-console-producer.sh --broker-list <broker_ip>:9092 --topic <topic_name>
# run consumer (for doing it from beginnig add `--from-beginning` flag)
bin/kafka-console-consumer.sh --zookeeper <zookeeper_ip>:2181 --topic <topic_name>

# NOTE 1: for local kafka service, use:
# `zookeeper` for <zookeeper_ip>
# `localhost` for <broker_ip>

# NOTE 2: if an error happens related to container name resolution, one of ways to solve it is to add a record to `/etc/hosts`, so the machine is recognised
# e.g. line like this needs to be added
# 127.0.0.1 linuxkit-025000000001 linuxkit-025000000001