kafka and zookeeper docker

docker-run-kafka-with-zookeeper.sh

# run zookeeper
docker run -p 2181:2181 --name zookeeper --rm wurstmeister/zookeeper

# run a kafka server
docker run -p 9092 \
  -e KAFKA_ADVERTISED_HOST_NAME=$YOUR_IP \
  -e KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 \
  -e KAFKA_BROKER_ID=1 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  --link zookeeper:zookeeper \
  --name kafka \
  --rm \
  wurstmeister/kafka
  
# optionally run a few more
docker run -p 9092 \
  -e KAFKA_ADVERTISED_HOST_NAME=$YOUR_IP \
  -e KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 \
  -e KAFKA_BROKER_ID=2 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  --link zookeeper:zookeeper \
  --name kafka-2 \
  --rm \
  wurstmeister/kafka

docker run -p 9092 \
  -e KAFKA_ADVERTISED_HOST_NAME=$YOUR_IP \
  -e KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 \
  -e KAFKA_BROKER_ID=3 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  --link zookeeper:zookeeper \
  --name kafka-3 \
  --rm \
  wurstmeister/kafka
  
# get the kafka cli tools from https://kafka.apache.org/downloads
curl -o kafka-binaries.tgz http://apache.mirrors.lucidnetworks.net/kafka/2.2.1/kafka_2.12-2.2.1.tgz
gunzip -c kafka-binaries.tgz | tar xopf -
cd kafka_2*

# list all topics
bin/kafka-topics.sh --list --zookeeper 

# create a topic
bin/kafka-topics.sh --create --zookeeper $YOUR_IP:2181 --replication-factor 4 --partitions 10 --topic test

# describe a topic
bin/kafka-topics.sh --describe --zookeeper $YOUR_IP:2181 --topic test

# add a bunch of messages/records to a topic
for x in {1..1000000}; do echo $x; sleep 1; done | bin/kafka-console-producer.sh \
  --broker-list $YOUR_IP:KAFKA_PORT1,$YOUR_IP:KAFKA_PORT2,$YOUR_IP:KAFKA_PORT3 \
  --topic failover-test

bin/kafka-console-consumer.sh --bootstrap-server $YOUR_IP:32780 --topic test --from-beginning

for x in {1..1000000}; do echo $x; sleep 1; done | bin/kafka-console-producer.sh \
  --broker-list $YOUR_IP:$KAFKA_PORT1 \
  --topic failover-test