Simulate Apache Kafka changes safely and cleanly before doing it in production

a.sh

make run

# Play around with Kafka
# Example decreasing topic partitions

# In another terminal
docker exec -t -i my_container_name /bin/bash
kb=/kafka_2.12-1.1.1/bin
topic=test

# Create a topic
"$kb/kafka-topics.sh" --zookeeper localhost:2181 --create --topic $topic --partitions 10 --replication-factor 3
"$kb/kafka-topics.sh" --zookeeper localhost:2181 --describe --topic $topic

# Push some message
echo "Hello world" | "$kb/kafka-console-producer.sh" --broker-list localhost:9092,localhost:9093,localhost:9094 --topic $topic

# Backup the data into file
"$kb/kafka-console-consumer.sh" --bootstrap-server localhost:9092,localhost:9093,localhost:9094 --topic $topic --from-beginning --timeout-ms 500 > /tmp/$topic.log
cat /tmp/$topic.log

# Delete topic
"$kb/kafka-topics.sh" --delete --zookeeper 127.0.0.1:2181 --topic $topic
"$kb/kafka-topics.sh" --zookeeper localhost:2181 --describe --topic $topic

# Recreate topic with appropriate partitions
"$kb/kafka-topics.sh" --zookeeper localhost:2181 --create --topic $topic --partitions 1 --replication-factor 3
"$kb/kafka-topics.sh" --zookeeper localhost:2181 --describe --topic $topic

# Restore data
cat /tmp/$topic.log | "$kb/kafka-console-producer.sh" --broker-list localhost:9092,localhost:9093,localhost:9094 --topic $topic

# Check no data loss
"$kb/kafka-console-consumer.sh" --bootstrap-server localhost:9092,localhost:9093,localhost:9094 --topic $topic --from-beginning --timeout-ms 500

Dockerfile

FROM debian:9

# Wget, ps, java
RUN apt-get update && apt-get install -y wget procps default-jdk

ENV KAFKA_VERSION=1.1.1
ENV KAFKA_DIRECTORY=kafka_2.12-${KAFKA_VERSION}

# Current version
RUN wget https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/${KAFKA_DIRECTORY}.tgz \
    && tar -xzf ${KAFKA_DIRECTORY}.tgz

COPY simulation.sh .
RUN chmod +x simulation.sh

# Let's do it with 3 brokers
# Our production setup brokers id is 1,2,3
RUN cp ${KAFKA_DIRECTORY}/config/server.properties ${KAFKA_DIRECTORY}/config/server-1.properties \
    && cp ${KAFKA_DIRECTORY}/config/server.properties ${KAFKA_DIRECTORY}/config/server-2.properties \
    && sed -i 's/broker.id=0/broker.id=1/g' ${KAFKA_DIRECTORY}/config/server.properties \
    && sed -i 's/broker.id=0/broker.id=2/g' ${KAFKA_DIRECTORY}/config/server-1.properties \
    && sed -i 's/broker.id=0/broker.id=3/g' ${KAFKA_DIRECTORY}/config/server-2.properties \
    && sed -i 's/#listeners=PLAINTEXT:\/\/:9092/listeners=PLAINTEXT:\/\/:9093/g' ${KAFKA_DIRECTORY}/config/server-1.properties \
    && sed -i 's/#listeners=PLAINTEXT:\/\/:9092/listeners=PLAINTEXT:\/\/:9094/g' ${KAFKA_DIRECTORY}/config/server-2.properties \
    && sed -i 's/log.dirs=\/tmp\/kafka-logs/log.dirs=\/tmp\/kafka-logs-1/g' ${KAFKA_DIRECTORY}/config/server-1.properties \
    && sed -i 's/log.dirs=\/tmp\/kafka-logs/log.dirs=\/tmp\/kafka-logs-2/g' ${KAFKA_DIRECTORY}/config/server-2.properties

CMD [ "./simulation.sh" ]

Makefile

NS ?= my_namespace
VERSION ?= 1.0.0
ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
VOLUMES =
IMAGE_NAME ?= my_image_name
CONTAINER_NAME ?= my_container_name


.PHONY: help build run stop rm default

help:
	@echo ''
	@echo 'Usage: make [TARGET]'
	@echo 'Targets:'
	@echo '  build    	build docker image'
	@echo '  run    	docker container'
	@echo '  stop    	stop docker container'
	@echo '  rm    	remove docker image'
	@echo ''

build: Dockerfile
	docker build \
		-t $(NS)/$(IMAGE_NAME):$(VERSION) \
		-f Dockerfile \
		--label version=$(VERSION) .

run:
	docker run --rm --name $(CONTAINER_NAME) $(VOLUMES) $(NS)/$(IMAGE_NAME):$(VERSION)

stop:
	docker stop $(CONTAINER_NAME)

rm:
	docker rm $(CONTAINER_NAME)

default: build

simulation.sh

#!/bin/bash

"${KAFKA_DIRECTORY}/bin/zookeeper-server-start.sh" "${KAFKA_DIRECTORY}/config/zookeeper.properties" &
"${KAFKA_DIRECTORY}/bin/kafka-server-start.sh" "${KAFKA_DIRECTORY}/config/server.properties" --override delete.topic.enable=true &
"${KAFKA_DIRECTORY}/bin/kafka-server-start.sh" "${KAFKA_DIRECTORY}/config/server-1.properties" --override delete.topic.enable=true &
"${KAFKA_DIRECTORY}/bin/kafka-server-start.sh" "${KAFKA_DIRECTORY}/config/server-2.properties" --override delete.topic.enable=true