Resizing a Kafka cluster via bash

gistfile1.sh

#!/bin/bash -xe

# Parse server.properties to get Zookeeper hosts
ZK_HOST=$(grep 'zookeeper.connect=' /etc/kafka/server.properties | cut -d'=' -f 2)

# Find the most newly created directory of Kafka in the /opt directory
KAFKA_PATH=$(find /opt/ -maxdepth 1 -name kafka-2* -print0 | xargs -0 ls -d -1 -t | head -n1)

# Manually configured list of brokers to rebalance across (zero-indexed)
BROKER_LIST="0,1,2"

# record all topics
$KAFKA_PATH/bin/kafka-topics.sh --zookeeper $ZK_HOST --list > ~/repartition/all_topics

# generate json for listing which topics to reassign
grep -v deletion ~/repartition/all_topics | tr '\n' ' ' | head --bytes -1 | jq -R 'split(" ") | reduce .[] as $topic ([]; . + [{"topic": $topic }]) | {"topics": . , "version": 1}' > ~/repartition/all_topics.json

# generate the json for reassigning topics
$KAFKA_PATH/bin/kafka-reassign-partitions.sh --zookeeper $ZK_HOST --topics-to-move-json-file ~/repartition/all_topics.json --broker-list "$BROKER_LIST" --generate | tail -n 1 > ~/repartition/reassignment.json

# actually do the reassignment
$KAFKA_PATH/bin/kafka-reassign-partitions.sh --zookeeper $ZK_HOST --reassignment-json-file ~/repartition/reassignment.json --execute

# watch to see when reassignment is completed
watch -n 5 "$KAFKA_PATH/bin/kafka-reassign-partitions.sh --zookeeper $ZK_HOST --reassignment-json-file ~/repartition/reassignment.json --verify | grep progress"