Reassign Kafka partitions, possibly increasing the replication factor

spread_partitions_ring.sh

#!/bin/bash
#
# Spreads the topic among the active brokers.
# Can increase the replication factor; by default sets it to 3.
#
# Usage: $0 <ZOOKEEPER> <TOPIC> <REPLICATION_FACTOR>

set -eu ${DEBUG:+-x}

KAFKA_HOME="${KAFKA_HOME:-/opt/kafka}"
ZOOKEEPER="$1"
TOPIC="$2"
REP_FACTOR="${3:-3}"
BROKERS="${4:-}"

WORKDIR=''
trap '[ -z "$WORKDIR" ] || rm -rf "$WORKDIR"' HUP PIPE INT QUIT TERM EXIT
WORKDIR=$(mktemp -d ${TMPDIR:-/tmp}/ssl_for_k8s.XXXXXXXX)
cd "$WORKDIR"


list_zk_ids () {
    $KAFKA_HOME/bin/zookeeper-shell.sh "$ZOOKEEPER" ls "$1" \
        | tail -n 1 \
        | egrep -o "[[:digit:]]+" \
        | sort -n \
        | tr '\n' ' '
}

join_by () { local IFS="$1"; shift; echo -n "$*"; }
rotate () { local first="$1"; shift; echo "$@" "$first"; }


JSON_OUT='reassign.json'


if [ -z "$BROKERS" ]; then
    BROKERS="$(list_zk_ids /brokers/ids)"
fi
echo "Brokers: $BROKERS"

echo "Gathering the reassignment plan..."
declare -a PLAN
PARTITIONS="$(list_zk_ids "/brokers/topics/$TOPIC/partitions")"

for partition in $PARTITIONS; do
    PLAN[$partition]=$(echo -en "\n  {\"topic\": \"$TOPIC\", \"partition\": $partition, \"replicas\": [$(join_by ',' $(echo $BROKERS | cut -d' ' "-f1-$REP_FACTOR"))]}")
    BROKERS=$(rotate $BROKERS)
done


echo -n '{"partitions": ['      > "$JSON_OUT"
join_by ','  "${PLAN[@]}"      >> "$JSON_OUT"
echo -en '\n],\n"version": 1}' >> "$JSON_OUT"

if [ -n "${DRY_RUN:-}" ]; then
    echo "The plan is:"
    cat "$JSON_OUT"
    exit 0
fi

echo "Starting the reassignment..."
"$KAFKA_HOME/bin/kafka-reassign-partitions.sh" \
    --zookeeper "$ZOOKEEPER" \
    --reassignment-json-file "$JSON_OUT" \
    --execute

while ! "$KAFKA_HOME/bin/kafka-reassign-partitions.sh" \
    --zookeeper "$ZOOKEEPER" \
    --reassignment-json-file "$JSON_OUT" \
    --verify > /dev/null; do
    echo 'Operation in progress...'
    sleep 5s
done

echo 'Reassignment complete.'

"$KAFKA_HOME/bin/kafka-preferred-replica-election.sh" \
    --zookeeper "$ZOOKEEPER" \
    --path-to-json-file "$JSON_OUT"
echo 'Done.'