Simple script to dump all kafka topics to json files

kafka-dump.sh

#!/usr/bin/env bash

# Overview:
# 1) Get list of all topic names
# 2) Iterate list, dumping each one to json

# Pre-reqs:
# 1) kafkacat
# 2) jq

mkdir ./output

broker="<BROKER>"

topics=$(kafkacat -b ${broker} -L -J | jq -r '.topics[].topic' | sort)

for topic in $topics; do
    # Ignore "private"/"internal" topics. Adjust as needed.
    if [[ $topic == "_"* ]]; then
        continue
    fi
    echo "Dumping $topic"
    
    kafkacat -b ${broker} -C -J -e -q -o beginning -t "${topic}" > "./output/$topic.json"
done

echo "Done!"