kafka snippets updated to work w ubuntu 12.04 and kafka 0.9

kafka-snippets

#!/bin/bash -xe
# requires jq 1.5 (or at least > 1.3) and kafkacat

PATH=$PATH:/usr/lib/kafka/bin
topic="hound-staging.retriever-mutation"

which kafkacat || echo 'no kafkacat found, bye!' && exit 1
which jq || echo 'no jq found, bye!' && exit 1

# make sure jq is v 1.5
if [[ "$(jq --version |tr -d 'jq-' |tr -d '\n')" == "1.5" ]] ; then echo "wrong version of jq" && exit 1 ; fi

# set zk host list
ZK_HOST=zk1:2181,zk2:2181,zk3:2181/kafka

# how many brokers
kafkacat -L -b localhost:9092 -t $topic "we have `grep brokers |tr -d ':'` live"

# record all topics
kafka-topics --zookeeper $ZK_HOST --list > all_topics

# prune internal topics, generate json for listing which topics to reassign
grep -v -e __consumer_offset -e deletion all_topics | tr '\n' ' ' | head --bytes -1 | jq -R 'split(" ") | reduce .[] as $topic ([]; . + [{"topic": $topic }]) | {"topics": . , "version": 1}' > all_topics.json

# generate the json for reassigning topics, the tool outputs current state then proposed state so take only the last line.  For a real cluster resize, you'd include all the brokers, but in this case I'm retiring brokers 0, 1 and 2.
kafka-reassign-partitions.sh --zookeeper $ZK_HOST --topics-to-move-json-file all_topics.json --broker-list "1004,1005,1006" --generate | tail -n 1 > reassignment.json

# actually do the reassignment
kafka-reassign-partitions.sh --zookeeper $ZK_HOST --topics-to-move-json-file all_topics.json --reassignment-json-file reassignment.json --execute

# watch to see when reassignment is completed
watch -n 5 kafka-reassign-partitions.sh --zookeeper $ZK_HOST --topics-to-move-json-file all_topics.json --reassignment-json-file reassignment.json --verify

# watch status while taking nodes up and down
watch -n 1 kafkacat -L -b localhost:9092 -t $topic

# dump kafka metadata / configs
kafkacat -X list -b localhost:9092