Useful kafka-* commands

useful-kafka-commands.sh

############################
# kafka-topics
############################

#Prints topics with offline partitions
kafka-topics --zookeeper <HOSTNAME>:<PORT> --describe --unavailable-partitions |\
sed -r 's/^\s+Topic:\s+(.*)\s+Partition:.*$/\1/'

#Prints offline partitions
kafka-topics --zookeeper <HOSTNAME>:<PORT> --describe --unavailable-partitions |\
sed -r 's/^\s+Topic:\s+(.*)\s+Partition:\s+([0-9]+).*$/\1-\2/'


############################
# kafka-producer-perf-test
############################
kafka-producer-perf-test --producer-props bootstrap.servers=<HOSTNAME>:<PORT> acks=-1 \
--topic <TOPIC> --num-records 10000000 --throughput -1 --record-size 1024 --print-metrics

#Command with properties to tune performance
kafka-producer-perf-test --producer.config <PRODUCER_PROPERTIES_FILE> --topic <TOPIC> \
--throughput 5000 --num-records 100000000 --record-size 3200 --print-metrics \
--producer-props bootstrap.servers=<HOSTNAME>:<PORT> client.id=<CLIENT_ID> acks=all max.in.flight.requests.per.connection=5 \
batch.size=300000 linger.ms=500 buffer.memory=256000000 



############################
# kafka-consumer-perf-test
############################
kafka-consumer-perf-test --bootstrap-server <HOSTNAME>:<PORT> --topic <TOPIC> \
--messages 10000000 --print-metrics

############################
# kafka-configs
############################

#updating topic configurations
kafka-configs --zookeeper <HOSTNAME>:<PORT> --alter --entity-name <TOPIC NAME> --entity-type topics \
--add-config <CONFIG_PROPERTY>=<VALUE>

#updating all topics' replica placement constraints
kafka-topics --bootstrap-server node1:9091 --command-config ~/ssl_command.config --list | \
xargs -L1 kafka-configs --command-config ~/ssl_command.config --bootstrap-server node1:9091 --entity-type topics --alter --replica-placement ~/testcases-ansible/sync-replication.json --entity-name


#####################################
# kafka-run-class kafka.tools.JmxTool
#####################################
kafka-run-class kafka.tools.JmxTool --jmx-url service:jmx:rmi:///jndi/rmi://localhost:7772/jmxrmi \
--object-name kafka.producer:type=producer-metrics,client-id=yamato \
--attributes record-queue-time-avg,\
batch-size-avg,\
bufferpool-wait-ratio,\
request-size-avg,\
request-latency-avg,\
record-retry-rate,\
record-error-rate,\
buffer-available-bytes,\
produce-throttle-time-avg,\
compression-rate-avg  \
--report-format properties