Kafka tools.md

create topic

kafka-topics --create --zookeeper localhost:2181 --topic foo-topic --replication-factor 1 --partitions 1

list topics

kafka-topics --list --zookeeper localhost:2181

describe topics

kafka-topics --describe --zookeeper localhost:2181

change topic config

kafka-topics --zookeeper localhost:2181 --alter --topic some-topic --partitions 40

change topic retention

kafka-topics --zookeeper localhost:2181 --alter --topic some-topic --config retention.ms=432000000

list consumer groups

kafka-consumer-groups --bootstrap-server localhost:9092 --list

list consumer groups including old fashion zookeeper consumer groups

kafka-consumer-groups --zookeeper localhost:2181 --list

describe offsets

kafka-consumer-groups --bootstrap-server localhost:9092 --describe --group SomeConsumerGroup

write to topic

kafka-console-producer --topic some-topic --broker-list localhost:9092

write to topic with compression

kafka-console-producer --topic some-topic --broker-list localhost:9092

read from topic

kafka-console-consumer --bootstrap-server localhost:9092 --topic some-topic --from-beginning

read from topic in avro

kafka-avro-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic some-topic

shift topic offset for consumer group (preview)

kafka-consumer-groups --bootstrap-server localhost:9092  --group SomeConsumerGropu --shift-by -4 --reset-offsets --topic some-topic

shift topic offset for consumer group (real execution)

kafka-consumer-groups --bootstrap-server localhost:9092  --group SomeConsumerGropu --shift-by -4 --reset-offsets --topic some-topic --execute

shift topic offset for consumer group to given date (preview)

kafka-consumer-groups --bootstrap-server localhost:9092  --group SomeConsumerGroupt --to-datetime '2019-02-02T00:00:00.000' --reset-offsets --topic some-topic

shift topic offsets for consumer group to earliest

kafka-consumer-groups --bootstrap-server localhost:9092 --group SomeGroput --reset-offsets --to-earliest --topic some-topic --execute

shift topic offsets for consumer group to specific offset

kafka-consumer-groups --bootstrap-server localhost:9092 --group SomeGroput --reset-offsets --to-offset 18406 --topic some-topic --execute

how to manually create consumer group in console

kafka-console-consumer --bootstrap-server localhost:9092 --topic some-topic --group SomeConsumerGroup

save messages to file

kafka-console-consumer --bootstrap-server localhost:9092 --topic some-topic --from-beginning > messages.txt

find topic files on disk

find /var/lib/kafka/ -maxdepth 1 -type d -name '_schema*'

mark topic for deletion

kafka-topics --delete --zookeeper localhost:2181 --topic some-topic

mark all topics to delete

#!/bin/bash

TOPIC_NAMES=$(kafka-topics --zookeeper localhost:2181 --list)

for TOPIC in $TOPIC_NAMES
do
    kafka-topics --zookeeper localhost:2181 --delete --topic $TOPIC
done

trim topic using kafka retention

kafka-topics --zookeeper localhost:2181 --alter --topic _schemas --config retention.ms=1000
kafka-topics --zookeeper localhost:2181 --alter --topic _schemas --config cleanup.policy=delete
# wait and check if topic is empty
# reset to previous settings
kafka-topics --zookeeper localhost:2181 --alter --topic _schemas --delete-config retention.ms
kafka-topics --zookeeper localhost:2181 --alter --topic _schemas --config cleanup.policy=compact

read segment file

kafka-run-class kafka.tools.DumpLogSegments --deep-iteration --print-data-log --files 00000000000000000000.log