Kafka benchmark commands

consumer.sh

## Consumer Throughput: Single consumer thread, no compression
## Consumer Throughput: 3 consumer thread, no compression

bin/kafka-consumer-perf-test.sh --topic benchmark-3-3-none \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--messages 15000000 \
--threads 1

manage topic.sh

#可以在zk串后面指定path，修改命名空间 例如 --zookeeper zk_host:port/chroot 

# create
bin/kafka-topics.sh --zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1

# list
bin/kafka-topics.sh --list --zookeeper localhost:2181

# detail
bin/kafka-topics.sh --zookeeper localhost:2181 --topic request --describe

#delete
bin/kafka-topics.sh --zookeeper localhost:2181  --delete --topic adserver

# see consmuer
 bin/kafka-consumer-groups.sh --bootstrap-server prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092  --group CommittableSourceConsumerAdServer --describe

# offset
bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 --topic api-request-qa

# consumer
bin/kafka-console-consumer.sh --bootstrap-server localhost:4500  --topic ssp-druid --partition 1 --offset 246870


#modify 
## retention
bin/kafka-topics.sh --zookeeper localhost:13003 --alter --topic MyTopic --config retention.ms=1000


## partition
bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic my_topic_name --partitions 40
## offset
@see https://github.com/foobaar/kafka-offset-manager
## replica
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --topics-to-move-json-file /root/test.json --broker-list kafka1:9092,kafka2:9092,kafka3:9092,kafka4:9092  --generate
{"version":1,
  "partitions":[
     {"topic":"signals","partition":0,"replicas":[0,1,2]},
     {"topic":"signals","partition":1,"replicas":[0,1,2]},
     {"topic":"signals","partition":2,"replicas":[0,1,2]}
]}
kafka-reassign-partitions --zookeeper localhost:2181 --reassignment-json-file increase-replication-factor.json --execute
@see https://stackoverflow.com/questions/37960767/how-to-change-the-replicas-of-kafka-topic

producer.sh

# see: https://gist.github.com/jkreps/c7ddb4041ef62a900e6c
# see: https://kafka.apache.org/documentation/#producerconfigs

## Producer Throughput: Single producer thread, no replication, no compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-1-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196



1989159 records sent, 397831.8 records/sec (37.94 MB/sec), 744.2 ms avg latency, 1042.0 max latency.
2616566 records sent, 523313.2 records/sec (49.91 MB/sec), 711.1 ms avg latency, 1183.0 max latency.
2107976 records sent, 421595.2 records/sec (40.21 MB/sec), 100.9 ms avg latency, 461.0 max latency.
2817837 records sent, 563567.4 records/sec (53.75 MB/sec), 295.3 ms avg latency, 605.0 max latency.
2621580 records sent, 524316.0 records/sec (50.00 MB/sec), 357.9 ms avg latency, 685.0 max latency.
2427126 records sent, 485425.2 records/sec (46.29 MB/sec), 762.4 ms avg latency, 1171.0 max latency.
15000000 records sent, 487582.889091 records/sec (46.50 MB/sec), 508.79 ms avg latency, 1183.00 ms max latency, 445 ms 50th, 1096 ms 95th, 1156 ms 99th, 1182 ms 99.9th.

## Producer Throughput: Single producer thread, no replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-lz4

bin/kafka-producer-perf-test.sh --topic benchmark-1-1-lz4 \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=lz4 \
batch.size=8196

3423330 records sent, 684666.0 records/sec (65.29 MB/sec), 2.8 ms avg latency, 166.0 max latency.
3405545 records sent, 681109.0 records/sec (64.96 MB/sec), 1.3 ms avg latency, 8.0 max latency.
3951806 records sent, 790361.2 records/sec (75.37 MB/sec), 0.9 ms avg latency, 12.0 max latency.
3199207 records sent, 639841.4 records/sec (61.02 MB/sec), 1.3 ms avg latency, 7.0 max latency.
15000000 records sent, 693417.159763 records/sec (66.13 MB/sec), 1.54 ms avg latency, 166.00 ms max latency, 1 ms 50th, 4 ms 95th, 13 ms 99th, 78 ms 99.9th.

## Producer Throughput: Single producer thread, 3x asynchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196


1802501 records sent, 360500.2 records/sec (34.38 MB/sec), 692.6 ms avg latency, 968.0 max latency.
2060456 records sent, 412008.8 records/sec (39.29 MB/sec), 1056.8 ms avg latency, 1343.0 max latency.
2120073 records sent, 424014.6 records/sec (40.44 MB/sec), 592.4 ms avg latency, 1099.0 max latency.
2126094 records sent, 425218.8 records/sec (40.55 MB/sec), 832.2 ms avg latency, 1413.0 max latency.
2118694 records sent, 423738.8 records/sec (40.41 MB/sec), 1062.3 ms avg latency, 1442.0 max latency.
2120396 records sent, 424079.2 records/sec (40.44 MB/sec), 452.8 ms avg latency, 900.0 max latency.
1995188 records sent, 399037.6 records/sec (38.06 MB/sec), 1196.3 ms avg latency, 1498.0 max latency.
15000000 records sent, 410419.174784 records/sec (39.14 MB/sec), 842.00 ms avg latency, 1498.00 ms max latency, 912 ms 50th, 1336 ms 95th, 1466 ms 99th, 1490 ms 99.9th.


## Producer Throughput: Single producer thread, 3x asynchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-lz4

bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-lz4 \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=lz4 \
batch.size=8196

3218270 records sent, 643654.0 records/sec (61.38 MB/sec), 13.3 ms avg latency, 225.0 max latency.
3908145 records sent, 781629.0 records/sec (74.54 MB/sec), 1.1 ms avg latency, 14.0 max latency.
4114542 records sent, 822908.4 records/sec (78.48 MB/sec), 1.0 ms avg latency, 7.0 max latency.
15000000 records sent, 766871.165644 records/sec (73.13 MB/sec), 3.64 ms avg latency, 225.00 ms max latency, 1 ms 50th, 5 ms 95th, 100 ms 99th, 218 ms 99.9th.

## Producer Throughput: Single producer thread, 3x synchronous replication, no compression ack=-1 (the leader will wait for the full set of in-sync replicas to acknowledge the record. )

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

#org.apache.kafka.common.errors.TimeoutException: Expiring 74 record(s) for benchmark-1-3-none-0: 30130 ms has passed since last append
651251 records sent, 130250.2 records/sec (12.42 MB/sec), 1886.6 ms avg latency, 3665.0 max latency.
725348 records sent, 145069.6 records/sec (13.83 MB/sec), 4184.9 ms avg latency, 4351.0 max latency.
738372 records sent, 147674.4 records/sec (14.08 MB/sec), 4125.5 ms avg latency, 4170.0 max latency.
683760 records sent, 136752.0 records/sec (13.04 MB/sec), 4248.8 ms avg latency, 4446.0 max latency.
677914 records sent, 135582.8 records/sec (12.93 MB/sec), 4501.4 ms avg latency, 4551.0 max latency.
692862 records sent, 138572.4 records/sec (13.22 MB/sec), 4352.6 ms avg latency, 4427.0 max latency.
626558 records sent, 125311.6 records/sec (11.95 MB/sec), 4537.2 ms avg latency, 4853.0 max latency.
661337 records sent, 132267.4 records/sec (12.61 MB/sec), 4819.1 ms avg latency, 4984.0 max latency.
700927 records sent, 140185.4 records/sec (13.37 MB/sec), 4383.7 ms avg latency, 4599.0 max latency.
683020 records sent, 136604.0 records/sec (13.03 MB/sec), 4382.7 ms avg latency, 4484.0 max latency.
701002 records sent, 140200.4 records/sec (13.37 MB/sec), 4352.1 ms avg latency, 4458.0 max latency.
689310 records sent, 137862.0 records/sec (13.15 MB/sec), 4380.9 ms avg latency, 4425.0 max latency.
706404 records sent, 141280.8 records/sec (13.47 MB/sec), 4341.8 ms avg latency, 4385.0 max latency.
664372 records sent, 132874.4 records/sec (12.67 MB/sec), 4288.7 ms avg latency, 4608.0 max latency.
706550 records sent, 141310.0 records/sec (13.48 MB/sec), 4539.0 ms avg latency, 4764.0 max latency.
733340 records sent, 146668.0 records/sec (13.99 MB/sec), 4161.5 ms avg latency, 4270.0 max latency.
717282 records sent, 143456.4 records/sec (13.68 MB/sec), 4181.3 ms avg latency, 4253.0 max latency.
717282 records sent, 143456.4 records/sec (13.68 MB/sec), 4256.1 ms avg latency, 4303.0 max latency.
720834 records sent, 144166.8 records/sec (13.75 MB/sec), 4169.4 ms avg latency, 4212.0 max latency.
681984 records sent, 136396.8 records/sec (13.01 MB/sec), 4271.9 ms avg latency, 4503.0 max latency.
588892 records sent, 117778.4 records/sec (11.23 MB/sec), 4881.3 ms avg latency, 5160.0 max latency.
15000000 records sent, 137782.799195 records/sec (13.14 MB/sec), 4268.74 ms avg latency, 5160.00 ms max latency, 4320 ms 50th, 4935 ms 95th, 5133 ms 99th, 5158 ms 99.9th.


## Producer Throughput: Single producer thread, 3x synchronous replication, zstandard compression ack =-1

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-lz4

bin/kafka-producer-perf-test.sh --topic benchmark-1-3-lz4 \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=lz4 \
batch.size=8196

## Producer Throughput: 3 producer thread, no replication, no compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--partitions 1 \
--replication-factor 1 \
--topic benchmark-3-0-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-0-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

5267325 records sent, 1053465.0 records/sec (100.47 MB/sec), 4.6 ms avg latency, 123.0 max latency.
5935216 records sent, 1187043.2 records/sec (113.21 MB/sec), 3.7 ms avg latency, 14.0 max latency.
15000000 records sent, 1069518.716578 records/sec (102.00 MB/sec), 8.45 ms avg latency, 208.00 ms max latency, 4 ms 50th, 11 ms 95th, 164 ms 99th, 204 ms 99.9th.

## Producer Throughput: 3 producer thread, no replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--partitions 1 \
--replication-factor 1 \
--topic benchmark-3-0-lz4

bin/kafka-producer-perf-test.sh --topic benchmark-3-0-lz4 \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=lz4 \
batch.size=8196


1:
3696834 records sent, 739366.8 records/sec (70.51 MB/sec), 1.5 ms avg latency, 117.0 max latency.
3775478 records sent, 755095.6 records/sec (72.01 MB/sec), 0.9 ms avg latency, 13.0 max latency.
4358877 records sent, 871775.4 records/sec (83.14 MB/sec), 0.9 ms avg latency, 4.0 max latency.
15000000 records sent, 813581.385258 records/sec (77.59 MB/sec), 1.09 ms avg latency, 117.00 ms max latency, 1 ms 50th, 2 ms 95th, 7 ms 99th, 45 ms 99.9th. 
2:
2424598 records sent, 484919.6 records/sec (46.25 MB/sec), 2.4 ms avg latency, 135.0 max latency.
2515248 records sent, 503049.6 records/sec (47.97 MB/sec), 1.9 ms avg latency, 50.0 max latency.
1815538 records sent, 363107.6 records/sec (34.63 MB/sec), 1.2 ms avg latency, 15.0 max latency.
2625103 records sent, 525020.6 records/sec (50.07 MB/sec), 1.2 ms avg latency, 13.0 max latency.
3003159 records sent, 600631.8 records/sec (57.28 MB/sec), 0.9 ms avg latency, 8.0 max latency.
2191916 records sent, 438383.2 records/sec (41.81 MB/sec), 0.9 ms avg latency, 5.0 max latency.
15000000 records sent, 488822.264225 records/sec (46.62 MB/sec), 1.40 ms avg latency, 135.00 ms max latency, 1 ms 50th, 3 ms 95th, 11 ms 99th, 76 ms 99.9th.



## Producer Throughput: 3 producer thread, 3x asynchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x asynchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-lz4

bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-lz4 \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=lz4 \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x synchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x synchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper prod-us-kafka1:2181,prod-us-kafka2:2181,prod-us-kafka3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-lz4

bin/kafka-producer-perf-test.sh --topic benchmark-3-3-lz4 \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=prod-us-kafka1:9092,prod-us-kafka2:9092,prod-us-kafka3:9092 \
buffer.memory=67108864 \
compression.type=lz4 \
batch.size=8196