Kafka benchmark commands

consumer.sh

## Consumer Throughput: Single consumer thread, no compression
## Consumer Throughput: 3 consumer thread, no compression

bin/kafka-consumer-perf-test.sh --topic benchmark-3-3-none \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--messages 15000000 \
--threads 1

producer.sh

# see: https://gist.github.com/jkreps/c7ddb4041ef62a900e6c
# see: https://kafka.apache.org/documentation/#producerconfigs

# Producer Throughput: Single producer thread, no replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-1-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

# Producer Throughput: Single producer thread, no replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-1-1-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: Single producer thread, 3x asynchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: Single producer thread, 3x asynchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: Single producer thread, 3x synchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-none

bin/kafka-producer-perf-test.sh --topic benchmark-1-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: Single producer thread, 3x synchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-1-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: 3 producer thread, no replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-0-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, no replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-3-0-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x asynchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x asynchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x synchronous replication, no compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-none

bin/kafka-producer-perf-test.sh --topic benchmark-3-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196

## Producer Throughput: 3 producer thread, 3x synchronous replication, zstandard compression

bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-zstd

bin/kafka-producer-perf-test.sh --topic benchmark-3-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196

Hi,
Where you put the thread numbers? I cant see any differences between script for 3 producer threads and single producer thread.

Thanks

I want to ask that.where is the number of threads?

Same query. Where is number of threads?

I suppose that multiple produer threads are implemented by running the producer script several times.

On Kafka 0.10.1 "--num-records 15000000" syntax not found. Can anyone please suggest what should I use instead of it.

Where can I found documentation for what all the options mean?

I suppose that multiple producer threads are implemented by running the producer script several times.

You can run it on multiple servers

Where can I found documentation for what all the options mean?

--topic - The topic on which performance test will write to
--num-records - The total number of records to send for the test
--throughput - The maximum throughput for the test
--producer-props - The producer properties
bootstrap.servers - The detail of Kafka brokers
--record-size - The size of each record in bytes
batch.size - The batch size in bytes

I believe the zookeeper switch has deprecated for 'kafka-consumer-perf-test.bat' . (I am running remotely from kafka path CLI)
Running the below from CLI I am getting the error.

kafka-consumer-perf-test --topic test --bootstrap-server test:9093 --messages 10 --threads 1 --consumer.config C:\kafka_2.12-2.5.0\config\consumer.properties --group test --timeout 100000 --print-metrics

Exception in thread "main" java.util.IllegalFormatConversionException: f != java.lang.Integer
at java.util.Formatter$FormatSpecifier.failConversion(Formatter.java:4302)
at java.util.Formatter$FormatSpecifier.printFloat(Formatter.java:2806)
at java.util.Formatter$FormatSpecifier.print(Formatter.java:2753)
at java.util.Formatter.format(Formatter.java:2520)
at java.util.Formatter.format(Formatter.java:2455)
at java.lang.String.format(String.java:2940)
at scala.collection.immutable.StringLike.format(StringLike.scala:354)
at scala.collection.immutable.StringLike.format$(StringLike.scala:353)
at scala.collection.immutable.StringOps.format(StringOps.scala:33)
at kafka.utils.ToolsUtils$.$anonfun$printMetrics$3(ToolsUtils.scala:60)
at kafka.utils.ToolsUtils$.$anonfun$printMetrics$3$adapted(ToolsUtils.scala:58)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at kafka.utils.ToolsUtils$.printMetrics(ToolsUtils.scala:58)
at kafka.tools.ConsumerPerformance$.main(ConsumerPerformance.scala:82)
at kafka.tools.ConsumerPerformance.main(ConsumerPerformance.scala)

I have tried without the timeout it failed . It seems the process did not fetch the messages and stopped before doing so. Please help.