Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Kafka benchmark commands
## Consumer Throughput: Single consumer thread, no compression
## Consumer Throughput: 3 consumer thread, no compression
bin/kafka-consumer-perf-test.sh --topic benchmark-3-3-none \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--messages 15000000 \
--threads 1
# see: https://gist.github.com/jkreps/c7ddb4041ef62a900e6c
# see: https://kafka.apache.org/documentation/#producerconfigs
# Producer Throughput: Single producer thread, no replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-none
bin/kafka-producer-perf-test.sh --topic benchmark-1-1-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
# Producer Throughput: Single producer thread, no replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-1-1-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: Single producer thread, 3x asynchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-none
bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: Single producer thread, 3x asynchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: Single producer thread, 3x synchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-none
bin/kafka-producer-perf-test.sh --topic benchmark-1-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: Single producer thread, 3x synchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-1-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: 3 producer thread, no replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-none
bin/kafka-producer-perf-test.sh --topic benchmark-3-0-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: 3 producer thread, no replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-3-0-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x asynchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-none
bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x asynchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x synchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-none
bin/kafka-producer-perf-test.sh --topic benchmark-3-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x synchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-3-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
@littleali

This comment has been minimized.

Copy link

commented Sep 5, 2017

Hi,
Where you put the thread numbers? I cant see any differences between script for 3 producer threads and single producer thread.

Thanks

@Jane870103

This comment has been minimized.

Copy link

commented Dec 13, 2017

I want to ask that.where is the number of threads?

@continuum-sagar-priyadarshi

This comment has been minimized.

Copy link

commented Jan 23, 2018

Same query. Where is number of threads?

@link3280

This comment has been minimized.

Copy link

commented Mar 5, 2018

I suppose that multiple produer threads are implemented by running the producer script several times.

@Lawands

This comment has been minimized.

Copy link

commented Feb 14, 2019

On Kafka 0.10.1 "--num-records 15000000" syntax not found. Can anyone please suggest what should I use instead of it.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.