Instantly share code, notes, and snippets.

Embed
What would you like to do?
Kafka benchmark commands
## Consumer Throughput: Single consumer thread, no compression
## Consumer Throughput: 3 consumer thread, no compression
bin/kafka-consumer-perf-test.sh --topic benchmark-3-3-none \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--messages 15000000 \
--threads 1
# see: https://gist.github.com/jkreps/c7ddb4041ef62a900e6c
# see: https://kafka.apache.org/documentation/#producerconfigs
# Producer Throughput: Single producer thread, no replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-none
bin/kafka-producer-perf-test.sh --topic benchmark-1-1-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
# Producer Throughput: Single producer thread, no replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 1 \
--partitions 1 \
--topic benchmark-1-1-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-1-1-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: Single producer thread, 3x asynchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-none
bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: Single producer thread, 3x asynchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3a-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-1-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: Single producer thread, 3x synchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-none
bin/kafka-producer-perf-test.sh --topic benchmark-1-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: Single producer thread, 3x synchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-1-3-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-1-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: 3 producer thread, no replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-none
bin/kafka-producer-perf-test.sh --topic benchmark-3-0-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: 3 producer thread, no replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--partitions 1 \
--topic benchmark-3-0-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-3-0-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x asynchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-none
bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x asynchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3a-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-3-3a-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x synchronous replication, no compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-none
bin/kafka-producer-perf-test.sh --topic benchmark-3-3-none \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=none \
batch.size=8196
## Producer Throughput: 3 producer thread, 3x synchronous replication, zstandard compression
bin/kafka-topics.sh --create \
--zookeeper kafka-zk-1:2181,kafka-zk-2:2181,kafka-zk-3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic benchmark-3-3-zstd
bin/kafka-producer-perf-test.sh --topic benchmark-3-3-zstd \
--num-records 15000000 \
--record-size 100 \
--throughput 15000000 \
--producer-props \
acks=-1 \
bootstrap.servers=kafka-kf-1:9092,kafka-kf-2:9092,kafka-kf-3:9092 \
buffer.memory=67108864 \
compression.type=zstd \
batch.size=8196
@littleali

This comment has been minimized.

littleali commented Sep 5, 2017

Hi,
Where you put the thread numbers? I cant see any differences between script for 3 producer threads and single producer thread.

Thanks

@Jane870103

This comment has been minimized.

Jane870103 commented Dec 13, 2017

I want to ask that.where is the number of threads?

@continuum-sagar-priyadarshi

This comment has been minimized.

continuum-sagar-priyadarshi commented Jan 23, 2018

Same query. Where is number of threads?

@link3280

This comment has been minimized.

link3280 commented Mar 5, 2018

I suppose that multiple produer threads are implemented by running the producer script several times.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment