Kafka Benchmark Commands

benchmark-commands.md

Benchmark commands

Topic Setup

kafka-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic part-1-rep-1 \
  --partitions 1 \
  --replication-factor 1
kafka-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic part-1-rep-3 \
  --partitions 1 \
  --replication-factor 3
kafka-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic part-3-rep-1 \
  --partitions 3 \
  --replication-factor 1
kafka-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic part-3-rep-3 \
  --partitions 3 \
  --replication-factor 3
kafka-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic part-6-rep-1 \
  --partitions 6 \
  --replication-factor 1
kafka-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic part-6-rep-3 \
  --partitions 6 \
  --replication-factor 3

Delete a Topic

kafka-topics --bootstrap-server cp-kafka.kafka:9092 --delete --topic $(NAME)

Producer

Single thread, no replication

kafka-producer-perf-test \
  --topic part-6-rep-3 \
  --num-records 50000000 \
  --record-size 100 \
  --throughput -1 \
  --producer-props \
  acks=1 \
  bootstrap.servers=cp-kafka.kafka:9092 \
  buffer.memory=67108864 \
  batch.size=8196

Single-thread, async 3x replication

kafk-topics \
  --bootstrap-server cp-kafka.kafka:9092 \
  --create \
  --topic test \
  --partitions 6 \
  --replication-factor 3
kafka-producer-perf-test \
  --topic test \
  --num-records 50000000 \
  --record-size 100 \
  --throughput -1 \
  --producer-props acks=1 \
  bootstrap.servers=cp-kafka.kafka:9092 \
  buffer.memory=67108864 \
  batch.size=8196

Single-thread, sync 3x replication

kafka-producer-perf-test \
  --topic test \
  --num-records 50000000 \
  --record-size 100 \
  --throughput -1 \
  --producer-props acks=1 \
  bootstrap.servers=cp-kafka.kafka:9092 \
  buffer.memory=67108864 batch.size=64000

Three Producers, 3x async replication

kafka-producer-perf-test \
  --topic test \
  --num-records 50000000 \
  --record-size 100 \
  --throughput -1 \
  --producer-props acks=1 \
  bootstrap.servers=cp-kafka.kafka:9092 \
  buffer.memory=67108864 \
  batch.size=8196

Throughput Versus Stored Data

kafka-producer-perf-test \
  --topic test \
  --num-records 50000000 \
  --record-size 100 \
  --throughput -1 \
  --producer-props acks=1 \
  bootstrap.servers=cp-kafka.kafka:9092 \
  buffer.memory=67108864 batch.size=8196

Effect of message size

for i in 10 100 1000 10000 100000; do
  echo ""
  echo $i
  kafka-producer-perf-test \
    --topic test \
    --num-records $((1000*1024*1024/$i))\
    --record-size $i\
    --throughput -1 \
    --producer-props acks=1 \
    bootstrap.servers=cp-kafka.kafka:9092 \
    buffer.memory=67108864 \
    batch.size=128000
done;

Consumer

Consumer throughput

kafka-consumer-perf-test \
  --bootstrap-server cp-kafka.kafka:9092 \
  --messages 50000000 \
  --topic test \
  --threads 1

3 Consumers

On three servers, run:

kafka-consumer-perf-test \
  --bootstrap-server cp-kafka.kafka:9092 \
  --messages 50000000 \
  --topic test \
  --threads 1

End-to-end Latency

USAGE: java kafka.tools.EndToEndLatency
  broker_list
  topic
  num_messages
  producer_acks
  message_size_bytes
  [optional] properties_file

kafka-run-class \
  kafka.tools.EndToEndLatency \
  cp-kafka.kafka:9092 \
  end-to-end-latency \
  10000 \
  all \
  2500

Producer and consumer

kafka-run-class \
  org.apache.kafka.tools.ProducerPerformance \
kafka-producer-perf-test \
  --topic test \
  --num-records 50000000 \
  --record-size 100 \
  --throughput -1 \
  --producer-props acks=1 \
  bootstrap.servers=cp-kafka.kafka:9092 \
  buffer.memory=67108864 \
  batch.size=8196

kafka-consumer-perf-test \
  --bootstrap-server cp-kafka.kafka:9092 \
  --messages 50000000 \
  --topic test \
  --threads 1

server-config.properties