#################################################################################### | |
# DESCRIPTION: Kafka performance test. One producer sending 1 million messages with | |
# record sizes between 10, 100, 500, 1000, 2000, 5000, 10000 to 1 topic with 1 | |
# partition. | |
# PRECONDITIONS: Kafka and Zookeeper services must be running. | |
#################################################################################### | |
THREADCOUNT=1 | |
TOPICS=1 | |
PARTITIONS=1 | |
MESSAGES=1000000 | |
cd /opt/kafka_2.11-0.10.0.1/ | |
# write csv header | |
echo "messages,Average nMsgs/sec,Average nKBs/sec,Average latency (ms),threadCount,topicCount,partitions,messageSize" > ~/here2.dat | |
# run perf test | |
for MSGSIZE in 10 100 500 1000 2000 5000 10000; do | |
echo "Running 20 iterations for message size = $MSGSIZE" | |
for i in `seq 1 20`; do | |
TOPICNAME=iantest-$MSGSIZE-$i | |
#create topic | |
sudo service kafka stop >& /dev/null | |
ssh kafkanodeb sudo service kafka stop >& /dev/null | |
ssh kafkanodec sudo service kafka stop >& /dev/null | |
sudo rm -rf /tmp/kafka-logs | |
ssh kafkanodeb sudo rm -rf /tmp/kafka-logs | |
ssh kafkanodec sudo rm -rf /tmp/kafka-logs | |
sudo rm -rf /tmp/zookeeper/version-2 | |
ssh kafkanodeb sudo rm -rf /tmp/zookeeper/version-2 | |
ssh kafkanodec sudo rm -rf /tmp/zookeeper/version-2 | |
ssh kafkanodeb sudo service kafka start >& /dev/null | |
ssh kafkanodec sudo service kafka start >& /dev/null | |
sudo service kafka start >& /dev/null | |
sleep 10 | |
/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper kafkanodea:2181,kafkanodeb:2181,kafkanodec:2181 --create --partitions $PARTITIONS --replication-factor 3 --topic $TOPICNAME --config compression.type=uncompressed &> /dev/null | |
/opt/kafka_2.11-0.10.0.1/bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --topic $TOPICNAME --num-records 1000000 --record-size $MSGSIZE --throughput -1 --producer-props bootstrap.servers=localhost:9092 acks=all | tail -n 1 | tr -d "(" | tr -d ")" | awk -F " " '{printf "%s,%s,%s,%s,",$1,$4,$6*1000,$8}' >> ~/here2.dat | |
echo "$THREADCOUNT,$TOPICS,$PARTITIONS,$MSGSIZE" >> ~/here2.dat | |
echo -n "." | |
done | |
done | |
#delete topics | |
for i in `seq 1 20`; do | |
for MSGSIZE in 10 100 500 1000 2000 5000 10000; do | |
TOPICNAME="iantest-$MSGSIZE-$i" | |
/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper kafkanodea:2181,kafkanodeb:2181,kafkanodec:2181 --delete --topic $TOPICNAME | |
done | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment