KafkaThroughputParameterizedByMessageSize.sh

####################################################################################
# DESCRIPTION: Kafka performance test. One producer sending 1 million messages with
# record sizes between 10, 100, 500, 1000, 2000, 5000, 10000 to 1 topic with 1
# partition.
# PRECONDITIONS: Kafka and Zookeeper services must be running.
####################################################################################

THREADCOUNT=1
TOPICS=1
PARTITIONS=1
MESSAGES=1000000

cd /opt/kafka_2.11-0.10.0.1/ 

# write csv header
echo "messages,Average nMsgs/sec,Average nKBs/sec,Average latency (ms),threadCount,topicCount,partitions,messageSize" > ~/here2.dat

# run perf test
for MSGSIZE in 10 100 500 1000 2000 5000 10000; do
echo "Running 20 iterations for message size = $MSGSIZE"
for i in `seq 1 20`; do
TOPICNAME=iantest-$MSGSIZE-$i
#create topic

sudo service kafka stop >& /dev/null
ssh kafkanodeb sudo service kafka stop >& /dev/null
ssh kafkanodec sudo service kafka stop >& /dev/null
sudo rm -rf /tmp/kafka-logs
ssh kafkanodeb sudo rm -rf /tmp/kafka-logs
ssh kafkanodec sudo rm -rf /tmp/kafka-logs
sudo rm -rf /tmp/zookeeper/version-2
ssh kafkanodeb sudo rm -rf /tmp/zookeeper/version-2
ssh kafkanodec sudo rm -rf /tmp/zookeeper/version-2
ssh kafkanodeb sudo service kafka start >& /dev/null
ssh kafkanodec sudo service kafka start >& /dev/null
sudo service kafka start >& /dev/null

sleep 10
/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper kafkanodea:2181,kafkanodeb:2181,kafkanodec:2181 --create --partitions $PARTITIONS --replication-factor 3 --topic $TOPICNAME --config compression.type=uncompressed &> /dev/null

/opt/kafka_2.11-0.10.0.1/bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --topic $TOPICNAME --num-records 1000000 --record-size $MSGSIZE --throughput -1 --producer-props bootstrap.servers=localhost:9092 acks=all | tail -n 1 | tr -d "(" | tr -d ")" | awk -F " " '{printf "%s,%s,%s,%s,",$1,$4,$6*1000,$8}' >> ~/here2.dat
echo "$THREADCOUNT,$TOPICS,$PARTITIONS,$MSGSIZE" >> ~/here2.dat
echo -n "."
done
done
		

#delete topics
for i in `seq 1 20`; do
for MSGSIZE in 10 100 500 1000 2000 5000 10000; do
TOPICNAME="iantest-$MSGSIZE-$i"
/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper kafkanodea:2181,kafkanodeb:2181,kafkanodec:2181 --delete --topic $TOPICNAME
done
done