partitions_vs_tput-kafka.sh

# init csv file
echo "numPartitions, sizeMsg, topicCount, numMessages, Average nMsgs/sec, Average nKBs/sec, Average latency (ms), eth0 TX kB/s, eth0 RX kB/s" >> partitions-kafka.csv

for NUMPARTITIONS in 1 2 3 4 5 6 7 8 9 10 15 20 25 30 35 40 45 50 60 70 80 90 100 150 200 250 300 350 400; do

~/cleanup.sh >& /dev/null
sleep 5

TOPIC='iantest-'$NUMPARTITIONS
MESSAGES=1000000
MSGSIZE=100

/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic $TOPIC --partitions $NUMPARTITIONS --replication-factor 3

for i in `seq 1 10`; do
echo -n "$NUMPARTITIONS,$MSGSIZE," >> partitions-kafka.csv

R1=`cat /sys/class/net/eth0/statistics/rx_bytes`
T1=`cat /sys/class/net/eth0/statistics/tx_bytes`
SECONDS=0

/opt/kafka_2.11-0.10.0.1/bin/kafka-producer-perf-test.sh --topic $TOPIC --num-records $MESSAGES --record-size $MSGSIZE --throughput -1 --producer-props bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=8196 acks=all | tail -n 1 | tr -d "(" | tr -d ")" | awk -F " " '{printf "%s,%s,%s,%s,",$1,$4,$6*1000,$8}' >> partitions-kafka.csv

duration=$SECONDS
R2=`cat /sys/class/net/eth0/statistics/rx_bytes`
T2=`cat /sys/class/net/eth0/statistics/tx_bytes`
TBPS=`expr $T2 - $T1`
RBPS=`expr $R2 - $R1`
TKBPS=`expr $TBPS / 1024 / $duration`
RKBPS=`expr $RBPS / 1024 / $duration`
echo -n "$NUMPARTITIONS, $MESSAGES, $MSGSIZE" >> partitions-kafka.csv
echo ", $TKBPS, $RKBPS" >> partitions-kafka.csv

tail -n 1 partitions-kafka.csv

done
done