Skip to content

Instantly share code, notes, and snippets.

@iandow
Created January 1, 2017 21:45
Show Gist options
  • Save iandow/625d783333a53b592f0381e6b37ee9ab to your computer and use it in GitHub Desktop.
Save iandow/625d783333a53b592f0381e6b37ee9ab to your computer and use it in GitHub Desktop.
# init csv file
echo "numPartitions, sizeMsg, topicCount, numMessages, Average nMsgs/sec, Average nKBs/sec, Average latency (ms), eth0 TX kB/s, eth0 RX kB/s" >> partitions-kafka.csv
for NUMPARTITIONS in 1 2 3 4 5 6 7 8 9 10 15 20 25 30 35 40 45 50 60 70 80 90 100 150 200 250 300 350 400; do
~/cleanup.sh >& /dev/null
sleep 5
TOPIC='iantest-'$NUMPARTITIONS
MESSAGES=1000000
MSGSIZE=100
/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic $TOPIC --partitions $NUMPARTITIONS --replication-factor 3
for i in `seq 1 10`; do
echo -n "$NUMPARTITIONS,$MSGSIZE," >> partitions-kafka.csv
R1=`cat /sys/class/net/eth0/statistics/rx_bytes`
T1=`cat /sys/class/net/eth0/statistics/tx_bytes`
SECONDS=0
/opt/kafka_2.11-0.10.0.1/bin/kafka-producer-perf-test.sh --topic $TOPIC --num-records $MESSAGES --record-size $MSGSIZE --throughput -1 --producer-props bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=8196 acks=all | tail -n 1 | tr -d "(" | tr -d ")" | awk -F " " '{printf "%s,%s,%s,%s,",$1,$4,$6*1000,$8}' >> partitions-kafka.csv
duration=$SECONDS
R2=`cat /sys/class/net/eth0/statistics/rx_bytes`
T2=`cat /sys/class/net/eth0/statistics/tx_bytes`
TBPS=`expr $T2 - $T1`
RBPS=`expr $R2 - $R1`
TKBPS=`expr $TBPS / 1024 / $duration`
RKBPS=`expr $RBPS / 1024 / $duration`
echo -n "$NUMPARTITIONS, $MESSAGES, $MSGSIZE" >> partitions-kafka.csv
echo ", $TKBPS, $RKBPS" >> partitions-kafka.csv
tail -n 1 partitions-kafka.csv
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment