consumer.sh

# Run these commands to measure consumer throughput in Kafka
	MESSAGES=1000000
	echo "numMsgs, sizeMsg, start.time, end.time, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec" >> consumer.csv 
	for MSGSIZE in 10 100 1000 2000 4000 6000 8000 10000; do
	for i in `seq 1 20`; do
	TOPIC='iantest-'$MSGSIZE
	echo $TOPIC
	/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic $TOPIC --config compression.type=uncompressed
	/opt/kafka_2.11-0.10.0.1/bin/kafka-producer-perf-test.sh --topic $TOPIC --num-records $MESSAGES --record-size $MSGSIZE --throughput -1 --producer-props bootstrap.servers=localhost:9092 buffer.memory=67108864 batch.size=8196 acks=all >& /dev/null
	echo -n "$MESSAGES, $MSGSIZE, " >> consumer.csv 
	# consume messages
	/opt/kafka_2.11-0.10.0.1/bin/kafka-consumer-perf-test.sh --zookeeper localhost:2181 --messages $MESSAGES --topic $TOPIC --threads 1 --hide-header >> consumer.csv 
	# cleanup
	/opt/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --zookeeper kafkanodea:2181,kafkanodeb:2181,kafkanodec:2181 --delete --topic $TOPIC >& /dev/null
	~/cleanup.sh >& /dev/null
	done
	done

# Run these commands to measure consumer throughput in MapR Streams

	MESSAGES=1000000
	echo "numMsgs, sizeMsg, Total time (ms), Total bytes received, Total messages received, Average nKBs/sec, Average nMsgs/sec" >> consumer-mapr.csv
	for MSGSIZE in 10 100 1000 2000 4000 6000 8000 10000; do
	maprcli stream create -path /user/mapr/iantest -produceperm p -consumeperm p -topicperm p -compression off
	echo -n "$MESSAGES, $MSGSIZE, " >> consumer-mapr.csv
	mapr perfproducer -path /user/mapr/iantest -ntopics 1 -npart 1 -nmsgs $MESSAGES -msgsz $MSGSIZE >& /dev/null
	# consume messages
	mapr perfconsumer -path /user/mapr/iantest -ntopics 1 -npart 1 -nmsgs $MESSAGES | tail -n 10 | tr "\n" ":" | awk -F ":" '{printf "%s, %s, %s, %s, %s\n",$2,$4,$6,$8,$10}' >> consumer-mapr.csv
	# cleanup
	maprcli stream delete -path /user/mapr/iantest >& /dev/null
	done