kafka failed broker tests

Commands.sh

# Asynchronous (batched) producer:
./sequence_generate.sh 20000 10000 | bin/kafka-console-producer.sh --zookeeper analytics1003:2181 --topic test6

# Synchronous producer: 
./sequence_generate.sh 20000 10000 | bin/kafka-producer-shell.sh --props config/producer.properties --topic test6

# Consumer
bin/kafka-consumer-shell.sh --props config/consumer.properties --topic test6 | tee -a /tmp/test6.log

# sequence check
cat /tmp/test6.log | egrep '^consumed:' | awk '{print $3}' | sort -g | ~/bin/sequence_check.sh 1 20000

# To check on consumer group state in ZooKeeper for a topic:
bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --group test-consumer-group --zkconnect analytics1003:2181 --topic test6

Results for 10 producers, 10 brokers, 1 failed broker

Producer an01
consumed: 2012-08-07_20:28:30.145622121 3522 analytics1001
consumed: 2012-08-07_20:28:31.189350269 3864 analytics1001

Reconfigured in  1.043728148 seconds
Lost 36/342 ~= 10%

Producer an02
consumed: 2012-08-07_20:28:30.318517732 1249 analytics1002
consumed: 2012-08-07_20:28:31.132631454 1345 analytics1002

Reconfigured in .814113722 seconds
Lost 11/96 ~= 11%

Producer an03
consumed: 2012-08-07_20:28:30.732243744 4199 analytics1003
consumed: 2012-08-07_20:28:31.161735699 4395 analytics1003

Reconfigured in .429491955 seconds
Lost 20/196 ~= 10%

Producer an04
consumed: 2012-08-07_20:28:30.480141625 3984 analytics1004
consumed: 2012-08-07_20:28:31.155894440 4322 analytics1004

Reconfigured in .675752815
Lost 36/338 ~= 10%

Producer an05
consumed: 2012-08-07_20:28:30.234519818 3132 analytics1005
consumed: 2012-08-07_20:28:31.113172131 3518 analytics1005

Reconfigured in .878652313
Lost 31/386 ~= %8

Producer an06
consumed: 2012-08-07_20:28:30.038108932 1476 analytics1006
consumed: 2012-08-07_20:28:31.163637631 1610 analytics1006

Reconfigured in 1.125528699
Lost 12/134 ~= 9%

Producer an07
consumed: 2012-08-07_20:28:30.660764408 2131 analytics1007
consumed: 2012-08-07_20:28:31.175204389 2197 analytics1007

Reconfigured in .514439981
Lost 9/66 ~= 13%

Producer an08
consumed: 2012-08-07_20:28:30.336355526 1570 analytics1008
consumed: 2012-08-07_20:28:31.139607063 1701 analytics1008

Reconfigured in .803251537
Lost 13/131 ~= 10%

Producer an09
consumed: 2012-08-07_20:28:30.033838061 1784 analytics1009
consumed: 2012-08-07_20:28:30.962274566 2059 analytics1009

Reconfigured in .928436505
Lost 34/275 ~= 12%

Producer an10
consumed: 2012-08-07_20:28:30.649331993 1018 analytics1010
consumed: 2012-08-07_20:28:31.016440719 1050 analytics1010

Reconfigured in .367108726
Lost  7/32 ~= 21%

sequence_check.sh

#!/bin/bash

# Checks a contiguos ordered sequence on stdin for gaps.
# Prints out missing sequence numbers.
# 
# Usage: $0 <start> <end>


start=$1
: ${start:=1}
end=$2
: ${end:=20000}

i=$start

read num
while true; do
	if [ -z ${num} ]; then
		echo "Missing $i";
	elif [ ${num} -eq $i ]; then
		read num
	else
		echo "Missing $i"
	fi
	
	if [ $i -ge $end ]; then
		exit 0;
	fi
	
	((i+=1))
done

sequence_generate.sh

#!/bin/bash

# Generates sequenced log messages, stopping every <mod> for permission to continue.
#
# Usage:   $0 <limit> <mod>


limit=$1
: ${limit:=20000}

mod=$2
: ${mod:=1000}

echo "Generating $limit messages, stopping every $mod for permission to continue..." >&2
for (( i=1; $i<=$limit; i=(( $i+1 )) ))do
	message="$(date +%Y-%m-%d_%H:%M:%S.%N) $i $(hostname)"
	echo $message;
	if [ $(( $i % $mod )) -eq 0 ]; then
		echo $message >&2
		echo -n "Continue? (y/n) " >&2
		read answer
		if [ $answer != "y" ]; then
			echo "Exiting" >&2
			exit 0;
		fi
	fi
done

exit 0;