Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
kafka failed broker tests
# Asynchronous (batched) producer:
./sequence_generate.sh 20000 10000 | bin/kafka-console-producer.sh --zookeeper analytics1003:2181 --topic test6
# Synchronous producer:
./sequence_generate.sh 20000 10000 | bin/kafka-producer-shell.sh --props config/producer.properties --topic test6
# Consumer
bin/kafka-consumer-shell.sh --props config/consumer.properties --topic test6 | tee -a /tmp/test6.log
# sequence check
cat /tmp/test6.log | egrep '^consumed:' | awk '{print $3}' | sort -g | ~/bin/sequence_check.sh 1 20000
# To check on consumer group state in ZooKeeper for a topic:
bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --group test-consumer-group --zkconnect analytics1003:2181 --topic test6
Producer an01
consumed: 2012-08-07_20:28:30.145622121 3522 analytics1001
consumed: 2012-08-07_20:28:31.189350269 3864 analytics1001
Reconfigured in 1.043728148 seconds
Lost 36/342 ~= 10%
Producer an02
consumed: 2012-08-07_20:28:30.318517732 1249 analytics1002
consumed: 2012-08-07_20:28:31.132631454 1345 analytics1002
Reconfigured in .814113722 seconds
Lost 11/96 ~= 11%
Producer an03
consumed: 2012-08-07_20:28:30.732243744 4199 analytics1003
consumed: 2012-08-07_20:28:31.161735699 4395 analytics1003
Reconfigured in .429491955 seconds
Lost 20/196 ~= 10%
Producer an04
consumed: 2012-08-07_20:28:30.480141625 3984 analytics1004
consumed: 2012-08-07_20:28:31.155894440 4322 analytics1004
Reconfigured in .675752815
Lost 36/338 ~= 10%
Producer an05
consumed: 2012-08-07_20:28:30.234519818 3132 analytics1005
consumed: 2012-08-07_20:28:31.113172131 3518 analytics1005
Reconfigured in .878652313
Lost 31/386 ~= %8
Producer an06
consumed: 2012-08-07_20:28:30.038108932 1476 analytics1006
consumed: 2012-08-07_20:28:31.163637631 1610 analytics1006
Reconfigured in 1.125528699
Lost 12/134 ~= 9%
Producer an07
consumed: 2012-08-07_20:28:30.660764408 2131 analytics1007
consumed: 2012-08-07_20:28:31.175204389 2197 analytics1007
Reconfigured in .514439981
Lost 9/66 ~= 13%
Producer an08
consumed: 2012-08-07_20:28:30.336355526 1570 analytics1008
consumed: 2012-08-07_20:28:31.139607063 1701 analytics1008
Reconfigured in .803251537
Lost 13/131 ~= 10%
Producer an09
consumed: 2012-08-07_20:28:30.033838061 1784 analytics1009
consumed: 2012-08-07_20:28:30.962274566 2059 analytics1009
Reconfigured in .928436505
Lost 34/275 ~= 12%
Producer an10
consumed: 2012-08-07_20:28:30.649331993 1018 analytics1010
consumed: 2012-08-07_20:28:31.016440719 1050 analytics1010
Reconfigured in .367108726
Lost 7/32 ~= 21%
#!/bin/bash
# Checks a contiguos ordered sequence on stdin for gaps.
# Prints out missing sequence numbers.
#
# Usage: $0 <start> <end>
start=$1
: ${start:=1}
end=$2
: ${end:=20000}
i=$start
read num
while true; do
if [ -z ${num} ]; then
echo "Missing $i";
elif [ ${num} -eq $i ]; then
read num
else
echo "Missing $i"
fi
if [ $i -ge $end ]; then
exit 0;
fi
((i+=1))
done
#!/bin/bash
# Generates sequenced log messages, stopping every <mod> for permission to continue.
#
# Usage: $0 <limit> <mod>
limit=$1
: ${limit:=20000}
mod=$2
: ${mod:=1000}
echo "Generating $limit messages, stopping every $mod for permission to continue..." >&2
for (( i=1; $i<=$limit; i=(( $i+1 )) ))do
message="$(date +%Y-%m-%d_%H:%M:%S.%N) $i $(hostname)"
echo $message;
if [ $(( $i % $mod )) -eq 0 ]; then
echo $message >&2
echo -n "Continue? (y/n) " >&2
read answer
if [ $answer != "y" ]; then
echo "Exiting" >&2
exit 0;
fi
fi
done
exit 0;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment