Skip to content

Instantly share code, notes, and snippets.

@ottomata
Created August 28, 2015 20:31
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save ottomata/978e252d09ece47cd714 to your computer and use it in GitHub Desktop.
# Usage:
# eventlogging_processor_bench.sh <num_processors> <time_to_run> <consumer_group>
#
eventlogging_path=/home/otto/EventLogging/server
export PYTHONPATH=$eventlogging_path
num_processors=${1:-1}
time_to_run=${2:-10}
consumer_group=${3:-"$USER-00"}
processor="python $eventlogging_path/bin/eventlogging-processor --sid $consumer_group '%q %{recvFrom}s %{seqId}d %t %h %{userAgent}i' 'kafka:///kafka1012.eqiad.wmnet:9092,kafka1013.eqiad.wmnet:9092,kafka1014.eqiad.wmnet:9092,kafka1018.eqiad.wmnet:9092,kafka1020.eqiad.wmnet:9092,kafka1022.eqiad.wmnet:9092?topic=eventlogging-client-side&zookeeper_connect=conf1001.eqiad.wmnet,conf1002.eqiad.wmnet,conf1003.eqiad.wmnet/kafka/eqiad&auto_commit_enable=False&auto_offset_reset=-2' stdout://"
function process {
for i in $(seq 1 $num_processors); do
echo "Starting processor $i: $processor" >&2
timeout $time_to_run python $eventlogging_path/bin/eventlogging-processor --sid $consumer_group '%q %{recvFrom}s %{seqId}d %t %h %{userAgent}i' 'kafka:///kafka1012.eqiad.wmnet:9092,kafka1013.eqiad.wmnet:9092,kafka1014.eqiad.wmnet:9092,kafka1018.eqiad.wmnet:9092,kafka1020.eqiad.wmnet:9092,kafka1022.eqiad.wmnet:9092?topic=eventlogging-client-side&zookeeper_connect=conf1001.eqiad.wmnet,conf1002.eqiad.wmnet,conf1003.eqiad.wmnet/kafka/eqiad&auto_commit_enable=False&auto_offset_reset=-2' stdout:// 2>/tmp/el-${i}.log &
done
}
echo "Consuming raw events from the beginning of eventlogging-client-side with $num_processors processors running for $time_to_run seconds"
start=$(date +%s)
processed_count=$(process | wc -l)
end=$(date +%s)
time_taken=$(($end-$start))
process_rate=$(bc <<< "scale=10; $processed_count/$time_taken")
echo "Using $num_processors processors, processed $processed_count events in $time_taken seconds: $process_rate events/sec"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment