Test the performance of the logstash-input-kafka plugin.

gistfile1.md

Old

1 Thread:
2.86MiB 0:00:32 [  91KiB/s] [  91KiB/s]

4 Threads:
2.86MiB 0:00:31 [93.1KiB/s] [93.1KiB/s]

1 Decorated Thread:
2.86MiB 0:00:41 [71.2KiB/s] [71.2KiB/s]

4 Decorated Threads: 
2.86MiB 0:00:44 [66.4KiB/s] [66.4KiB/s]

New

1 Thread:
2.86MiB 0:00:32 [89.6KiB/s] [89.6KiB/s]

2 Threads:
2.86MiB 0:00:23 [ 126KiB/s] [ 126KiB/s]

4 Threads:
2.86MiB 0:00:19 [ 147KiB/s] [ 147KiB/s]

1 Decorated Thread:
2.86MiB 0:00:43 [66.6KiB/s] [66.6KiB/s]

2 Decorated Threads:
2.86MiB 0:00:28 [ 101KiB/s] [ 101KiB/s]

4 Decorated Threads:
2.86MiB 0:00:23 [ 124KiB/s] [ 124KiB/s]

perf_test_logstash_kafka_input.sh

#!/usr/bin/env bash

# ./logstash-2.2.2
# ./kafka_2.10-0.8.2.2

KAFKA=./kafka_2.10-0.8.2.2
LOGSTASH=./logstash-2.2.2
$KAFKA/bin/zookeeper-server-start.sh -daemon $KAFKA/config/zookeeper.properties
sleep 5
$KAFKA/bin/kafka-server-start.sh -daemon $KAFKA/config/server.properties
sleep 5
$KAFKA/bin/kafka-topics.sh --zookeeper localhost:2181 --topic perf --create --partitions 4 --replication-factor 1
sleep 5
$KAFKA/bin/kafka-producer-perf-test.sh --messages 3000000 --topics perf --broker-list localhost:9092
# Normal
$LOGSTASH/bin/logstash --quiet -e 'input {kafka { zk_connect => "localhost:2181" topic_id => "perf" reset_beginning => true auto_offset_reset => 'smallest' consumer_threads => 1 codec => plain } } output { stdout{codec => dots} }' | pv -SWbart -s 2928k > /dev/null
# Decorate
$LOGSTASH/bin/logstash --quiet -e 'input {kafka { zk_connect => "localhost:2181" topic_id => "perf" decorate_events => true reset_beginning => true auto_offset_reset => 'smallest' consumer_threads => 1 codec => plain } } output { stdout{codec => dots} }' | pv -SWbart -s 2928k > /dev/null

$KAFKA/bin/kafka-server-stop.sh
ps ax | grep -i 'zookeeper' | grep -v grep | awk '{print $1}' | xargs kill -9
rm -r /tmp/zookeeper /tmp/kafka-logs