Logstash 1.4.2 Kafka 0.8.2.0 Quickstart

00 - Get Logstash and logstash-kafka

#!/bin/bash
curl -O https://download.elasticsearch.org/logstash/logstash/logstash-1.4.2.tar.gz
tar xvf logstash*
cd logstash*
GEM_HOME=vendor/bundle/jruby/1.9 GEM_PATH= java -Xmx500m -jar vendor/jar/jruby-complete-1.7.11.jar -S gem install logstash-kafka
cp -R vendor/bundle/jruby/1.9/gems/logstash-kafka-*-java/{lib/logstash/*,spec/*} {lib/logstash/,spec/}
# test install
USE_JRUBY=1 bin/logstash rspec spec/**/kafka*.rb
cd ..

01 - Get Kafka

#!/bin/bash
curl -O http://www.gtlib.gatech.edu/pub/apache/kafka/0.8.2.0/kafka_2.10-0.8.2.0.tgz
tar xvf kafka*

02 - Start Zookeeper and Kafka

#!/bin/bash
cd kafka*
rm -rf /tmp/zoo*
screen -d -m bash -c "bin/zookeeper-server-start.sh config/zookeeper.properties"
rm -rf /tmp/kafka*
screen -d -m bash -c "bin/kafka-server-start.sh config/server.properties"
cd ..

03 - logstash-kafka demos

#!/bin/bash
function anykey {
  read -rsp $"Press any key to $1" -n1 key
  echo ""
}
anykey "run Kafka output demo"
logstash*/bin/logstash -e 'input { stdin {} } output { kafka {topic_id => "test"} stdout {} }'
anykey "view Kafka output messages"
kafka*/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning

anykey "test Kafka input for logstash then press ctrl-a ctrl-d and run 04 - Query Elasticsearch"
screen logstash*/bin/logstash -e 'input { kafka {topic_id => "test" reset_beginning => true auto_offset_reset => "smallest" } } output { stdout {} elasticsearch { embedded => true } }'

04 - Query Elasticsearch

#!/bin/bash
curl -XGET 'http://localhost:9200/logstash-'`date "+%Y.%m.%d"`'/logs/_search?pretty=true' -d '{ "query" : { "query_string" : { "query" : "*" } } }'

05 - Tear Down Screen

#!/bin/bash
for s in $(screen -ls| head -n4 | tail -3 | tail -r | egrep -o "\d{2,}"); do screen -r $s; done;