LogStash Demo

Logstash Demo

Files and commands for Logstash demo

Logstash Demo - bootstrap.sh

#!/bin/sh

# update apt
apt-get update

# install java, apache, curl
apt-get -y install openjdk-7-jre curl apache2

# install logstash
cd /tmp
wget https://download.elasticsearch.org/logstash/logstash/logstash-1.4.0.tar.gz
tar zxvf logstash-1.4.0.tar.gz
cp -R logstash-1.4.0 /opt/

echo "Done !"

Logstash Demo - apachemon.conf

input {
  file {
    path => "/var/log/apache2/*.log"
  }
}

filter {
  if [path] =~ "access" {
    mutate { replace => { type => "apache_access" } }
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
  } else if [path] =~ "error" {
    mutate { replace => { type => "apache_error" } }
  } else {
    mutate { replace => { type => "random_logs" } }
  }
}

output {
  elasticsearch { embedded => true }
  stdout { codec => json }
}

Logstash Demo - quick examples

# StdIO
    /opt/logstash-1.4.0/
    bin/logstash -e 'input { stdin { } } output { stdout { codec => rubydebug } }'


# embedded elasticsearch

1. Start it 

    bin/logstash -e 'input { stdin { } } output { elasticsearch { embedded => true } }'

2. Query elastic search

    curl 'http://localhost:9200/_search?pretty'
    curl 'http://localhost:9200/_search?pretty&q=is'


# Starting kibana

    bin/logstash-web&
    
# Monitoring apache

    bin/logstash -f /vagrant/apachemon.conf
    

Logstash Demo - twittermon.conf

input {
  twitter {
    consumer_key => ""
    consumer_secret => ""
    keywords => "bieber"
    oauth_token => ""
    oauth_token_secret => ""
    type => "bieber_tweet"
  }
}

filter {
  
}

output {
  elasticsearch { embedded => true }
  stdout { codec => json }
}

Logstash Demo - Vagrantfile

Vagrant.configure(2) do |config| 
  config.vm.box = "hashicorp/precise32"  
  
  config.vm.network "forwarded_port", guest: 80, host: 8080
  config.vm.network "forwarded_port", guest: 9200, host: 9200
  config.vm.network "forwarded_port", guest: 9292, host: 9292
  
  
  config.vm.provider :virtualbox do |vb|
    vb.customize ["modifyvm", :id, "--memory", "512"]
  end
  
  config.vm.provision "shell", path: "bootstrap.sh"
  
end