An example logstash setup

README.md

Logstash proof of concept setup

Basic vagrant box setup which follows How To Use Logstash and Kibana To Centralize Logs On Ubuntu 14.04 for the setup steps.

elastic-search.sh

#!/bin/bash

wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add -
echo 'deb http://packages.elasticsearch.org/elasticsearch/1.1/debian stable main' | sudo tee /etc/apt/sources.list.d/elasticsearch.list
sudo apt-get --quiet update

sudo apt-get --quiet -y install openjdk-7-jdk elasticsearch

IP=$( ifconfig | awk -v OFS="\n" '{ print $2, $NF }' | grep 192.168 | awk -F':' '{print $2}' )

sudo tee /etc/elasticsearch/elasticsearch.yml <<EOF > /dev/null
cluster:
  name: logstash_cluster
script.disable_dynamic: true

# Needed because of the limitted networking within Vagrant
network.host: "${IP}"
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["192.168.70.12", "192.168.70.11"]
EOF
sudo service elasticsearch restart
sudo update-rc.d elasticsearch defaults 95 10


if [ ! -d /usr/share/elasticsearch/plugins/HQ ]; then
  cd /usr/share/elasticsearch/bin
  sudo ./plugin -install royrusso/elasticsearch-HQ
fi

forwarder.sh

#!/bin/bash

sudo apt-get install sshpass
system=$( ifconfig | grep 70.11 )
if [ -z "$system" ]; then
    sshpass -p 'vagrant' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q vagrant@192.168.70.11:/etc/pki/tls/certs/logstash-forwarder.crt /tmp
    sshpass -p 'vagrant' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q vagrant@192.168.70.11:/etc/pki/tls/private/logstash-forwarder.key /tmp
    ls /tmp
else
  cp /etc/pki/tls/certs/logstash-forwarder.crt /tmp
  cp /etc/pki/tls/private/logstash-forwarder.key /tmp
fi

wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add -
echo 'deb http://packages.elasticsearch.org/logstashforwarder/debian stable main' | sudo tee /etc/apt/sources.list.d/logstashforwarder.list
sudo apt-get update
sudo apt-get install logstash-forwarder
cd /etc/init.d/; sudo wget https://raw.github.com/elasticsearch/logstash-forwarder/master/logstash-forwarder.init -O logstash-forwarder
sudo chmod +x logstash-forwarder
sudo update-rc.d logstash-forwarder defaults

sudo mkdir -p /etc/pki/tls/certs
sudo cp /tmp/logstash-forwarder.crt /etc/pki/tls/certs/

sudo tee /etc/logstash-forwarder <<EOF >/dev/null
{
  "network": {
    "servers": [ "192.168.70.11:5000" ],
    "timeout": 15,
    "ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt",
    "ssl key": "/etc/pki/tls/certs/logstash-forwarder.key"
  },
  "files": [
    {
      "paths": [
        "/var/log/syslog",
        "/var/log/auth.log"
       ],
      "fields": { "type": "syslog" }
    }
   ]
}
EOF

sudo service logstash-forwarder restart

logstash.sh

#!/bin/bash

echo 'deb http://packages.elasticsearch.org/logstash/1.4/debian stable main' | sudo tee /etc/apt/sources.list.d/logstash.list
sudo apt-get --quiet update
sudo apt-get install logstash

if [ ! -d /etc/pki/tls/certs ]; then
  sudo mkdir -p /etc/pki/tls/certs
  sudo mkdir /etc/pki/tls/private
  cd /etc/pki/tls; sudo openssl req -x509 -batch -nodes -days 3650 -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt
fi

sudo tee /etc/logstash/conf.d/01-lumberjack-input.conf <<EOF >/dev/null
input {
  lumberjack {
    port => 5000
    type => "logs"
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}
EOF

sudo tee /etc/logstash/conf.d/10-syslog.conf <<EOF >/dev/null
filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
}
EOF

sudo tee /etc/logstash/conf.d/30-lumberjack-output.conf <<EOF >/dev/null
output {
  elasticsearch {
        host => "192.168.70.11"
        cluster => "logstash_cluster"
  }
  stdout { codec => rubydebug }
}
EOF

sudo ln -s /etc/elasticsearch/elasticsearch.yml /var/log/logstash/elasticsearch.yml

sudo service logstash restart

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  config.vm.box = "ubuntu/trusty64"
  config.vm.define "frontend" do |frontend|
    frontend.vm.hostname =  "frontend.log.local"
    frontend.vm.network "private_network", ip: "192.168.70.10"
    frontend.vm.provision "shell", path: "forwarder.sh"
  end
  config.vm.define "server1" do |server1|
    server1.vm.hostname =  "server1.log.local"
    server1.vm.network "private_network", ip: "192.168.70.11"
#    server1.vm.provision "shell", path: "elastic-search.sh"
#    server1.vm.provision "shell", path: "logstash.sh"
    server1.vm.provision "shell", path: "forwarder.sh"
    server1.vm.provider "virtualbox" do |vb|
      vb.customize ["modifyvm", :id, "--memory", "1024"]
    end
  end
  config.vm.define "server2" do |server2|
    server2.vm.hostname =  "server2.log.local"
    server2.vm.network "private_network", ip: "192.168.70.12"
    server2.vm.provision "shell", path: "elastic-search.sh"
    server2.vm.provision "shell", path: "forwarder.sh"
    server2.vm.provider "virtualbox" do |vb|
      vb.customize ["modifyvm", :id, "--memory", "1024"]
    end
  end
  config.vm.define "client1" do |client1|
    client1.vm.network "private_network", ip: "192.168.70.21"
    client1.vm.hostname =  "client1.log.local"
    client1.vm.provision "shell", path: "forwarder.sh"
  end
end