Basic logstash conf

ansible.cfg

# http://docs.ansible.com/intro_configuration.html
[defaults]
hostfile = hosts.ini

base.conf

input {
    generator {
        message => '127.0.0.1 - - [08/Aug/2009:00:01:01 -0000] "GET /elasticsearch/logstash/tree/82fdc11f2e35b52b05f773be52ba3c0e8072c509 HTTP/1.1" 200 1420 "https://github.com/elasticsearch/logstash/tree/82fdc11f2e35b52b05f773be52ba3c0e8072c509" "curl/7.37.1"'
        type => "generated"
        count => 1
    }
    tcp { codec => json port => 33333 type => "tcp" }
    stdin { codec => json type => "stdin" }
}
filter {
        grok { match => { "message" => "%{COMBINEDAPACHELOG}" } }
        date { match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ] }
    }
    else if [type] == "tcp" {
        if [action] == "login" {
            mutate { remove_field => "secret" }
        }
    }
    else { 
        # default to stdin
        mutate { gsub => [ "message", "flume", "logstash" ] }
    }

}

output {
    stdout { codec => rubydebug }
}

hosts.ini

[logstash]
localhost ansible_ssh_port=2222 ansible_ssh_user=vagrant ansible_ssh_pass=vagrant

logstash.sh.j2

#!/bin/bash

/opt/{{logstash_fullname}}/bin/logstash "$@"

playbook-logstash.yml

---
- hosts: logstash
  vars:
    logstash_dir: /opt/logstash
    logstash_version: 1.4.2
    logstash_fullname: logstash-{{logstash_version}}
  sudo: True
  user: root
  tasks:
  - name: download openjdk-7-jdk
    apt: name=openjdk-7-jdk state=present update_cache=yes

  - name: download {{logstash_fullname}}
    get_url: >
      url=https://download.elasticsearch.org/logstash/logstash/{{logstash_fullname}}.tar.gz 
      dest={{logstash_dir}}/{{logstash_fullname}}.tar.gz 
      sha256sum=d5be171af8d4ca966a0c731fc34f5deeee9d7631319e3660d1df99e43c5f8069
      mode=0644

  - name: extract {{logstash_fullname}}
    unarchive: >
        copy=False 
        src={{logstash_dir}}/{{logstash_fullname}}.tar.gz 
        dest={{logstash_dir}}

  - name: update logstash.conf
    copy: src=base.conf dest={{logstash_dir}}/base.conf
    notify: restart logstash

  - name: update upstart-logstash.conf
    copy: src=upstart-logstash.conf dest=/etc/init/logstash.conf
    notify: restart logstash

  - name: set up logstash symlink/shellscript
    template: src=logstash.sh.j2 dest=/usr/local/bin/logstash mode=0755

  handlers:
  - name: restart logstash
    service: name=logstash state=restarted

upstart-logstash.conf

# logstash
# this should log to /var/log/upstart/logstash.log
# if not, configure with http://upstart.ubuntu.com/cookbook/#console-log

description     "logstash instance"

setuid logstash
setgid logstash

start on virtual-filesystems or runlevel [2345]
stop on runlevel [!2345]

# Respawn it if the process exits
respawn
respawn limit 5 30
limit nofile 65550 65550

env LS_HEAP_SIZE=1875m
env HOME=/home/logstash

exec /opt/logstash/bin/logstash -f /opt/logstash/base.conf

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

VAGRANTFILE_API_VERSION = "2"

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  config.vm.box = "hashicorp/precise64"
  config.vm.network :forwarded_port, guest: 9200, host: 9200
  config.vm.network :forwarded_port, guest: 9300, host: 9300
  config.vm.network :forwarded_port, guest: 9201, host: 9201

   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--memory", "256"]
     vb.customize ["modifyvm", :id, "--cpus", "1"]
   end
end