Logstash configuration (playing around)

__etc__logstash__conf.d__local.conf

#
# Resources:
# - Logstash manual at http://logstash.net/
# - Grok Pattern Debugger at http://grokdebug.herokuapp.com/
# - Other patterns at /opt/logstash/pattern/
#

input {

    syslog {
        type => syslog
        host => "localhost"
        port => 5000
    }

    file {
        type => "apache_access"
        exclude => ["*.gz","*.zip","*.tgz"]
        path => [ "/var/log/apache2/access.log" ]
        sincedb_path => "/dev/null"
    }

    file {
        type => "apache_error"
        exclude => ["*.gz","*.zip","*.tgz"]
        path => [ "/var/log/rsyslog2/error.log" ]
        sincedb_path => "/dev/null"
    }

    file {
        type => "dpkg_log"
        path => [ "/var/log/dpkg.log" ]
        sincedb_path => "/dev/null"
    }
}

filter {
    if [type] == "syslog" {
        if [program] =~ /^postfix\// {
            # Using postfix.conf from https://gist.github.com/poolski/9911628

            # Parse generic Postfix Syslog line
            grok {
                patterns_dir => ["/etc/logstash/patterns"]
                match => [ "program", "^%{COMPID}" ]
                add_field => [ "_parsed", "postfix_compid" ]
            }

            # Parse Postfix components using hand crafted patterns
            grok {
                patterns_dir => ["/etc/logstash/patterns"]
                match => { "message" => "^%{PF}" }
                add_field => [ "_parsed", "postfix_pf" ]
            }
            # Parse Postfix using default Key-Value match
            if ! ("postfix_pf" in [_parsed]) {
                grok {
                    patterns_dir => ["/etc/logstash/patterns"]
                    match => { "message" => "^%{QUEUEID:qid}:" }
                    add_field => [ "_parsed", "postfix_qid" ]
                }
                kv {
                    source => "message"
                    trim => "<>\[\],"
                    add_field => [ "_parsed", "postfix_kv" ]
                }
            }
        }

    } else if [type] == "apache_access" {
        grok {
            match => { "message" => "%{COMBINEDAPACHELOG}" }
        }
        date {
            match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
        }

    } else if [type] == "apache_error" {
        grok {
            match => { "message" => "%{COMBINEDAPACHELOG}" }
        }
        date {
            match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
        }
    } else if [type] == "dpkg_log" {
        grok {
            patterns_dir => ["/etc/logstash/patterns"]
            match => { "message" => "^%{DPKG_LOG}$" }
            add_field => [ "_parsed", "dpkg_log" ]
        }
    }
}

output {
    stdout {
        codec => "rubydebug"
    }
    elasticsearch {
        embedded => true
    }
}


# vim: syntax=logstash ts=4 sw=4 expandtab