logstash

custompatterns

POSTFIXHOST (%{HOST:remotehostname}\[%{IP:remoteip}\])
POSTFIXQID (?:<postfix_queue_id>[0-9A-F]{11}|NOQUEUE):
RSYSLOGRELPMESSAGE (%{SYSLOG5424PRI}%{TIMESTAMP_ISO8601:syslog5424_ts} %{SYSLOGHOST:loghost} %{SYSLOGPROG}: %{GREEDYDATA:message})

logstash.conf

input {
    relp {
        port => 5142
        type => 'rsyslogrelp'
        debug => true
    }
}

filter {
    if [type] == "rsyslogrelp" {
        syslog_pri { }
        grok {
            patterns_dir => "/etc/logstash/patterns"
            match   => [ 'message', '%{RSYSLOGRELPMESSAGE}' ]
            overwrite => [ "message" ]
        }
    }

    # silence spammy snmpd
    if [program] == 'snmpd' and [message] =~ '^error on subcontainer' {
        drop {}
    }

    # fix non fqdns
    if [loghost] == "nue1d0" or [loghost] == "nue1" {
       mutate { replace => [ "loghost", "%{loghost}.datavibe.net" ] }
    }
    if [loghost] == "eeqj" {
       mutate { replace => [ "loghost", "%{loghost}.com" ] }
    }
    if [loghost] == "admin" {
       mutate { replace => [ "loghost", "%{loghost}.eeqj.de" ] }
    }

    if [program] == 'sshd' {
        grok {
            patterns_dir => "/etc/logstash/patterns"
            match => [ 'message', 'from %{IP:remoteip} port' ]
            tag_on_failure => []
        }
    }

    if [program] =~ 'dovecot' {
        grok {
            patterns_dir => "/etc/logstash/patterns"
            match => [ 'message', 'rip=%{IP:remoteip}' ]
            tag_on_failure => []
        }
    }
    if [program] =~ '^postfix\/' {
        grok {
            patterns_dir => "/etc/logstash/patterns"
            match => [ 'message', '%{POSTFIXQID} %{GREEDYDATA:message}' ]
            overwrite => [ "message" ]
            tag_on_failure => []
        }
    }

    if [program] == 'postfix/smtpd' {
        grok {
            patterns_dir => "/etc/logstash/patterns"
            match => ['message', '(?<postfix_event>connect|disconnect) from %{POSTFIXHOST}' ]
            add_field => [ 'event', 'postfix_smtpd_%{postfix_event}' ]
            remove_field => 'postfix_event'
            tag_on_failure => []
        }
        grok {
            patterns_dir => "/etc/logstash/patterns"
            match => ['message', '^reject: %{WORD} from %{POSTFIXHOST}:' ]
            tag_on_failure => []
        }
 
    }

    if [remoteip] != '' {
        geoip {
                source => 'remoteip'
        }
    }
}

output {
    stdout {
        codec => rubydebug 
    }
    file {
        codec           => json
        flush_interval  => 0
        path => "/opt/logstash/logarchive/%{+YYYYMM}/%{+YYYYMMdd}.%{loghost}.json"
    }
}