LogStash Configuration

logstash.conf

input {
    tcp {
        type => "syslog"
        port => 8514
    }
}

filter {

## DISCARD IMPROPERLY FORMATTED MESSAGES
    grep {
        match => [ "@message", "^\<[0-9]+\>[A-Za-z]{3}\s+[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} \S+ \S+:" ]
        add_tag => "syslog"
    }

## DECODE SYSLOG MESSAGE
    grok {
        tags => "syslog"
        pattern => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST:logsource} %{GREEDYDATA:message_remainder}"
    }
    grok {
        tags => "syslog"
        pattern => "%{SYSLOGTIMESTAMP} %{HOSTSHORT:hostname}"
        patterns_dir => "/etc/logstash/pattern.d"
    }
    mutate {
        tags => "syslog"
        replace => [ "@message", "%{syslog_pri}"]
    }
    ## Syslog Priority Parser Plugin
    #syslog_pri {
    #    tags => "syslog"
    #}
    ## Restore @message
    mutate {
        tags => "syslog"
        replace => [ "@message", "%{message_remainder}" ]
        rename => [ "syslog_facility", "facility", "syslog_severity", "severity" ]
    }
    mutate {
        tags => "syslog"
        remove => [ "message_remainder", "syslog_facility_code", "syslog_severity_code", "syslog_pri" ]
    }
## DETERMINE PATH ##
    grok {
        tags => "syslog"
        pattern => "^%{SYSLOGPROG}: %{GREEDYDATA:message_remainder}"
        add_tag => "%{program}"
    }
    mutate {
        tags => "syslog"
        replace => [ "@message", "%{message_remainder}" ]
    }
## OSSEC
    grok {
        tags => "ossec"
        patterns_dir => "/etc/logstash/pattern.d"
        pattern => "\"component\": \"\(%{HOSTSHORT:src}"
    }
    mutate {
        tags => "ossec"
        add_field => [ "json_data", "%{message_remainder}" ]
        add_tag => [ "json", "pull_message" ]
    }
## APACHE
    grok {
        tags => "apache"
        patterns_dir => "/etc/logstash/pattern.d"
        pattern => "%{HOST:vhost} %{BOOKINGAPACHELOG}"
    }
## nginx
    grok {
        tags => "nginx"
        pattern => "%{HOST:vhost} %{GREEDYDATA:json_data}"
    }
    mutate {
        tags => [ "nginx_parsed" ]
        add_tag => "json"
    }
## GLOBAL JSON Parser
    json {
        tags => "json"
        json_data => "cee"
        add_tag => "parsed"
    }
## GLOBAL CLEANUP
    # Remove the message_remainder
    mutate {
        remove => [ "message_remainder" ]
    }
    # Remove the json_data field only if the parse was successful
    mutate {
        tags => [ "json", "parsed" ]
        remove => [ "json_data" ]
    }
    # Pull message from the message field
    mutate {
        tags => "pull_message"
        rename => [ "message", "@message" ]
    }
}

output {
    #stdout { debug => true }
    elasticsearch {
        embedded => false
        cluster  => "log-ng"
        host     => "logsearch-cluster.example.com"
    }
}