A configuration for a logstash agent. This logstash agent collects logs from apache access, apache error, and syslog via UDP.

logstash-shipper

input {
   tcp {
        port => 5000
        type => "syslog"
        host => "127.0.0.1"
    }
    udp {
        port => 5000
        type => "syslog"
        host => "127.0.0.1"
    }
    udp {
        port => 5001
        host => "127.0.0.1"
        type => "apache"
        buffer_size => 8192
        format => "json_event"
    }
    udp {
        port => 5002
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "_SERVER_NAME_" ]
        format => "plain"
    }
    udp {
        port => 5003
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "_SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5004
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "logging._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5010
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "ajenti._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5011
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "graphite._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5012
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "graylog._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5013
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "logstash._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5014
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "rabbit._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5015
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "sensu._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
    udp {
        port => 5016
        host => "127.0.0.1"
        type => "apache-error"
        tags => [ "Apache", "Error", "supervisor._SERVER_NAME_._TLD_" ]
        format => "plain"
    }
}
filter {

    # SYSLOG PROCESSING

    grok {
        type => "syslog"
        pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ]
        add_field => [ "received_at", "%{@timestamp}" ]
        add_field => [ "received_from", "%{@source_host}" ]
    }
    syslog_pri {
        type => "syslog"
    }
    date {
        type => "syslog"
        match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
    mutate {
        type => "syslog"
        exclude_tags => "_grokparsefailure"
        replace => [ "@source_host", "%{syslog_hostname}" ]
        replace => [ "@message", "%{syslog_message}" ]
    }
    mutate {
        type => "syslog"
        remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
    }

    # APACHE ACCESS LOG PROCESSING

    date {
         type => "apache"
         timestamp => "ISO8601"
    }
    mutate {
          type   => "apache"
          remove => [ "timestamp" ]
    }

    # APACHE ERROR LOG PROCESSING

    grok {
        type => "apache-error"
        pattern => [ "%{GENERICAPACHEERROR}" ]
        patterns_dir => "/usr/local/logstash/conf/grok_patterns"
    }
    date {
        type => "apache-error"
        timestamp => "EEE MMM dd HH:mm:ss yyyy"
    }

    mutate {
        type => "apache-error"
        replace => [ "severity", "%{apacheseverity}" ]
        remove => [ "apacheseverity", "timestamp" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "ajenti._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "ajenti._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "graphite._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "graphite._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "graylog._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "graylog._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "logstash._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "logstash._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "rabbit._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "rabbit._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "sensu._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "sensu._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "supervisor._SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "supervisor._SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "logging._TLD_" ]
        replace => [ "@source_host", "logging._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "_SERVER_NAME_._TLD_" ]
        replace => [ "@source_host", "_SERVER_NAME_._TLD_" ]
    }

    mutate {
        type => "apache-error"
        tags => [ "Apache", "Error", "_SERVER_NAME_" ]
        replace => [ "@source_host", "_SERVER_NAME_" ]
    }

}

output {
    redis {
        host => "###REDIS_IP###"
        data_type => "list"
        key => "logstash"
    }
}