gistfile1.txt

input {
        file {
                path => "/var/log/nginx/*access.log"
                type => "nginx"
                sincedb_path => "/var/log/.sincedb"
        }
}
input {
        udp {
                type => "syslog"
                port => "514"
        }
}
filter {
        if [type] == "syslog" {
                dns {
                        reverse => [ "host" ] action => "replace"
                }
                mutate {
                        add_tag => [ "syslog-UDP" ]
                }
                if [host] =~ /.*?(nsvpx).*?(everythingshouldbevirtual.local)?/ {
                        mutate {
                                add_tag => [ "Netscaler", "Ready" ]
                        }
                }
                if [host] =~ /.*?(pfsense).*?(everythingshouldbevirtual.local)?/ {
                        mutate {
                                add_tag => [ "PFSense", "Ready" ]
                        }
                }
                if "Ready" not in [tags] {
                        mutate {
                                add_tag => [ "syslog" ]
                        }
                }
        }
}
filter {
        if [type] == "syslog" {
                mutate {
                        remove_tag => "Ready"
                }
        }
}
filter {
        if "syslog" in [tags] {
                grok {
                        match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
                        add_field => [ "received_at", "%{@timestamp}" ]
                        add_field => [ "received_from", "%{host}" ]
                }
                syslog_pri { }
                date {
                        match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
                }
                if !("_grokparsefailure" in [tags]) {
                        mutate {
                                replace => [ "@source_host", "%{syslog_hostname}" ]
                                replace => [ "@message", "%{syslog_message}" ]
                        }
                }
                mutate {
                        remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
                }
                if "_grokparsefailure" in [tags] {
                        drop { }
                }
        }
}
filter {
        if "syslog" in [tags] {
                if [syslog_program] == "haproxy" {
                        grok {
                                break_on_match => false
                                match => [
                                        "message", "%{HAPROXYHTTP}",
                                        "message", "%{HAPROXYTCP}"
                                ]
                                add_tag => [ "HAProxy" ]
                        }
                        geoip {
                                source => "client_ip"
                                target => "geoip"
                                add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                                add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
                        }
                        mutate {
                                convert => [ "[geoip][coordinates]", "float" ]
                        }
                        mutate {
                                replace => [ "host", "%{@source_host}" ]
                        }
                        mutate {
                                add_field => [ "bytes_read_int", "%{bytes_read}" ]
                                convert => [ "bytes_read_int", "integer" ]
                        }
                }
        }
}
filter {
        if "Netscaler" in [tags] {
                grok {
                        break_on_match => true
                        match => [
                                "message", "<%{POSINT:syslog_pri}> %{DATE_US}:%{TIME} GMT %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:netscaler_message} : %{DATA} %{INT:netscaler_spcbid} - %{DATA} %{IP:netscaler_client_ip} - %{DATA} %{INT:netscaler_client_port} - %{DATA} %{IP:netscaler_vserver_ip} - %{DATA} %{INT:netscaler_vserver_port} %{GREEDYDATA:netscaler_message} - %{DATA} %{WORD:netscaler_session_type}",
                                "message", "<%{POSINT:syslog_pri}> %{DATE_US}:%{TIME} GMT %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:netscaler_message}"
                        ]
                }
                syslog_pri { }
                mutate {
                        replace => [ "@source_host", "%{host}" ]
                }
                mutate {
                        replace => [ "@message", "%{netscaler_message}" ]
                }
                geoip {
                        source => "netscaler_client_ip"
                        target => "geoip"
                        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
                }
                mutate {
                        convert => [ "[geoip][coordinates]", "float" ]
                }
        }
}
filter {
    if "PFSense" in [tags] {
        grok {
            add_tag => [ "firewall" ]
            match => [ "message", "<(?<evtid>.*)>(?<datetime>(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:[0-5][0-9])) (?<prog>.*?): (?<msg>.*)" ]
        }
        mutate {
            gsub => ["datetime","  "," "]
        }
        date {
            match => [ "datetime", "MMM dd HH:mm:ss" ]
        }
        mutate {
            replace => [ "message", "%{msg}" ]
        }
        mutate {
            remove_field => [ "msg", "datetime" ]
        }
    }
    if [prog] =~ /^pf$/ {
        mutate {
            add_tag => [ "packetfilter" ]
        }
        multiline {
            pattern => "^\s+|^\t\s+"
            what => "previous"
        }
        mutate {
            remove_field => [ "msg", "datetime" ]
            remove_tag => [ "multiline" ]
        }
        grok {
            match => [ "message", "rule (?<rule>.*)\(.*\): (?<action>pass|block) .* on (?<iface>.*): .* proto (?<proto>TCP|UDP|IGMP|ICMP) .*\n\s*(?<src_ip>(\d+\.\d+\.\d+\.\d+))\.?(?<src_port>(\d*)) [<|>] (?<dest_ip>(\d+\.\d+\.\d+\.\d+))\.?(?<dest_port>(\d*)):" ]
        }
    }
    if [prog] =~ /^dhcpd$/ {
        if [message] =~ /^DHCPACK|^DHCPREQUEST|^DHCPOFFER/ {
            grok {
                match => [ "message", "(?<action>.*) (on|for|to) (?<src_ip>[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]) .*(?<mac_address>[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]).* via (?<iface>.*)" ]
            }
        }
        if [message] =~ /^DHCPDISCOVER/ {
            grok {
                match => [ "message", "(?<action>.*) from (?<mac_address>[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]).* via (?<iface>.*)" ]
            }
        }
        if [message] =~ /^DHCPINFORM/ {
            grok {
                match => [ "message", "(?<action>.*) from (?<src_ip>.*).* via (?<iface>.*)" ]
            }
        }
   }
   if "_grokparsefailure" in [tags] {
        drop { }
   }

}
filter {
        if "PFSense" in [tags] {
                mutate {
                        replace => [ "@source_host", "%{host}" ]
                }
                mutate {
                        replace => [ "@message", "%{message}" ]
                }
        }
}
filter {
        if [type] == "nginx" {
                grok {
                        pattern => "%{COMBINEDAPACHELOG}"
                }
        }
}
output {
        if [type] != "nginx" {
                elasticsearch {
                        cluster => "logstash-cluster"
                        flush_size => 1
                        manage_template => true
                        template => "/opt/logstash/lib/logstash/outputs/elasticsearch/elasticsearch-template.json"
        }       }
}
output {
        if [type] == "nginx" {
                redis {
                        host => "logstash"
                        data_type => "list"
                        key => "logstash"
                }
        }
}
output {
        if "HAProxy" in [tags] {
                statsd {
                        host => "graphite"
                        port => 8125
                        gauge => [ "haproxy.bytes.read", "%{bytes_read}" ]
                        increment => "haproxy.http.status.code.%{http_status_code}"
                        timing => [ "haproxy.time.backend.connect", "%{time_backend_connect}" ]
                        timing => [ "haproxy.time.backend.response", "%{time_backend_response}" ]
                        timing => [ "haproxy.time.duration", "%{time_duration}" ]
                        timing => [ "haproxy.time.queue", "%{time_queue}" ]
                }
        }
}