indexer.conf

input {
  courier {
    port => 6379
    ssl_certificate => "/opt/logstash/ssl/logstash-forwarder.crt"
    ssl_key => "/opt/logstash/ssl/logstash-forwarder.key"
  }
}

filter {

#Dropping all headings
        if [message] =~ "^#" {drop {}}

#Grok parsing - Using custom patterns to account for null values.
#Also - We may change all date parsing to use ISO8601
        #ELMAH
        if [type] == "ELMAHLog" {
                 multiline {
                        pattern => "^%{TIMESTAMP_ISO8601}"
                        what => "previous"
                        negate => true
                 }
                 grok {
                        match => ["message", "(?m)(%{TIMESTAMP_ISO8601:log_times                                                                                                                                   tamp})\t(?<Elmah_Error_Id>(.*?))\t(?<Application>(.*?))\t(?<Host>(.*?))\t(?<Type                                                                                                                                   >(.*?))\t(?<Source>(.*?))\t(?<Elmah_Error_Message>(.*?))\t(?<User>(.*?))\t(?<HTT                                                                                                                                   P_Status_Code>(.*?))\t(?<Sequence>(.*?))\t(?<Ignored>(.*?))\t(%{GREEDYDATA:ELMAH                                                                                                                                   _XML})"]
                        add_tag => "ELMAH"
                 }

                        date {
                match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss.SSS" ]
                timezone => "Etc/UCT"
                        }
                        mutate {
                        replace => ["host", "%{Host}"]
                        remove_field => ["Host"]
                        replace => ["message", "%{ELMAH_XML}"]
                        remove_field => ["ELMAH_XML"]
                        remove_tag => ["multiline"]
                        remove_field => ["Ignored"]
                        }
         }

        #HTTPErr
        if [type] == "HTTPLog" {
                 grok {
                        match => { "message" => "%{TIMESTAMP_ISO8601:log_timesta                                                                                                                                   mp} %{IP:Client_IP} %{NUMBER:Client_Port} %{IP:Server_IP} %{NUMBER:Server_Port}                                                                                                                                    %{NOTSPACE:Client_Version} %{NOTSPACE:HTTP_Request_Method} %{NOTSPACE:HTTPERR_Cl                                                                                                                                   ient_URI} %{NOTSPACE:HTTP_Status_Code} %{NOTSPACE:Server_Site_Id} %{NOTSPACE:HTT                                                                                                                                   P_Error_Message} (%{NOTSPACE:Server_Queue_Name})"}
                        add_tag => "HTTP" }
                 if [HTTP_Error_Message] == "Timer_ConnectionIdle" {drop {}}
                                 date {
                match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
                timezone => "Etc/UCT"
                        }
        }

        #IIS
                if [type] == "IISLog" {
                grok {
                      match => [
                        "message","(%{TIMESTAMP_ISO8601:log_timestamp})\s(?<Serv                                                                                                                                   er_Sitename>(.*?))\s(?<Server_IP>(.*?))\s(?<HTTP_Request_Method>(.*?))\s(?<IIS_C                                                                                                                                   lient_Query>(.*?))\s(?<Client_Stem>(.*?))\s(?<Server_Port>(.*?))\s(?<Client_ID>(                                                                                                                                   .*?))\s(?<Client_IP>(.*?))\s(?<User_Agent>(.*?))\s(?<Referer>(.*?))\s(?<HTTP_Sta                                                                                                                                   tus_Code>(?:[4-5_.-]+[0-9_.-]+[0-9_.-]))\s(?<Server_Substatus>(.*?))\s(?<Server_                                                                                                                                   Win32_Status>(.*?))\s(%{GREEDYDATA:Request_Time_Elapsed})",
                        "message","(%{TIMESTAMP_ISO8601:log_timestamp})\s(?<Serv                                                                                                                                   er_IP>(.*?))\s(?<HTTP_Request_Method>(.*?))\s(?<IIS_Client_Query>(.*?))\s(?<Clie                                                                                                                                   nt_Stem>(.*?))\s(?<Server_Port>(.*?))\s(?<Client_ID>(.*?))\s(?<Client_IP>(.*?))\                                                                                                                                   s(?<User_Agent>(.*?))\s(?<HTTP_Status_Code>(?:[4-5_.-]+[0-9_.-]+[0-9_.-]))\s(?<S                                                                                                                                   erver_Substatus>(.*?))\s(?<Server_Win32_Status>(.*?))\s(%{GREEDYDATA:Request_Tim                                                                                                                                   e_Elapsed})"
                                ]

                      add_tag => "IIS"}
                                date {
                match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
                timezone => "Etc/UCT"
                        }
        }

        #Matching dates


    #Removing timestamp field (covered by @timestamp date filter), message field                                                                                                                                    (covered by allXML), and multiline tag.
    mutate {
                remove_field => ["log_timestamp"]
                remove_field => ["offset"]
    }
}
#When the service starts, disregard any log files older than 7 days.
        ruby {
                code => "event.cancel if event.timestamp < (Time.now - (86400 * 2))"
        }

output {
        #Only exporting files that parsed properly.
        if "_grokparsefailure" not in [tags] {

                elasticsearch {
                   host => "localhost"
                }
}
}