Logstash 2.1.x config for log4net logs.

Logstash-log4net-README.md

<!- .... ->
<log4net>    
    <appender name="RollingLogFileAppenderLogstash" type="log4net.Appender.RollingFileAppender">
      <encoding value="utf-8" />
      <!--该目录必需有 IIS用户 写权限-->
      <file value="X:/var/log/[app_name]/logfile.log" />
      <appendToFile value="true" />
      <rollingStyle value="Date" />
      <lockingModel type="log4net.Appender.FileAppender+MinimalLock" />
      <datePattern value="yyyy-MM-dd" />
      <layout type="log4net.Layout.PatternLayout">
        <conversionPattern value="%level %date{ISO8601} %logger [%thread] [%C] [%property{requestId}] [%property{log4net:HostName}] %message %exception %newline" />
      </layout>
    </appender>
 
    <root>
      <appender-ref ref="RollingLogFileAppenderLogstash" />
    </root>
</log4net>
<!- .... ->

input { 
    file {
        path => "X:\var\log\[app_name]\logfile.log"
        type => "log4net"
        codec => multiline {
                    pattern => "^(DEBUG|WARN|ERROR|INFO|FATAL)"
                    negate => true
                    what => previous
        }
    }
}
filter {
   
  if [type] == "log4net" {
   
    grok {      
        match => [ "message", "(?m)%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:timestamp} %{DATA:logger} \[%{NUMBER:threadId}\] \[%{DATA:class}\] \[%{DATA:requestId}\] \[%{IPORHOST:tempHost}\] %{GREEDYDATA:tempMessage}" ]
        overwrite => ["message","timestamp"]
    }
     
    date {      
        match => ["timestamp","yyyy-MM-dd HH:mm:ss,SSS"]
        remove_field => ["timestamp"]
    }
    mutate {
        replace => [ "message" , "%{tempMessage}" ]
        replace => [ "host" , "%{tempHost}" ]
        remove_field => [ "tempMessage" ]
        remove_field => [ "tempHost" ]
    }
     
    grok {
        match => [ "message", "(?<message>[^\r\n]*)\r?(\n(?<exception>.*))?"]
        overwrite => ["message"]
    }
     
  }
     
}
 
output {
     
    elasticsearch { 
        hosts => [ "192.168.99.100:9200" ]
        index => "logstash1-%{+YYYY.MM.dd}"
        template_overwrite => true
    }
     
    stdout { codec => rubydebug }
}

• ref: http://www.ben-morris.com/using-logstash-elasticsearch-and-log4net-for-centralized-logging-in-windows/

logstash.conf

input {
    file {
        path => ".../Logs/*.log"
        type => "log4net"
        sincedb_path => ".../since.db"
        codec => multiline {
            pattern => "^%{TIMESTAMP_ISO8601} "
            negate => true
            what => previous
        }
        # start_position => "beginning" - does not work with the codec in logstash 2.1
    }
}

filter {
    grok {
        match => ["path", "(?<filename>[^/]+?)(-\[(?<processid>\d+)\])?\.log" ]
        add_field => [ "process", "%{host}:%{processid}" ]
    }

    if [type] == "log4net" {
        grok {
            match => ["message", "%{TIMESTAMP_ISO8601:timestamp} \[%{NUMBER:threadid}\] %{WORD:level}\s*%{DATA:class} %{DATA:NDC} - %{GREEDYDATA:message}"]
            overwrite => ["message","timestamp"]
        }

        date {
            match => ["timestamp","yyyy-MM-dd HH:mm:ss,SSS"]
            remove_field => ["timestamp"]
        }

        grok {
            match => [ "message", "(?<message>[^\r\n]*)\r?(\n(?<exception>.*))?"]
            overwrite => ["message"]
        }
    }

    kv {
        trim => "\r\(\)"
        trimkey => "\r\(\)"
        source => "message"
        # target => "props"
        value_split => ":" 
        allow_duplicate_values => false
    }

    mutate {
        convert => { 
            "processid" => "integer"
            "threadid" => "integer" 
            "Elapsed" => "integer"
            "ItemId" => "integer"
        }
    }
}

output {
    elasticsearch { 
        hosts => [ "127.0.0.1:9200" ]
        template_overwrite => true
    }

    stdout { codec => rubydebug }
}