Using split and multiline to handle weird cases of munged multiline logs

logstash-split-multiline.conf

input {
    # Generate some test trace events -- Note: multiline requires that a second event MUST start before the previous one is pushed though the pipe.
    generator {
        type => "trace"
        lines => [
"First line blah1",
"  subsequent line 2",
"  subsequent line 3",
"First line blah2
  subsequent line 2
  subsequent line 3",
"First line blah3\n  subsequent line 2\n  subsequent line 3",
"First line blah4 to flush"
         ]
        count => 1
    }
}

filter {
  if [type] == "trace" {
    split { }
    multiline {
       pattern => "^First line"
       what => "previous"
       negate => true
    }
  }

}

output {
    stdout { codec => rubydebug }
}

Results

]$ /opt/logstash/bin/logstash -f logstash-split-multiline.conf
Using milestone 2 filter plugin 'split'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones {:level=>:warn}
{
       "message" => "First line blah2",
      "@version" => "1",
    "@timestamp" => "2014-10-31T17:55:38.237Z",
          "type" => "trace",
          "host" => "LRC1-RNC1-1-LMU-0",
      "sequence" => 0
}
{
       "message" => "  subsequent line 2",
      "@version" => "1",
    "@timestamp" => "2014-10-31T17:55:38.237Z",
          "type" => "trace",
          "host" => "LRC1-RNC1-1-LMU-0",
      "sequence" => 0
}
{
       "message" => "  subsequent line 3",
      "@version" => "1",
    "@timestamp" => "2014-10-31T17:55:38.237Z",
          "type" => "trace",
          "host" => "LRC1-RNC1-1-LMU-0",
      "sequence" => 0
}
{
       "message" => "First line blah1\n  subsequent line 2\n  subsequent line 3",
      "@version" => "1",
    "@timestamp" => "2014-10-31T17:55:38.234Z",
          "type" => "trace",
          "host" => "LRC1-RNC1-1-LMU-0",
      "sequence" => 0,
          "tags" => [
        [0] "multiline"
    ]
}
{
       "message" => "First line blah3\\n  subsequent line 2\\n  subsequent line 3",
      "@version" => "1",
    "@timestamp" => "2014-10-31T17:55:38.237Z",
          "type" => "trace",
          "host" => "LRC1-RNC1-1-LMU-0",
      "sequence" => 0
}