Logstash JSON filter

00-logstash.conf

input {
  file {
    type => syslog
    path => "/var/log/messages"
  }
}

filter {
  grok {
    type => syslog
    pattern => ["%{SYSLOGLINE}"]
    named_captures_only => true
  }

  grep {
    type => syslog
    match => ["program", "jsontest"]
    drop => false
    add_tag => json
  }

  json {
    type => syslog
    tags => json

    # parse JSON in "message" field,
    # put resulting structure in "data" field
    message => data
  }
}

output {
  stdout {
    debug => true
  }
}

01-non-json-filtered.out

{
         "@source" => "file://carrera.databits.net//var/log/messages",
           "@type" => "syslog",
           "@tags" => [],
         "@fields" => {
            "timestamp" => [
            [0] "Dec 31 00:53:11"
        ],
        "timestamp8601" => [],
             "facility" => [],
             "priority" => [],
            "logsource" => [
            [0] "carrera"
        ],
              "program" => [
            [0] "test"
        ],
                  "pid" => [],
              "message" => [
            [0] "{\"level\": \"info\", \"message\": \"foo bar\"}"
        ]
    },
      "@timestamp" => "2011-12-31T06:53:12.816000Z",
    "@source_host" => "carrera.databits.net",
    "@source_path" => "//var/log/messages",
        "@message" => "Dec 31 00:53:11 carrera test: {\"level\": \"info\", \"message\": \"foo bar\"}"
}

02-json-filtered.out

{
         "@source" => "file://carrera.databits.net//var/log/messages",
           "@type" => "syslog",
           "@tags" => [
        [0] "json"
    ],
         "@fields" => {
            "timestamp" => [
            [0] "Dec 31 00:53:04"
        ],
        "timestamp8601" => [],
             "facility" => [],
             "priority" => [],
            "logsource" => [
            [0] "carrera"
        ],
              "program" => [
            [0] "jsontest"
        ],
                  "pid" => [],
              "message" => [
            [0] "{\"level\": \"info\", \"message\": \"foo bar\"}"
        ],
                 "data" => {
              "level" => "info",
            "message" => "foo bar"
        }
    },
      "@timestamp" => "2011-12-31T06:53:04.807000Z",
    "@source_host" => "carrera.databits.net",
    "@source_path" => "//var/log/messages",
        "@message" => "Dec 31 00:53:04 carrera jsontest: {\"level\": \"info\", \"message\": \"foo bar\"}"
}