Using nxlog to rename fields to match logstash syslog input. Yes, I could have just sent the raw message to the logstash syslog input, but this taught me a lot about nxlog and some of this is not well documented.

logstash filter for nxlog json

filter {
  if [nxtags] == "nxlogsyslog" {

    mutate {
      add_field => [ "type", "%{nxtags}" ]
      replace => [ "host", "%{Hostname}" ]
    }

    # Parse the date 2014-02-27 14:57:04 - Needed to set timezone value to nxlog server's timezone
    date {
      locale => "en"
      timezone => "Etc/GMT"
      match => [ "EventTime", "YYYY-MM-dd HH:mm:ss" ]
    }

    mutate {
      remove_field => [ "Hostname", "EventTime" ]
    }
  }
}

logstash.inputs.conf

input {
  tcp {
    # have to use json_lines because sometimes nxlog puts two json messages in one tcp packet.
    codec => json_lines { charset => CP1252 }
    port => "7001"
    tags => [ "tcpjson" ]
  }
}

nxlog.conf

<Extension _syslog>
    Module      xm_syslog
</Extension>

<Extension json>
    Module      xm_json
</Extension>
   
<Input messages>
    Module      im_file
    File        "/var/log/messages"
   SavePos     TRUE
    Exec        parse_syslog_bsd();
# rewrite fields for to match naming of logstash syslog input
# just doing rename_field("Message","message"); didn't work
# also setting host, tags or type here didn't work - had to put in logstash filter
    Exec        $foo = $Message; delete($Message); rename_field("foo","message");
    Exec        $foo = lc($SyslogSeverity); delete($SyslogSeverity); rename_field("foo","syslog_severity");
    Exec        $foo = $SyslogSeverityValue; delete($SyslogSeverityValue); rename_field("foo","syslog_severity_code");
    Exec        $foo = lc($Severity); delete($Severity); rename_field("foo","severity_label");
    Exec        $foo = $SeverityValue; delete($SeverityValue); rename_field("foo","severity");
    Exec        $foo = lc($SyslogFacility); delete($SyslogFacility); rename_field("foo","syslog_facility");
    Exec        $foo = $SyslogFacilityValue; delete($SyslogFacilityValue); rename_field("foo","syslog_facility_code");
    Exec        $foo = $SourceName; delete($SourceName); rename_field("foo","sysloghost");
    Exec        $foo = $ProcessID; delete($ProcessID); rename_field("foo","pid");
    Exec        delete($EventReceivedTime);
    Exec        $nxtags = "nxlogsyslog";
# Lastly, convert to JSON
    Exec        to_json();
</Input>

# Outputs

<Output logstash_tcp>
  Module om_tcp
  Port 7001
  Host 10.1.1.1
</Output>

########################################
# Routes                               #
########################################

<Route to_logstash>
  Path messages => logstash_tcp
</Route>

Thanks for this gist, the 'json_lines' seems to have solved my encoding problems :)