gistfile1.txt

The config:

input {
  udp {
    port => 515
    type => esxi
  }
}

filter {
  if ("esxi" in [type]) {
    grok {
      match => { "message" => "(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGHOST:logsource}) (?:%{SYSLOGPROG}): (?<messagebody>(?:\[(?<esxi_thread_id>[0-9A-Z]{8,8}) %{DATA:esxi_loglevel}
 \'%{DATA:esxi_service}\'\] %{GREEDYDATA:esxi_message}|%{GREEDYDATA}))" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
  fingerprint {
    source => ["message"]
    target => "fingerprint"
    key => "123456"
    concatenate_sources => true
  }
}



output {
  elasticsearch { host => localhost cluster => tandyuklogs document_id => "%{fingerprint}" }
}




The log lines:

<166>2014-11-11T03:07:22.765Z esxi-1.tandyuk.com Rhttpproxy: [208C6B90 verbose 'Proxy Req 44421'] New proxy client SSL(TCP(local=109.169.6.71:443, peer=109.169.6.72:57398))
<166>2014-11-11T03:06:52.806Z esxi-1.tandyuk.com Rhttpproxy: [20885B90 verbose 'Proxy Req 44420'] The client closed the stream, not unexpectedly.
<166>2014-11-11T03:06:52.794Z esxi-1.tandyuk.com Hostd: [39009B90 error 'Default' opID=HB-host-10@53268-e07039d7-3e] Unable to parse MinRamPerCpu value:
<166>2014-11-11T03:06:32.796Z esxi-1.tandyuk.com Rhttpproxy: [20885B90 verbose 'Proxy Req 44419'] The client closed the stream, not unexpectedly.


and their raw outputs from kibana:

{
  "_index": "logstash-2014.11.11",
  "_type": "syslog",
  "_id": "Xc6nEFFeTtaHdXa9fl0JqQ",
  "_score": null,
  "_source": {
    "message": "<166>2014-11-11T03:07:22.765Z esxi-1.tandyuk.com Rhttpproxy: [208C6B90 verbose 'Proxy Req 44421'] New proxy client SSL(TCP(local=109.169.6.71:443, peer=109.169.6.72:57398))\n",
    "@version": "1",
    "@timestamp": "2014-11-11T03:06:46.532Z",
    "type": "syslog",
    "host": "2001:1b40:5000:22:0:0:0:71",
    "tags": [
      "_grokparsefailure",
      "esxi"
    ],
    "syslog_severity_code": 5,
    "syslog_facility_code": 1,
    "syslog_facility": "user-level",
    "syslog_severity": "notice",
    "timestamp8601": "2014-11-11T03:07:22.765Z",
    "logsource": "esxi-1.tandyuk.com",
    "program": "Rhttpproxy",
    "messagebody": "[208C6B90 verbose 'Proxy Req 44421'] New proxy client SSL(TCP(local=109.169.6.71:443, peer=109.169.6.72:57398))",
    "esxi_thread_id": "208C6B90",
    "esxi_loglevel": "verbose",
    "esxi_service": "Proxy Req 44421",
    "esxi_message": "New proxy client SSL(TCP(local=109.169.6.71:443, peer=109.169.6.72:57398))",
    "received_at": "2014-11-11 03:06:46 UTC",
    "received_from": "2001:1b40:5000:22:0:0:0:71"
  },
  "sort": [
    1415675206532,
    1415675206532
  ]
}

{
  "_index": "logstash-2014.11.11",
  "_type": "syslog",
  "_id": "5QZCqCzjQ4ygEmS01ZjRLw",
  "_score": null,
  "_source": {
    "message": "<166>2014-11-11T03:06:52.806Z esxi-1.tandyuk.com Rhttpproxy: [20885B90 verbose 'Proxy Req 44420'] The client closed the stream, not unexpectedly.\n",
    "@version": "1",
    "@timestamp": "2014-11-11T03:06:16.574Z",
    "type": "syslog",
    "host": "2001:1b40:5000:22:0:0:0:71",
    "tags": [
      "_grokparsefailure",
      "esxi"
    ],
    "syslog_severity_code": 5,
    "syslog_facility_code": 1,
    "syslog_facility": "user-level",
    "syslog_severity": "notice",
    "timestamp8601": "2014-11-11T03:06:52.806Z",
    "logsource": "esxi-1.tandyuk.com",
    "program": "Rhttpproxy",
    "messagebody": "[20885B90 verbose 'Proxy Req 44420'] The client closed the stream, not unexpectedly.",
    "esxi_thread_id": "20885B90",
    "esxi_loglevel": "verbose",
    "esxi_service": "Proxy Req 44420",
    "esxi_message": "The client closed the stream, not unexpectedly.",
    "received_at": "2014-11-11 03:06:16 UTC",
    "received_from": "2001:1b40:5000:22:0:0:0:71"
  },
  "sort": [
    1415675176574,
    1415675176574
  ]
}

{
  "_index": "logstash-2014.11.11",
  "_type": "syslog",
  "_id": "Pnk99xSpQ3m_WLbpZYcJQw",
  "_score": null,
  "_source": {
    "message": "<166>2014-11-11T03:06:52.794Z esxi-1.tandyuk.com Hostd: [39009B90 error 'Default' opID=HB-host-10@53268-e07039d7-3e] Unable to parse MinRamPerCpu value:\n",
    "@version": "1",
    "@timestamp": "2014-11-11T03:06:16.562Z",
    "type": "syslog",
    "host": "2001:1b40:5000:22:0:0:0:71",
    "tags": [
      "_grokparsefailure",
      "esxi"
    ],
    "syslog_severity_code": 5,
    "syslog_facility_code": 1,
    "syslog_facility": "user-level",
    "syslog_severity": "notice",
    "timestamp8601": "2014-11-11T03:06:52.794Z",
    "logsource": "esxi-1.tandyuk.com",
    "program": "Hostd",
    "messagebody": "[39009B90 error 'Default' opID=HB-host-10@53268-e07039d7-3e] Unable to parse MinRamPerCpu value:",
    "received_at": "2014-11-11 03:06:16 UTC",
    "received_from": "2001:1b40:5000:22:0:0:0:71"
  },
  "sort": [
    1415675176562,
    1415675176562
  ]
}

{
"_index": "logstash-2014.11.11",
"_type": "esxi",
"_id": "220a5aba33c436efc08904ddb502695088fd5972",
"_score": null,
"_source": {
"message": "<166>2014-11-11T12:16:29.463Z esxi-1.tandyuk.com Rhttpproxy: [FFEFE430 verbose 'Proxy Req 45351'] The client closed the stream, not unexpectedly.\n",
"@Version": "1",
"@timestamp": "2014-11-11T12:15:51.160Z",
"type": "esxi",
"host": "2001:1b40:5000:22:0:0:0:71",
"timestamp8601": "2014-11-11T12:16:29.463Z",
"logsource": "esxi-1.tandyuk.com",
"program": "Rhttpproxy",
"messagebody": "[FFEFE430 verbose 'Proxy Req 45351'] The client closed the stream, not unexpectedly.",
"esxi_thread_id": "FFEFE430",
"esxi_loglevel": "verbose",
"esxi_service": "Proxy Req 45351",
"esxi_message": "The client closed the stream, not unexpectedly.",
"received_at": "2014-11-11 12:15:51 UTC",
"received_from": "2001:1b40:5000:22:0:0:0:71",
"syslog_severity_code": 5,
"syslog_facility_code": 1,
"syslog_facility": "user-level",
"syslog_severity": "notice",
"fingerprint": "220a5aba33c436efc08904ddb502695088fd5972",
"tags": [
"_grokparsefailure"
]
},
"sort": [
1415708151160,
1415708151160
]
}