CollectD to ELK

README.md

This is an example of using ELK to parse and view collectd data.

Caveat - I haven't fully tested this mapping yet, it doesn't take into account any other fields that may be added with other collectd plugins, just the ones I have specified below.

collectd.conf

Hostname "YOUR_HOST_HERE"
FQDNLookup false
LoadPlugin syslog
LoadPlugin cpu
LoadPlugin df
LoadPlugin entropy
LoadPlugin interface
LoadPlugin java
LoadPlugin load
LoadPlugin memory
LoadPlugin network
LoadPlugin processes
LoadPlugin swap
LoadPlugin uptime
LoadPlugin users

<Plugin df>
	Disk "xvda"
        MountPoint "/"
        FSType "ext4"
        ReportReserved "true"
</Plugin>
<Plugin interface>
	Interface "eth0"
</Plugin>
<Plugin network>
	Server "LS_HOST_HERE" "PORT_HERE"
</Plugin>
<Plugin syslog>
	LogLevel info
</Plugin>
<Include "/etc/collectd/collectd.conf.d">
	Filter "*.conf"
</Include>

es_mapping_template.json

PUT _template/collectd
{
  "order": 0,
  "template": "collectd-*",
  "settings": {
    "index.refresh_interval": "30s",
    "index.number_of_replicas": "0",
    "index.number_of_shards": "1"
  },
  "mappings": {
    "_default_": {
      "properties": {
        "@timestamp": {
          "type": "date",
          "format": "dateOptionalTime"
        },
        "@version": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "collectd_type": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "host": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "longterm": {
          "type": "double",
          "index": "not_analyzed",
          "doc_values": true
        },
        "midterm": {
          "type": "double",
          "index": "not_analyzed",
          "doc_values": true
        },
        "plugin": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "plugin_instance": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "rx": {
          "type": "long",
          "index": "not_analyzed",
          "doc_values": true
        },
        "shortterm": {
          "type": "double",
          "index": "not_analyzed",
          "doc_values": true
        },
        "tx": {
          "type": "long",
          "index": "not_analyzed",
          "doc_values": true
        },
        "type": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "type_instance": {
          "type": "string",
          "index": "not_analyzed",
          "doc_values": true
        },
        "value": {
          "type": "double",
          "index": "not_analyzed",
          "doc_values": true
        }
      }
    }
  }
}

logstash.conf

input {
  udp {
    port => PORT_HERE
    buffer_size => 1452
    codec => collectd { }
    type => "collectd"
  }
}

output {
  elasticsearch {
      host => "ES_HOST_HERE"
      protocol => "http"
    }
}