Quick notes on using Fluentd

22-fluentd.conf

# rsyslog conf referenced in tasks.yml
*.* @127.0.0.1:42185

general_info.md

• Fluentd quickstart: http://docs.fluentd.org/articles/quickstart
• Some guides: http://www.fluentd.org/guides

kibana-dashboard-for-fluentd.json

{ "index" : { "_index" : ".kibana", "_type" : "index-pattern", "_id" : "syslog-*" } }
{"title":"syslog-*","timeFieldName":"@timestamp","customFormats":"{}","fields":"[{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"host\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_source\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_index\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"@version\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"message\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"name\":\"_type\",\"count\":0,\"scripted\":false},{\"type\":\"date\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"@timestamp\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"ident\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_id\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"host.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"ident.raw\",\"count\":0,\"scripted\":false},{\"type\":\"geo_point\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"geoip.location\",\"count\":0,\"scripted\":false}]"}
{ "index" : { "_index" : ".kibana", "_type" : "index-pattern", "_id" : "http-access*" } }
{"title":"http-access*","timeFieldName":"@timestamp","customFormats":"{}","fields":"[{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"agent.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_source\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"code.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"remote\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"user.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"@version\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"method.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"name\":\"_type\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_id\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"host.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"path\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"size.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"referer.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"host\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_index\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"code\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"path.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"agent\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"size\",\"count\":0,\"scripted\":false},{\"type\":\"date\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"@timestamp\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"method\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"referer\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"remote.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"user\",\"count\":0,\"scripted\":false},{\"type\":\"geo_point\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"geoip.location\",\"count\":0,\"scripted\":false}]"}
{ "index" : { "_index" : ".kibana", "_type" : "index-pattern", "_id" : "http-error*" } }
{"title":"http-error*","timeFieldName":"@timestamp","customFormats":"{}","fields":"[{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_source\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"log_level.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_index\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"pid\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"@version\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"message\",\"count\":0,\"scripted\":false},{\"type\":\"date\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"@timestamp\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"name\":\"_type\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"pid.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"tid.raw\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":false,\"analyzed\":false,\"name\":\"_id\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"log_level\",\"count\":0,\"scripted\":false},{\"type\":\"string\",\"indexed\":true,\"analyzed\":true,\"doc_values\":false,\"name\":\"tid\",\"count\":0,\"scripted\":false},{\"type\":\"geo_point\",\"indexed\":true,\"analyzed\":false,\"doc_values\":false,\"name\":\"geoip.location\",\"count\":0,\"scripted\":false}]"}
{ "index" : { "_index" : ".kibana", "_type" : "visualization", "_id" : "HTTP-Access" } }
{"title":"HTTP Access","visState":"{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"mode\":\"stacked\",\"defaultYExtents\":false,\"spyPerPage\":10},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"code.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"http-access*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}
{ "index" : { "_index" : ".kibana", "_type" : "visualization", "_id" : "HTTP-Errors" } }
{"title":"HTTP Errors","visState":"{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"mode\":\"stacked\",\"defaultYExtents\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"log_level.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"http-error*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}
{ "index" : { "_index" : ".kibana", "_type" : "visualization", "_id" : "HTTP-Remotes" } }
{"title":"HTTP Remotes","visState":"{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"mode\":\"stacked\",\"defaultYExtents\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"significant_terms\",\"schema\":\"group\",\"params\":{\"field\":\"remote.raw\",\"size\":10}}],\"listeners\":{}}","description":"","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"http-access*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"}}
{ "index" : { "_index" : ".kibana", "_type" : "dashboard", "_id" : "Default-Dashboard" } }
{"title":"Default Dashboard","hits":0,"description":"","panelsJSON":"[{\"id\":\"HTTP-Access\",\"type\":\"visualization\",\"size_x\":12,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"HTTP-Errors\",\"type\":\"visualization\",\"size_x\":12,\"size_y\":3,\"col\":1,\"row\":5},{\"id\":\"HTTP-Remotes\",\"type\":\"visualization\",\"size_x\":12,\"size_y\":3,\"col\":1,\"row\":8}]","version":1,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}"}}

tasks.yml

---
# Getting fluentd installed on a ubuntu (trusty) server
#- name: Dump all vars for testing
#  template:
#    src: dumpall.j2
#    dest: '/tmp/ansible.json'

##
# Fluentd Install
##
- name: Install Fluentd Dependencies
  apt:
    pkg: '{{ item }}'
    state: installed
  with_items:
   - apt-transport-https
   - build-essential
   - libcurl4-gnutls-dev

- name: Add Fluentd Key
  apt_key:
    url: 'http://packages.treasuredata.com/GPG-KEY-td-agent'
    state: present

- name: Add Fluentd Repository
  apt_repository:
    repo: 'deb [arch=amd64] http://packages.treasuredata.com/2/ubuntu/trusty/ trusty contrib'
    state: present

- name: Install Fluentd
  apt:
    pkg: td-agent
    state: installed
    update_cache: yes

- name: Add td-agent user to group adm
  user:
    name: td-agent
    groups: 'adm'

##
# Fluentd Conf
##
- name: Add Fluentd Conf
  template:
    src: td-agent.conf.j2
    dest: /etc/td-agent/td-agent.conf
    owner: root
    group: root

- name: Install TD Agent Plugins
  gem: >
      name={{ item }}
      executable=/usr/sbin/td-agent-gem
      state=latest
      user_install=no
  with_items:
    - fluent-plugin-elasticsearch
  notify:
    - Restart Fluentd

- name: Restart Fluentd
  service:
    name: td-agent
    state: restarted

##
# Rsyslog
##
- name: Add Syslog Conf
  copy:
    src: 22-fluentd.conf
    dest: /etc/rsyslog.d/22-fluentd.conf
    owner: root
    group: root

- name: Restart Rsyslog
  service:
    name: rsyslog
    state: restarted

td-agent.conf

# I believe this is a working td-agent.conf file
# Mine was generated from templates and some structured yaml data


##
# Nginx
##
<source>
  type tail
  path /var/log/nginx/access.log,/var/log/nginx/*access.log
  pos_file /var/log/td-agent/http-access.log.pos
  tag http.access.default
  format nginx
</source>

<source>
  type tail
  path /var/log/nginx/error.log,/var/log/nginx/*error.log
  pos_file /var/log/td-agent/http-error.log.pos
  tag http.error.default
  format /^(?<time>[^ ]+ [^ ]+) \[(?<log_level>.*)\] (?<pid>\d*).(?<tid>[^:]*): (?<message>.*)$/
</source>

##
# Apache [httpd|apache]
##
<source>
  type tail
  path /var/log/apache2/access.log,/var/log/apache2/*access.log
  pos_file /var/log/td-agent/http-access.log.pos
  tag http.access.default
  format apache2
</source>

<source>
  type tail
  path /var/log/apache2/error.log,/var/log/apache2/*error.log
  pos_file /var/log/td-agent/http-error.log.pos
  tag http.error.default
  format apache_error
</source>

##
# Application
##
<source>
  type tail
  path /var/www/myapp/logs/*.log
  pos_file /var/log/td-agent/app.log.pos
  tag app.web1                              # WEB SERVER NAME
  format json
</source>

<source>
  type forward
</source>

##
# Syslog
##
<match syslog.**>
  type elasticsearch
  logstash_format true
  flush_interval 10s
  host localhost        # HOST!
  port 9200
  logstash_prefix syslog
  type_name fluentd
</match>

##
# Http
##
<match http.access.**>
  type elasticsearch
  logstash_format true
  flush_interval 10s
  host localhost        # HOST!
  port 9200
  logstash_prefix http-access
  type_name fluentd
</match>

<match http.error.**>
  type elasticsearch
  logstash_format true
  flush_interval 10s
  host localhost        # HOST!
  port 9200
  logstash_prefix http-error
  type_name fluentd
</match>

##
# Application
##
<match app.**>          # WEB SERVER NAME
  type elasticsearch
  logstash_format true
  flush_interval 10s
  host localhost        # HOST!
  port 9200
  logstash_prefix app
  type_name fluentd
</match>

Notes:

• Any td-agent.conf mentions of "localhost" was because I tested with a localhost elasticsearch - that should instead of the IP address of your elasticsearch server.