A fluent.conf for OpenStack in need of a position file.

fluent.conf

# Nova compute
<source>
  @type tail
  path /var/log/nova/nova-compute.log
  tag nova.compute

  # The first line of a multiline log entry begins with a timestamp, YYYY-MM-DD
  # HH:MM:SS, and an optional sub-second value using 3 or 6 digits, followed by
  # pid, level, and class. To determine a "first line" we only need to match
  # the beginning part of it, so for OpenStack log lines, those four fields are
  # sufficient.  NOTE: if we don't explicitly enumerate the time field with
  # digits, any line that begins with the same pattern of spaces will match,
  # but then time format processing will not get the proper timestamp in the
  # log entry itself.
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/

  # As documented at http://docs.fluentd.org/articles/in_tail (see the Java log
  # line example), we only need to use one format line to match multi line log
  # entries AND single line log entries at the same time.  Note that the context
  # field is optional, and the message field will hold the rest of the current
  # line and all subsequent log lines which do not match the above "first line"
  # pattern.
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/

  time_format %F %T.%L
</source>

# Nova API
<source>
  @type tail
  path /var/log/nova/nova-api.log
  tag nova.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Nova Cert
<source>
  @type tail
  path /var/log/nova/nova-cert.log
  tag nova.cert
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Nova Conductor
<source>
  @type tail
  path /var/log/nova/nova-conductor.log
  tag nova.conductor
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Nova Consoleauth
<source>
  @type tail
  path /var/log/nova/nova-consoleauth.log
  tag nova.consoleauth
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Nova Scheduler
<source>
  @type tail
  path /var/log/nova/nova-scheduler.log
  tag nova.scheduler
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Neutron Openvswitch Agent
<source>
  @type tail
  path /var/log/neutron/openvswitch-agent.log
  tag neutron.openvswitch
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Neutron Server
<source>
  @type tail
  path /var/log/neutron/server.log
  tag neutron.dhcp
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Neutron L3 Agent
<source>
  @type tail
  path /var/log/neutron/l3-agent.log
  tag neutron.l3
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Neutron Metadata Agent
<source>
  @type tail
  path /var/log/neutron/metadata-agent.log
  tag neutron.metadata
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Keystone
<source>
  @type tail
  path /var/log/keystone/keystone.log
  tag keystone
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Glance API
<source>
  @type tail
  path /var/log/glance/api.log
  tag glance.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Glance Registry
<source>
  @type tail
  path /var/log/glance/registry.log
  tag glance.registry
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Cinder API
<source>
  @type tail
  path /var/log/cinder/api.log
  tag cinder.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Cinder Scheduler
<source>
  @type tail
  path /var/log/cinder/scheduler.log
  tag cinder.scheduler
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Cinder Volume
<source>
  @type tail
  path /var/log/cinder/volume.log
  tag cinder.volume
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Cinder Backup
<source>
  @type tail
  path /var/log/cinder/backup.log
  tag cinder.backup
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Heat API CloudFormation
<source>
  @type tail
  path /var/log/heat/heat-api-cfn.log
  tag heat.api.cfn
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Heat API CloudWatch
<source>
  @type tail
  path /var/log/heat/heat-api-cloudwatch.log
  tag heat.api.cloudwatch
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Heat API
<source>
  @type tail
  path /var/log/heat/heat-api.log
  tag heat.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Heat Engine
<source>
  @type tail
  path /var/log/heat/heat-engine.log
  tag heat.engine
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer Agent Notification
<source>
  @type tail
  path /var/log/ceilometer/agent-notification.log
  tag ceilometer.agent.notification
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer Alarm Evaluator
<source>
  @type tail
  path /var/log/ceilometer/alarm-evaluator.log
  tag ceilometer.alarm.evaluator
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer Alarm Notifier
<source>
  @type tail
  path /var/log/ceilometer/alarm-notifier.log
  tag ceilometer.alarm.notifier
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer API
<source>
  @type tail
  path /var/log/ceilometer/api.log
  tag ceilometer.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer DB Sync
<source>
  @type tail
  path /var/log/ceilometer/ceilometer-dbsync.log
  tag ceilometer.dbsync
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer Central
<source>
  @type tail
  path /var/log/ceilometer/central.log
  tag ceilometer.central
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer Collector
<source>
  @type tail
  path /var/log/ceilometer/collector.log
  tag ceilometer.collector
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ceilometer Compute
<source>
  @type tail
  path /var/log/ceilometer/compute.log
  tag ceilometer.compute
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Sahara All
<source>
  @type tail
  path /var/log/sahara/sahara-all.log
  tag sahara.all
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Trove API
<source>
  @type tail
  path /var/log/trove/trove-api.log
  tag trove.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Trove Conductor
<source>
  @type tail
  path /var/log/trove/trove-conductor.log
  tag trove.conductor
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Trove Task Manager
<source>
  @type tail
  path /var/log/trove/trove-taskmanager.log
  tag trove.taskmanager
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ironic API
<source>
  @type tail
  path /var/log/ironic/ironic-api.log
  tag ironic.api
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

# Ironic Conductor
<source>
  @type tail
  path /var/log/ironic/ironic-conductor.log
  tag ironic.conductor
  format multiline
  format_firstline /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})? [^ ]* [^ ]* [^ ]*/
  format1 /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3,6})?) (?<pid>[^ ]*) (?<loglevel>[^ ]*) (?<class>[^ ]*)( \[(?<context>[^\]]*)\])? (?<message>.*)/
  time_format %F %T.%L
</source>

<filter *.**>
  @type record_transformer
  <record>
    service ${tag}
    hostname "#{Socket.gethostname}"
    tags "$LIST_OF_ADMIN_ASSIGNED_TAGS"
  </record>
</filter>

<match **>
  @type forward
  heartbeat_type tcp
  flush_interval 1s
  <server>
    name $FQDN_OF_LOGGING_SERVER
    host $FQDN_OF_LOGGING_SERVER
    port 4000
  </server>
</match>