Fluentd configuration for nginx, PHP-FPM, and syslog forwarding to AWS CloudWatch Logs

td-agent.conf

## This configuration file expects certain environment variables to be available.
## You can add them to /etc/default/td-agent:
## AWS_REGION
## AWS_VPC_ID
## AWS_INSTANCE_ID
<source>
  @type syslog
  port 42185
  bind 127.0.0.1
  tag rsyslog
  <parse>
    @type regexp
    expression /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\- ]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
    time_format %b %d %H:%M:%S
  </parse>
</source>

## This is for capturing nginx access logs, with the virtual host at the beginning of the line in double quotes.
## This corresponds to the nginx log_format of:
## "$host" $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"
<filter rsyslog.local7.info>
  @type parser
  key_name message
  <parse>
    @type regexp
    expression /^"(?<host>[^"]*)" (?<remote>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?: "(?<forwarder>[^\"]*)")?)?/
    time_format %d/%b/%Y:%H:%M:%S %z
    time_key time
  </parse>
</filter>
<match rsyslog.local7.info>
  @type rewrite_tag_filter
  <rule>
    key     host
    pattern ^(.+)$
    tag     "nginx.access.#{ENV['AWS_INSTANCE_ID']}.$1"
  </rule>
</match>
<match rsyslog.local7.*>
  @type rewrite_tag_filter
  <rule>
    key     ident
    pattern ^nginx_(.+)$
    tag     "nginx.error.#{ENV['AWS_INSTANCE_ID']}.$1"
  </rule>
</match>
<filter nginx.error.**>
  @type parser
  key_name message
  <parse>
    @type multiline
    format_firstline /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[\w+\] (?<pid>\d+).(?<tid>\d+): /
    format1 /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)/
  </parse>
</filter>
<match {rsyslog.user.**,rsyslog.daemon.**}>
  @type rewrite_tag_filter
  <rule>
    key     ident
    pattern ^(php|ool www)$
    tag     "php.error.#{ENV['AWS_INSTANCE_ID']}.$1"
  </rule>
</match>
<filter php.error.**>
  @type record_transformer
  remove_keys ident
</filter>

<match nginx.**>
  @type cloudwatch_logs
  log_group_name "nginx.#{ENV['AWS_VPC_ID']}"
  use_tag_as_stream true
  auto_create_stream true
  region "#{ENV['AWS_REGION']}"
  <buffer>
    @type memory
    flush_interval 10s
    overflow_action drop_oldest_chunk
  </buffer>
</match>
<match php.error.**>
  @type cloudwatch_logs
  log_group_name "php-fpm.#{ENV['AWS_VPC_ID']}"
  use_tag_as_stream true
  auto_create_stream true
  region "#{ENV['AWS_REGION']}"
  <buffer>
    @type memory
    flush_interval 10s
    overflow_action drop_oldest_chunk
  </buffer>
</match>

<label @FLUENT_LOG>
  <match fluent.*>
    @type null
  </match>
</label>
<filter *.**>
  @type record_transformer
  <record>
    facility ${tag_parts[1]}
    severity ${tag_parts[2]}
  </record>
</filter>
<match *.**>
  @type cloudwatch_logs
  log_group_name "syslog.#{ENV['AWS_VPC_ID']}"
  log_stream_name "#{ENV['AWS_INSTANCE_ID']}"
  auto_create_stream true
  region "#{ENV['AWS_REGION']}"
  <buffer>
    @type memory
    flush_interval 10s
    overflow_action drop_oldest_chunk
  </buffer>
</match>