Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Fluentd configuration for nginx, PHP-FPM, and syslog forwarding to AWS CloudWatch Logs
## This configuration file expects certain environment variables to be available.
## You can add them to /etc/default/td-agent:
## AWS_REGION
## AWS_VPC_ID
## AWS_INSTANCE_ID
<source>
@type syslog
port 42185
bind 127.0.0.1
tag rsyslog
<parse>
@type regexp
expression /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\- ]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
time_format %b %d %H:%M:%S
</parse>
</source>
## This is for capturing nginx access logs, with the virtual host at the beginning of the line in double quotes.
## This corresponds to the nginx log_format of:
## "$host" $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"
<filter rsyslog.local7.info>
@type parser
key_name message
<parse>
@type regexp
expression /^"(?<host>[^"]*)" (?<remote>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?: "(?<forwarder>[^\"]*)")?)?/
time_format %d/%b/%Y:%H:%M:%S %z
time_key time
</parse>
</filter>
<match rsyslog.local7.info>
@type rewrite_tag_filter
<rule>
key host
pattern ^(.+)$
tag "nginx.access.#{ENV['AWS_INSTANCE_ID']}.$1"
</rule>
</match>
<match rsyslog.local7.*>
@type rewrite_tag_filter
<rule>
key ident
pattern ^nginx_(.+)$
tag "nginx.error.#{ENV['AWS_INSTANCE_ID']}.$1"
</rule>
</match>
<filter nginx.error.**>
@type parser
key_name message
<parse>
@type multiline
format_firstline /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[\w+\] (?<pid>\d+).(?<tid>\d+): /
format1 /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)/
</parse>
</filter>
<match {rsyslog.user.**,rsyslog.daemon.**}>
@type rewrite_tag_filter
<rule>
key ident
pattern ^(php|ool www)$
tag "php.error.#{ENV['AWS_INSTANCE_ID']}.$1"
</rule>
</match>
<filter php.error.**>
@type record_transformer
remove_keys ident
</filter>
<match nginx.**>
@type cloudwatch_logs
log_group_name "nginx.#{ENV['AWS_VPC_ID']}"
use_tag_as_stream true
auto_create_stream true
region "#{ENV['AWS_REGION']}"
<buffer>
@type memory
flush_interval 10s
overflow_action drop_oldest_chunk
</buffer>
</match>
<match php.error.**>
@type cloudwatch_logs
log_group_name "php-fpm.#{ENV['AWS_VPC_ID']}"
use_tag_as_stream true
auto_create_stream true
region "#{ENV['AWS_REGION']}"
<buffer>
@type memory
flush_interval 10s
overflow_action drop_oldest_chunk
</buffer>
</match>
<label @FLUENT_LOG>
<match fluent.*>
@type null
</match>
</label>
<filter *.**>
@type record_transformer
<record>
facility ${tag_parts[1]}
severity ${tag_parts[2]}
</record>
</filter>
<match *.**>
@type cloudwatch_logs
log_group_name "syslog.#{ENV['AWS_VPC_ID']}"
log_stream_name "#{ENV['AWS_INSTANCE_ID']}"
auto_create_stream true
region "#{ENV['AWS_REGION']}"
<buffer>
@type memory
flush_interval 10s
overflow_action drop_oldest_chunk
</buffer>
</match>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment