Created
December 15, 2022 20:41
-
-
Save chrisjchandler/35d58ce8391b7192df5bb6a4cc1108da to your computer and use it in GitHub Desktop.
fluentd config with multiple destination configurations templated
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<source> | |
@type tail | |
path /var/log/myapp/*.log | |
pos_file /var/log/fluentd-myapp.pos | |
tag myapp.* | |
format json | |
</source> | |
<match myapp.elasticsearch> | |
@type elasticsearch | |
host elasticsearch.mycompany.com | |
port 9200 | |
logstash_format true | |
logstash_prefix myapp | |
logstash_dateformat %Y%m%d | |
include_tag_key true | |
type_name access_log | |
tag_key @log_name | |
flush_interval 10s | |
</match> | |
<match myapp.kafka> | |
@type kafka | |
brokers kafka1.mycompany.com:9092,kafka2.mycompany.com:9092 | |
default_topic myapp-logs | |
output_data_type json | |
<buffer> | |
flush_mode interval | |
flush_interval 10s | |
retry_type exponential_backoff | |
retry_forever true | |
</buffer> | |
</match> | |
<match myapp.s3> | |
@type s3 | |
aws_key_id YOUR_AWS_ACCESS_KEY | |
aws_sec_key YOUR_AWS_SECRET_KEY | |
s3_bucket myapp-logs | |
s3_region us-east-1 | |
path logs/ | |
time_format %Y%m%dT%H%M%S | |
time_slice_format %Y%m%d | |
utc | |
buffer_type file | |
buffer_path /var/log/fluentd-s3-myapp | |
flush_interval 10s | |
retry_limit 17 | |
retry_wait 1.0 | |
num_threads 1 | |
</match> | |
$fluentd -c fluent.conf |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment