Skip to content

Instantly share code, notes, and snippets.

@devster31
Created June 23, 2015 00:41
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save devster31/cf5b4a730436d5b3cca8 to your computer and use it in GitHub Desktop.
Save devster31/cf5b4a730436d5b3cca8 to your computer and use it in GitHub Desktop.
simple fluentd config for apache and syslog
## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
@type forward
@id forward_input
</source>
## built-in UNIX socket input
#<source>
# @type unix
#</source>
# HTTP input
# http://localhost:8888/<tag>?json=<json>
<source>
@type http
@id http_input
port 8888
</source>
## File input
## read apache logs with tag=apache.access
<source>
@type tail
format apache2
path /var/log/apache2/access.log
pos_file /opt/fluent/apache_access.log.pos
tag orig.apache.access
</source>
<match orig.apache.access>
@type rewrite_tag_filter
remove_tag_prefix orig.
rewriterule1 host 127.0.0.1 local.${tag}
rewriterule2 host !^127.0.0.1$ geo.${tag}
</match>
<match geo.apache.access>
@type geoip
geoip_lookup_key host
<record>
# enable_key_country_code country
# enable_key_city city
# enable_key_latitude lat
# enable_key_longitude lon
country ${country_code['host']}
city ${city['host']}
# coordinates '[${longitude["host"]},${latitude["host"]}]'
coordinates '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
</record>
remove_tag_prefix geo.
add_tag_prefix external.
</match>
#<filter apache.access>
# @type record_transformer
# <record>
# coordinates '["${lon}","${lat}"]'
# </record>
#</filter>
<match **.apache.access>
@type elasticsearch
logstash_format true
logstash_prefix phab_apache_access
logstash_dateformat %Y.%U
host localhost
port 9200
index_name fluentd
type_name apache_logs
include_tag_key true
tag_key tag
</match>
<source>
@type tail
format /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\] \[pid (?<pid>[^\]]*)\] \[client (?<client>[^\]]*)\] (?<message>.*)$/
tag apache.error
pos_file /opt/fluent/apache_error.log.pos
path /var/log/apache2/error.log
</source>
<match apache.error>
@type elasticsearch
logstash_format true
logstash_prefix phab_apache_errors
logstash_dateformat %Y.%U
host localhost
port 9200
index_name apache
type_name apache_logs
</match>
## match tag=apache.access and write to file
#<match apache.access>
# @type file
# path /var/log/fluent/access
#</match>
<source>
@type syslog
port 42185
tag phab.syslog
</source>
<match phab.syslog.**>
@type elasticsearch
logstash_format true
logstash_prefix phab_syslog
logstash_dateformat %Y.%U
host localhost
port 9200
index_name fluentd
type_name syslog
</match>
# Listen HTTP for monitoring
# http://localhost:24220/api/plugins
# http://localhost:24220/api/plugins?type=TYPE
# http://localhost:24220/api/plugins?tag=MYTAG
<source>
@type monitor_agent
@id monitor_agent_input
port 24220
</source>
# Listen DRb for debug
<source>
@type debug_agent
@id debug_agent_input
bind 127.0.0.1
port 24230
</source>
## match tag=debug.** and dump to console
<match debug.**>
@type stdout
@id stdout_output
</match>
# match tag=system.** and forward to another fluent server
#<match system.**>
# @type forward
# @id forward_output
#
# <server>
# host 192.168.0.11
# </server>
# <secondary>
# <server>
# host 192.168.0.12
# </server>
# </secondary>
#</match>
## match tag=myapp.** and forward and write to file
#<match myapp.**>
# @type copy
# <store>
# @type forward
# buffer_type file
# buffer_path /var/log/fluent/myapp-forward
# retry_limit 50
# flush_interval 10s
# <server>
# host 192.168.0.13
# </server>
# </store>
# <store>
# @type file
# path /var/log/fluent/myapp
# </store>
#</match>
## match fluent's internal events
#<match fluent.**>
# @type null
#</match>
## match not matched logs and write to file
#<match **>
# @type file
# path /var/log/fluent/else
# compress gz
#</match>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment