fluentd config

td-agent-graphite.conf

# nginx log -> elasticsearch 
# nginx log -> rabbitmq 
# nginx log -> norikra       -> email
# nginx log -> norikra       -> elasticsearch 
# spring metrcis log -> graphite

# metrics log ago
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_marksweep.count","count":"2"}
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_marksweep.time","count":"213"}
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_scavenge.count","count":"114"}
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_scavenge.time","count":"672"}
 
# metrics log after 
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_marksweep.count":2}
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_marksweep.time":213}
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_scavenge.count":114}
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_scavenge.time":672}

<source> 
  type tail 
  path /var/log/foo/bar.log 
  pos_file /var/log/td-agent/foo-bar.log.pos 
  tag foo.bar 
  format /(?<date>[^ ]*) (?<times>[^ ]*) (?<level>[^ ]*) (([^=]+))=(?<type>([^=]+))(?:,|$) (([^=]+))=(?<name>([^=]+))(?:,|$) (([^=]+))=(?<count>([^=]+))(?:,|$)/ 
</source>

                            
<source>
  type tail
  path /home/opdev/logs/nginx/opdev.pinpoint.korea.ncsoft.corp_access.log
  pos_file /var/log/td-agent/opdev.pinpoint.access.log.pos
  format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
  time_format %d/%b/%Y:%H:%M:%S %z
  tag nginx.access
</source>

<filter nginx.access>
  type grep
  exclude1 path css|js|png|gif|jpg|jpeg
</filter>

<match nginx.access>
  type copy
  <store>
  type stdout
  </store>
  <store>
  type norikra
  norikra 127.0.0.1:26571
  buffer_queue_limit 64
  retry_limit 17
  remove_tag_prefix nginx
  target_map_tag true
  </store>

  <store>
  type elasticsearch
  host 127.0.0.1
  port 9200
  logstash_format true
  type_name pinpoint
  logstash_prefix nginx_access
  </store>
  
  <store>
     #rabbitmq 내에서 routing key는 tag 네임으로 정의해서 bind 해서 사용해야 수집가능   
      type amqp
      host 127.0.0.1
      port 5672
      user guest
      password guest
      exchange fluentd_exchange
      exchange_type topic
      exchange_durable true
      payload_only false
      content_type text/x-json  #application/json
  </store>
</match>

<source>
  type norikra
  norikra 127.0.0.1:26571
  <fetch>
    method sweep
    tag query_name
    tag_prefix norikra.query
    interval 10s
  </fetch>
</source>
 
<match norikra.query.access**>
 type copy
 <store>
  type elasticsearch
  host 172.20.54.152
  port 9200
  logstash_format true
  type_name pinpoint
  logstash_prefix norikra_access
  </store>

  <store>
  type mail
  host mail.host.com
  port 25
  from norikra@ncsoft.com
  to cowboy76@ncsoft.com
  subject SUBJECT: %s
  subject_out_keys tag
  out_keys tag,requests,remote
  </store>
</match>
           
           
<source>
   type tail
   path /home/opdev/GPORTAL/logs/metrics.out
   pos_file /var/log/td-agent/metrics.out.pos
   format /(?<logtime>[^ ]* [^ ]*) (?<level>[^ ]*) (([^=]+))=(?<type>([^=]+))(?:,|$) (([^=]+))=(?<name>([^=]+))(?:,|$) (([^=]+))=(?<count>([^=]+))(?:,|$)/
   time_format %Y-%m-%d %H:%M:%S
   tag event.log
</source>
<match event.log>
  type copy
  <store>
  type map
  map ([["graphite.#{Socket.gethostname}", time, {record["name"] => record["count"].to_i}]])
  multi true
  </store>
</match>
<match graphite **>
  type copy
  <store>
  type stdout
  </store>
  <store>
  type graphite
  host 127.0.0.1
  port 2003
  name_keys gc.ps_marksweep.count,gc.ps_marksweep.time,gc.ps_scavenge.count,gc.ps_scavenge.time,heap.committed,heap.init,heap.used,threads.daemon,threads.peak,threads.totalStarted
  </store>
</match>