/etc/td-agent/td-agent.conf 예제(apache to elasticsearch)

td-agent.conf

###################################
## Input

<source>
  type tail
  format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<timestamp>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time_micro_sec>[^ ]*))?$/
  path /home/ubuntu/log_maker/tmp/log.access
  pos_file /var/log/td-agent/apache.access.pos
  tag apache.access
</source>


###################################
## OUTPUT

<match apache.access>
  type extract_query_params
  key path
  add_tag_prefix extracted.
  only uid, m
</match>

<match extracted.apache.access>
  type typecast
  item_types size:integer,code:integer,response_time_micro_sec:integer,id:integer
  prefix typed
</match>

<match typed.extracted.apache.access>
  type hostname
  key_name server_host
  add_prefix hostname
</match>


<match hostname.typed.extracted.apache.access>
  type elasticsearch
  hosts ec2-xxx.compute-1.amazonaws.com:9200
  type_name access-log
  logstash_format true
  logstash_prefix logstash
  utc_index false

  # buffer
  buffer_type file
  buffer_path /var/log/td-agent/buffer/apache-access-elasticsearch.*.buffer
  buffer_chunk_limit 8m
  buffer_queue_limit 10000
  flush_interval 60
  retry_limit 17
</match>