oranie

<source>
  type tail
  #LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
  format /^(?<XFF-host>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
  time_format %d/%b/%Y:%H:%M:%S %z
  path /var/log/httpd/access_log_sym
  tag apache.access
  pos_file /var/log/td-agent/tmp/access.log.pos
</source>

oranie

unmatched_rate:0.0 2xx_rate:0.0 3xx_rate:0.0 4xx_rate:0.0 5xx_rate:0.0

oranie

<source>
  type tail
  tag app_csv_log_file_name
  format /^(?<column1>[^,]*),(?<column2>[^ ]*),(?<column3>[^ ]*),(?<column4>[^ ]*),(?<column5>[^ ]*)?$/

  path /var/log/app_log_dir/app_csv_log_file
  pos_file /var/log/td-agent/tmp/app_csv_log.pos
</source>

<match *.**>

oranie

2012-03-22 20:49:03 +0900: process finished code=0
2012-03-22 20:49:04 +0900: starting fluentd-0.10.15
2012-03-22 20:49:04 +0900: reading config file path="/etc/td-agent/td-agent.conf"
2012-03-22 20:49:04 +0900: adding source type="forward"
2012-03-22 20:49:04 +0900: adding match pattern="*.**" type="forward"
2012-03-22 20:49:04 +0900: adding forwarding server 'aggregate' host="192.168.1.1" port=24224 weight=60
2012-03-22 20:49:04 +0900: listening fluent socket on 0.0.0.0:24224
2012-03-22 20:49:23 +0900: detached forwarding server 'aggregate' host="192.168.1.1" port=24224 phi=8.432087244706564
2012-03-22 20:49:24 +0900: failed to flush the buffer, retrying. error="no nodes are available" instance=23577120
  2012-03-22 20:49:24 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/plugin/out_forward.rb:137:in `write_objects'

oranie

2012-04-07 13:12:00 +0900: emit transaction failed  error="closed stream"
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/plugin/buf_file.rb:31:in `write'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/plugin/buf_file.rb:31:in `<<'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/buffer.rb:168:in `block in emit'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/1.9.1/monitor.rb:201:in `mon_synchronize'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/buffer.rb:163:in `emit'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/output.rb:517:in `block in emit'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.15/lib/fluent/event.rb:52:in `call'
  2012-04-07 13:12:00 +0900: /usr/lib64/fluent/r

oranie

h3 {
 font-size: 100%;
 font-weight: bold;
 margin: 0.6em 10% 0 0.4em;
 border-left: 8px solid gray;
 border-bottom: solid;
 border-color: gray;
 padding: 0px 0.5em 0.1em 0.5em;
}
h4 {

oranie

#  /usr/lib64/fluent/ruby/bin/fluent-gem search -rd fluentd-plugin
/usr/lib64/fluent/ruby/lib/ruby/1.9.1/yaml.rb:56:in `<top (required)>':
It seems your ruby installation is missing psych (for YAML output).
To eliminate this warning, please install libyaml and reinstall your ruby.

*** REMOTE GEMS ***

oranie

# yum install td-agent
Loaded plugins: security
Setting up Install Process
Resolving Dependencies
--> Running transaction check
---> Package td-agent.x86_64 0:1.1.4.4-0 will be installed
--> Processing Dependency: /usr/bin/ruby for package: td-agent-1.1.4.4-0.x86_64
--> Processing Dependency: td-libyaml for package: td-agent-1.1.4.4-0.x86_64
--> Running transaction check
---> Package ruby.x86_64 0:1.8.7.352-4.el6_2 will be installed

oranie

2012-05-08T21:03:05+09:00       fluent.warn     {"message":"retrysucceeded.","instance":-610783388}
2012-05-08T21:03:16+09:00       fluent.warn     {"message":"Size of theemitted data exceeds buffer_chunk_limit."}
2012-05-08T21:03:16+09:00       fluent.warn     {"message":"This may occurproblems in the output plugins ``at this server.``"}
2012-05-08T21:03:16+09:00       fluent.warn     {"message":"To avoid problems,set a smaller number to the buffer_chunk_limit"}
2012-05-08T21:03:16+09:00       fluent.warn     {"message":"in the forwardoutput ``at the log forwarding server.``"}

oranie

<source>
  type forward
  port 24224
</source>

<match apache.access.**>
    type copy
    <store>
        compress gzip
        type file