mongo_up_config

gistfile1.txt

<source>
  type tail
  format /^(?<XFF-host>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
  #format /^(?<XFF-host>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<TIME>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/

  time_format %d/%b/%Y:%H:%M:%S %z
  #path /var/log/td-agent/tmp/access_log_sym
  path /var/log/httpd/access_log
  #tag apache.access.localhost.127.0.0.1
  tag apache.access
  pos_file /var/log/td-agent/tmp/access.log.pos
</source>

<match apache.access.**>
    type copy
    <store>
        type mongo
        capped
        capped_size 1024m
        database apache
        collection access
        host localhost
        port 27017
        flush_interval 10s
    </store>
    <store>
        compress gzip
        type file
        path /var/log/td-agent/access_log
    </store>
    <store>
        type datacounter
        #unit minute
        count_interval 60
        count_key status
        tag apache_result.status.all
        aggregate all
        # patternX: X(1-9)
        pattern1 2xx ^2\d\d$
        pattern2 3xx ^3\d\d$
        pattern3 4xx ^4\d\d$  # pattern4 doesn't matches messages matches pattern[123]
        pattern4 5xx ^5\d\d$
    </store>
    <store>
        type datacounter
        #unit minute
        count_interval 60
        count_key response_time
        tag apache_result.response_time.all
        aggregate all
        # patternX: X(1-9)
        pattern1 0-100msec ^\d{1,5}$
        pattern2 100-200msec ^[1-2]\d{5}$
        pattern3 200-500msec ^[2-5]\d{5}$
        pattern4 500msec-1sec ^[5-9]\d{5}$
        pattern5 1sec_over ^\d{7,}$
    </store>
    <store>
        type flowcounter
        count_keys *
        aggregate  all
        tag        traffic.apache
    </store>
</match>

<match apache_result.**>
    type copy
    <store>
        type mongo
        capped
        capped_size 1024m
        database apache
        collection result
        host localhost
        port 27017
        flush_interval 10s
    </store>

    <store>
        type file
        time_slice_format %Y_%m_%d
        compress gzip
        path /var/log/td-agent/apache_result
    </store>
</match>

<match traffic.**>
    type copy
    <store>
        type mongo
        capped
        capped_size 1024m
        database apache
        collection traffic
        host localhost
        port 27017
        flush_interval 10s
    </store>
    <store>
        type file
        time_slice_format %Y_%m_%d
        compress gzip
        path /var/log/td-agent/traffic
    </store>
    <store>
        type growthforecast
        gfapi_url http://localhost:5125/api/
        service   localserver
        section   traffic.apache
        name_keys count,count_rate
    </store>
</match>

<match fluent.**>
  type file
  time_slice_format %Y_%m_%d
  compress gzip
  path /var/log/td-agent/fluent-app
</match>


<match *.**>
  type file
  time_slice_format %Y_%m_%d
  compress gzip
  path /var/log/td-agent/other_log
</match>