再現コード

hoge.rb

# coding: utf-8

require 'json'

rand = Random.new

i = 0
while true do
  i += 1
  obj = {
    "aaa" => rand.rand(1000),
    "bbb" => rand.rand(1000),
    "i" => i ,
    "timestamp" => Time.now.to_i
  }
  puts JSON.dump(obj)
  sleep 0.1
end

td-agent.conf

<source>
    type tail
    format json
    path /data/logs/test.log
    pos_file /var/log/td-agent/test.pos
    tag xxx.yyy.test
    time_key timestamp
</source>

<match xxx.yyy.test>
    type copy
    <store>
        type file
        path /data/output/out_test1.txt
        buffer_type file
        buffer_path /data/td-agent/s3/test/
        time_slice_format %Y-%m-%d-%H-%M
        time_slice_wait 10m
        time_key timestamp
        include_time_key
        time_as_epoch
        buffer_chunk_limit 2048m
        format json
        utc
    </store>
    <store>
        type file
        path /data/output/out_test2.txt
    </store>
</match>


<match **>
    type file
    path /data/td-agent/not-matched.log
    time_slice_format %Y%m%d
    time_slice_wait 1m
    utc
</match>

versions

2015-07-29 04:36:49 +0000 [info]: starting fluentd-0.12.14
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-mixin-config-placeholders' version '0.3.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-mixin-config-placeholders' version '0.2.4'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-mixin-plaintextformatter' version '0.2.6'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-bigquery' version '0.2.8'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-buffer-lightening' version '0.0.2'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-datacounter' version '0.4.1'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-dd' version '0.1.7'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-extract_query_params' version '0.0.5'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-flowcounter' version '0.1.8'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-flume' version '0.1.1'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-forest' version '0.1.4'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-format' version '0.0.1'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-growthforecast' version '0.2.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-jsonbucket' version '0.0.2'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-kinesis' version '0.3.4'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-lossycount' version '0.0.1'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-mongo' version '0.7.3'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-multiprocess' version '0.1.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-numeric-monitor' version '0.1.6'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-reassemble' version '0.0.7'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-record-modifier' version '0.3.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-record-reformer' version '0.3.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-rename-key' version '0.1.3'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-rewrite' version '0.0.12'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '1.4.1'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '1.2.1'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-s3' version '0.4.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-s3' version '0.3.5'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-scribe' version '0.10.10'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-tail-ex' version '0.1.0'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-td' version '0.10.20'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-td-monitoring' version '0.1.2'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-typecast' version '0.1.2'
2015-07-29 04:36:49 +0000 [info]: gem 'fluent-plugin-webhdfs' version '0.2.2'
2015-07-29 04:36:49 +0000 [info]: gem 'fluentd' version '0.12.14'
2015-07-29 04:36:49 +0000 [info]: gem 'fluentd' version '0.10.50'

ruby hoge.rb  > /data/logs/test.log

でログを出力してfluentdに食べさせる。

/data/output/out_test1.txt.2015-07-29-05_0.log には全てのレコードにtimestampが入っている

out_test2.txt の方には↓の様にtimestampが入ったり入らなかったり

2015-07-29T05:37:48+00:00       xxx.yyy.test    {"aaa":959,"bbb":680,"i":470,"timestamp":1438148268}
2015-07-29T05:37:48+00:00       xxx.yyy.test    {"aaa":857,"bbb":326,"i":471}
2015-07-29T05:37:48+00:00       xxx.yyy.test    {"aaa":624,"bbb":443,"i":472}
2015-07-29T05:37:48+00:00       xxx.yyy.test    {"aaa":192,"bbb":686,"i":473}
2015-07-29T05:37:48+00:00       xxx.yyy.test    {"aaa":210,"bbb":315,"i":474}

ちなみに、deep_copy true を入れると、out_test2.txtの方のtimestampは消えてしまう