fluentd.conf

<source>
  @type twitter
  consumer_key        ****
  consumer_secret     ****
  oauth_token         ****
  oauth_token_secret  ****
  tag                 input.twitter.sampling  # Required
  timeline            tracking                # Required (tracking or sampling or location or userstream)
  keyword             乃木坂,欅坂,秋元真夏,生田絵梨花,生駒里奈,伊藤かりん,伊藤純奈,伊藤万理華,井上小百合,衛藤美彩,川後陽菜,川村真洋,北野日奈子,齋藤飛鳥,斎藤ちはる,相楽伊織,桜井玲香,佐々木琴子,白石麻衣,新内眞衣,鈴木絢音,高山一実,寺田蘭世,中田花奈,中元日芽香,永島聖羅,西野七瀬,能條愛未,橋本奈々未,樋口日奈,深川麻衣,星野みなみ,堀未央奈,松村沙友理,山崎玲奈,若月佑美,渡辺みり愛,和田まあや,石森虹花,今泉佑唯,上村莉菜,尾関梨香,織田奈那,小池美波,小林由依,齋藤冬優花,佐藤詩織,志田愛佳,菅井友香,鈴本美愉,長沢菜々香,土生瑞穂,原田葵,平手友梨奈,守屋茜,米谷奈々未,渡辺梨加,渡邉理佐,長濱ねる             # Optional (keyword is priority than follow_ids)
  output_format       flat                    # Optional (nest or flat or simple[default])
</source>

<match input.twitter.sampling>
  @type copy
  <store>
    @type record_reformer
    renew_record true
    enable_ruby false
    keep_keys text,user_id_str,user_name,created_at,user_lang,user_time_zone,place,coordinates,time,tag,lang,geo,id_str,timestamp_ms
    tag filtered.${tag}
  </store>
  <store>
    @type s3

    aws_key_id AKI******
    aws_sec_key *********
    s3_bucket *******
    s3_region ap-northeast-1
    path sakamichi/
    buffer_path /var/log/td-agent/s3

    time_slice_format %Y%m%d
    s3_object_key_format %{path}%{time_slice}_%{index}_%{hostname}.%{file_extension}
    time_slice_wait 10m
    utc

    format json
    include_time_key true
    include_tag_key true

    buffer_chunk_limit 8m
  </store>
</match>

<match filtered.input.twitter.sampling>
  @type copy
  <store>
    @type elasticsearch
    host localhost
    port 9200
    type_name idle
    logstash_format true
  </store>
  <store>
    @type bigquery

    method insert    # default

    auth_method json_key
    json_key /home/tzm/.keys/****.json

    email *****@*****.iam.gserviceaccount.com

    project *****
    dataset *****
    auto_create_table true
    table   tweet_sakamichi_%{time_slice}

    time_format %s
    time_field  timestamp_ms

    field_integer timestamp_ms
    field_string  text,user_id_str,user_name,created_at,user_lang,user_time_zone,place,coordinates,time,tag,lang,geo,id_str
  </store>
</match>