fluent-plugin-google-cloud-storage-out config sample

td-agent.conf

# tail
<source>
  type tail
  format /^(?<date>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) (?<status>\S+) \[(?<role>\S+)\] (?<info>.*)$/
  path /tmp/application.log
  pos_file /var/log/td-agent/myapp.application.pos
  tag td.myapp.application
</source>

<match td.myapp.application>
  type copy

  # filter json to tag
  <store>
    type rewrite_tag_filter
    rewriterule1 info ^\{.*\}$ bq.myapp.json
  </store>

</match>

# make json
<match bq.myapp.json>
  type parser
  format json
  key_name info
  time_parse no
  tag bq.myapp.json.parse
</match>

# post to GCS
<match bq.myapp.json.parse>
  type copy

#  <store>
#    type stdout
#  </store>

  <store>
    type google_cloud_storage_out
    service_account_json_key_path /etc/td-agent/bigquery/My_First_Project-000000000.json
    bucket_id test_bucket
    path myapp.application/%Y/%m/%d/%H/${hostname}/${unique}.log
    unique_strategy chunk_id
    output_tag false
    output_time false
    buffer_path /var/log/td-agent/buffer/google_cloud_storage.myapp.application
#    flush_interval 10s
    buffer_chunk_limit 500m
    time_slice_wait 300s
    compress gzip
  </store>

</match>