td-agent.conf example

td-agent.conf

<source>
  @type tail
  @id input_tail_nginx_access_log
  format ltsv
  time_format %Y-%m-%d %H:%M:%S %z
  path /var/log/nginx/yukizuri.access.log
  read_from_head true
  rotate_wait 60
  pos_file /var/log/td-agent/yukizuri_access_log.pos
  tag yukizuri.log.nginx
</source>

<match yukizuri.log.nginx>
  @type bigquery
  @id bigquery_nginx_access_log
  method insert

  <buffer time>
    flush_interval 0.1  # flush as frequent as possible
    buffer_queue_limit 10240        # 1MB * 10240 -> 10GB!
    flush_thread_count 16
    timekey 1d
  </buffer>

  auth_method json_key
  json_key /etc/td-agent/.keys/bq-credential-for-fluentd-jsonkey.json

  project project-name
  dataset yukizuri_nginx_log
  auto_create_table true
  table nginx_access_log_%Y%m%d

  <inject>
    time_key time
    time_type string
    time_format %s
  </inject>

  schema [
    { "name": "time",    "type": "timestamp" },
    { "name": "local_time",    "type": "string" },
    { "name": "host",  "type": "string"  },
    { "name": "forwardedfor",  "type": "string"  },
    { "name": "req",  "type": "string"  },
    { "name": "status",    "type": "integer"  },
    { "name": "size",  "type": "integer"  },
    { "name": "referer",  "type": "string"  },
    { "name": "ua",    "type": "string" },
    { "name": "reqtime",   "type": "float" },
    { "name": "cache",    "type": "string" },
    { "name": "runtime",   "type": "float" },
    { "name": "vhost", "type": "string" },
    { "name": "id",    "type": "string"  }
]
</match>