td-agent.conf example
<source> | |
@type tail | |
@id input_tail_nginx_access_log | |
format ltsv | |
time_format %Y-%m-%d %H:%M:%S %z | |
path /var/log/nginx/yukizuri.access.log | |
read_from_head true | |
rotate_wait 60 | |
pos_file /var/log/td-agent/yukizuri_access_log.pos | |
tag yukizuri.log.nginx | |
</source> | |
<match yukizuri.log.nginx> | |
@type bigquery | |
@id bigquery_nginx_access_log | |
method insert | |
<buffer time> | |
flush_interval 0.1 # flush as frequent as possible | |
buffer_queue_limit 10240 # 1MB * 10240 -> 10GB! | |
flush_thread_count 16 | |
timekey 1d | |
</buffer> | |
auth_method json_key | |
json_key /etc/td-agent/.keys/bq-credential-for-fluentd-jsonkey.json | |
project project-name | |
dataset yukizuri_nginx_log | |
auto_create_table true | |
table nginx_access_log_%Y%m%d | |
<inject> | |
time_key time | |
time_type string | |
time_format %s | |
</inject> | |
schema [ | |
{ "name": "time", "type": "timestamp" }, | |
{ "name": "local_time", "type": "string" }, | |
{ "name": "host", "type": "string" }, | |
{ "name": "forwardedfor", "type": "string" }, | |
{ "name": "req", "type": "string" }, | |
{ "name": "status", "type": "integer" }, | |
{ "name": "size", "type": "integer" }, | |
{ "name": "referer", "type": "string" }, | |
{ "name": "ua", "type": "string" }, | |
{ "name": "reqtime", "type": "float" }, | |
{ "name": "cache", "type": "string" }, | |
{ "name": "runtime", "type": "float" }, | |
{ "name": "vhost", "type": "string" }, | |
{ "name": "id", "type": "string" } | |
] | |
</match> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment