Instantly share code, notes, and snippets.

@a-know /td-agent.conf Secret
Created Dec 26, 2017

Embed
What would you like to do?
td-agent.conf example
<source>
@type tail
@id input_tail_nginx_access_log
format ltsv
time_format %Y-%m-%d %H:%M:%S %z
path /var/log/nginx/yukizuri.access.log
read_from_head true
rotate_wait 60
pos_file /var/log/td-agent/yukizuri_access_log.pos
tag yukizuri.log.nginx
</source>
<match yukizuri.log.nginx>
@type bigquery
@id bigquery_nginx_access_log
method insert
<buffer time>
flush_interval 0.1 # flush as frequent as possible
buffer_queue_limit 10240 # 1MB * 10240 -> 10GB!
flush_thread_count 16
timekey 1d
</buffer>
auth_method json_key
json_key /etc/td-agent/.keys/bq-credential-for-fluentd-jsonkey.json
project project-name
dataset yukizuri_nginx_log
auto_create_table true
table nginx_access_log_%Y%m%d
<inject>
time_key time
time_type string
time_format %s
</inject>
schema [
{ "name": "time", "type": "timestamp" },
{ "name": "local_time", "type": "string" },
{ "name": "host", "type": "string" },
{ "name": "forwardedfor", "type": "string" },
{ "name": "req", "type": "string" },
{ "name": "status", "type": "integer" },
{ "name": "size", "type": "integer" },
{ "name": "referer", "type": "string" },
{ "name": "ua", "type": "string" },
{ "name": "reqtime", "type": "float" },
{ "name": "cache", "type": "string" },
{ "name": "runtime", "type": "float" },
{ "name": "vhost", "type": "string" },
{ "name": "id", "type": "string" }
]
</match>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment