Last active
March 6, 2016 02:54
-
-
Save tzmfreedom/8490a9899fd3a61c6eb7 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<source> | |
@type twitter | |
consumer_key **** | |
consumer_secret **** | |
oauth_token **** | |
oauth_token_secret **** | |
tag input.twitter.sampling # Required | |
timeline tracking # Required (tracking or sampling or location or userstream) | |
keyword 乃木坂,欅坂,秋元真夏,生田絵梨花,生駒里奈,伊藤かりん,伊藤純奈,伊藤万理華,井上小百合,衛藤美彩,川後陽菜,川村真洋,北野日奈子,齋藤飛鳥,斎藤ちはる,相楽伊織,桜井玲香,佐々木琴子,白石麻衣,新内眞衣,鈴木絢音,高山一実,寺田蘭世,中田花奈,中元日芽香,永島聖羅,西野七瀬,能條愛未,橋本奈々未,樋口日奈,深川麻衣,星野みなみ,堀未央奈,松村沙友理,山崎玲奈,若月佑美,渡辺みり愛,和田まあや,石森虹花,今泉佑唯,上村莉菜,尾関梨香,織田奈那,小池美波,小林由依,齋藤冬優花,佐藤詩織,志田愛佳,菅井友香,鈴本美愉,長沢菜々香,土生瑞穂,原田葵,平手友梨奈,守屋茜,米谷奈々未,渡辺梨加,渡邉理佐,長濱ねる # Optional (keyword is priority than follow_ids) | |
output_format flat # Optional (nest or flat or simple[default]) | |
</source> | |
<match input.twitter.sampling> | |
@type copy | |
<store> | |
@type record_reformer | |
renew_record true | |
enable_ruby false | |
keep_keys text,user_id_str,user_name,created_at,user_lang,user_time_zone,place,coordinates,time,tag,lang,geo,id_str,timestamp_ms | |
tag filtered.${tag} | |
</store> | |
<store> | |
@type s3 | |
aws_key_id AKI****** | |
aws_sec_key ********* | |
s3_bucket ******* | |
s3_region ap-northeast-1 | |
path sakamichi/ | |
buffer_path /var/log/td-agent/s3 | |
time_slice_format %Y%m%d | |
s3_object_key_format %{path}%{time_slice}_%{index}_%{hostname}.%{file_extension} | |
time_slice_wait 10m | |
utc | |
format json | |
include_time_key true | |
include_tag_key true | |
buffer_chunk_limit 8m | |
</store> | |
</match> | |
<match filtered.input.twitter.sampling> | |
@type copy | |
<store> | |
@type elasticsearch | |
host localhost | |
port 9200 | |
type_name idle | |
logstash_format true | |
</store> | |
<store> | |
@type bigquery | |
method insert # default | |
auth_method json_key | |
json_key /home/tzm/.keys/****.json | |
email *****@*****.iam.gserviceaccount.com | |
project ***** | |
dataset ***** | |
auto_create_table true | |
table tweet_sakamichi_%{time_slice} | |
time_format %s | |
time_field timestamp_ms | |
field_integer timestamp_ms | |
field_string text,user_id_str,user_name,created_at,user_lang,user_time_zone,place,coordinates,time,tag,lang,geo,id_str | |
</store> | |
</match> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment