Twitterデータ(DLできるやつ)からニコニコのマイリストしたログを抽出

extract-niconico-mylist-from-twitter-data.py

import json
import datetime
import sys
dir = sys.argv[1]
def jsloader(o):
    with open(dir + "/" + o["fileName"], "r") as f:
        r = f.read()
        prefix = "window." + o["globalName"] + " = "
        if not r.startswith(prefix):
            raise Exception("???")
        return json.loads(r[len(prefix):])
manifest = jsloader({"fileName": "data/manifest.js", "globalName": "__THAR_CONFIG"})
print(manifest)

mylists = []

for tweetsFile in manifest["dataTypes"]["tweet"]["files"]:
    tweets = jsloader(tweetsFile)
    for tweet in tweets:
        tweet = tweet["tweet"]
        if tweet["source"] != '<a href="http://www.nicovideo.jp/" rel="nofollow">niconico ニコレポ連携</a>':
            continue
        if tweet["full_text"].startswith("【マイリスト】"):
            # print(tweet["created_at"])
            mylists.append({
                "niconico_id": tweet["entities"]["hashtags"][-1]["text"],
                "tweet_id": tweet["id"],
                "created_at": datetime.datetime.strptime(tweet["created_at"] ,'%a %b %d %H:%M:%S %z %Y').isoformat(),
            })

mylists.sort(key=lambda x:x["created_at"])

with open("extracted_mylists.json", "w") as f:
    json.dump(mylists, f, indent="\t")