archive.py

import json
import pathlib
import tweepy
import pickle
from collections import defaultdict
import pandas as pd
from dateutil.parser import parse as parse_time


def load_api():
    with open("credentials.json) as fp:
        creds = json.load(fp)
    
    auth = tweepy.OAuthHandler(creds['appKey'], creds['appSecret'])
    auth.set_access_token(creds['userToken'], creds['userSecret'])
    api = tweepy.API(auth)
    return api


def load_twitter_json(file_path):
    with open(file_path) as fp:
        data = fp.read()
        return json.loads(data[data.index("["):])
    
    
def load_tweets(archive_path="/mnt/data/twitter_data/data/"):
    path = pathlib.Path(archive_path) / "tweet.js"
    my_tweets = []
    for tweet in load_twitter_json(path):
        tweet = tweet['tweet']
        tweet['created_at'] = parse_time(tweet['created_at'])
        my_tweets.append(tweet)
    return sorted(my_tweets, key=lambda t: t['created_at'])


def load_likes(archive_path="/mnt/data/twitter_data/data/"):
    path = pathlib.Path(archive_path) / "like.js"
    my_likes = []

    for like in load_twitter_json(path):
        like = like['like']
        my_likes.append(like)
        
    return my_likes