Script to archive all your own Tweets

tweet_archiver.py

#!/usr/bin/python3

import tweepy,os,sys,json,requests
from datetime import datetime

consumer_key="FILL_ME_IN"
consumer_secret="SETEC_ASTRONOMY"

auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
home = os.environ["HOME"]
tokens = home+"/.twitter.tokens"

new_files = []

def get_media(mediaurl):
    global new_files
    medianame = mediaurl.split("/")[-1]
    medianame = medianame.split("?")[0]
    if os.path.exists(medianame):
        return
    print("\t"+mediaurl+" -> "+medianame)
    response = requests.get(mediaurl)
    out = open(medianame,"wb")
    out.write(response.content)
    out.close()
    new_files.append(medianame)

try:
	f = open(tokens,"r")
	key = f.readline().strip()
	secret = f.readline().strip()
	auth.set_access_token(key,secret)
	f.close()
except:
	redirect_url = auth.get_authorization_url()
	os.system("firefox '"+redirect_url+"'")
	verifier = input("Verifier: ")
	try:
		auth.get_access_token(verifier)
	except tweepy.TweepError:
		print("Error! Failed to get access token.")
	f = open(tokens,"w")
	f.write(auth.access_token+"\n")
	f.write(auth.access_token_secret+"\n")

api = tweepy.API(auth)

#print(api.configuration())

#public_tweets = api.user_timeline(user_id="floemuc",trim_user=True,include_rts=False,tweet_mode="extended")
public_tweets = tweepy.Cursor(api.user_timeline,user_id="floemuc",trim_user=True,include_rts=False,tweet_mode="extended").items()
# TODO: automatically move the Cursor backwards when it expires
#public_tweets = tweepy.Cursor(api.search_full_archive,label="testing",query="from:floemuc",toDate="202006251935").items()

for tweet in public_tweets:

    data = tweet._json
    date = datetime.strptime(data["created_at"],"%a %b %d %H:%M:%S %z %Y")
    fname = "tweet-"+date.strftime("%Y%m%d-%H%M%S")+".json"
    if "retweeted_status" in data:
        continue
    
    #'Thu Jan 06 18:37:58 +0000 2022'
    print(fname)
    f = open(fname,"w")
    json.dump(data,f,indent=2,ensure_ascii=False)
    f.close()

    new_files.append(fname)

    # plain tweet (with media)
    if "media" in data["entities"]:
        media = data["entities"]["media"]
        for m in media:
            mediaurl = m["media_url_https"]
            get_media(mediaurl)

    try:
        media = data["extended_entities"]["media"]
        for m in media:
            try:
                vi = m["video_info"]["variants"]
                for var in vi:
                   get_media(var["url"])
            except:
                pass
    except:
        pass

# TODO: write to tmpfile, then run svn add --target tmpfile && svn commit -m $DATE
print(new_files)