Extraction_Example.py

#tweets is the html data as a bs4 (beautifulsoup) object
def writeTweets(tweets):
    newTweetRecords = []
    for tweet in tweets:
        try:
            if tweet.find("a", {"class" : "js-action-profile-promoted"}):
                continue
            text = tweet.find("p", {"class" : "tweet-text"}).get_text()
            date = tweet.find("span", {"class" : "_timestamp"})["data-time-ms"]
            tweetId = tweet['data-item-id']
            tweetRecord = {"text" : text, "date" : date, "tweetId" : tweetId}
            newTweetRecords.append(tweetRecord)
        except:
                print("Unable to process tweet")