A simple twiiter program to collect a tweet on a query using tweepy.

collect_tweet.py

import tweepy
import json
from sqlite_utils import Database

def authenticate():
    # twitter API credentials
    cred = json.load(open('.cred.json', 'r'))
    auth = tweepy.OAuthHandler(consumer_key=cred['api_key'], consumer_secret=cred['api_secret_key'])
    auth.set_access_token(key=cred['access_token'], secret=cred['access_token_secret'])
    api = tweepy.API(auth)
    return api

def get_data(obj, field):
    if '.' in field:
        first, second = field.split('.')
        inner_obj = getattr(obj, first)
        return getattr(inner_obj, second)
    else:
        return getattr(obj, field)


def process_tweet(tweet, q):
    data = {}
    fields = ['full_text', 'source', 'id_str', 'retweet_count', 'lang', 'user.name', 'user.screen_name',
             'is_quote_status', 'created_at', 'in_reply_to_user_id_str', 'in_reply_to_status_id_str']
    if hasattr(tweet, 'retweeted_status'):
        obj = status.retweeted_status
    else:
        obj = tweet
    for field in fields:
        data[field] = get_data(obj, field)
    data["query"] = q
    return data

def store_tweets():
    db = Database("tweets_storage.db")
    table = db['tweets']
    for tweet in tweets:
        try:
            table.insert(tweet, pk='id_str')
        except Exception as exc:
            print(exc)
    
def collect_tweets():
    api = authenticate()
    tweets = []
    query = "Nihilism"
    cursor = tweepy.Cursor(api.search,
                           q=query,
                           lang='en',
                           tweet_mode='extended',
                           include_entities=True).items()
    count = 0
    for status in cursor:
        tweets.append(process_tweet(status, q=query))
        if count == 100:
            count = 0
            store_tweets(tweets)
            tweets = []
        else:
            count += 1


collect_tweets()