Final program to run on system for collecting twitter data.

FinalDataRecorder.py

# Done using Twitter API search  hourly implementation.
import datetime
from google.cloud import firestore
from TwitterAPI import TwitterAPI
import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
from time import sleep

#Global variables
companies =['microsoft','tesla','spacex']

# Use the application default credentials
cred = credentials.Certificate('DataExtractor-c129a8b176a8.json')
default_app = firebase_admin.initialize_app(cred)
db = firestore.client()

#twitter keys
consumer_key = "9z7ncIgRrWVjOHf4j2ig54kTe"
consumer_secret = "Vn3ZGJdGcyZ0dSY4NEMzdX9L7ka9cBzVE4gfVdv9xiJIo2SCv5"
access_key = "1046215934229151745-1jV00mypxAdXCprcm6AiGIDkWffcpH"
access_secret = "8IUsF25hR9HkuOMGl3pwCEuSVqqz851KgpdY2LYiScQYH"
#twitter authentication
api = TwitterAPI(consumer_key, 
                 consumer_secret,
                 access_key,
                 access_secret)


#twitter search parameters
while true :
  for company in companies:
      search_text = company
      search ="#"+search_text
      tweet_count =100
      public_tweets =api.request('statuses/filter', {'track': search })

      for tweet in public_tweets:
            collection_name = search_text+'-data'
            tweet_text =tweet['text']
            tweet_lang =tweet['lang']
            tweet_tags =tweet['entities']['hashtags']
            tweet_name = tweet['user']['screen_name']
            tweet_id =tweet['id']
            tweet_sensitive =tweet['possibly_sensitive']
            tweet_retweet_count = tweet['retweet_count']
            tweet_if_retweeted =tweet['retweeted']
            tweet_favorite_count = tweet['favorite_count']
            tweet_if_favorite=tweet['favorited']
            tweet_timestamp = tweet['created_at']
            user_follower_count =tweet['followers_count']
            verified_user =tweet['user']['verified']
            tweet_reply_id=tweet['in_reply_to_status_id']
            tweet_status =tweet['metadata']['result_type']
            
            if tweet['in_reply_to_status_id']:
                  doc_ref = db.collection(collection_name).document(tweet['in_reply_to_status_id_str'])
                  try:
                        doc = doc_ref.get()

                        print ("the type is ",type(doc.to_dict()))
                        #olddb = doc.to_dict();
                        #new_tweet_cnt = olddb['retweet_count'] + 1
                        #db.collection(collection_name).doc(tweet['in_reply_to_status_id']).set({u'Retweet count':new_tweet_cnt }, {merge: true})
                  except google.cloud.exceptions.NotFound:
                        print(u'No such document!')

            if tweet_lang == "en" :
                  print(str(tweet_id)+'===>>'+tweet_text)
                  data = {
                        u'Tweet Timestamp' : tweet_timestamp,
                        u'Tweet Entity tags': tweet_tags
                        u'Favorite count' : tweet_favorite_count,
                        u'Favorited' : tweet_if_favorite,
                        u'Tweet Reply ID' : tweet_reply_id,
                        u'Tweet Sentivitivity' : tweet_sensitive,
                        u'Retweet count' : tweet_retweet_count,
                        u'Retweeted' : tweet_if_retweeted,
                        u'Username' : tweet_name,
                        u'Tweet Status' : tweet_status,
                        u'User followers_count':user_follower_count,
                        u'Verified User' : verified_user,
                        u'Tweet text' : tweet_text
                       
                  }

                  db.collection(collection_name).document(str(tweet_id)).set(data)
  sleep(3600)