tweepy_scraper

tweet_logger.py

#!/usr/bin/env python
import sys 
from datetime import datetime
import os
import json
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener

consumer_key = os.environ['CONSUMER_KEY']
consumer_secret_key = os.environ['CONSUMER_SECRET_KEY']
access_token = os.environ['ACCESS_TOKEN']
access_secret = os.environ['ACCESS_SECRET']


class TweetTaker(StreamListener):
    def on_data(self, data):
        try:
            with open('tweets_' + '_'.join(track_list)  + '.json', 'a') as f:
                f.write(data)
                if printTweet:
                    print(str(datetime.now())[:10], json.loads(data)['text'])
                return True
        except BaseException as e:
            pass
        return True
    def on_error(self, status):
        print(status)
        return True

if __name__ == '__main__':
    printTweet = True
    track_list = sys.argv[1].split(',')
    auth = OAuthHandler(consumer_key, consumer_secret_key)
    auth.set_access_token(access_token, access_secret)
    l = TweetTaker()
    stream = Stream(auth, l)
    stream.filter(track=track_list)