twitter_streaming_unicode_truncated.py

import twitter, json, sys, csv

# == OAuth Authentication ==
CONSUMER_KEY = ''
CONSUMER_SECRET = ''
OAUTH_TOKEN = ''
OAUTH_TOKEN_SECRET = ''

auth = twitter.oauth.OAuth(OAUTH_TOKEN,
  OAUTH_TOKEN_SECRET,
  CONSUMER_KEY,
  CONSUMER_SECRET)
twitter_api = twitter.Twitter(auth=auth)

csvfile = open('election2020.csv', 'w')
csvwriter = csv.writer(csvfile,delimiter ='|')

q = "election2020"

# clean up our data so we can write unicode to CSV
def clean(val):
    clean = ""
    if val:
        val = val.replace('|', ' ')
        val = val.replace('\n', ' ')
        val = val.replace('\r', ' ')
        clean = val
    return clean

print ('Filtering the public timeline for keyword="%s"' % (q))
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
stream = twitter_stream.statuses.filter(track=q)
for tweet in stream:
    # print json.dumps(tweet)
    try:
        if tweet['truncated']:
            tweet_text = tweet['extended_tweet']['full_text']
        else:
            tweet_text = tweet['text']
        csvwriter.writerow([tweet['id_str'],
                            tweet['created_at'],
                            clean(tweet['user']['screen_name']),
                            clean(tweet_text),
                            tweet['user']['created_at'],
                            tweet['user']['followers_count'],
                            tweet['user']['friends_count'],
                            tweet['user']['statuses_count'],
                            clean(tweet['source']),
                            clean(tweet['user']['location']),
                            tweet['user']['geo_enabled'],
                            tweet['user']['lang'],
                            clean(tweet['user']['time_zone'])
                            ])
        print (tweet_text)
    except Exception as err:
        print (err)
        pass