download_semeval.py

import tweepy
from keys import *
import json
import time
import sys

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)


def download_tweets(_file, tweet_sentiment=True, topic=False, topic_sentiment=False):
	"""
	Takes file with one tweet per line, on the format:

	105121481662541824	#dexter	negative	neutral

	"""

	with open(_file) as infile:
		res = {}

		for line in infile:
			line = line.split()

			_id = int(line.pop(0))

			try:
				tweet_object = api.get_status(_id)._json
			except:
				print("Tweet not found.")
				continue

			if line[-1] == "topic":
				line = line[:-2]
				line.append("off-topic")

			text_topic_sentiment = line.pop()
			text_tweet_sentiment = line.pop()
			text_topic = " ".join(line)

			res["text"] = tweet_object.get("text")
			res["tweet_sentiment"] = text_tweet_sentiment

			if topic:
				res["topic"] = text_topic

			if topic_sentiment:
				res["topci_sentiment"] = text_topic_sentiment

			with open("out.json", "a") as f:
				f.write(json.dumps(res) + "\n")
			print json.dumps(res)


if __name__ == "__main__":
	download_tweets("twitter-train-B-tweet-and-topics.txt")