Mining the Social Web, Example 1-3

ex13.py

import twitter

#gist https://gist.github.com/1592859
twitter_api=twitter.Twitter(domain="api.twitter.com", api_version='1')
WORLD_WOE_ID = 1 # The Yahoo! Where On Earth ID for the entire world
world_trends = twitter_api.trends._(WORLD_WOE_ID) # get back a callable
print [ trend for trend in world_trends()[0]['trends'] ] # call the callable and iterate through the trends returned

twitter_search = twitter.Twitter(domain="search.twitter.com")
search_results = []

for page in range(1,6):
    search_results.append(twitter_search.search(q="barackobama", rpp=100, page=page))

import json
print json.dumps(search_results, sort_keys=True, indent=1)

tweets = []
tweets = [ r['text'] \
    for result in search_results \
        for r in result['results'] ]

words = []
for t in tweets:
    words += [ w for w in t.split() ]

print len(words) # total words
print len(set(words)) # unique words

print 1.0*len(set(words)) / len(words) # lexical diversity
print 1.0*sum([ len(t.split()) for t in tweets ]) / len(tweets) # avg words per tweet

import cPickle
f = open("myData.pickle", "wb")
cPickle.dump(words, f)
f.close()