Query the Twitter Public Timeline ~every couple minutes. For each user listed, save their status and location.

scrape_twitter_timeline.py

#Query the Twitter Public Timeline ~every couple minutes. For each user listed, save their status and location.
#I used this to examine what % of twitter users provide resolvable locations, and potentially to analyse 
#those locations (I had an idea I might try passing them thru Yahoo Geolocation API to get a country by 
#dcountry breakdown, but in the end I didn't)

import twitter
import time


#set up files
statuspath="F:\\TWITDATA\\TIMELINE\\"+time.strftime("%b_%d_%Y__%H-%M", time.localtime())+"statuses.txt"
locpath="F:\\TWITDATA\\GEOGRAPHY\\"+time.strftime("%b_%d_%Y__%H-%M", time.localtime())+"locations.txt"

#access to Twitter public timeline
api=twitter.Api()



#loop...
for y in range(10000):
    timeline=api.GetPublicTimeline()  #public timeline requires no authentication

    #open files for writing
    statusfile=open(statuspath, 'a')
    locfile=open(locpath, 'a')

    #file headers
    statusfile.write("\n\n"+time.asctime(time.localtime())+"\n\n")
    locfile.write("\n\n"+time.asctime(time.localtime())+"\n\n")

    #write status info to files
    for s in timeline:
        #status
        statusfile.write(s.user.screen_name+"\n POST:\n")
        try:
            statusfile.write(s.text)
            
        except:
            statusfile.write("***NO MSG OR MSG NOT RETRIEVABLE***")
        
        statusfile.write("\nCREATED AT:\n"+s.created_at)
        statusfile.write("\nStatuss ID: "+str(s.id)+"\n\n")

        #write location
        locfile.write("\n"+s.user.screen_name+"\n")
        try:
            if s.user.location=="":
                locfile.write("***NO LOCATION***\n\n")
            else:
                locfile.write(s.user.location+"\n\n")
                
        except:
            locfile.write("***NO LOCATION***\n\n")

    statusfile.write("\n\n=============FIN================\n\n\n")
    locfile.write("\n\n=============FIN================\n\n\n")
    
    #close files
    statusfile.close()
    locfile.close()

    #sleep..
    time.sleep(110)