Skip to content

Instantly share code, notes, and snippets.

@gdsaxton
Last active May 4, 2016 23:21
Show Gist options
  • Save gdsaxton/0dbfa236a7fadb52c93e to your computer and use it in GitHub Desktop.
Save gdsaxton/0dbfa236a7fadb52c93e to your computer and use it in GitHub Desktop.
Loop over list of Twitter user screen_names, create dictionary of users' follower IDs and friend IDs.
for id in screen_names_unique[42:44]:
screen_name = id
print '\n', screen_name
cursor = "-1"
print 'followers list first cursor', cursor
d = get_followers_ids (screen_name, cursor)
limits = t.get_application_rate_limit_status()
follower_limit = limits['resources']['followers']['/followers/ids']['remaining']
print '# followers/ids API calls remaining', follower_limit
if not d:
followers_list = 'protected'
friends_list = 'protected'
org_dict_protected = {'screen_name': screen_name, 'followers_list': followers_list, 'friends_list': friends_list}
#org_list.append(org_dict_protected)
if screen_name not in org_dict:
org_dict[screen_name] = {'followers_list': followers_list, 'friends_list': friends_list}
continue
else:
pass
followers_list = d['ids']
print 'len(followers_list)', len(followers_list)
cursor = d['next_cursor']
print 'followers list second cursor', cursor
count = 2
while cursor != 0 :
print "SLEEPING FOR 75 SECONDS................................"
time.sleep(75)
print 'trying to get another set of 5,000 for page %s' % count
limits = t.get_application_rate_limit_status()
follower_limit = limits['resources']['followers']['/followers/ids']['remaining']
if follower_limit > 1:
print "follower_limit OK...continuing"
#continue #RETURN TO START OF WHILE LOOP
else:
time.sleep(450) #5 minute sleep
#time.sleep(900) #15 minute sleep
#continue
d = get_followers_ids (screen_name, cursor)
followers_list.extend(d['ids'])
cursor = d['next_cursor']
print 'followers list cursor %s' % count, cursor
print 'len(followers_list)', len(followers_list)
count +=1
limits = t.get_application_rate_limit_status()
follower_limit = limits['resources']['followers']['/followers/ids']['remaining']
print '# followers/ids API calls remaining', follower_limit
if follower_limit > 1:
print "follower_limit OK...continuing"
#continue #RETURN TO START OF WHILE LOOP
else:
time.sleep(450) #5 minute sleep
#time.sleep(900) #15 minute sleep
#continue
limits = t.get_application_rate_limit_status()
follower_limit = limits['resources']['followers']['/followers/ids']['remaining']
if follower_limit > 0:
print "follower_limit OK...continuing"
#continue #RETURN TO START OF WHILE LOOP
else:
print "SLEEPING FOR 10 MINUTES....."
time.sleep(600) #10 minute sleep
#continue
print "DONE WITH FOLLOWERS LIST, NOW LET'S GET FRIENDS LIST"
cursor = "-1"
print 'friends list first cursor', cursor
d = get_friends_ids (screen_name, cursor)
if not d:
friends_list = 'no friends'
org_dict_protected = {'screen_name': screen_name, 'followers_list': followers_list, 'friends_list': friends_list}
org_list.append(org_dict_protected)
if screen_name not in org_dict:
org_dict[screen_name] = {'followers_list': followers_list, 'friends_list': friends_list}
continue
else:
pass
friends_list = d['ids']
print 'len(friends_list)', len(friends_list)
cursor = d['next_cursor']
print 'friends list second cursor', cursor
count = 2
while cursor != 0 :
print 'trying to get another set of 5,000 for page %s' % count
d = get_friends_ids (screen_name, cursor)
friends_list.extend(d['ids'])
cursor = d['next_cursor']
print 'friends list cursor %s' % count, cursor
print 'len(friends_list)', len(friends_list)
count +=1
limit = t.get_lastfunction_header('x-rate-limit-remaining')
friend_limit = limits['resources']['friends']['/friends/ids']['remaining']
print '# friends/ids API calls remaining', limit, friend_limit
if friend_limit > 1:
print "friend_limit OK...continuing"
#continue
else:
time.sleep(900) #15 minute sleep
"WAKING UP.....NOW TRYING TO CONTINUE"
continue
#org_dict_for_list = {'screen_name': screen_name, 'followers_list': followers_list, 'friends_list': friends_list}
#org_list.append(org_dict_for_list)
if screen_name not in org_dict:
org_dict[screen_name] = {'followers_list': followers_list, 'friends_list': friends_list}
@amaltese
Copy link

amaltese commented May 4, 2016

Hi! Total neophyte here, but I am hoping to use your code snippets to scrape some data. However, in this instance, I can't seem to figure out where to actually input the main screen name(s) that I want to gather data for. Any guidance is appreciated. Thanks for making all of this public!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment