Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#!/usr/bin/python
import json, urllib2
'''
retrieves all ideas from all categories for all agencies, and
normalizes category names from unique numerical ids to shared category
names so they can be compared across agencies. each idea in the list
contains the following keys:
data structure:
agency_ideas = {each_agency : {
'transparency': [idea_objects...],
'participation' = [idea_objects...],
'collaboration' : [idea_objects...],
'innovation' = [idea_objects...]
}
each idea_object has the following keys:
"voteCount", "author", "authorInfo" : {"voteCount", "url", "id":,
"name":, "ideaCount": }, "url", "text", "tags", "id", "commentCount",
"authorId", "timestamp", "title", "myVote", "categoryID"
note: the text of the comments is not included in what's returned by
the API.
'''
# API keys for each agency's ideascale site
agencies = {
"usaid": "your key here...",
"comm":"your key here...",
"dod": "your key here...",
"ed": "your key here...",
"energy": "your key here...",
"nasa":"your key here...",
'dot': "your key here...",
"int": "your key here...",
"va": "your key here...",
"treas": "your key here...",
"gsa": "your key here...",
"opm": "your key here...",
"labor": "your key here...",
"doj": "your key here...",
"ssa": "your key here...",
"state": "your key here...",
"nsf": "your key here...",
"hud": "your key here...",
"epa": "your key here...",
"sba": "your key here...",
"dhs": "your key here...",
"nrc": "your key here...",
"ostp": "your key here...",
}
# category ids for each category for each agency (the category IDs are
# unique for each agency, but the categories they map to are the
# same).
cat_id = {
'usaid': {11832: 'transparency' , 11833: 'participation', 11834:'collaboration', 11835: 'innovation', 11836: 'site_feedback'},
'comm': {11860: 'transparency', 11861: 'participation', 11862: 'collaboration', 11863: 'innovation', 11864: 'site_feedback'},
'dod': {11865: 'transparency', 11866: 'participation', 11867: 'collaboration', 11868: 'innovation', 11869: 'site_feedback'},
'ed': {11870: 'transparency', 11871: 'participation', 11872: 'collaboration', 11873: 'innovation', 11874: 'site_feedback'},
'energy': {11808: 'transparency', 11809: 'participation', 11810: 'collaboration', 11811: 'innovation', 11812: 'site_feedback'},
'nasa': {11571: 'transparency', 11572: 'participation', 11573: 'collaboration', 11928: 'innovation', 11929: 'site_feedback'},
'dot': {11908: 'transparency', 11909: 'participation', 11910: 'collaboration', 11911: 'innovation', 11912: 'site_feedback'},
'int': {11883: 'transparency', 11884: 'participation', 11885: 'collaboration', 11886: 'innovation', 11887: 'site_feedback'},
'va': {11918: 'transparency', 11919: 'participation', 11920: 'collaboration', 11921: 'innovation', 11922: 'site_feedback'},
'treas': {11913: 'transparency', 11914: 'participation', 11915: 'collaboration', 11916: 'innovation', 11917: 'site_feedback'},
'gsa': {11827: 'transparency', 11828: 'participation', 11829: 'collaboration', 11830: 'innovation', 11831: 'site_feedback'},
'opm': {11580: 'transparency', 11581: 'participation', 11582: 'collaboration', 11937: 'innovation', 11938: 'site_feedback'},
'labor': {11893: 'transparency', 11894: 'participation', 11895: 'collaboration', 11896: 'innovation', 11897: 'site_feedback'},
'doj': {11888: 'transparency', 11889: 'participation', 11890: 'collaboration', 11891: 'innovation', 11892: 'site_feedback'},
'ssa': {11653: 'transparency', 11654: 'participation', 11655: 'collaboration', 11941: 'innovation', 11942: 'site_feedback'},
'state': {11903: 'transparency', 11904: 'participation', 11905: 'collaboration', 11906: 'innovation', 11907: 'site_feedback'},
'nsf': {11577: 'transparency', 11578: 'participation', 11579: 'collaboration', 11930: 'innovation', 11931: 'site_feedback'},
'hud': {11878: 'transparency', 11879: 'participation', 11880: 'collaboration', 11881: 'innovation', 11882: 'site_feedback'},
'epa': {11813: 'transparency', 11814: 'participation', 11815: 'collaboration', 11816: 'innovation', 11817: 'site_feedback'},
'sba': {11650: 'transparency', 11651: 'participation', 11652: 'collaboration', 11939: 'innovation', 11940: 'site_feedback'},
'dhs': {11923: 'transparency', 11924: 'participation', 11925: 'collaboration', 11926: 'innovation', 11927: 'site_feedback'},
'nrc': {11583: 'transparency', 11584: 'participation', 11585: 'collaboration', 11935: 'innovation', 11936: 'site_feedback'},
'ostp': {12295: 'transparency', 12296: 'participation', 12297: 'collaboration', 12299: 'innovation', 122300: 'site_feedback'},
}
agency_ideas = {}
for agency in agencies.keys():
key = agencies[agency]
api_base_url = "http://api.ideascale.com/akira/api/ideascale.ideas.getRecentIdeas"
ideas = {}
for category_id, category_name in cat_id[agency].iteritems():
arguments = "?categoryID=%s&apiKey=%s" % (category_id, key)
api_call = api_base_url+arguments
url = urllib2.urlopen(api_call)
js = json.loads(url.read())
ideas[category_name] = js['response']['ideas']
agency_ideas[agency] = ideas
save = open('agency_data.json', 'w')
json.dump(agency_ideas, save)
save.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.