Parse appropriations from Cato XML
| import os | |
| import json | |
| import re | |
| import csv | |
| from bs4 import BeautifulSoup | |
| #format numbers | |
| def commafy(x): | |
| if type(x) not in [type(0), type(0L)]: | |
| raise TypeError("Parameter must be an integer.") | |
| if x < 0: | |
| return '-' + intWithCommas(-x) | |
| result = '' | |
| while x >= 1000: | |
| x, r = divmod(x, 1000) | |
| result = ",%03d%s" % (r, result) | |
| return "%d%s" % (x, result) | |
| #use govtrack's bioguide spreadheet to get state and formal name of lawmakers. we need two lookup tables because they could use 1 of 2 kinds of ids | |
| #the csv comes from https://github.com/unitedstates/congress-legislators | |
| #i have a simple pythons script that converts current_legislators from yaml to csv | |
| bioguide = {} | |
| fbioguide = csv.DictReader(open('bioguide.csv','r')) | |
| for line in fbioguide: | |
| bioguide[line['bioguide']] = line | |
| bioguide_lis = {} | |
| fbioguide = csv.DictReader(open('bioguide.csv','r')) | |
| for line in fbioguide: | |
| bioguide_lis[line['lis']] = line | |
| def removeNonAscii(s): return "".join(filter(lambda x: ord(x)<128, s)) | |
| def process_sponsor(x): | |
| if not x.find('sponsor'): | |
| return (None,None,None,None) | |
| (sponsor_name,sponsor_id) = (x.find('sponsor').string, x.find('sponsor')['name-id']) | |
| b = None | |
| if sponsor_id in bioguide: | |
| b = bioguide[sponsor_id] | |
| elif sponsor_id in bioguide_lis: | |
| b = bioguide_lis[sponsor_id] | |
| if b: | |
| govtrack = b['govtrack'] | |
| state = b['state'] | |
| sponsor_name = b['term_type'].title() + '. ' + b['official_full'] + ' (' + b['party'][0].upper() + '-' + state + ')' | |
| sponsor_name = removeNonAscii(sponsor_name) | |
| else: | |
| govtrack = '' | |
| state = '' | |
| return (sponsor_name,sponsor_id,govtrack,state) | |
| os.system('mkdir output') | |
| os.system('rm dl -rf') | |
| os.system('mkdir dl') | |
| os.system('wget http://deepbills.cato.org/download -O dl/dl.zip') | |
| os.system('unzip dl/dl.zip -d dl') | |
| os.system('chmod 755 -R dl') #weird permissions issue with Cato ZIP file | |
| billlist = [] | |
| reg = re.compile('(\d+)(\w+\d+)') | |
| dir_files = os.listdir('dl/bills') | |
| for f in dir_files: | |
| #split the filename into the congress and bill number to generate link to govtrack | |
| m = reg.match(f.replace('.xml','')) | |
| link = m.groups() | |
| (congress,billnum) = link | |
| #if there's another file representing the passed version of the bill, skip this one | |
| if f.endswith('ih.xml'): | |
| if '%s%seh.xml' % (congress,billnum) in dir_files: | |
| continue | |
| if '%s%srh.xml' % (congress,billnum) in dir_files: | |
| continue | |
| elif f.endswith('is.xml'): | |
| if '%s%ses.xml' % (congress,billnum) in dir_files: | |
| continue | |
| if '%s%srs.xml' % (congress,billnum) in dir_files: | |
| continue | |
| if f.endswith('ih.xml'): | |
| if '%s%seh.xml' % (congress,billnum) in dir_files: | |
| continue | |
| if '%s%srh.xml' % (congress,billnum) in dir_files: | |
| continue | |
| elif f.endswith('is.xml'): | |
| if '%s%ses.xml' % (congress,billnum) in dir_files: | |
| continue | |
| if '%s%srs.xml' % (congress,billnum) in dir_files: | |
| continue | |
| s = removeNonAscii(open('dl/bills/'+f,'r').read()) | |
| x = BeautifulSoup(s,"xml") | |
| title = x.find('title') | |
| try: | |
| title = title.string | |
| except: | |
| pass | |
| if not title: | |
| if x.find('short-title'): | |
| title = x.find('short-title').string | |
| elif x.find('official-title'): | |
| title = x.find('official-title').string | |
| a3 = [] #there can be man appropriations in a bill; this is a list of them | |
| amount = 0 #the running tally of total cost | |
| indefinite = False #does "such sums as necessary" appear anywhere? if so, that changes our dollar amount | |
| for a in x.findAll('funds-and-year'): | |
| print a | |
| if a.string: a3.append(a.string) | |
| if a.get('amount'): | |
| if a['amount']=='indefinite': | |
| indefinite=True | |
| else: | |
| try: | |
| amount += int(a['amount']) | |
| except: | |
| print a, 'amount couldnt be parsed as int' | |
| try: | |
| introdate = x.find('date').string | |
| except: | |
| introdate = None | |
| lastaction = [] | |
| for a in x.findAll('action-date'): | |
| try: | |
| lastaction.append(a['date']) | |
| except: | |
| pass | |
| if x.find('attestation-date') and x.find('attestation-date').get('date') and len(x.find('attestation-date').get('date'))==8: | |
| #special case for getting date from passed bills | |
| #="20130730" chamber="House">Passed the House of Representatives July 30, 2013.</attestation-date> | |
| lastaction.append( x.find('attestation-date').get('date') ) | |
| lastaction.sort() | |
| if len(lastaction)>0: | |
| actiondate = lastaction[-1] | |
| try: | |
| actiondate = actiondate[:4]+'-'+actiondate[4:6]+'-'+actiondate[6:] | |
| except: | |
| pass | |
| else: | |
| actiondate = None | |
| actionall = ';'.join(lastaction) | |
| status = None | |
| try: | |
| status = x.resolution['resolution-stage'] | |
| except: | |
| try: | |
| status = x.bill['bill-stage'] | |
| status = status.replace('-',' ') | |
| except: | |
| pass | |
| pass | |
| purpose = x.find('property',{'name':'purpose'}) | |
| if purpose: purpose = purpose.string | |
| (sponsor_name,sponsor_id,govtrack,state) = process_sponsor(x) | |
| if not sponsor_name and (f.endswith('es.xml') or f.endswith('eh.xml')): | |
| #passed bills dont include sponsors, so open up the file showing its earlier version to grab it | |
| earlier_version = None | |
| if f.endswith('eh.xml'): | |
| status = 'Passed the House' | |
| if '%s%sih.xml' % (congress,billnum) in dir_files: | |
| earlier_version = '%s%sih.xml' % (congress,billnum) | |
| elif '%s%srh.xml' % (congress,billnum) in dir_files: | |
| earlier_version = '%s%srh.xml' % (congress,billnum) | |
| elif f.endswith('is.xml'): | |
| status = 'Passed the Senate' | |
| if '%s%sis.xml' % (congress,billnum) in dir_files: | |
| earlier_version = '%s%sis.xml' % (congress,billnum) | |
| elif '%s%srs.xml' % (congress,billnum) in dir_files: | |
| earlier_version = '%s%srs.xml' % (congress,billnum) | |
| if earlier_version: | |
| s_earlier = removeNonAscii(open('dl/bills/'+earlier_version,'r').read()) | |
| x_earlier = BeautifulSoup(s_earlier,"xml") | |
| (sponsor_name,sponsor_id,govtrack,state) = process_sponsor(x_earlier) | |
| #only list bills with money | |
| if amount>0 or indefinite: | |
| descrip = '; '.join(a3) | |
| if indefinite: | |
| if amount==0: | |
| display_amount = 'such sums as necessary'; | |
| else: | |
| display_amount = '$'+commafy(amount) + ' + such sums as necessary'; | |
| else: | |
| display_amount = '$'+commafy(amount); | |
| d = {'id':f,'title':title,'funds-and-year':descrip,'amount':amount, 'indefinite': indefinite, 'display_amount': display_amount, 'sponsorid':sponsor_id, 'sponsorname':sponsor_name, 'status':status, 'actiondate':actiondate,'introdate': introdate, 'actionall': actionall, 'purpose': purpose, 'congress': congress, 'billnum': billnum, 'govtrack': govtrack, 'state': state} | |
| billlist.append(d) | |
| #order them by how recently anything has happened with them | |
| billlist.sort(key=lambda k: k['actiondate'],reverse=True) | |
| fshort = open('output/short.json','w') | |
| fshort.write('catoshort(' + json.dumps(billlist[:5]) + ');') | |
| fshort.close() | |
| flong = open('output/long.json','w') | |
| flong.write('cato(' + json.dumps(billlist) + ');') | |
| flong.close() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment