Add to YourNextMP all of the Labour MPs confirmed as standing

gistfile1.py

import requests
import json
import time
import re


###############################################
##                                           ##
##  Add to YourNextMP all of the Labour MPs  ##
##  confirmed as standing                    ##
##                                           ##
###############################################

twfy_api_key = 'theyworkforyou API key'
ynmp_login = 'yournextmp username'
ynmp_password = 'yournextmp password'

source = 'Email from Labour: https://gist.github.com/symroe/2d4a5b112428adc64d32'

# big list of names, taken from the source above
not_standing = [u'Bob Ainsworth', u'Joe Benton', u'Hazel Blears', u'David Blunkett', u'Martin Caton', u'Tony Cunningham', u'John Denham', u'Frank Dobson', u'Frank Doran', u'Hywel Francis', u'Peter Hain', u'Dai Havard', u'David Heyes', u'Glenda Jackson', u'Siân James', u'Tessa Jowell', u'Andrew Love', u'Anne McGuire', u'Andrew Miller', u'Austin Mitchell', u'George Mudie', u'Meg Munn', u'Dawn Primarolo', u'Nick Raynsford', u'Lindsay Roy', u'Joan Ruddock', u'Jack Straw', u'Gerry Sutcliffe', u'Joan Walley', u'Dave Watts', u'Mike Wood', u'Shaun Woodward']

# fetch TheyWorkForYou details of Labour MPs who are _not_ not standing
r = requests.get('http://www.theyworkforyou.com/api/getMPs?party=labour&output=js&key=%s' % twfy_api_key)
labour_mps = json.loads(r.text)
labour_mps_standing = [x for x in labour_mps if x['name'] not in not_standing]

# fetch all constituency IDs from MapIt
r = requests.get('http://mapit.mysociety.org/areas/WMC')
constituencies = json.loads(r.text)
constituencies = {x['name']: id_ for id_, x in constituencies.items()}

# rejig standing labour MPs by constituency ID
labour_mps_by_constituency_id = {constituencies[x['constituency']]: x for x in labour_mps_standing}

ynmp_api_template = 'http://yournextmp.popit.mysociety.org/api/v0.1/posts/%s?embed=membership.person'

# I wrote a thing to figure these out, by outputting
# the name of the current Labour MP
alias = {
    u'Christopher Leslie': u'Chris Leslie',
    u'Stephen McCabe': u'Steve McCabe',
    u'Jonathan Reynolds': u'Jonny Reynolds',
    u'Helen Jones': u' Helen Jones',
    u'Steve Rotheram': u'Steve Rotherham',
    u'Robert Flello': u'Rob Flello',
    u'Alan Meale': u'Joseph Alan Meale',
    u'Jim Sheridan': u'James Sheridan',
    u'John McDonnell': u'John Martin McDonnell',
    u'Edward Miliband': u'Ed Miliband',
    u'Diana Johnson': u'Diana R. Johnson',
    u'Jimmy Hood': u'Jim Hood',
    u'Chi Onwurah': u'Chinyelu Susan Onwurah',
    u'William Bain': u'Willie Bain',
    u'Andy Slaughter': u'Andrew Slaughter',
    u'Edward Balls': u'Ed Balls',
    u'Stephen Pound': u'Steve Pound',
    u'Brian H Donohoe': u'Brian H. Donohoe',
    u'Huw Irranca-Davies': u'Huw Irranca Davies',
}

# not sure if you actually need to do this multiple times?
def get_csrf(page):
    r = s.get(page, verify=False)
    return re.search('csrfmiddlewaretoken\' value=\'(.*?)\'', r.text).group(1)

ynmp_data = {}
with requests.Session() as s:
    # Log in to yournextmp
    login_page = 'https://yournextmp.com/accounts/login/'
    token = get_csrf(login_page)
    r = s.post(login_page, data={'login': ynmp_login, 'password': ynmp_password, 'csrfmiddlewaretoken': token}, verify=False, headers={'Referer': login_page})    
    time.sleep(0.5)
    for constituency_id, mp in labour_mps_by_constituency_id.items():
        # fetch constituency json from YNMP API
        r = requests.get(ynmp_api_template % constituency_id)
        time.sleep(0.5)
        ynmp_data[constituency_id] = json.loads(r.text)['result']
        # look up candidate alias
        mp_name = alias.get(mp['name'], mp['name'])
        candidate_found = False
        candidates = ynmp_data[constituency_id]['memberships']
        for candidate in candidates:
            # try and find candidate by name in YNMP data
            if candidate['person_id']['name'] == mp_name:
                candidate_found = True
                if '2015' not in candidate['person_id']['party_memberships']:
                    # candidate is there, but YNMP doesn't know they're standing
                    page = 'https://yournextmp.com/constituency/%s/' % constituency_id
                    token = get_csrf(page)
                    print '%s _is_ standing in %s (%s)' % (mp_name, mp['constituency'], constituency_id)
                    payload = {
                        'csrfmiddlewaretoken': token,
                        'person_id': candidate['person_id']['id'],
                        'mapit_area_id': constituency_id,
                        'source': source,
                    }
                    s.headers.update({
                        'Referer': page
                    })
                    # update YNMP accordingly
                    t = s.post('https://yournextmp.com/candidacy', data=payload, verify=False)
                    break
        if not candidate_found:
            # YNMP doesn't appear to know about this candidate at all
            print '%s needs adding to %s (%s)' % (mp_name, mp['constituency'], constituency_id)

# now let's check YNMP knows about the MPs who are not standing again
labour_mps_not_standing = [x for x in labour_mps if x['name'] in not_standing]
labour_mps_not_standing_by_constituency_id = {constituencies[x['constituency']]: x for x in labour_mps_not_standing}

for constituency_id, mp in labour_mps_not_standing_by_constituency_id.items():
    r = requests.get(ynmp_api_template % constituency_id)
    ynmp_data[constituency_id] = json.loads(r.text)['result']
    mp_name = alias.get(mp['name'], mp['name'])
    candidate_found = False
    candidates = ynmp_data[constituency_id]['memberships']
    for candidate in candidates:
        if candidate['person_id']['name'] == mp_name:
            # we found the candidate on YNMP...
            candidate_found = True
            if candidate['person_id']['standing_in'].get('2015', '') is not None:
                # ...but they're not marked as not standing.
                page = 'https://yournextmp.com/constituency/%s/' % constituency_id
                print '%s _is not_ standing in %s (%s)' % (mp_name, mp['constituency'], constituency_id)
                break
            else:
                # ...and they're known to be not standing
                print '%s is known not to be standing in %s (%s)' % (mp_name, mp['constituency'], constituency_id)
    if not candidate_found:
        # it's possible no action needs to be taken here -
        # perhaps it's an alias problem. Check these manually.
        print '%s not found - maybe it\'s fine? %s (%s)' % (mp_name, mp['constituency'], constituency_id)