parlamento_loader.py

import logging
import tortilla
from pci import Popit, Mapit

popit = Popit(
    instance='parlamento',
    host='popit.mysociety.org',
    api_key='a279f48f86fb3177df49f4ef9bc2484a5f6f57c2',
)

mapit = Mapit(
    base_endpoint='http://mapit.openpolis.it/'
)

op_api = tortilla.wrap('http://api.openpolis.it/op/1.0', debug=True)
op_api_auth = ('admin', 'Vakka94')
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)


def prepare_organizations():

    parlamento = popit.organizations.post(data={
        'id': 'parlamento',
        'name': 'Parlamento della Repubblica',
        'other_names': [
            {'name': 'Parlamento', 'note': 'short_name'},
        ]
    })['result']
    print 'Created: %s' % parlamento

    camera = popit.organizations.post(data={
        'id': 'camera',
        'name': 'Camera dei Deputati',
        'parent_id': 'parlamento',
        'other_names': [
            {'name': 'Camera', 'note': 'short_name'},
        ]
    })['result']
    print 'Created: %s' % camera

    senato = popit.organizations.post(data={
        'id': 'senato',
        'name': 'Senato della Repubblica',
        'parent_id': 'parlamento',
        'other_names': [
            {'name': 'Senato', 'note': 'short_name'},
        ]
    })['result']
    print 'Created: %s' % senato


def slugify(name):
    return name.strip().replace(' ', '-').lower()


def create_person(uri, info):
    data={
        'name': u'{last_name} {first_name}'.format(**info),
        'gender': {'M': 'male', 'F': 'female'}[info['sex'].upper()],
        'image': 'http://politici.openpolis.it/politician/picture?content_id=%d' % info['op_politician_id'],
        'birth_date': info['birth_date'].split("T")[0] if info['birth_date'] else '',
        'identifiers':[
            {'identifier': uri, 'scheme': 'op'}
        ]
    }
    print data
    return popit.persons.post(data=data)['result']


def import_parlamento():

    print 'Import della parlamento dalle api.openpolis.it'

    prepare_organizations()

    person_db = {}

    camera_circoscrizioni = dict([(slugify(obj['name']), obj) for obj in mapit.areas.get('CIRCCAM').values()])

    for charge in op_api.decisionmakers.get(params={'context': 'deputato', 'limit': 1500}, auth=op_api_auth)['members']:
        if charge['politician_uri'] not in person_db:
            person_db[charge['politician_uri']] = create_person(charge['politician_uri'], charge['politician'])['id']

        data = {
            'label': charge['charge'],
            'role': 'Deputato',
            'person_id': person_db[charge['politician_uri']],
            'organization_id': 'camera',
            'start_date': charge['date_start'],
            'end_date': charge.get('date_end', None),
        }
        if 'constituency' in charge:
            area_id = slugify(charge['constituency'])
            if area_id in camera_circoscrizioni:
                data['area'] = {
                    'id': '%s-%s' % ('camera', area_id),
                    'name': camera_circoscrizioni[area_id]['name'],
                    'identifier': '%sarea/%s' % (mapit.base_endpoint, camera_circoscrizioni[area_id]['id'])
                }

        charge_result = popit.memberships.post(data=data)['result']
        print 'Created: %s' % charge_result

    senato_circoscrizioni = dict([(slugify(obj['name']), obj) for obj in mapit.areas.get('REG').values()])

    for charge in op_api.decisionmakers.get(params={'context': 'senatore', 'limit': 1500}, auth=op_api_auth)['members']:

        if charge['politician_uri'] not in person_db:
            person_db[charge['politician_uri']] = create_person(charge['politician_uri'], charge['politician'])['id']

        data={
            'label': charge['charge'],
            'role': 'Senatore',
            'person_id': person_db[charge['politician_uri']],
            'organization_id': 'senato',
            'start_date': charge['date_start'],
            'end_date': charge.get('date_end', None),
        }

        if 'constituency' in charge and 'a vita' not in charge['constituency']:
            area_id = slugify(charge['constituency'])
            if area_id in senato_circoscrizioni:
                data['area'] = {
                    'id': '%s:%s' % ('camera', area_id),
                    'name': senato_circoscrizioni[area_id]['name'],
                    'identifier': '%sarea/%s' % (mapit.base_endpoint, senato_circoscrizioni[area_id]['id'])
                }

        charge_result = popit.memberships.post(data=data)['result']
        print 'Created: %s' % charge_result


def reset(popit):

    for chamber in ['parlamento', 'camera', 'senato']:
        popit.organizations(chamber).delete()

    for charge in ['camera-president', 'camera-member', 'senato-president', 'senato-member', 'senato-lifetime-member']:
        popit.posts(charge).delete()

    while True:
        memberships_results = popit.memberships.get(params={'page': 1})
        for membership in memberships_results['result']:
            popit.memberships(membership['id']).delete()
        if memberships_results['total'] < memberships_results['per_page']:
            break

    while True:
        persons_results = popit.persons.get(params={'page': 1})
        for person in persons_results['result']:
            popit.persons(person['id']).delete()
        if persons_results['total'] < persons_results['per_page']:
            break


def load():
    reset(popit)
    import_parlamento()

if __name__ == '__main__':
    load()