replace_images.py

#!/usr/bin/env python

import csv
import hashlib
import os
from os.path import dirname, exists, join
import re
import shutil
from urlparse import urljoin

import requests

image_directory = join(dirname(__file__), 'images')

md5sum_to_new_url = {} 

# First get all the images currently in
# candidates.democracyclub.org.uk using its API, and find their MD5sum:

url = 'https://candidates.democracyclub.org.uk/api/v0.9/images/?format=json&page_size=200'

while url:
    r = requests.get(url)
    data = r.json()
    for i in data['results']:
        new_image_url = urljoin(
            'https://candidates.democracyclub.org.uk',
            i['image_url'])
        md5sum_to_new_url[i['md5sum']] = new_image_url
    url = data.get('next')

# Now get the EveryPolitician JSON:

ep_json_url = 'https://raw.githubusercontent.com/everypolitician/everypolitician-data/master/data/UK/Commons/ep-popolo-v1.0.json'

ep_ynmp_image_urls = set()

r = requests.get(ep_json_url)
ep_data = r.json()
for p in ep_data['persons']:
    image_urls = []
    if 'image' in p:
        image_urls.append(p['image'])
    for image in p.get('images', []):
        image_urls.append(image['url'])
    ep_ynmp_image_urls.update(
        i_url for i_url in image_urls
        if re.search(r'yournextmp\.popit\.mysociety\.org.*/image/', i_url))

if not exists(image_directory):
    os.makedirs(image_directory)

ynmp_column = 'YNMP Image URL'
cdc_column = 'Candidates.DC Image URL'

with open('url-correspondence.csv', 'wb') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=[ynmp_column, cdc_column])
    for i_url in ep_ynmp_image_urls:
        target_filename = join(image_directory, hashlib.md5(i_url).hexdigest())
        if not exists(target_filename):
            r = requests.get(i_url, stream=True)
            with open(target_filename, 'wb') as f:
                shutil.copyfileobj(r.raw, f)
        with open(target_filename, 'rb') as f:
            md5sum = hashlib.md5(f.read()).hexdigest()
        if md5sum in md5sum_to_new_url:
            cdc_url = md5sum_to_new_url[md5sum]
            print md5sum, "=>", cdc_url
            writer.writerow({
                ynmp_column: i_url,
                cdc_column: cdc_url,
            })
        else:
            print "Couldn't find:", md5sum

url-correspondence.csv