Turkish elections

download.py

#!/usr/bin/env python3
# ---------------------------------------------------------------------------
# download.py - Download Turkish election data
#
# Written by Alexander Shpilkin <ashpilkin@gmail.com>, Feb 2012
#
# To the extent possible under law, the author has dedicated all copyright
# and related and neighboring rights to this software to the public domain
# worldwide.  This software is distributed without any warranty.
#
# See <http://creativecommons.org/publicdomain/zero/1.0/> for the complete
# text of the CC0 Public Domain Dedication.
# ---------------------------------------------------------------------------

from sys import stdout
import os.path
from shutil import copyfileobj
from bs4 import BeautifulSoup
from http.cookiejar import CookieJar
import urllib.request
from urllib.parse import urlencode

def progress(fmt, *args, **named):
    if hasattr(stdout, 'isatty') and stdout.isatty():
        print('\r\033[2K' + fmt.format(*args, **named), end='')
    else:
        print(fmt.format(*args, **named), end=('' if fmt.endswith('\n') else None))
    stdout.flush()

def fnescape(filename):
    return filename.replace('/', '_SLASH_')

PAGE_URL = 'https://sonuc.ysk.gov.tr/module/sonuc.jsf'
FORCE = False

opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(CookieJar()))
opener.addheaders.append(('Referer', PAGE_URL))

progress("[*] Fetching district list")
with opener.open(PAGE_URL) as response:
    root_soup = BeautifulSoup(response.read())

districts = ((tag.string.title(), tag['value']) for tag
             in root_soup.find(id='j_id41:j_id42:j_id62:cmbSecimCevresi')('option')[1:])

empty_terrs = []
for dname, dvalue in districts:
    params = {
        'AJAXREQUEST': '_viewRoot',
        'j_id41': 'j_id41',
        'j_id41:j_id42:j_id110:inputSandikBas': '',
        'j_id41:j_id42:j_id119:inputSandikBit': '',
        'j_id41:j_id42:j_id46:sandikTuruRadio': 0,
        'j_id41:j_id42:j_id62:cmbSecimCevresi': dvalue,
        'j_id41:j_id42:j_id62:j_id71': 'j_id41:j_id42:j_id62:j_id71',
        'j_id41:j_id42:j_id86:cmbMahalle': '',
        'javax.faces.ViewState': root_soup.find(id='javax.faces.ViewState')['value']
    }
    progress("[*] {0}: Fetching territory list", dname)
    with opener.open(PAGE_URL, urlencode(params).encode('latin-1')) as response:
        ajax_soup = BeautifulSoup(response.read())
    del params['j_id41:j_id42:j_id62:j_id71']

    terrs = ((tag.string.title(), tag['value']) for tag
             in ajax_soup.find(id='j_id41:j_id42:j_id97:cmbIlceSecimKurulu')('option')[1:])
    for tname, tvalue in terrs:
        filename = '{0} - {1}.xls'.format(fnescape(dname), fnescape(tname))
        if not FORCE and os.path.exists(filename):
            progress("[+] {0}/{1}: Cached in \"{2}\"", dname, tname, filename)
            continue

        params['j_id41:j_id42:j_id97:cmbIlceSecimKurulu'] = tvalue
        params['j_id41:j_id42:j_id97:j_id106'] = 'j_id41:j_id42:j_id97:j_id106'
        progress("[*] {0}/{1}: (1/3) Selecting territory", dname, tname)
        with opener.open(PAGE_URL, urlencode(params).encode('latin-1')) as response:
            response.read()
        del params['j_id41:j_id42:j_id97:j_id106']

        params['j_id41:j_id42:j_id131'] = 'j_id41:j_id42:j_id131'
        progress("[*] {0}/{1}: (2/3) Fetching results", dname, tname)
        with opener.open(PAGE_URL, urlencode(params).encode('latin-1')) as response:
            data_soup = BeautifulSoup(response.read())
        del params['j_id41:j_id42:j_id131']

        link_elt = data_soup.find(id='j_id41:tabloBilgileriPanel:excelLink')
        if 'href' in link_elt.attrs:
            excel_url = 'https://sonuc.ysk.gov.tr' + link_elt['href']
            progress("[*] {0}/{1}: (3/3) Downloading Excel report", dname, tname)
            with opener.open(excel_url) as response, open(filename, 'wb') as output:
                copyfileobj(response, output)
            progress("[+] {0}/{1}: Saved to \"{2}\"\n", dname, tname, filename)
        else:
            empty_terrs.append("{0}/{1}".format(dname, tname))
            progress("[-] {0}/{1}: Warning: No data available\n", dname, tname)

    progress("[+] {0}: Download complete\n", dname)

progress("[+] Download complete\n")
for terr in empty_terrs:
    progress("[-] Warning: No data for {0}\n".format(terr))