yunruse/byelection.py

## byelection.py
'''
UK by-election analysis by Mia yun Ruse. Public domain.

As seen on:
https://commons.wikimedia.org/wiki/File:UK_By-elections.png
https://en.wikipedia.org/wiki/List_of_United_Kingdom_by-elections_(2010–present)

Data is sourced from:
https://www.parliament.uk/about/how/elections-and-voting/by-elections/

Historical data before the 2005 election is available (but unused) from:
https://researchbriefings.files.parliament.uk/documents/CBP-7529/CBP-7529.pdf#page=45
'''

from dataclasses import dataclass
from typing import List, Optional, Tuple
from datetime import datetime, timedelta
import csv
import re

import numpy as np
from matplotlib import pyplot
from matplotlib.dates import AutoDateLocator

pyplot.rc('figure', figsize=(10, 5), dpi=200)
pyplot.rc('figure.subplot', left=0.1, right=0.95)
pyplot.rc('savefig', pad_inches=0.05)

START_DATE = "2005-05-05"
END_DATE = "2022-02-10"


@dataclass
class Election:
    date: datetime
    elected: str
    party: str
    swing: Optional[str]


@dataclass
class ByElection:
    constituency: str
    date_leaving: datetime
    MP_leaving: Optional[str]
    reason: str
    election: Optional[Election]


REASON_MP = re.compile(r'(.+?) (?:against|of) (.+?)(?:, closed)? (\d.+)')
REASON = re.compile(r'(.+?) (\d.+?)')
ELECTION_RESULT = re.compile(r'(.+?) \((.+) (.+?)\)')
DATE_FMTS = [
    '%Y-%m-%d',
    '%d.%m.%y',
    '%d.%m.%Y',
    '%d %B %Y',
    '%d %B %Y.',
    '%d %b %Y',
    '%d %b %Y.',
]


def date(string):
    for fmt in DATE_FMTS:
        try:
            return datetime.strptime(string, fmt)
        except ValueError:
            continue


with open('elections.txt') as f:
    ELECTIONS = [date(line.strip()) for line in f.readlines()]


byelections: List[ByElection] = []
waits: List[Tuple[timedelta, ByElection]] = []

with open('byelections.txt') as f:
    for (
        constituency,
        reasons,
        election_date,
        election_result
    ) in csv.reader(f, delimiter='\t'):
        election_date = date(election_date)
        result = ELECTION_RESULT.match(election_result)
        if election_date is None or result is None:
            election = None
        else:
            candidate, party, swing = result.groups()
            swing = swing.lower()
            election = Election(election_date, candidate, party, swing)

        if match := REASON_MP.match(reasons):
            reason, MP_leaving, date_leaving = match.groups()
        elif match := REASON.match(reasons):
            reason, date_leaving = match.groups()
            MP_leaving = None
        else:
            raise ValueError('Weird election string:' + reasons)

        byelection = ByElection(
            constituency,
            date(date_leaving),
            MP_leaving,
            reason,
            election,
        )
        byelections.append(byelection)
        if byelection.election and byelection.date_leaving:
            wait = byelection.election.date - byelection.date_leaving
            waits.append((wait, byelection))

if __name__ == '__main__':

    COLORS = {
        'Resignation': 'tab:orange',
        'Death': 'tab:blue',
        'Successful recall petition': 'tab:green',
    }
    seen_reasons = set()
    pyplot.title('Wait for UK by-elections')

    for wait, byelection in waits:
        r = byelection.reason

        x = [byelection.date_leaving, byelection.election.date]
        y = [wait.days]*2
        col = COLORS[r]
        label = None
        if r not in seen_reasons:
            seen_reasons.add(r)
            label = r
        pyplot.plot(x, y, label=label, c=col)
        pyplot.scatter(x, y, s=8, marker='x', c=col)

    # prediction
    xs = [b.election.date for _, b in waits]
    x_ords = [x.toordinal() for x in xs]
    ys = [w.days for w, _ in waits]
    trend = np.polyfit(x_ords, ys, deg=1)
    pyplot.plot(xs, np.poly1d(trend)(x_ords), dashes=[3, 2],
                label='Trendline (all)', c='tab:gray', lw=1)

    # vertical data
    pyplot.ylabel('Duration of seat being empty / days')
    for i, d in enumerate(ELECTIONS):
        label = 'General elections' if i == 0 else ''
        pyplot.axvline(d, c='tab:gray', label=label)

    # horizontal data
    loc = AutoDateLocator()
    YEARLY = 0
    loc.intervald = {YEARLY: [1]}
    pyplot.gca().xaxis.set_major_locator(loc)
    pyplot.xlabel('Date')
    pyplot.xlim(date(START_DATE), date(END_DATE))

    pyplot.legend()
    pyplot.savefig('byelection.png')

## byelections.txt
North Shropshire	Resignation of Rt Hon Owen Paterson, 5 November 2021	16 December 2021
Southend West	Death of Sir David Amess, 15 October 2021
Old Besley and Sidcup	Death of James Brokenshire, 7 October 2021	2 December 2021	Louie French (Conservative hold)
Batley and Spen	Resignation of Tracy Brabin, 13 May 2021	1 July 2021	Kim Leadbeater (Labour hold)
Chesham and Amersham	Death of Dame Cheryl Gillan, 4 April 2021	17 June 2021	Sarah Green (Liberal Democrats gain)
Airdrie and Shotts	Resignation of Neil Gray, 24 March 2021	13 May 2021	Anum Qaisar-Javed (SNP hold)
Hartlepool	Resignation of Mike Hill, 16 March 2021	6 May 2021	Jill Mortimer (Conservative gain)
Brecon and Radnorshire	Successful recall petition against Chris Davies, closed 20 June 2019	1 August 2019	Jane Dodds (Liberal Democrats win)
Peterborough	Successful recall petition against Fiona Onasanya, closed 1 May 2019	6 June 2019	Lisa Forbes (Labour Win)
Newport West	Death of Paul Flynn, 17 February 2019	4 April 2019	Ruth Jones (Labour hold)
Lewisham East	Resignation of Heidi Alexander, 9 May 2018	14 June 2018	Janet Daby (Labour hold)
Manchester Gorton	Death of Sir Gerald Kaufman 26.02.17	This seat was contested at the General Election on 8 June 2017	Afzal Khan (Labour hold)
Copeland	Resignation of Jamie Reed 23.01.17	23 February 2017	Trudy Harrison (Conservative gain)
Stoke-on-Trent Central	Resignation of Tristram Hunt 23.01.17	23 February 2017	Gareth Snell (Labour hold)
Sleaford and North Hykeham	Resignation of Stephen Phillips 04.11.16	8 December 2016	Caroline Johnson (Conservative hold)
Richmond Park	Resignation of Zac Goldsmith 25.10.16	1 December 2016	Sarah Olney (Liberal Democrat gain)
Witney	Resignation of David Cameron 12.09.16	20 October 2016	Robert Courts (Conservative hold)
Batley and Spen	Death of Jo Cox 16.06.16	20 October 2016	Tracy Brabin (Labour hold)
Tooting	Resignation of Sadiq Khan 09.05.2016	16 June 2016	Rosena Allin-Khan (Labour hold)
Ogmore	Resignation of Huw Irranca-Davies 23.03.2016	5 May 2016	Chris Elmore (Labour hold)
Sheffield, Brightside and Hillsborough	Death of Harry Harpham 04.02.2016	5 May 2016	Gill Furniss (Labour hold)
Oldham West and Royton	Death of Michael Meacher 21.10.2015	3 December 2015	Jim McMahon (Labour hold)
Rochester and Strood	Resignation of Mark Reckless 30.09.14	20 November 2014	Mark Reckless (UK Independence Party Win)
Heywood and Middleton	Death of Jim Dobbin 07.09.14	9 October 2014	Liz McInnes (Labour Hold)
Clacton	Resignation of Douglas Carswell 29.08.14	9 October 2014	Douglas Carswell (UK Independence Party Win)
Newark	Resignation of Patrick Mercer 30.04.14	5 June 2014	Robert Jenrick (Conservative Win)
Wythenshawe and Sale East	Death of Paul Goggins 07.01.14	13 February 2014	Michael Kane (Labour Hold)
South Shields	Resignation of David Miliband 15.04.13	2 May 2013	Emma Lewell-Buck (Labour Hold)
Eastleigh	Resignation of Chris Huhne 05.02.13	28 February 2013	Mike Thornton (Lib Dem Hold)
Mid-Ulster	Resignation of Martin McGuiness 02.01.13	7 March 2013	Francie Molloy (Sinn Fein Hold)
Rotherham	Resignation of Rt Hon Denis MacShane 05.11.12	29 November 2012	Sarah Champion (Labour Hold)
Manchester Central	Resignation of Tony Lloyd 22.10.12	15 November 2012	Lucy Powell (Labour Hold)
Cardiff South and Penarth	Resignation of Rt Hon Alun Michael 22.10.12	15 November 2012	Stephen Doughty (Labour/Co-op Hold)
Middlesbrough	Death of Sir Stuart Bell 13.10.12	29 November 2012	Andy McDonald (Labour Hold)
Croydon North	Death of Malcolm Wicks 29.09.12	29 November 2012	Steve Reed (Labour Hold)
Corby	Resignation of Louise Mensch 29.08.12	15 November 2012	Andy Sawford (Labour/Co-op Win)
Bradford West	Resignation of Marsha Singh 02.03.12	29 March 2012	George Galloway (Respect Win)
Feltham and Heston	Death of Alan Keen 14.11.11	15 December 2011	Seema Malhotra (Labour Hold)
Inverclyde	Death of David Cairns 09.05.11	30 June 2011	Iain McKenzie (Labour Hold)
Leicester South	Resignation of Sir Peter Soulsby 01.04.11	5 May 2011	Jon Ashworth (Labour hold)
Barnsley Central	Resignation of Eric Illsley 08.02.11	3 March 2011	Dan Jarvis (Labour hold)
Belfast West	Resignation of Mr Gerry Adams 26.01.11	9 June 2011	Paul Maskey (Sinn Fein Hold)
Oldham East and Saddleworth	Previous election declared void 05.11.10	13 January 2011	Debbie Abrahams (Labour hold)
Strangford	Resignation of Iris Robinson (DUP) 15 January 2009.	No by-election called.	Jim Shannon (elected at 2010 general election)
North West Leicestershire	Death of David Taylor (Lab/Co-op) 26 December 2009.	No by-election called.	Andrew Bridgen (elected at 2010 general election)
Glasgow North East	Resignation of Rt Hon Michael Martin (Speaker) 22 June 2009.	12 November 2009	William Bain (Labour win)
Norwich North	Resignation of Dr Ian Gibson (Lab) 8 June 2009.	23 July 2009	Chloe Smith (Conservative gain)
Glenrothes	Death of John MacDougall (Lab) 12 August 2008.	6 November 2008	Lindsay Roy (Labour hold)
Glasgow East	Resignation of David Marshall (Lab) 30 June 2008.	24 July 2008	John Mason (SNP gain)
Haltemprice and Howden	Resignation of Rt Hon David Davis (Con) 18 June 2008.	10 July 2008	David Davis (Conservative hold)
Henley	Resignation of Boris Johnson (Con) 4 June 2008.	26 June 2008	John Howell (Conservative hold)
Crewe and Nantwich	Death of Gwyneth Dunwoody (Lab) 17 April 2008.	22 May 2008	Edward Timpson (Conservative gain)
Ealing Southall	Death of Piara Khabra (Lab) 19 June 2007.	19 July 2007	Virendra Sharmar (Labour hold)
Sedgefield	Resignation of Rt Hon Tony Blair (Lab) 27 June 2007.	19 July 2007	Phil Wilson (Labour hold)
Bromley and Chislehurst	Death of Rt Hon Eric Forth (Con) 17 May 2006.	29 June 2006	Bob Neill (Conservative hold)
Blaenau Gwent	Death of Peter Law (Ind) 25 Apr 2006.	29 June 2006	Dai Davies (Independent hold)
Dunfermline and West Fife	Death of Rachel Squire (Lab) 5 January 2006.	9 Feb 2006	Willie Rennie (Lib Dem gain)
Livingston	Death of Rt Hon Robin Cook (Lab) 6 August 2005.	29 Sep 2005	Jim Devine (Labour hold)
Cheadle	Death of Patsy Calton (LD) 29 May 2005.	14 Jul 2005	Mark Hunter (Lib Dem hold)

## elections.txt
2005-05-05
2010-04-06
2015-05-07
2017-06-08
2019-12-12
	'''
	UK by-election analysis by Mia yun Ruse. Public domain.

	As seen on:
	https://commons.wikimedia.org/wiki/File:UK_By-elections.png
	https://en.wikipedia.org/wiki/List_of_United_Kingdom_by-elections_(2010–present)

	Data is sourced from:
	https://www.parliament.uk/about/how/elections-and-voting/by-elections/

	Historical data before the 2005 election is available (but unused) from:
	https://researchbriefings.files.parliament.uk/documents/CBP-7529/CBP-7529.pdf#page=45
	'''

	from dataclasses import dataclass
	from typing import List, Optional, Tuple
	from datetime import datetime, timedelta
	import csv
	import re

	import numpy as np
	from matplotlib import pyplot
	from matplotlib.dates import AutoDateLocator

	pyplot.rc('figure', figsize=(10, 5), dpi=200)
	pyplot.rc('figure.subplot', left=0.1, right=0.95)
	pyplot.rc('savefig', pad_inches=0.05)

	START_DATE = "2005-05-05"
	END_DATE = "2022-02-10"


	@dataclass
	class Election:
	date: datetime
	elected: str
	party: str
	swing: Optional[str]


	@dataclass
	class ByElection:
	constituency: str
	date_leaving: datetime
	MP_leaving: Optional[str]
	reason: str
	election: Optional[Election]


	REASON_MP = re.compile(r'(.+?) (?:against\|of) (.+?)(?:, closed)? (\d.+)')
	REASON = re.compile(r'(.+?) (\d.+?)')
	ELECTION_RESULT = re.compile(r'(.+?) \((.+) (.+?)\)')
	DATE_FMTS = [
	'%Y-%m-%d',
	'%d.%m.%y',
	'%d.%m.%Y',
	'%d %B %Y',
	'%d %B %Y.',
	'%d %b %Y',
	'%d %b %Y.',
	]


	def date(string):
	for fmt in DATE_FMTS:
	try:
	return datetime.strptime(string, fmt)
	except ValueError:
	continue


	with open('elections.txt') as f:
	ELECTIONS = [date(line.strip()) for line in f.readlines()]


	byelections: List[ByElection] = []
	waits: List[Tuple[timedelta, ByElection]] = []

	with open('byelections.txt') as f:
	for (
	constituency,
	reasons,
	election_date,
	election_result
	) in csv.reader(f, delimiter='\t'):
	election_date = date(election_date)
	result = ELECTION_RESULT.match(election_result)
	if election_date is None or result is None:
	election = None
	else:
	candidate, party, swing = result.groups()
	swing = swing.lower()
	election = Election(election_date, candidate, party, swing)

	if match := REASON_MP.match(reasons):
	reason, MP_leaving, date_leaving = match.groups()
	elif match := REASON.match(reasons):
	reason, date_leaving = match.groups()
	MP_leaving = None
	else:
	raise ValueError('Weird election string:' + reasons)

	byelection = ByElection(
	constituency,
	date(date_leaving),
	MP_leaving,
	reason,
	election,
	)
	byelections.append(byelection)
	if byelection.election and byelection.date_leaving:
	wait = byelection.election.date - byelection.date_leaving
	waits.append((wait, byelection))

	if __name__ == '__main__':

	COLORS = {
	'Resignation': 'tab:orange',
	'Death': 'tab:blue',
	'Successful recall petition': 'tab:green',
	}
	seen_reasons = set()
	pyplot.title('Wait for UK by-elections')

	for wait, byelection in waits:
	r = byelection.reason

	x = [byelection.date_leaving, byelection.election.date]
	y = [wait.days]*2
	col = COLORS[r]
	label = None
	if r not in seen_reasons:
	seen_reasons.add(r)
	label = r
	pyplot.plot(x, y, label=label, c=col)
	pyplot.scatter(x, y, s=8, marker='x', c=col)

	# prediction
	xs = [b.election.date for _, b in waits]
	x_ords = [x.toordinal() for x in xs]
	ys = [w.days for w, _ in waits]
	trend = np.polyfit(x_ords, ys, deg=1)
	pyplot.plot(xs, np.poly1d(trend)(x_ords), dashes=[3, 2],
	label='Trendline (all)', c='tab:gray', lw=1)

	# vertical data
	pyplot.ylabel('Duration of seat being empty / days')
	for i, d in enumerate(ELECTIONS):
	label = 'General elections' if i == 0 else ''
	pyplot.axvline(d, c='tab:gray', label=label)

	# horizontal data
	loc = AutoDateLocator()
	YEARLY = 0
	loc.intervald = {YEARLY: [1]}
	pyplot.gca().xaxis.set_major_locator(loc)
	pyplot.xlabel('Date')
	pyplot.xlim(date(START_DATE), date(END_DATE))

	pyplot.legend()
	pyplot.savefig('byelection.png')
	North Shropshire Resignation of Rt Hon Owen Paterson, 5 November 2021 16 December 2021
	Southend West Death of Sir David Amess, 15 October 2021
	Old Besley and Sidcup Death of James Brokenshire, 7 October 2021 2 December 2021 Louie French (Conservative hold)
	Batley and Spen Resignation of Tracy Brabin, 13 May 2021 1 July 2021 Kim Leadbeater (Labour hold)
	Chesham and Amersham Death of Dame Cheryl Gillan, 4 April 2021 17 June 2021 Sarah Green (Liberal Democrats gain)
	Airdrie and Shotts Resignation of Neil Gray, 24 March 2021 13 May 2021 Anum Qaisar-Javed (SNP hold)
	Hartlepool Resignation of Mike Hill, 16 March 2021 6 May 2021 Jill Mortimer (Conservative gain)
	Brecon and Radnorshire Successful recall petition against Chris Davies, closed 20 June 2019 1 August 2019 Jane Dodds (Liberal Democrats win)
	Peterborough Successful recall petition against Fiona Onasanya, closed 1 May 2019 6 June 2019 Lisa Forbes (Labour Win)
	Newport West Death of Paul Flynn, 17 February 2019 4 April 2019 Ruth Jones (Labour hold)
	Lewisham East Resignation of Heidi Alexander, 9 May 2018 14 June 2018 Janet Daby (Labour hold)
	Manchester Gorton Death of Sir Gerald Kaufman 26.02.17 This seat was contested at the General Election on 8 June 2017 Afzal Khan (Labour hold)
	Copeland Resignation of Jamie Reed 23.01.17 23 February 2017 Trudy Harrison (Conservative gain)
	Stoke-on-Trent Central Resignation of Tristram Hunt 23.01.17 23 February 2017 Gareth Snell (Labour hold)
	Sleaford and North Hykeham Resignation of Stephen Phillips 04.11.16 8 December 2016 Caroline Johnson (Conservative hold)
	Richmond Park Resignation of Zac Goldsmith 25.10.16 1 December 2016 Sarah Olney (Liberal Democrat gain)
	Witney Resignation of David Cameron 12.09.16 20 October 2016 Robert Courts (Conservative hold)
	Batley and Spen Death of Jo Cox 16.06.16 20 October 2016 Tracy Brabin (Labour hold)
	Tooting Resignation of Sadiq Khan 09.05.2016 16 June 2016 Rosena Allin-Khan (Labour hold)
	Ogmore Resignation of Huw Irranca-Davies 23.03.2016 5 May 2016 Chris Elmore (Labour hold)
	Sheffield, Brightside and Hillsborough Death of Harry Harpham 04.02.2016 5 May 2016 Gill Furniss (Labour hold)
	Oldham West and Royton Death of Michael Meacher 21.10.2015 3 December 2015 Jim McMahon (Labour hold)
	Rochester and Strood Resignation of Mark Reckless 30.09.14 20 November 2014 Mark Reckless (UK Independence Party Win)
	Heywood and Middleton Death of Jim Dobbin 07.09.14 9 October 2014 Liz McInnes (Labour Hold)
	Clacton Resignation of Douglas Carswell 29.08.14 9 October 2014 Douglas Carswell (UK Independence Party Win)
	Newark Resignation of Patrick Mercer 30.04.14 5 June 2014 Robert Jenrick (Conservative Win)
	Wythenshawe and Sale East Death of Paul Goggins 07.01.14 13 February 2014 Michael Kane (Labour Hold)
	South Shields Resignation of David Miliband 15.04.13 2 May 2013 Emma Lewell-Buck (Labour Hold)
	Eastleigh Resignation of Chris Huhne 05.02.13 28 February 2013 Mike Thornton (Lib Dem Hold)
	Mid-Ulster Resignation of Martin McGuiness 02.01.13 7 March 2013 Francie Molloy (Sinn Fein Hold)
	Rotherham Resignation of Rt Hon Denis MacShane 05.11.12 29 November 2012 Sarah Champion (Labour Hold)
	Manchester Central Resignation of Tony Lloyd 22.10.12 15 November 2012 Lucy Powell (Labour Hold)
	Cardiff South and Penarth Resignation of Rt Hon Alun Michael 22.10.12 15 November 2012 Stephen Doughty (Labour/Co-op Hold)
	Middlesbrough Death of Sir Stuart Bell 13.10.12 29 November 2012 Andy McDonald (Labour Hold)
	Croydon North Death of Malcolm Wicks 29.09.12 29 November 2012 Steve Reed (Labour Hold)
	Corby Resignation of Louise Mensch 29.08.12 15 November 2012 Andy Sawford (Labour/Co-op Win)
	Bradford West Resignation of Marsha Singh 02.03.12 29 March 2012 George Galloway (Respect Win)
	Feltham and Heston Death of Alan Keen 14.11.11 15 December 2011 Seema Malhotra (Labour Hold)
	Inverclyde Death of David Cairns 09.05.11 30 June 2011 Iain McKenzie (Labour Hold)
	Leicester South Resignation of Sir Peter Soulsby 01.04.11 5 May 2011 Jon Ashworth (Labour hold)
	Barnsley Central Resignation of Eric Illsley 08.02.11 3 March 2011 Dan Jarvis (Labour hold)
	Belfast West Resignation of Mr Gerry Adams 26.01.11 9 June 2011 Paul Maskey (Sinn Fein Hold)
	Oldham East and Saddleworth Previous election declared void 05.11.10 13 January 2011 Debbie Abrahams (Labour hold)
	Strangford Resignation of Iris Robinson (DUP) 15 January 2009. No by-election called. Jim Shannon (elected at 2010 general election)
	North West Leicestershire Death of David Taylor (Lab/Co-op) 26 December 2009. No by-election called. Andrew Bridgen (elected at 2010 general election)
	Glasgow North East Resignation of Rt Hon Michael Martin (Speaker) 22 June 2009. 12 November 2009 William Bain (Labour win)
	Norwich North Resignation of Dr Ian Gibson (Lab) 8 June 2009. 23 July 2009 Chloe Smith (Conservative gain)
	Glenrothes Death of John MacDougall (Lab) 12 August 2008. 6 November 2008 Lindsay Roy (Labour hold)
	Glasgow East Resignation of David Marshall (Lab) 30 June 2008. 24 July 2008 John Mason (SNP gain)
	Haltemprice and Howden Resignation of Rt Hon David Davis (Con) 18 June 2008. 10 July 2008 David Davis (Conservative hold)
	Henley Resignation of Boris Johnson (Con) 4 June 2008. 26 June 2008 John Howell (Conservative hold)
	Crewe and Nantwich Death of Gwyneth Dunwoody (Lab) 17 April 2008. 22 May 2008 Edward Timpson (Conservative gain)
	Ealing Southall Death of Piara Khabra (Lab) 19 June 2007. 19 July 2007 Virendra Sharmar (Labour hold)
	Sedgefield Resignation of Rt Hon Tony Blair (Lab) 27 June 2007. 19 July 2007 Phil Wilson (Labour hold)
	Bromley and Chislehurst Death of Rt Hon Eric Forth (Con) 17 May 2006. 29 June 2006 Bob Neill (Conservative hold)
	Blaenau Gwent Death of Peter Law (Ind) 25 Apr 2006. 29 June 2006 Dai Davies (Independent hold)
	Dunfermline and West Fife Death of Rachel Squire (Lab) 5 January 2006. 9 Feb 2006 Willie Rennie (Lib Dem gain)
	Livingston Death of Rt Hon Robin Cook (Lab) 6 August 2005. 29 Sep 2005 Jim Devine (Labour hold)
	Cheadle Death of Patsy Calton (LD) 29 May 2005. 14 Jul 2005 Mark Hunter (Lib Dem hold)