parseData.py

import csv
import json
import pygame
import random

# Data source:
# http://ec.europa.eu/eurostat/data/database?p_p_id=NavTreeportletprod_WAR_NavTreeportletprod_INSTANCE_nPqeVbPXRmWQ&p_p_lifecycle=0&p_p_state=normal&p_p_mode=view&p_p_col_id=column-2&p_p_col_pos=1&p_p_col_count=2#
# The three consituent graphs can be generated by tweaking the constants
# uses "abbreviations.json" from https://github.com/samayo/country-json/blob/master/src/country-by-abbreviation.json
# only with an added "uk" for united kingdom, since the original uses GB

FILENAME="gov_10dd_edpt1.tsv"

COUNTRIES = ["NL", "UK", "DE", "FI", "RO", "CY", "DK"]
UNIT = "MIO_EUR"
#UNIT = "PC_GDP"
ITEM = "GD"
#ITEM = "B1GQ"

#SECTOR = "S1"
SECTOR = "S13"


MIN_YEAR = 1995
MAX_YEAR = 2018
if UNIT == "PC_GDP":
    MIN_VALUE=0
    MAX_VALUE=110 #3000000

    Y_FORMAT = "{}%"
    Y_SCALE = 12.5
else:
    MIN_VALUE=0
    if (ITEM == "B1GQ"):
        MAX_VALUE=3500000
    else:
        MAX_VALUE=500000#2250000

    Y_FORMAT = "€{:,},000,000"
    Y_SCALE = 100000#250000

SCREEN_WIDTH = 1800
SCREEN_HEIGHT = 950


MIN_SCREEN_X = 50
MAX_SCREEN_X = 1800
MIN_SCREEN_Y = 950
MAX_SCREEN_Y = 0


LEGEND_X=20
LEGEND_Y = 700

COLORS = (
        (200, 0, 0),
        (0, 200, 0),
        (0, 0, 200),
        (150, 150, 0),
        (0, 150, 150),
        (150, 0, 150),
        (0xff, 0xa5, 0)
    )

def map_value(value, oldmin, oldmax, newmin, newmax):
    return newmin + (newmax - newmin) * (value - oldmin) / (oldmax - oldmin) 

with open(FILENAME) as f:
    data = tuple(csv.reader(f, delimiter="\t"))

data = [[i[0].split(","), *(j.strip() for j in i[1:])] for i in data]

with open("abbreviations.json") as f:
    abbreviations = json.load(f)
    abbreviations = {i["abbreviation"]:i["country"] for i in abbreviations}

print("data length: "+str(len(data)))
print("\t".join(str(i) for i in data[0]))



mappedData = [[i for i in data if i[0][1]==SECTOR and i[0][3]==country and i[0][0]==UNIT and i[0][2]==ITEM]
              for country in COUNTRIES]
print ("mapped data length: "+str(len(mappedData[0])))

t = [set() for i in data[0][0]]
for dat in data[1:]:
    for i, key in enumerate(dat[0]):
        t[i].add(key)

t2 = [list(i) for i in t]
for i in t2:
    i.sort()

i = 0
while (i <= max(len(f) for f in t2)):
    for index, da in enumerate(t2):
        if i < len(da):
            l = [dat for dat in data if dat[0][index]==da[i]]
            s=da[i]+" ("+str(len(l))+")"
        elif i == len(da):
            s = "TOTAL: "+str(len(da))
        else:
            s=""
        
        print (s+(" "*(20-len(s))), end="")
    i+=1
    print()

try:
    pygame.init()
    font = pygame.font.SysFont("Tahoma", 18)
    screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
    screen.fill((255,255,255))
    colors = {}

    for year in range(MIN_YEAR, MAX_YEAR):
        if (year % 2 == 0):
            pygame.draw.line(screen, (200, 200, 200),
                             (int(map_value(year, MIN_YEAR, MAX_YEAR, MIN_SCREEN_X, MAX_SCREEN_X)),
                              0),
                             (int(map_value(year, MIN_YEAR, MAX_YEAR, MIN_SCREEN_X, MAX_SCREEN_X)),
                              SCREEN_WIDTH)
                             )
        surf = font.render(str(year),
                           1,
                           (0, 0, 0))
        screen.blit(surf,
                    (int(map_value(year, MIN_YEAR, MAX_YEAR, MIN_SCREEN_X, MAX_SCREEN_X) - surf.get_width()/2),
                     0))

    y = MIN_VALUE
    while y < MAX_VALUE:
        pygame.draw.line(screen, (200, 200, 200),
                         (MIN_SCREEN_X,
                          int(map_value(y, MIN_VALUE, MAX_VALUE, MIN_SCREEN_Y, MAX_SCREEN_Y))),
                          (MAX_SCREEN_X,
                           int(map_value(y, MIN_VALUE, MAX_VALUE, MIN_SCREEN_Y, MAX_SCREEN_Y)))
                         )
        surf = font.render(Y_FORMAT.format(y),
                           1,
                           (0, 0, 0))
        screen.blit(surf,
                    (MAX_SCREEN_X - surf.get_width(),
                     int(map_value(y, MIN_VALUE, MAX_VALUE, MIN_SCREEN_Y, MAX_SCREEN_Y))))

        y+=Y_SCALE
        
    
    for index, filteredData in enumerate(mappedData):
        if (len(filteredData)!=1):
            raise ValueError(len(filteredData))
        color = COLORS[index]
        colors[COUNTRIES[index]]=color

        last_year = None
        last_value = None
        
        for year in range(MIN_YEAR, MAX_YEAR):
            column = data[0].index(str(year))
            try:
                value = float(filteredData[0][column])
            except ValueError:
                continue
            #print(value)
            
            pygame.draw.circle(screen, color,
                               (int(map_value(year, MIN_YEAR, MAX_YEAR, MIN_SCREEN_X, MAX_SCREEN_X)),
                                int(map_value(value, MIN_VALUE, MAX_VALUE, MIN_SCREEN_Y, MAX_SCREEN_Y))),
                               4
                               )

            if (year != MIN_YEAR and last_year is not None and last_value is not None):
                pygame.draw.aaline(screen, color,
                                 (int(map_value(year, MIN_YEAR, MAX_YEAR, MIN_SCREEN_X, MAX_SCREEN_X)),
                                int(map_value(value, MIN_VALUE, MAX_VALUE, MIN_SCREEN_Y, MAX_SCREEN_Y))),
                                 (int(map_value(last_year, MIN_YEAR, MAX_YEAR, MIN_SCREEN_X, MAX_SCREEN_X)),
                                int(map_value(last_value, MIN_VALUE, MAX_VALUE, MIN_SCREEN_Y, MAX_SCREEN_Y))),
                                 )
            last_year = year
            last_value = value
                                 
            
    for index, (country, color) in enumerate(sorted(colors.items(), key=lambda i:i[1])):
        screen.fill(color, (LEGEND_X, LEGEND_Y+20*index, 15, 15))
        screen.blit(font.render(abbreviations[country.upper()], 1, (0,0,0)),
                    (LEGEND_X+20, LEGEND_Y+20*index))

    pygame.image.save(screen, UNIT+"_"+ITEM+"_"+SECTOR+".png")
    
    pygame.display.flip()

    running = True
    while running:
        for event in pygame.event.get():
            if event.type==pygame.QUIT:
                running=False
                               
finally:
    pygame.quit()