Project for Udacity Visualization

EpaReader.R

library(ggplot2)
library(tidyr)
library(dplyr)
library(RColorBrewer)
library(GGally)
library(scales)
library(memisc)
library(gridExtra)
library(reshape2)
library(data.table)

select <- dplyr::select 

#setwd('C:/Users/federico/Documents/Potencials Avaluacions/Blog/EPA/')
setwd('/Volumes/KINGSTON/D3JS/Unemployment/')



inicio = 2005
fin = 2015

#Fase uno: generamos los valores para cada anyo

for (j in seq(inicio,fin)){
  epa.fixed <- read.fwf(paste(paste("datos_t4", toString(j), sep = ""), ".txt", sep = ""), width = c(3,2,2,5,1,2,2,1,1,2,2,2,1,1,2,3,1,3,2,2,2,3,1,2,1,2,3,2,1,1,1,2,2,1,1,1,2,1,1,1,2,2,2,3,3,2,3,1,2,4,4,4,1,4,4,2,1,1,1,2,4,1,1,2,2,1,1,1,1,2,1,1,2,1,1,1,3,1,1,2,1,2,2,2,1,1,1,2,3,1,2,2,7))
  
  dimnames(epa.fixed)[[2]] <- c("CICLO","CCAA","PROV","NVIVI","NIVEL","NPERS","EDAD5","RELPP1","SEXO1","NCONY","NPADRE","NMADRE","RELLMILI","ECIV1","PRONA1","REGNA1","NAC1","EXREGNA1","ANORE1","NFORMA","RELLB1","EDADEST","CURSR","NCURSR","CURSNR","NCURNR","HCURNR","RELLB2","TRAREM","AYUDFA","AUSENT","RZNOTB","VINCUL","NUEVEM","OCUP1","ACT1","SITU","SP","DUCON1","DUCON2","DUCON3","TCONTM","TCONTD","DREN","DCOM","PROEST","REGEST","PARCO1","PARCO2","HORASP","HORASH","HORASE","EXTRA","EXTPAG","EXTNPG","RZDIFH","TRAPLU","OCUPLU1","ACTPLU1","SITPLU","HORPLU","MASHOR","DISMAS","RZNDISH","HORDES","BUSOTR","BUSCA","DESEA","FOBACT","NBUSCA","ASALA","EMBUS","ITBU","DISP","RZNDIS","EMPANT","DTANT","OCUPA","ACTA","SITUA","OFEMP","SIDI1","SIDI2","SIDI3","SIDAC1","SIDAC2","MUN1","PRORE1","REPAIRE1","TRAANT","AOI","CSE","FACTOREL")
  
  dim(epa.fixed)
  names(epa.fixed)
  str(epa.fixed)
#  table(epa.fixed$EDAD5)
  
  
  epa.fixed$provincia <- factor(epa.fixed$PROV)
  epa.fixed$ccaa <- factor(epa.fixed$CCAA)
  #Atur
  epa.fixed$actiu <- ifelse((epa.fixed$AOI == 3) | (epa.fixed$AOI == 4) | (epa.fixed$AOI == 5) | (epa.fixed$AOI == 6), 1,0)
  epa.fixed$aturat <- ifelse((epa.fixed$AOI == 5) | (epa.fixed$AOI == 6),1,0)
  
  epa.fixed$actiu2 <- ifelse((epa.fixed$AOI == 3) | (epa.fixed$AOI == 4) | (epa.fixed$AOI == 5) | (epa.fixed$AOI == 6) | (epa.fixed$AOI == 7), 1,0)
  epa.fixed$aturat2 <- ifelse((epa.fixed$AOI == 5) | (epa.fixed$AOI == 6) | (epa.fixed$AOI == 7),1,0)
  
  epa.fixed$aturvllarg <- ifelse(epa.fixed$aturat == 1 & (epa.fixed$ITBU == 7 | epa.fixed$ITBU == 8),1,0)
  epa.fixed$aturllarg <- ifelse((epa.fixed$aturat == 1) & ((epa.fixed$ITBU == 7) | (epa.fixed$ITBU == 8) | (epa.fixed$ITBU == 5) | (epa.fixed$ITBU == 6)),1,0)
  
  epa.fixed$home <- ifelse((epa.fixed$SEXO == 1),1,0)
  
  epa.fixed$Educ1 <- ifelse((epa.fixed$NFORMA == "AN") | (epa.fixed$NFORMA == "P1") | (epa.fixed$NFORMA == "P2") | (epa.fixed$NFORMA == "S1"), 1, 0)
  epa.fixed$Educ2 <- ifelse((epa.fixed$NFORMA == "SG") | (epa.fixed$NFORMA == "SP"), 1, 0)
  epa.fixed$Educ3 <- ifelse((epa.fixed$NFORMA == "SU"), 1, 0)
  
  
  epa.fixed$jove <- ifelse((epa.fixed$EDAD5 == 16 | epa.fixed$EDAD5 == 20 | epa.fixed$EDAD5 == 25),1,0)
  epa.fixed$gran <- ifelse((epa.fixed$EDAD5 == 45 | epa.fixed$EDAD5 == 50 | epa.fixed$EDAD5 == 55 | epa.fixed$EDAD5 == 60),1,0)
  
  epa.fixed$noef <- ifelse((epa.fixed$CURSR == 3 & epa.fixed$CURSNR == 3 & (epa.fixed$AOI == 5 | epa.fixed$AOI == 6 | epa.fixed$AOI == 7 | epa.fixed$AOI == 8 | epa.fixed$AOI == 9) & (epa.fixed$EDAD5 == 16 | epa.fixed$EDAD5 == 20 | epa.fixed$EDAD5 == 25)),1,0)
  epa.fixed$noefvul <- ifelse((epa.fixed$noef == 1 & epa.fixed$Educ1 == 1),1,0)
  
  epa.fixed$ocupado <- ifelse((epa.fixed$AOI == 3 | epa.fixed$AOI == 4),1,0)
  epa.fixed$emppub <- ifelse((epa.fixed$ocupado == 1 & epa.fixed$SITU == 8),1,0)
  
  y = 2005
  for (i in  seq(133, 173, 4)){
    if (epa.fixed$CICLO == i){
      epa.fixed$year <- y
    }
    y = y + 1
  }
  
  desocupats.prov <- epa.fixed %>%
    filter(!is.na(AOI)) %>%
    mutate(aturjove = aturat * jove,
           aturgran = aturat * gran,
           atureso = aturat * Educ1,
           aturpost = aturat * Educ2,
           aturuniv = aturat * Educ3,
           aturhome = aturat * home,
           joveactiu = jove * actiu,
           homeactiu = home * actiu,
           donaactiu = (1-home) * actiu,
           esoactiu = Educ1 * actiu,
           postactiu = Educ2 * actiu,
           univactiu = Educ3 * actiu,
           aturdona = aturat * (1-home))%>%
    group_by(provincia) %>%
    summarise(year = mean(year),
              Actius = sum(actiu * FACTOREL/100),
              Aturats = sum(aturat * FACTOREL/100),
              AturatsLlT = sum(aturllarg * FACTOREL/100),
              AturatsVLlT = sum(aturvllarg * FACTOREL/100),
              Joves = sum(joveactiu * FACTOREL/100),
              Joves2 = sum(jove * FACTOREL/100),
              Grans = sum(gran * FACTOREL/100),
              Homes = sum(homeactiu * FACTOREL/100),
              Dones = sum(donaactiu * FACTOREL/100),
              Aturat.Joves = sum(aturjove * FACTOREL/100),
              Aturat.Grans = sum(aturgran * FACTOREL/100),
              Aturat.Eso = sum(atureso * FACTOREL/100),
              Aturat.Post = sum(aturpost * FACTOREL/100),
              Aturat.Univ = sum(aturuniv * FACTOREL/100),
              Aturat.Homes = sum(aturhome * FACTOREL/100),
              Aturat.Dones = sum(aturdona * FACTOREL/100),
              EsoMenys = sum(esoactiu * FACTOREL/100),
              PostObl = sum(postactiu * FACTOREL/100),
              Univ = sum(univactiu * FACTOREL/100),
              Noefs = sum(noef * FACTOREL/100),
              Noefs.Vul = sum(noefvul * FACTOREL/100),
              Ocupados = sum(ocupado * FACTOREL/100),
              Emp.publicos = sum(emppub * FACTOREL/100),
              obs = sum(FACTOREL/100),
              n = n())%>%
    mutate(tasa.atur = Aturats/Actius * 100,
           tasa.aturvlt = AturatsVLlT/Actius * 100,
           tasa.aturjove = Aturat.Joves / Joves * 100,
           tasa.jovesaturats = Aturat.Joves / Aturats * 100,
           tasa.aturgran = Aturat.Grans / Grans * 100,
           tasa.gransaturats = Aturat.Grans / Aturats * 100,
           tasa.atureso = Aturat.Eso / EsoMenys * 100,
           tasa.esoaturats = Aturat.Eso / Aturats * 100,
           tasa.aturpost = Aturat.Post / PostObl * 100,
           tasa.postaturats = Aturat.Post / Aturats * 100,
           tasa.aturuniv = Aturat.Univ / Univ * 100,
           tasa.univaturats = Aturat.Univ / Aturats * 100,
           tasa.aturdona = Aturat.Dones / Dones * 100,
           tasa.donaaturtats = Aturat.Dones / Aturats * 100,
           tasa.aturhome = Aturat.Dones / Homes * 100,
           tasa.homeaturats = Aturat.Homes / Aturats * 100,
           tasa.noefs = Noefs / Joves2 * 100,
           tasa.noefsvul = Noefs.Vul / Joves2 * 100,
           tasa.publico = Emp.publicos / Ocupados * 100) %>%
    arrange(provincia)
  
  desocupats.ccaa <- epa.fixed %>%
    filter(!is.na(AOI)) %>%
    mutate(aturjove = aturat * jove,
           aturgran = aturat * gran,
           atureso = aturat * Educ1,
           aturpost = aturat * Educ2,
           aturuniv = aturat * Educ3,
           aturhome = aturat * home,
           joveactiu = jove * actiu,
           homeactiu = home * actiu,
           donaactiu = (1-home) * actiu,
           esoactiu = Educ1 * actiu,
           postactiu = Educ2 * actiu,
           univactiu = Educ3 * actiu,
           aturdona = aturat * (1-home))%>%
    group_by(ccaa) %>%
    summarise(year = mean(year),
              Actius = sum(actiu * FACTOREL/100),
              Aturats = sum(aturat * FACTOREL/100),
              AturatsLlT = sum(aturllarg * FACTOREL/100),
              AturatsVLlT = sum(aturvllarg * FACTOREL/100),
              Joves = sum(joveactiu * FACTOREL/100),
              Joves2 = sum(jove * FACTOREL/100),
              Grans = sum(gran * FACTOREL/100),
              Homes = sum(homeactiu * FACTOREL/100),
              Dones = sum(donaactiu * FACTOREL/100),
              Aturat.Joves = sum(aturjove * FACTOREL/100),
              Aturat.Grans = sum(aturgran * FACTOREL/100),
              Aturat.Eso = sum(atureso * FACTOREL/100),
              Aturat.Post = sum(aturpost * FACTOREL/100),
              Aturat.Univ = sum(aturuniv * FACTOREL/100),
              Aturat.Homes = sum(aturhome * FACTOREL/100),
              Aturat.Dones = sum(aturdona * FACTOREL/100),
              EsoMenys = sum(esoactiu * FACTOREL/100),
              PostObl = sum(postactiu * FACTOREL/100),
              Univ = sum(univactiu * FACTOREL/100),
              Noefs = sum(noef * FACTOREL/100),
              Noefs.Vul = sum(noefvul * FACTOREL/100),
              Ocupados = sum(ocupado * FACTOREL/100),
              Emp.publicos = sum(emppub * FACTOREL/100),
              obs = sum(FACTOREL/100),
              n = n())%>%
    mutate(tasa.atur = Aturats/Actius * 100,
           tasa.aturvlt = AturatsVLlT/Actius * 100,
           tasa.aturjove = Aturat.Joves / Joves * 100,
           tasa.jovesaturats = Aturat.Joves / Aturats * 100,
           tasa.aturgran = Aturat.Grans / Grans * 100,
           tasa.gransaturats = Aturat.Grans / Aturats * 100,
           tasa.atureso = Aturat.Eso / EsoMenys * 100,
           tasa.esoaturats = Aturat.Eso / Aturats * 100,
           tasa.aturpost = Aturat.Post / PostObl * 100,
           tasa.postaturats = Aturat.Post / Aturats * 100,
           tasa.aturuniv = Aturat.Univ / Univ * 100,
           tasa.univaturats = Aturat.Univ / Aturats * 100,
           tasa.aturdona = Aturat.Dones / Dones * 100,
           tasa.donaaturtats = Aturat.Dones / Aturats * 100,
           tasa.aturhome = Aturat.Dones / Homes * 100,
           tasa.homeaturats = Aturat.Homes / Aturats * 100,
           tasa.noefs = Noefs / Joves2 * 100,
           tasa.noefsvul = Noefs.Vul / Joves2 * 100,
           tasa.publico = Emp.publicos / Ocupados * 100) %>%
    arrange(ccaa)

  
  
  
  desocupats.nac <- epa.fixed %>%
    filter(!is.na(AOI)) %>%
    mutate(aturjove = aturat * jove,
           aturgran = aturat * gran,
           atureso = aturat * Educ1,
           aturpost = aturat * Educ2,
           aturuniv = aturat * Educ3,
           aturhome = aturat * home,
           joveactiu = jove * actiu,
           homeactiu = home * actiu,
           donaactiu = (1-home) * actiu,
           esoactiu = Educ1 * actiu,
           postactiu = Educ2 * actiu,
           univactiu = Educ3 * actiu,
           aturdona = aturat * (1-home))%>%
    summarise(year = mean(year),
              Actius = sum(actiu * FACTOREL/100),
              Aturats = sum(aturat * FACTOREL/100),
              AturatsLlT = sum(aturllarg * FACTOREL/100),
              AturatsVLlT = sum(aturvllarg * FACTOREL/100),
              Joves = sum(joveactiu * FACTOREL/100),
              Joves2 = sum(jove * FACTOREL/100),
              Grans = sum(gran * FACTOREL/100),
              Homes = sum(homeactiu * FACTOREL/100),
              Dones = sum(donaactiu * FACTOREL/100),
              Aturat.Joves = sum(aturjove * FACTOREL/100),
              Aturat.Grans = sum(aturgran * FACTOREL/100),
              Aturat.Eso = sum(atureso * FACTOREL/100),
              Aturat.Post = sum(aturpost * FACTOREL/100),
              Aturat.Univ = sum(aturuniv * FACTOREL/100),
              Aturat.Homes = sum(aturhome * FACTOREL/100),
              Aturat.Dones = sum(aturdona * FACTOREL/100),
              EsoMenys = sum(esoactiu * FACTOREL/100),
              PostObl = sum(postactiu * FACTOREL/100),
              Univ = sum(univactiu * FACTOREL/100),
              Noefs = sum(noef * FACTOREL/100),
              Noefs.Vul = sum(noefvul * FACTOREL/100),
              Ocupados = sum(ocupado * FACTOREL/100),
              Emp.publicos = sum(emppub * FACTOREL/100),
              obs = sum(FACTOREL/100),
              n = n())%>%
    mutate(tasa.atur = Aturats/Actius * 100,
           tasa.aturvlt = AturatsVLlT/Actius * 100,
           tasa.aturjove = Aturat.Joves / Joves * 100,
           tasa.jovesaturats = Aturat.Joves / Aturats * 100,
           tasa.aturgran = Aturat.Grans / Grans * 100,
           tasa.gransaturats = Aturat.Grans / Aturats * 100,
           tasa.atureso = Aturat.Eso / EsoMenys * 100,
           tasa.esoaturats = Aturat.Eso / Aturats * 100,
           tasa.aturpost = Aturat.Post / PostObl * 100,
           tasa.postaturats = Aturat.Post / Aturats * 100,
           tasa.aturuniv = Aturat.Univ / Univ * 100,
           tasa.univaturats = Aturat.Univ / Aturats * 100,
           tasa.aturdona = Aturat.Dones / Dones * 100,
           tasa.donaaturtats = Aturat.Dones / Aturats * 100,
           tasa.aturhome = Aturat.Dones / Homes * 100,
           tasa.homeaturats = Aturat.Homes / Aturats * 100,
           tasa.noefs = Noefs / Joves2 * 100,
           tasa.noefsvul = Noefs.Vul / Joves2 * 100,
           tasa.publico = Emp.publicos / Ocupados * 100)
  
  desocupats.nac$ccaa = factor("Agregado")
  desocupats.ccaa = rbind(desocupats.ccaa, desocupats.nac)
  drops <- c("ccaa")
  desocupats.nac <- desocupats.nac[,!(names(desocupats.nac) %in% drops)]
  
  desocupats.nac$provincia = factor("Agregado")
  desocupats.prov = rbind(desocupats.prov, desocupats.nac)
  drops <- c("provincia")
  desocupats.nac <- desocupats.nac[,!(names(desocupats.nac) %in% drops)]
  
  
  desocupats.ccaa$ccaa <- factor(desocupats.ccaa$ccaa,
                                 levels = c(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,51,52,"Agregado"),
                                 labels = c("Andalucia", "Aragon", "Asturias", "Baleares", "Canarias", "Cantabria", "Castilla-Leon", "Castilla-La Mancha", "Catalunya", "Comunitat Valenciana", "Extremadura", "Galicia", "Madrid", "Murcia", "Navarra", "Pais Vasco", "La Rioja", "Ceuta", "Melilla", "Espanya"))
  
  desocupats.ccaa$auxccaa <- as.numeric(desocupats.ccaa$ccaa)
  
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==1] <- 1
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==2]<-2
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==6]<-3
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==8]<-4
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==7]<-5
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==9]<-6
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==18]<-7
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==13]<-8
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==15]<-9
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==10]<-10
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==11]<-11
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==12]<-12
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==4]<-13
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==5]<-14
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==17]<-15
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==16]<-16
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==3]<-17
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==14]<-18
  desocupats.ccaa$id2[desocupats.ccaa$auxccaa==20]<-20
  
  
  desocupats.prov$provincia <- factor(desocupats.prov$provincia,
                                      levels = c(seq(1,52),"Agregado"),
                                      labels = c("Alava", "Albacete", "Alicante", "Almeria", "Avila", "Badajoz", "Baleares", "Barcelona", "Burgos", "Caceres", "Cadiz", "Castellon", "Ciudad Real", "Cordoba", "La Coruna", "Cuenca", "Girona", "Granada", "Guadalajara", "Guipuzcoa", "Huelva", "Huesca", "Jaen", "Leon", "Lleida", "La Rioja", "Lugo", "Madrid", "Malaga", "Murcia", "Navarra", "Orense", "Asturias", "Palencia", "Las Palmas", "Pontevedra", "Salamanca", "Santa Cruz de Tenerife", "Cantabria", "Segovia", "Sevilla", "Soria", "Tarragona", "Teruel", "Toledo", "Valencia", "Valladolid", "Vizcaya", "Zamora", "Zaragoza", "Ceuta",  "Melilla", "Espanya"))
  
  desocupats.prov$auxprov <- as.numeric(desocupats.prov$prov)
  
  desocupats.prov$id2[desocupats.prov$auxprov==15]<-40
  desocupats.prov$id2[desocupats.prov$auxprov==1]<-48
  desocupats.prov$id2[desocupats.prov$auxprov==2]<-13
  desocupats.prov$id2[desocupats.prov$auxprov==3]<-35
  desocupats.prov$id2[desocupats.prov$auxprov==4]<-1
  desocupats.prov$id2[desocupats.prov$auxprov==33]<-51
  desocupats.prov$id2[desocupats.prov$auxprov==5]<-18
  desocupats.prov$id2[desocupats.prov$auxprov==6]<-38
  desocupats.prov$id2[desocupats.prov$auxprov==7]<-44
  desocupats.prov$id2[desocupats.prov$auxprov==8]<-27
  desocupats.prov$id2[desocupats.prov$auxprov==9]<-19
  desocupats.prov$id2[desocupats.prov$auxprov==10]<-39
  desocupats.prov$id2[desocupats.prov$auxprov==11]<-2
  desocupats.prov$id2[desocupats.prov$auxprov==39]<-12
  desocupats.prov$id2[desocupats.prov$auxprov==12]<-36
  desocupats.prov$id2[desocupats.prov$auxprov==51]<-31
  desocupats.prov$id2[desocupats.prov$auxprov==13]<-14
  desocupats.prov$id2[desocupats.prov$auxprov==14]<-3
  desocupats.prov$id2[desocupats.prov$auxprov==16]<-15
  desocupats.prov$id2[desocupats.prov$auxprov==17]<-28
  desocupats.prov$id2[desocupats.prov$auxprov==18]<-4
  desocupats.prov$id2[desocupats.prov$auxprov==19]<-16
  desocupats.prov$id2[desocupats.prov$auxprov==20]<-49
  desocupats.prov$id2[desocupats.prov$auxprov==21]<-5
  desocupats.prov$id2[desocupats.prov$auxprov==22]<-9
  desocupats.prov$id2[desocupats.prov$auxprov==23]<-6
  desocupats.prov$id2[desocupats.prov$auxprov==26]<-47
  desocupats.prov$id2[desocupats.prov$auxprov==35]<-45
  desocupats.prov$id2[desocupats.prov$auxprov==24]<-20
  desocupats.prov$id2[desocupats.prov$auxprov==25]<-29
  desocupats.prov$id2[desocupats.prov$auxprov==27]<-41
  desocupats.prov$id2[desocupats.prov$auxprov==28]<-33
  desocupats.prov$id2[desocupats.prov$auxprov==29]<-7
  desocupats.prov$id2[desocupats.prov$auxprov==52]<-32
  desocupats.prov$id2[desocupats.prov$auxprov==30]<-52
  desocupats.prov$id2[desocupats.prov$auxprov==31]<-34
  desocupats.prov$id2[desocupats.prov$auxprov==32]<-42
  desocupats.prov$id2[desocupats.prov$auxprov==34]<-21
  desocupats.prov$id2[desocupats.prov$auxprov==36]<-43
  desocupats.prov$id2[desocupats.prov$auxprov==37]<-22
  desocupats.prov$id2[desocupats.prov$auxprov==38]<-46
  desocupats.prov$id2[desocupats.prov$auxprov==40]<-23
  desocupats.prov$id2[desocupats.prov$auxprov==41]<-8
  desocupats.prov$id2[desocupats.prov$auxprov==42]<-24
  desocupats.prov$id2[desocupats.prov$auxprov==43]<-30
  desocupats.prov$id2[desocupats.prov$auxprov==44]<-10
  desocupats.prov$id2[desocupats.prov$auxprov==45]<-17
  desocupats.prov$id2[desocupats.prov$auxprov==46]<-37
  desocupats.prov$id2[desocupats.prov$auxprov==47]<-25
  desocupats.prov$id2[desocupats.prov$auxprov==48]<-50
  desocupats.prov$id2[desocupats.prov$auxprov==49]<-26
  desocupats.prov$id2[desocupats.prov$auxprov==50]<-11
  desocupats.prov$id2[desocupats.prov$auxprov==53]<-53
  
  
  write.csv(desocupats.ccaa, file = paste(paste("aturCCAA-",toString(j),sep = ""), ".csv", sep = ""))
  write.csv(desocupats.prov, file = paste(paste("aturProv-",toString(j),sep = ""), ".csv", sep = ""))
}

##Fase dos: Agarramos las tablas de cada anyo y las juntamos
aux <- lapply(Sys.glob("aturCCAA-*.csv"), read.csv)

aturCCAA <-aux[[1]]
for (j in seq(2,(fin - inicio + 1))){
  aturCCAA <- rbind(aturCCAA, aux[[j]])
}
write.csv(aturCCAA, file = "aturCCAA.csv")


aux <- lapply(Sys.glob("aturProv-*.csv"), read.csv)

aturProv <-aux[[1]]
for (j in seq(2,fin - inicio + 1)){
  aturProv <- rbind(aturProv, aux[[j]])
}
write.csv(aturProv, file = "aturProv.csv")

###Leemos el csv y con spread lo transformamos para que sea utilizable con las barras.

aturCCAA <- read.csv("aturCCAA.csv", header = TRUE, sep = ",", quote = "\"",
         dec = ".", fill = TRUE, comment.char = "")

#sels <- c("year", "ccaa", "tasa.atur")
#aturCCAA <- aturCCAA[,(names(aturCCAA) %in% sels)]

tasa.aturCCAA <- aturCCAA %>%
  select(ccaa, year, tasa.atur) %>%
  spread(year, tasa.atur)

write.csv(tasa.aturCCAA, file = "tasaaturCCAA.csv")

aturProv <- read.csv("aturProv.csv", header = TRUE, sep = ",", quote = "\"",
                     dec = ".", fill = TRUE, comment.char = "")

#sels <- c("year", "ccaa", "tasa.atur")
#aturCCAA <- aturCCAA[,(names(aturCCAA) %in% sels)]

tasa.aturProv <- aturProv %>%
  select(provincia, year, tasa.atur) %>%
  spread(year, tasa.atur)

write.csv(tasa.aturProv, file = "tasaaturProv.csv")

###Total Unemployed
ltunemployedCCAA <- aturCCAA %>%
  select(ccaa, year, AturatsLlT) %>%
  spread(year, AturatsLlT)

write.csv(ltunemployedCCAA, file = "ltaturCCAA.csv")

aturProv <- read.csv("aturProv.csv", header = TRUE, sep = ",", quote = "\"",
                     dec = ".", fill = TRUE, comment.char = "")

ltunemployedProv <- aturProv %>%
  select(provincia, year, AturatsLlT) %>%
  spread(year, AturatsLlT)

write.csv(ltunemployedProv, file = "ltaturProv.csv")

index.html

<!DOCTYPE html>
<html lang="en">
	<head>
		<meta charset="utf-8">
		<title>Bar Chart, Framed</title>
		<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.6/d3.min.js"></script>
		<style type="text/css">
			
			body {
				margin: 0;
				background-color: lightGray;
				font-family: Helvetica, Arial, sans-serif;
			}

			div {
				background-color: #F2F2F2;
			}

			#UnemploymentRate {
				width: 700px;
				margin-left: auto;
				margin-right: auto;
				margin-top: 50px;
				padding: 50px;
				background-color: white;
				box-shadow: 3px 3px 5px 6px #ccc;
			}

			#Unemployed {
				width: 700px;
				margin-left: auto;
				margin-right: auto;
				margin-top: 50px;
				padding: 50px;
				background-color: white;
				box-shadow: 3px 3px 5px 6px #ccc;
			}

			h1 {
				margin: 0;
				font-family: Helvetica, Arial, sans-serif;
				font-size: 48px;
				font-weight: bold;
				color: #A01E0C;
				border-bottom: solid 8px #A01E0C;
			}

			h2 {
				font-family: Helvetica, Arial, sans-serif;
				font-size: 24px;
				font-weight: bold;
				color: black;
				text-align:justify;				
			}

			h3 {
				font-family: Helvetica, Arial, sans-serif;
				font-size: 16px;
				font-weight: normal;
				font-style: italic;
				color: black;				
			}


			p {
				font-family: Helvetica;
				font-size: 12px;
				font-weight: normal;
				color: black;
				text-align:justify;
				line-height: 1.3;
				margin: 15px 0 10px 0;
				margin-top: 0em;
   				margin-bottom: 0.5em;
			}

			a:link {
				text-decoration: none;
				color: gray;
			}

			a:hover {
				text-decoration: underline;
			}

			a:visited {
				color: gray;
			}

			a:active {
				color: steelBlue;
			}

			svg {
				background-color: white;
			}

			g.bar text {
				font-size: 10px;
				font-weight: bold;
				text-anchor: end;
				opacity: 0;
			}

			g.bar{
				cursor:pointer;
			}

			g.bar:hover rect {
				fill: #A01E0C;
			}

			g.bar:hover text {
				opacity: 1;
			}

			g.highlight path {
				stroke: rgb(205,10,30);
				stroke-width: 3;
			}
			
			g.area text {
				font-size: 11px;
				font-weight: bold;
				text-anchor: end;
				opacity: 0;
			}

			g.area{
				cursor:pointer;
			}

			g.area:hover rect {
				fill: #A01E0C;
			}

			g.area:hover text {
				opacity: 1;
			}


			.axis path,
			.axis line {
				fill: none;
				stroke: black;
				shape-rendering: crispEdges;
			}
			
			.axis text {
				font-family: sans-serif;
				font-size: 11px;
			}

			.y.axis path,
			.y.axis line {
				opacity: 0;
			}

			.y2.axis path,
			.y2.axis line {
				opacity: 1;
			}

		</style>
	</head>
	<body>

		<div id="UnemploymentRate">

			<h1>Unemployment Rate in Spain</h1>

			<p>The following code shows the unemployment rate in Spain for the last quarter of 2015. The raw data comes from the <a href="www.ine.es">Spanish National Statistics Institute (INE)</a>. In order to convert the raw data into an csv output that could be easily managed by D3, I made use of R, particularly the <a href = "https://www.rstudio.com/wp-content/uploads/2015/02/data-wrangling-cheatsheet.pdf"> tidyr and dplyr packages</a>. The R code can be found in the gist. This is just a first step of a much ambicious that will try to characterize unemployment in Spain. So far, the code produces a bar chart with the unemployment rate by CCAA (first level of government) and by province (second level) and a stacked area chart with the total number of unemployed since 2005 by province or CCAA. The objective is to add more features to the html.</p>

			<input name="updateProv"
           	type="button" 
           	value="By Provincia"
           	onclick="updateProv()"
           	/>

           	<input name="updateCCAA"
           	type="button" 
           	value="By CCAA"
           	onclick="updateCCAA()"
           	/>


		</div>

		<div id="Unemployed">

			<h1>Total number of long term unemployed in Spain.</h1>
			<p>Long term unemployment is defined as people that has been actively seeking for a job but hasn't find any for at least two years.</p>


		</div>


		<script type="text/javascript">	


			var w = 700;
			var h = 600;
			var padding = [ 20, 10, 30, 136 ];  //Top, right, bottom, left


		//Bar Chart
			var widthScale = d3.scale.linear()
								.range([ 0, w - padding[1] - padding[3] ]);
			
			var heightScale = d3.scale.ordinal()
								.rangeRoundBands([ padding[0], h - padding[2] ], 0.1);



			var xAxis = d3.svg.axis()
							.scale(widthScale)
							.orient("bottom");

			var yAxis = d3.svg.axis()
							.scale(heightScale)
							.orient("left");



				//SVG goes into #UnemploymentRate instead of body

			function drawbars(data){	
				data.sort(function(a, b) {
					return d3.descending(+a['2015'],+b['2015']);
				});

				widthScale.domain([ 0, Math.ceil(d3.max(data, function(d) {
					return +d['2015'];
				})) ]);

				heightScale.domain(data.map(function(d) { return d.id1; } ));


				var svg = d3.select("#UnemploymentRate")
						.append("svg")
						.attr("width", w)
						.attr("height", h);

				//Bind data to groups (not bars directly)
				var groups = svg.selectAll("g")
								.data(data);

				groups.exit().remove();

				groups.enter().append("g")
								.attr("class", "bar");

				//Add a rect to each group
				var rects = groups.append("rect")
					.attr("x", padding[3])
					.attr("y", function(d) {
						return heightScale(d.id1);
					})
					.attr("width", 0)
					.attr("height", heightScale.rangeBand())
					.attr("fill", function(d) {
						if (d.id1 === "Espanya") {
							return "rgb(205, 10, 30)";}
						else{
							return "steelblue";}
					});
					//.attr("fill", "steelblue");

				                //Add a text element to each group
                groups.append("text")
                    .attr("x", function(d) {
                        return padding[3] + widthScale(d['2015']) - 3;
                    })
                    .attr("y", function(d) {
                        return heightScale(d.id1) + 10;
                    })
                    .text(function(d) {
                        return Math.round(d['2015']*100)/100 + "%";
                    });





                rects.transition()
                    .delay(function(d, i) {
                        return i * 50;
                    })
                    .duration(1000)
                    .attr("width", function(d) {
                        return widthScale(d['2015']);
                    });

                svg.append("g")
                    .attr("class", "x axis")
                    .attr("transform", "translate(" + padding[3] + "," + (h - padding[2]) + ")")
                    .call(xAxis);

                svg.append("g")
                    .attr("class", "y axis")
                    .attr("transform", "translate(" + padding[3] + ",0)")
                    .call(yAxis);



			//groups.exit().remove();


			}


		//Stack are chart method
			var stack = d3.layout.stack()
							.values(function(d) {
								return d.aturats;
							})
							.order("reverse");

			var dateFormat = d3.time.format("%Y");


			var xScale = d3.time.scale()
								.range([ padding[3], w - padding[1] - padding[3] ]);
			
			var yScale = d3.scale.linear()
								.range([ padding[0], h - padding[2] ]);

						//Configure axis generators
			var xAxis2 = d3.svg.axis()
							.scale(xScale)
							.orient("bottom")
							.ticks(15)
							.tickFormat(function(d) {
								return dateFormat(d);
							});

			var yAxis2 = d3.svg.axis()
							.scale(yScale)
							.orient("left")
							.ticks(5);

			var area = d3.svg.area()
				.x(function(d) {
					return xScale(dateFormat.parse(d3.format("")(d.x)));
				})
				.y0(function(d) {
					return yScale(d.y0);  //Updated
				})
				.y1(function(d) {
					return yScale(d.y0 + d.y);  //Updated
				});

			var color = d3.scale.category10();

			function populate_years(start, end, step){
    		var years = []; //empty years array
    		//YOUR CODE HERE
    		//_.range(start, end, step);
    		for (var yr = start; yr<= end; yr += step) {
        		if (yr !== 1942 && yr!==1946){
           			years.push(yr);
        			}
    			}
    
  		  	return years;  //return years array
			}



			function drawstacked(data){

			//d3.select("svg").remove();

			var svg2 = d3.select("#Unemployed")
					.append("svg")
					.attr("width", w)
					.attr("height", h);

			var years = populate_years(2005,2015,1);

			var dataset = [];

			for (var i = 0; i < data.length; i++) {


				//Create new object with id name and empty array
				dataset[i] = {
					identificador: data[i].id1,
					aturats: []
					};

				//Loop through all the quarters
				for (var j = 0; j < years.length; j++) {

				//Default value, used in case no value is present
				var amount = null;

				// If value is not empty
				if (data[i][years[j]]) {
					amount = +data[i][years[j]];
					}

				//Add a new object to the aturats data array
				//for this identificador
				dataset[i].aturats.push({
					x: years[j],
					y: amount
					});

					}

				}


				//Stack the data!
				stack(dataset);

				//Uncomment to log the original data to the console
				//console.log(data);

				//Uncomment to log the newly restructured dataset to the console
				//console.log(dataset);

			xScale.domain([ 
				d3.min(years, function(d) {
					return dateFormat.parse(d3.format("")(d));
				}),
				d3.max(years, function(d) {
					return dateFormat.parse(d3.format("")(d));
				})
			]);


			var totals = [];

			for (i = 0; i < years.length; i++) {
				totals[i] = 0;
				for (j = 0; j < dataset.length; j++) {
					totals[i] += dataset[j].aturats[i].y;
					}
				}

			yScale.domain([ d3.max(totals), 0 ]);


			var paths = svg2.selectAll("path")
					.data(dataset);
				paths.exit().remove();
				
				paths.enter()
					.append("path")
					//.filter(funcion(d){return d.identificador != "Espanya";})
					.attr("class", "area")
					.attr("d", function(d) {
						return area(d.aturats);
					})
					.attr("stroke", "none")
					.attr("fill", function(d, i) {
						return color(i);
					});

									//Append a title with the country name (so we get easy tooltips)
				paths.append("title")
					.text(function(d) {
						return d.identificador;
					});



				//Create axes
				svg2.append("g")
					.attr("class", "x axis")
					.attr("transform", "translate(0," + (h - padding[2]) + ")")
					.call(xAxis2);

				svg2.append("g")
					.attr("class", "y axis")
					.attr("transform", "translate(" + padding[3] + ",0)")
					.call(yAxis2);

				//paths.exit().remove();







			}

		function datafilter(d){
		data = data.filter(function(d) { return d.id1  != "Espanya";});
		return data;}


		function updateProv(){
		d3.select("svg").remove();
    	d3.csv("tasaaturProv2.csv", 
				//function(data) {
				//  console.log(data);})
				drawbars);
    	d3.select("svg").remove();
    	d3.csv("ltaturProv2.csv", 
				//function(data) {
				//  console.log(data);})
				drawstacked);
		}

		function updateCCAA(){
		d3.select("svg").remove();
    	d3.csv("tasaaturCCAA2.csv", 
				//function(data) {
				//  console.log(data);})
				drawbars);
    	d3.select("svg").remove();
    	d3.csv("ltaturCCAA2.csv", drawstacked);

		
		}

		</script>


		<p class="source"> Font: Spanish National Institute of Statistics <a href="http://www.ine.es/dyngs/INEbase/es/operacion.htm?c=Estadistica_C&cid=1254736176918&menu=resultados&secc=1254736030639&idp=1254735976595">Active Population Survey Microdata </a>
		</p>
	</body>
</html>

ltaturCCAA2.csv

ltaturProv2.csv

tasaaturCCAA2.csv

tasaaturProv2.csv

Summary

The raw data used for this visualization comes from the Spanish National Institute of Statistics and is a collection of individual observation of people regarding their employment status. We processed that data from 2005 to 2015, both at a Province and CCAA level, using an R script. The visualization has the objectives of showing how step was the increase in the total number of long term unemployed in Spain and how much variability does the unemployment rate in Spain has.

Design

Because unemployment rate has a zero lower bound, it was clear for us that a bar chart would be a good way of comparing it between different units. In order to change from CCAA (similar to a state in the US) to Province (lowest level of government supported by the raw data, similar to a county in the US) we added a button to update the data from one to the other. As for the total number of long term unemployed, we went for a stack area graph. This way we can see the dynamics of the phenomena and try to find similarities in the data, in order to understand better what factors explain the crisis.

We are hoping to add more features to the html, as we think of it as a permanent beta version. One of those potential features is a desagregation of the unemployment rate by age and by education attainment, as well as a map of the unemployment in spain on order to have a spatial understanding of the problem.

Feedback

My first design didn't have the option to go from CCAA to Province and instead showed both. This was ok, but it meant more information on the screen than what was necessary so instead we introduced an update button. Also, instead of a stack area we had the lines for all the units which was harder to see, so we followed the advise of stacking them.

Resources - list any sources you consulted to create your visualization

Books

• Scott Murray. Interactive Data Visualization for the Web
• Elijah Meeks - D3.js in Action
• Nick Qi Zhu - Data Visualization With D3.js Cookbook

Websites

• http://stackoverflow.com/questions/10615290/select-data-from-a-csv-before-loading-it-with-javascript-d3-library
• http://bl.ocks.org/mbostock/3808218
• https://bost.ocks.org/mike/circles/
• https://bost.ocks.org/mike/join/
• http://bl.ocks.org/nusstu/bd02dc90290019aa20fd
• https://s3.amazonaws.com/udacity-hosted-downloads/ud651/DataWranglingWithR.pdf