gistfile1.r

library(ggplot2)
library(RJSONIO)
library(foreign)

input_dir = "~/data/twitter-workshop/input/"

snow = read.csv(paste0(input_dir, "snow_pot.csv"))
props = read.dbf(paste0(input_dir, "SecondAuctionProperties.dbf"))

# Note: Need to remove a leading '\' from the MenByTract file before it will load
income = fromJSON(paste0(input_dir, "medianIncomeByTract.json"))
incomeEdu = fromJSON(paste0(input_dir, "medianIncomesByEduAndTract.json"))
incomeMen = fromJSON(paste0(input_dir, "medianIncomesMenByTract.json"))
incomeMenEdu = fromJSON(paste0(input_dir, "medianIncomesMenByEduAndTract.json"))
incomeWomen = fromJSON(paste0(input_dir, "medianIncomesWomenByTract.json"))
incomeWomenEdu = fromJSON(paste0(input_dir, "medianIncomesWomenByEduAndTract.json"))

all_tracts = lapply(income, function(county) 
  lapply(county$tracts, function(tract) as.data.frame(c(county[1:3],tract))))
all_tracts_temp = lapply(all_tracts, function(county) as.data.frame(do.call(rbind, county)))
income_df = as.data.frame(do.call(rbind, all_tracts_temp))
summary(income_df)
ff = as.numeric(income_df$medianIncome)
ff = as.numeric(as.character(income_df$medianIncome))
income_df$medianIncome = as.numeric(as.character(income_df$medianIncome))

# Income and Education
flatten_list <- function(income, numFields=NULL) {
  all_tracts = lapply(income, function(county) 
    lapply(county$tracts, function(tract) as.data.frame(c(county[1:3],tract))))
  all_tracts_temp = lapply(all_tracts, function(county) as.data.frame(do.call(rbind, county)))
  income_df = as.data.frame(do.call(rbind, all_tracts_temp))
  if (length(numFields) >0) {
    for (field in numFields) {
      income_df[,field] = as.numeric(as.character(income_df[,field]))
    }
  }
  income_df
}

income_df <- flatten_list(income, numFields = c("medianIncome"))
incomeEdu_df <- flatten_list(incomeEdu, numFields = c("PreHighSchool", "HighSchool", "SomeCollegeAssociate", "Bachelor", "Pro"))