Download all data from Christchurch City Council's property API

read-chc-data.R

# -----------------------------------------------------------------------------
# Setup
rm(list = ls())
library(magrittr)
library(jsonlite)
library(httr)
library(lubridate)
library(tidyverse)
library(scales)
source("clean-ggplot-theme.R")
# -----------------------------------------------------------------------------


# -----------------------------------------------------------------------------
# Explore Christchurch property data API

# API setup
base_url <- "https://api.ccc.govt.nz/properties/v1/sales?"
client_id <- ""     # Add your ID
client_secret <- "" # Add your secret

# API request function
chc_api_request <- function(start_date, end_date, query = "") {
  print(paste0("Requesting from ", start_date, " to ", end_date))
  params <- paste0("start_date=", start_date, "&end_date=", end_date, query)
  
  get_dat <- GET(url = paste0(base_url, params), 
                 config = add_headers(client_id = client_id, 
                                      client_secret = client_secret))
  
  result <- fromJSON(content(get_dat, as = "text")) 
  
  return(result)
}

# Request data for three days at a time, due to 1000 records limit
start_dates <- seq(ymd("2009-01-01"), ymd("2017-07-31"), by = "3 days")
end_dates <- start_dates + ddays(3)

dat_chc_pages <- list()
page_info <- tibble()
j <- 1
for (i in 1:length(start_dates)) {
  start_date <- start_dates[i]
  end_date <- end_dates[i]
  
  dat_chc_pages[[j]] <- chc_api_request(start_date, end_date)
  
  num_records <- ifelse(is.null(nrow(dat_chc_pages[[j]])), 0, nrow(dat_chc_pages[[j]]))
  print(paste0(num_records, " records retrieved"))
  
  page_info %<>% bind_rows(tibble(start = start_date,
                                  end = end_date, 
                                  num = num_records))
  
  j <- j + 1
  
  if (num_records == 1000) {
    # Limit reached, so try again with offset
    print("Trying again with offset ... ")
    dat_chc_pages[[j]] <- chc_api_request(start_date, end_date, "&offset=1000")
    num_records <- ifelse(is.null(nrow(dat_chc_pages[[j]])), 0, nrow(dat_chc_pages[[j]]))
    print(paste0(num_records, " records retrieved"))
    page_info %<>% bind_rows(tibble(start = start_date,
                                    end = end_date, 
                                    num = num_records))
    j <- j + 1
  }
}

# Combine paged data
dat_chc <- lapply(dat_chc_pages, as.data.frame)
dat_chc_combined <- rbind_pages(dat_chc)

# Write data
write_csv(dat_chc_combined, "dat-chc-combined.csv")