script to get electricity data from pjm.com

get_electricity_data.R

library(purrr)

data_dir <- here::here()
base_url <- "http://www.pjm.com"
url <- sprintf("%s/pub/operations/hist-meter-load", base_url)
years <- 1993L:2018L

# check for robots.txt
robotstxt::get_robotstxt(base_url)

.get_data <- function(year) {
  data_url <- sprintf("%s/%s-hourly-loads.xls", url, year)
  file_name <- basename(data_url)
  file_path <- file.path(data_dir, file_name)
  if (!file.exists(file_path)) {
    download.file(data_url, file_path)
    Sys.sleep(sample(5, 1)) # be nice
  }
}

get_data <- memoise::memoise(.get_data) # be nice and DRY

# test get_data on 1993
get_data(1993)

# get all years
walk(years, get_data)