scrape_stations_by_state

library(pacman)
p_load(curl, kirkegaard)


# fetch lists of stations -------------------------------------------------

for (i in 1:56) {
  message(i)
  
  #if i < 10, pad 0
  if (i < 10) {
    i = "0" + i
  }
  
  #make url
  url = "http://www.ncdc.noaa.gov/cdo-web/api/v2/stations?limit=1000&offset=1&datasetid=normal_ann&locationid=FIPS%3A" + i + "&sortfield=name"
  
  #make handle
  # l_headers = c('User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0',
  #               'Accept: */*',
  #               'Accept-Language: en-US,en;q=0.5',
  #               'token: 0x2a',
  #               'X-Requested-With: XMLHttpRequest',
  #               'Referer: http://www.ncdc.noaa.gov/cdo-web/datatools/normals',
  #               'Cookie: JSESSIONID=7A20CD2BCB4025788F05BCF72E1F0FDF; _ga=GA1.2.1535824352.1459267703; _gat=1; _ga=GA1.3.1535824352.1459267703; _gat_GSA_ENOR0=1; __utma=198104496.1535824352.1459267703.1459267703.1459267703.1; __utmb=198104496.2.10.1459267703; __utmc=198104496; __utmz=198104496.1459267703.1.1.utmcsr=t.co|utmccn=(referral)|utmcmd=referral|utmcct=/CJ7yyAO3rn; __utmt_GSA_CP=1',
  #               'Connection: keep-alive')
  #add headers
  handle = new_handle()
  handle_setheaders(handle, "User-Agent" = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0",
                    "Accept" = "*/*",
                    "Accept-Language" = "en-US,en;q=0.5",
                    "token" = "0x2a",
                    "X-Requested-With" = "XMLHttpRequest",
                    "Referer" = "http://www.ncdc.noaa.gov/cdo-web/datatools/normals",
                    "Cookie" = "JSESSIONID=7A20CD2BCB4025788F05BCF72E1F0FDF; _ga=GA1.2.1535824352.1459267703; _gat=1; _ga=GA1.3.1535824352.1459267703; _gat_GSA_ENOR0=1; __utma=198104496.1535824352.1459267703.1459267703.1459267703.1; __utmb=198104496.2.10.1459267703; __utmc=198104496; __utmz=198104496.1459267703.1.1.utmcsr=t.co|utmccn=(referral)|utmcmd=referral|utmcct=/CJ7yyAO3rn; __utmt_GSA_CP=1",
                    "Connection" = "keep-alive")

  #call curl
  curl_download(url = url, handle = handle, destfile = "state_" + i + ".json")
}