Parse JSON HTTP stream

json_http_stream.R

# This http data stream contains lines of minified JSON data. Therefore
# we can batch process this line by line. To speed things up, we actually
# process 100 lines at a time, by collapsing them into a JSON array.

library(jsonlite)
stopifnot(packageVersion("jsonlite") >= "0.9.11")

# note that open="r" results in line-by-line reading.
gzstream <- gzcon(url("http://78.46.48.103/sample/hourly_14.json.gz", open="r"))
batches <- list();
i <- 1;
while(length(records <- readLines(gzstream, n = 100))){
  message("Batch ", i, ": found ", length(records), " lines of json...")
  json <- paste0("[", paste0(records, collapse=","), "]")
  batches[[i]] <- fromJSON(json, validate=TRUE)
  i <- i+1
}
close(gzstream)
weather <- rbind.pages(batches)
rm(batches)