gistfile1.r

# Parses JSON results from WPT e.g. http://www.webpagetest.org/result/140506_W2_c46551bac76ab45d87e7e2f32c3159e6/?f=json
# Relies on JSON to be locally downloaded first (avoid's timeouts when results need to be retrieved from archive)
# Produces a data.frame with selected values from all test results

# url in code is really a path to a file, all.urls is a data frame with a list of files


require(RCurl)
require(stringr)
require(rjson) 

processResults <- function(url) {

	df <- data.frame()

	print(url)

	json <- fromJSON(file=url, method="C")

	for(i in 1:length(json$data$run)) {
		url <- json$data$url
		id <- json$data$id
		bwDown <- json$data$bwDown
		bwUp <- json$data$bwUp
		latency <- json$data$latency

		run <- i

		row <- json$data$run[[i]]

		ttfb <- row$firstView$TTFB
		if(is.null(ttfb)) ttfb <- NA

		render <- row$firstView$render
		if(is.null(render)) render <- NA

		visualComplete <- row$firstView$visualComplete
		if(is.null(visualComplete)) visualComplete <- NA

		loadTime <- row$firstView$loadTime
		if(is.null(loadTime)) loadTime <- NA

		speedIndex <- row$firstView$SpeedIndex
		if(is.null(speedIndex)) speedIndex <- NA

		requests <- length(row$firstView$requests)
		if(is.null(requests)) requests <- NA

		connections <- row$firstView$connections
		if(is.null(connections)) connections <- NA

		bytesIn <- row$firstView$bytesIn
		if(is.null(bytesIn)) bytesIn <- NA

		effectiveBps <- row$firstView$effectiveBps
		if(is.null(effectiveBps)) effectiveBps <- NA

		newrow <- data.frame(url, id, bwDown, bwUp, latency, run, ttfb, render, visualComplete, loadTime, speedIndex, requests, bytesIn, effectiveBps, stringsAsFactors=FALSE, check.rows=FALSE)

		df <- rbind(df, newrow)
	 }
	 return(df)
}

f <- do.call("rbind", apply(all.urls, 1, processResults))