wikiscrape.R

require(rvest)
require(dplyr)


scrape <- function(year, top) {
  possible <- c(10, 25, 100)
  if (!top %in% possible) stop("top must be 10, 25 or 100!")
  if (!year %in% 2012:2014) stop("data available only for years 2012 - 2014")
  
  targets <- top[1:which(possible == top)]
  separate <- sapply(possible, function(t) sub("rank", t, "tbody > tr[class='toprank']"))
  
  sel <- paste(separate, collapse = ", ")
  url <- sub("year", year, "https://tools.wmflabs.org/wikitrends/year.html")
  
  page <- html(url)
  clean <- function(text) sapply(strsplit(text, "\n"), "[", 1)
  
  item <- page %>%
    html_nodes(sel) %>%
    html_text() %>%
    clean() %>%
    gsub(pattern = ".*\\. ", replacement = "", x = .)
  
  language <- page %>%
    html_nodes("a[name]") %>%
    html_attr("name") %>%
    rep(., each = top)
  
  count <- page %>%
    html_nodes(sel) %>%
    html_nodes(".c") %>%
    html_text() %>%
    Filter(function(x) x != "View count", .) %>%
    gsub(pattern = " ", replacement = "", x = as.character(.)) %>%
    as.numeric()
  
  tbl_df(data.frame(language = language, item = item, count = count, stringsAsFactors = FALSE))
}