An RScript for webscraping the names and associated programming language of each Python, R, and SQL course available on DataCamp.

web_scrape_datacamp_courses.R

##################################################
# An RScript for webscraping the names and       #
# associated programming language of each        #
# Python, R, and SQL course                      #
# available on DataCamp.                         #
##################################################
# Last updated: March 21, 2019                   #
##################################################

## Load packages -----
library(dplyr)
library(tidyr)

## Functions -----
scrape_datacamp_courses <- function(my_url) {
  my_con <- file(my_url, "r")
  my_html <- readLines(my_con, -1)
  close(my_con)
  
  h4_loc <- grep("[<]h4 class[=]\"course[-]block[_][_]title\"[>]", my_html)
  courses <- unlist(lapply(as.list(my_html[h4_loc]), function(x) {trimws(gsub("<.*?>", "", gsub("&nbsp;", "", x)), which = "both")}))
  descr <- unlist(lapply(as.list(my_html[h4_loc+2]), function(x) {trimws(x, which = "both")}))
  
  df <- data.frame(Course = courses, Description = descr)
  
  return(df)
}

## Call the functions -----
py <- scrape_datacamp_courses(my_url = "https://www.datacamp.com/courses/tech:python")
py$Type <- "Python"
r <- scrape_datacamp_courses(my_url = "https://www.datacamp.com/courses/tech:r")
r$Type <- "R"
sql <- scrape_datacamp_courses(my_url = "https://www.datacamp.com/courses/tech:sql")
sql$Type <- "SQL"

## Join data -----
df <- union(py, r) %>%
  union(sql)

## Export data -----
write.csv(df, paste0("data_camp_courses_",Sys.Date(),".csv"))