come-follow-me-scraper.R

# simple scraper to compile schedule for 2021 Come Follow Me
# Source: https://www.churchofjesuschrist.org/study/manual/come-follow-me-for-individuals-and-families-doctrine-and-covenants-2021?lang=eng

library(tidyverse)
library(rvest)

# define list of weeks digits
weeks = c(c('01', '02', '03', '04', '05', '06', '07', '08', '09'), as.character(10:56))

# loop over 52 weeks, scraping title content
datalist = list()
for (i in 1:length(weeks)) {
  Sys.sleep(.5)
  print(i)
  url = sprintf('https://www.churchofjesuschrist.org/study/manual/come-follow-me-for-individuals-and-families-doctrine-and-covenants-2021/%s?lang=eng', weeks[i])
  page = read_html(url)
  
  dates = page %>% html_nodes('#title_number1') %>% html_text()
  title = page %>% html_nodes('#title1') %>% html_text()
  scripture = page %>% html_nodes('#title1 .scripture-ref') %>% html_text()
  subtitle = page %>% html_nodes('#subtitle1') %>% html_text()
  temp = tibble(dates = dates, title= title, scripture = scripture, subtitle = subtitle)
  datalist[[i]] = temp
}
raw = do.call(rbind, datalist)

# export
setwd("~/Downloads")
write_csv(raw, 'come-follow-me-2021.csv')