mega_church_scrape_v2.R

library(rvest)
library(dplyr)
library(readr)
library(purrr)

# Function to scrape one page
grab_page <- function(page) {
  url <- paste0("https://hirr.hartfordinternational.edu/research/megachurch-database/megachurches-by-state/?sf_paged=", page)
  
  page_content <- read_html(url)
  
  # Extract table rows
  rows <- page_content %>% html_nodes("table tbody tr")
  
  data <- map_df(rows, function(row) {
    cols <- row %>% html_nodes("td") %>% html_text(trim = TRUE)
    
    if (length(cols) == 5) {
      tibble(
        Church = cols[1],
        City = cols[2],
        State = cols[3],
        Attendance = cols[4],
        Denomination = cols[5]
      )
    } else {
      tibble(
        Church = NA,
        City = NA,
        State = NA,
        Attendance = NA,
        Denomination = NA
      )
    }
  })
  
  return(data)
}

# Scrape all 67 pages
total_pages <- 67
data_list <- map_df(1:total_pages, grab_page)

# Save as CSV
write_csv(data_list, "E://data/megachurches_v2.csv")

# Save as Excel
library(writexl)
write_xlsx(data_list, "megachurches.xlsx")

# Print first few rows
print(head(data_list))