scrap_CIS.R

setwd("...")
library(RSelenium)
library(netstat)
library(rvest)
library(purrr)

## -------------------------------------------

## URL list

urlbase = "https://analisis.cis.es/formulario.jsp?dwld=/Microdatos/MD"

# https://code.montera34.com/numeroteca/barometro_cis/-/blob/master/data/original/barometros-cis.csv
baros = read.csv("barometros-cis.csv")
baros$url = paste0(urlbase, baros$id, ".zip")

## Setting up Selenium

# Setting up RSelenium server etc
rd = rsDriver(browser = "chrome", chromever = NULL, port = free_port())
remo_dr = rd$client
remo_dr$setTimeout(type = "page load", milliseconds = 100000)

## FOR loop

for(i in 1:nrow(baros)){

  # Wait 5 seconds
  Sys.sleep(5)

  # Url and navigate
  url = baros$url[i]
  remo_dr$navigate(url)

  # Introduce email
  emailbox = remo_dr$findElement(using = "xpath", value = '//*[@id="Email"]')
  emailbox$sendKeysToElement(list("francisco.villamil@uc3m.es"))

  # Click on "investigación"
  clickbox = remo_dr$findElement(using = "id", value = "Terminos")
  clickbox$isElementEnabled()
  clickbox$isElementSelected()

  # Check from the HTML's DOM if the clickbanner already exists
  is_banner_loaded = remo_dr$getPageSource()[[1]] %>%
    read_html() %>%
    html_element(xpath = "//p/input[@id = 'Terminos']") %>%
    is_empty()

  # If not loaded, wait 5"
  if(is_banner_loaded){ Sys.sleep(5) }

  # Locate banner and click
  clickbanner = remo_dr$findElement(using = "xpath",
    value = "//p[@class = 'paceptacion']/label[@for = 'Terminos']")
  clickbanner$clickElement()

  # Check that it's clicked
  clickbox = remo_dr$findElement(using = "id", value = "Terminos")
  if(clickbox$isElementSelected()[[1]]){
    clickbox$sendKeysToElement(list(key = "enter"))
  } else {stop(paste("something wrong? i =", i))}

}


# # Shell loop
# for f in *.zip; do
#   echo "$f"
#   mkdir "${f%.zip}"
#   unzip "$f" -d "${f%.zip}"
#   rm -rvf "$f"
# done