lds-conference-speakers.py

  # import libraries
  from selenium import webdriver
  from selenium.webdriver.common.keys import Keys
  import time
  import pandas as pd

  # prepare the links
  years = list(range(1971, 2019))
  months = ['04', '10']

  links = []
  for year in years:
      for month in months:
          link = 'https://www.lds.org/general-conference/{}/{}?lang=eng'.format(year, month)
          links.append(link)

  # define the function
  def SessionInfo(link):
      driver = webdriver.Chrome(executable_path="/Users/erikgregorywebb/Downloads/chromedriver 2")
      driver.get(link)

      title_items = driver.find_elements_by_class_name("lumen-tile__title")
      speaker_items = driver.find_elements_by_class_name("lumen-tile__content")

      titles = []
      speakers = []
      for i in range(0, len(title_items)):
          titles.append(title_items[i].text)
          speakers.append(speaker_items[i].text)

      links = [link] * len(titles)
      session = [titles, speakers, links]
      data = pd.DataFrame(session).transpose()
      driver.close()
      return data

  # collect the data    
  all_sessions = pd.DataFrame()
  for link in links:
      time.sleep(3)
      session = SessionInfo(link)
      all_sessions = all_sessions.append(session, ignore_index=True) 

  # export
  all_sessions.to_csv("all-sessions.csv", index = False)