Instantly share code, notes, and snippets.

Embed
What would you like to do?
Scrape Devos
from selenium import webdriver
import time
driver = webdriver.Firefox()
url = "https://www.c-span.org/video/?421224-1/education-secretary-nominee-betsy-devos-testifies-confirmation-hearing"
driver.get(url)
time.sleep(2)
textboxes = driver.find_elements_by_css_selector('td')
for textbox in textboxes:
if textbox.text.startswith("Betsy"):
# print textbox.text
# now click the readmore in this textbox:
textbox.find_element_by_css_selector('span.link-show-full-text').click()
text = textbox.find_element_by_css_selector('p.short_transcript').text
print text
print ""
driver.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment