Selenium Scraping http://selenium-python.readthedocs.org/index.html

0.selenium.py

# file list: 
#   base.py
#   element.py (how to get content of element)
#   form.py          
#   javascript.py
#   selector.py
#   wait.py

base.py

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

# firefox
# driver = webdriver.Firefox()

# chrome headless
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gpu')
# options.add_argument('remote-debugging-port=9222')
driver = webdriver.Chrome(chrome_options=options)

driver.get("http://www.python.org")
if "Python" in driver.title:
    print("first aseert success")
else:
    print("ERROR: first assert failed")

elem = driver.find_element_by_name("q")
elem.send_keys("pycon")
elem.send_keys(Keys.RETURN)

if "No results found." not in driver.page_source:
    print("second aseert success")
else:
    print("ERROR: second assert failed")

driver.back()
driver.close()

element.py

# element html
element.get_attribute('innerHTML')
element.get_attribute('outerHTML')

# element text
element.text

# page html
page.source

form.py

# fill text element
element.send_keys("some text")
element.send_keys(" and some", Keys.ARROW_DOWN)
element.clear()

# select form
# http://selenium-python.readthedocs.org/navigating.html

# submit form
# Assume the button has the ID "submit" :)
driver.find_element_by_id("submit").click()
element.submit()

inspect_dom

driver.find_element_by_id('dvResultControls').get_attribute('innerHTML')
driver.find_element_by_id('aspnetForm').text

javascript.py

# run js
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

selector.py

# http://selenium-python.readthedocs.org/locating-elements.html

# id
login_form = driver.find_element_by_id('loginForm')

# name
username = driver.find_element_by_name('username')

# xpath
username = driver.find_element_by_xpath("//form[input/@name='username']")

# css
content = driver.find_element_by_css_selector('p.content')

# link text
link1 = driver.find_element_by_link_text('Continue')
link2 = driver.find_element_by_partial_link_text('Conti')

wait.py

# http://selenium-python.readthedocs.org/waits.html

###########################
#explicit wait
############################
from selenium.webdriver.support import expected_conditions as EC

wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.ID,'someid')))

############################
# implicit wait
##########################
from selenium import webdriver
driver = webdriver.Firefox()
driver.implicitly_wait(10) # seconds
driver.get("http://somedomain/url_that_delays_loading")
myDynamicElement = driver.find_element_by_id("myDynamicElement")
Next  Previous

Thank, so usefull. For who want understand Selenium basic function.

@ghost
My pleasure!