headless-scraper.py

# encoding: utf-8
from __future__ import unicode_literals

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import Select


class Scraper():
    def __init__(self):
        desired = dict(DesiredCapabilities.PHANTOMJS)
        desired["phantomjs.page.settings.userAgent"] = (
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
            "(KHTML, like Gecko) Chrome/15.0.87"
        )
        self.browser = webdriver.PhantomJS(desired_capabilities=desired)

    def __exit__(self, type, value, trackback):
        self.browser.quit()

    @property
    def html(self):
        return self.browser.page_source.encode('utf-8')

    def get(self, url):
        self.browser.get(url)

    def get_frame(self, selector):
        self.browser.switch_to.frame(selector)

    def by_id(self, selector):
        return self.browser.find_element_by_id(selector)

    def by_css(self, selector, multiple=False):
        if multiple:
            return self.browser.find_elements_by_css_selector(selector)
        else:
            return self.browser.find_element_by_css_selector(selector)

    def select_dropdown_option(self, element, value):
        select = Select(element)
        select.select_by_value(value)