fidelity_scraper.py

# low level webdriver functions

import os
import time

from selenium import webdriver
from selenium.common.exceptions import StaleElementReferenceException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


class FidelityWebdriver(object):
    def __init__(self, selenium_url='http://localhost:4444/wd/hub', caps=None, auto_login=True):
        self.logged_in = False
        self.username = 'fidelity-username'
        self.password = 'hunter42'
        self.trading_portfolio = 'Z03812123'

        if caps is None:
            caps = {
                'browserName': os.getenv('BROWSER', 'chrome'),
                'chromeOptions': {
                    'args': [
                        # '--no-sandbox',
                        '--user-agent=Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25'
                    ]
                }
            }

        try:
            self.driver = webdriver.Chrome(
                executable_path='/usr/bin/chromedriver',
                desired_capabilities=caps
            )
            # self.driver = webdriver.Remote(
            #     command_executor=selenium_url,
            #     desired_capabilities=caps
            # )
        except Exception:
            raise

        if auto_login:
            try:
                self.login()
            except Exception:
                raise

    def _wait_for_element(self, el_type, el_value):
        ec_type = getattr(By, el_type.upper())

        try:
            element = WebDriverWait(self.driver, 10).until(
                EC.presence_of_element_located((ec_type, el_value))
            )
        except Exception as e:
            raise ValueError("_Wait_for_element failed to detect element {}={}: {}".format(el_type, el_value, e))

        return element

    @staticmethod
    def link_has_gone_stale(old_id):
        try:
            # poll the link with an arbitrary call
            old_id.find_elements_by_id('dont matter')
            return False
        except StaleElementReferenceException:
            return True

    def login(self):
        # logs into fidelity
        # userId-input // password // fs-login-button
        if self.logged_in:
            return

        login_url = 'https://digital.fidelity.com/prgw/digital/login/full-page'
        self.driver.get(login_url)

        try:
            username_text = self._wait_for_element('ID', 'userId-input')
            password_text = self._wait_for_element('ID', 'password')
            submit_button = self._wait_for_element('ID', 'fs-login-button')
            device = self._wait_for_element('NAME', 'DEVICE_PRINT')
            print("device_name: {}".format(device))
        except Exception as e:
            raise ValueError("login failed to find username / password / submit elements: {}".format(e))

        try:
            username_text.send_keys(self.username)
            password_text.send_keys(self.password)
            self.driver.save_screenshot("pre-login.png")
            password_text.send_keys(Keys.ENTER)
            # submit_button.click()
        except Exception as e:
            raise ValueError("login failed to send_keys / click: {}".format(e))

        # while not self.link_has_gone_stale(submit_button):
        #     time.sleep(1)
        time.sleep(5)

        self.logged_in = True
        self.driver.save_screenshot("post-login.png")

    def logout_and_quit(self):
        try:
            self.driver.get('https://login.fidelity.com/ftgw/Fidelity/RtlCust/Logout/Init')
            self.driver.quit()
        except Exception:
            pass

    def get_price_of_symbol(self, symbol='IBM'):
        # given a symbol, pulls current stock price
        self.driver.save_screenshot("pre-get-ticker.png")

        price_url = 'https://snapshot.fidelity.com/fidresearch/snapshot/landing.jhtml#/research?symbol={}'.format(
            symbol)

        self.driver.get(price_url)

        try:
            price_string = self._wait_for_element('ID', 'summary-quote')
        except Exception as e:
            # self.logout_and_quit()
            raise ValueError("get_price_of_symbol failed to find price element: {}".format(e))

        print("hi")


f = FidelityWebdriver()
f.get_price_of_symbol(symbol='IBM')
print("fyad lol")
f.logout_and_quit()