driver.py

from selenium import webdriver

def driver():
    option = webdriver.ChromeOptions()
    option.add_argument(' — incognito')
    return webdriver.Chrome(executable_path='/path/to/chromedriver', chrome_options=option)

main.py

from mod.driver import driver
from mod.pre import pre
from mod.list import list
from mod.scroll import scroll
from mod.data import data
from mod.exp import exp

try:
    driver = driver()
    pre(driver)
    scroll(driver)
    list = list(driver)
    data = data(driver, list)
    exp(data)
except Exception as e:
    print(e)
    print('Hello, exception')

pre.py

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def pre(driver):
    driver.get('https://foo_url')
    login_page_link = driver.find_element_by_xpath('//div[@class="ctnLogin"]/p[@class="center"]/a')
    driver.get(login_page_link.get_attribute('href'))
    driver.find_element_by_id('userNameInput').send_keys('usuario')
    driver.find_element_by_id('passwordInput').send_keys('contrasena')
    driver.find_element_by_id('submitButton').submit()
    driver.get('https://foo_url2')

scroll.py

import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

def scroll(driver):
    old = 0
    new = driver.execute_script("window.scrollTo(0, document.body.scrollHeight);return document.body.scrollHeight;")
    while(old!=new):
        WebDriverWait(driver, 20).until(element_has_not_text((By.XPATH, '//div[@id="generic"]'), 'Loading Directory...'))
        old = new
        new = driver.execute_script("window.scrollTo(0, document.body.scrollHeight);return document.body.scrollHeight;")
    
    driver.execute_script("window.scrollTo(0, 0);return 1;")
    
    time.sleep(5)
    

class element_has_not_text(object):
  def __init__(self, locator, text):
    self.locator = locator
    self.text = text
  def __call__(self, driver):
    element = driver.find_element(*self.locator)   # Finding the referenced element
    if self.text != element.get_attribute("textContent"):
        return element
    else:
        return False