scraper_base_class

scraper_base_class.py

class ScraperException(Exception):
    """Starting point for Scraper exceptions."""
    pass


class ImdbScraperException(ScraperException):
    """Starting point for Scraper exceptions."""
    pass


class Scraper():
    """Class meant to be parent of various other scrapers.
    
    Attributes:
        chromedriver (chromedriver): a Chrome webdriver for Selenium.
        
    Methods:
        make_soup_with_selenium
        @staticmethod fetch_el_if_available
    """
    def __init__(self):
        driver_service = Service(ChromeDriverManager().install())
        self.chromedriver = webdriver.Chrome(service=driver_service)

    @staticmethod
    def fetch_el_if_available(soup: BeautifulSoup, element_type: str, class_type: str):
        """Returns element text if found, otherwise returns None.
        
        Args:
            soup (BeautifulSoup): a b24 soup.
            element_type (str): HTML type e.g. 'div'.
            class_type (str): the class of the desired element.
            
        Returns:
            element (str): text inside element.
        """
        element = soup.find(element_type, class_type)
        if element is not None:
            element = element.text

        return element