Make Soup with Selenium

make_soup_with_selenium.py

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

def make_soup_with_selenium(url: str, driver_service: Service) -> BeautifulSoup:
    """Return an HTML body from an URL.

    Args:
      url (str): string representation of a URL address.
      driver_service (Service): a Chrome webdriver.
    
    Returns:
      soup (BeautifulSoup): scraped webpage via bs4.
    """
    chromedriver = webdriver.Chrome(service=driver_service)
    chromedriver.maximize_window()
    chromedriver.get(url)
    sleep(5)  # We want to give the page some time to load up

    page_source = chromedriver.page_source
    soup = BeautifulSoup(page_source, 'lxml')

    return soup

# You can either use a driver like this or install one (Chromedriver, Safari, etc.)
driver_service = Service(ChromeDriverManager().install())