get_errors.py

# run the below command to download all the dependencies first
# pip install -r requirements.txt

# selenium dependencies
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from time import sleep
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

waiting_time = 1000
START_LINE = 11

# https://www.pythontutorial.net/python-basics/python-read-text-file/
urls = []
with open('urls.txt') as f:
    urls = f.readlines()

urls = urls[START_LINE - 1:]
d = DesiredCapabilities.CHROME
d['goog:loggingPrefs'] = {'browser': 'ALL'}
driver = webdriver.Chrome(ChromeDriverManager().install(), desired_capabilities=d)
for url in urls:
    # https://stackoverflow.com/questions/16346914/python-3-2-unicodeencodeerror-charmap-codec-cant-encode-character-u2013-i
    f = open(url[:-1] + ".txt", "w+", encoding='utf-8')
    driver.get("https://" + url)
    # https://stackoverflow.com/questions/20907180/getting-console-log-output-from-chrome-with-selenium-python-api-bindings
    for entry in driver.get_log('browser'):
        print(entry)
        f.write(str(entry))
    f.close()
    sleep(60)