jrosell/selenium_intercept_url.py

## selenium_intercept_url.py
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time


def intercept_requests(url, consent_css = "button[id*='onetrust-accept-btn-handler']"):
    print("Start ##############################")
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    driver = webdriver.Chrome(
        service=Service(ChromeDriverManager().install()),
        options=chrome_options
    )
    driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {'source': """
    window.intercepted_urls = [] || window.intercepted_urls;
    let oldXHROpen = window.XMLHttpRequest.prototype.open;
    window.XMLHttpRequest.prototype.open = function(method, url, async, user, password) {
        this.addEventListener('load', function() {
            console.log('intercepted '+url+': ' + this.responseText);
            window.intercepted_urls.push(url)
            console.log(window.intercepted_urls)
        });
        return oldXHROpen.apply(this, arguments);
    }
    """})
    driver.get(url)
    # wait = WebDriverWait(driver, 5)
    # wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div[style='bottom: 1rem'] button[id*='onetrust-accept-btn-handler']")))
    driver.find_element(By.CSS_SELECTOR, consent_css).click()

    time.sleep(15)
    urls = driver.execute_script(""" return window.intercepted_urls;""")
    driver.quit()
    print("Intercepted URLs:")
    print(urls)
    print("End ##############################")
    return urls
	from selenium import webdriver
	from selenium.webdriver.chrome.service import Service
	from webdriver_manager.chrome import ChromeDriverManager
	from selenium.webdriver.chrome.options import Options
	from selenium.webdriver.support.ui import WebDriverWait
	from selenium.webdriver.support import expected_conditions as EC
	from selenium.webdriver.common.by import By
	import time


	def intercept_requests(url, consent_css = "button[id*='onetrust-accept-btn-handler']"):
	print("Start ##############################")
	chrome_options = Options()
	chrome_options.add_argument("--headless")
	driver = webdriver.Chrome(
	service=Service(ChromeDriverManager().install()),
	options=chrome_options
	)
	driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {'source': """
	window.intercepted_urls = [] \|\| window.intercepted_urls;
	let oldXHROpen = window.XMLHttpRequest.prototype.open;
	window.XMLHttpRequest.prototype.open = function(method, url, async, user, password) {
	this.addEventListener('load', function() {
	console.log('intercepted '+url+': ' + this.responseText);
	window.intercepted_urls.push(url)
	console.log(window.intercepted_urls)
	});
	return oldXHROpen.apply(this, arguments);
	}
	"""})
	driver.get(url)
	# wait = WebDriverWait(driver, 5)
	# wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div[style='bottom: 1rem'] button[id*='onetrust-accept-btn-handler']")))
	driver.find_element(By.CSS_SELECTOR, consent_css).click()

	time.sleep(15)
	urls = driver.execute_script(""" return window.intercepted_urls;""")
	driver.quit()
	print("Intercepted URLs:")
	print(urls)
	print("End ##############################")
	return urls