Access Chrome's network tab (e.g. XHR requests) with Selenium

selenium_xhr_requests_via_performance_logging.py

#
# This small example shows you how to access JS-based requests via Selenium
# Like this, one can access raw data for scraping, 
# for example on many JS-intensive/React-based websites
#

from time import sleep

from selenium import webdriver
from selenium.webdriver import DesiredCapabilities

# make chrome log requests
capabilities = DesiredCapabilities.CHROME
capabilities["loggingPrefs"] = {"performance": "ALL"}  # newer: goog:loggingPrefs
driver = webdriver.Chrome(
    desired_capabilities=capabilities, executable_path="./chromedriver"
)

# fetch a site that does xhr requests
driver.get("https://sitewithajaxorsomething.com")
sleep(5)  # wait for the requests to take place

# extract requests from logs
logs_raw = driver.get_log("performance")
logs = [json.loads(lr["message"])["message"] for lr in logs_raw]

def log_filter(log_):
    return (
        # is an actual response
        log_["method"] == "Network.responseReceived"
        # and json
        and "json" in log_["params"]["response"]["mimeType"]
    )

for log in filter(log_filter, logs):
    request_id = log["params"]["requestId"]
    resp_url = log["params"]["response"]["url"]
    print(f"Caught {resp_url}")
    print(driver.execute_cdp_cmd("Network.getResponseBody", {"requestId": request_id}))

Hello guys, for Selenium 4.x use it

driver.options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
driver.get(url)

then just follow the steps from line 24+

Works for selenium 4.13.0

for Selenium 4.15 set the option：

options = webdriver.ChromeOptions()
options.set_capability(
            "goog:loggingPrefs", {"performance": "ALL"}
        )
driver = webdriver.Chrome(options=options)

Something I noticed is that you need to filter out Preflight requests.

if event['params']['type'] != 'Preflight':
    . . .

Otherwise, you might get this error:

{"code":-32000,"message":"No resource with given identifier found"}