tests_urls_on_page.py

import requests
import re
from bs4 import BeautifulSoup as bs

def test_urls_on_page(initial_page_url):
    r = requests.get(initial_page_url)
    assert r.status_code == 200
    content = r.content
    html_soup = bs(content, "html.parser")
    urls_found = html_soup.find_all("a", {"href": re.compile('http*')})

    successful = []
    not_found = []
    failed = []
    other = []

    for item in urls_found:
        url = item.get('href')
        try:
            r = requests.get(url)
            if r.status_code == 200:
                success.append(url)
            if r.status_code == 404:
                    not_found.append(url)
            else:
                other.append(url)
        except:
            fail.append(url)

    
    final_result = {
        'successfull': successful,
        'not_found': not_found,
        'failed': failed,
        'other': other,
    }

    return final_result