news.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service as ChromeService
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import fitz
import requests
import json

url = "https://www.mstcecommerce.com/auctionhome/container.jsp?title_id=Mine%20Block%20Summary&linkid=0&main_link=y&sublink=n&main_link_name=203&portal=mlcl"

options = webdriver.ChromeOptions()
options.add_argument('--headless')  # Run in headless mode

with webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options) as driver:
    driver.get(url)
    driver.implicitly_wait(4)
    print("Page URL:", driver.current_url)
    page_source = driver.page_source
    soup = BeautifulSoup(page_source, 'html.parser')
    pdf_links = []
    link = soup.find_all("a")
    hrefs = []
    for a in link:
        href = a['href']
        hrefs.append(href)
        print(href)
    print(len(hrefs))
    print(hrefs)