Scrape the page titles and URLs of Wikipedia search of "insects".

app.py

import requests
from bs4 import BeautifulSoup
import csv

BASE_URL = "https://en.wikipedia.org"
SEARCH_URL = "https://en.wikipedia.org/wiki/Special:Search?search=insects&fulltext=Search&ns0=1"

def get_wikipedia_links(search_url):
    r = requests.get(search_url)
    soup = BeautifulSoup(r.content, 'html.parser')
    
    links = []
    
    # Find search results on the Wikipedia page
    for div in soup.findAll('div', {'class': 'mw-search-result-heading'}):
        a = div.find('a')
        title = a.get_text()
        url = BASE_URL + a['href']
        links.append((title, url))
        
    return links

def save_to_csv(data, filename):
    with open(filename, 'w', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["Title", "URL"])
        for row in data:
            writer.writerow(row)

def main():
    links = get_wikipedia_links(SEARCH_URL)
    save_to_csv(links, 'wikipedia_insects_and_bugs.csv')
    print(f"Data saved to 'wikipedia_insects_and_bugs.csv'.")

if __name__ == "__main__":
    main()

More context about it, here: https://untalkedseo.com/pseo-data-scraping/