scraping google results exactly getting the number of search results for each keyword in csv

google_results_script.py

import requests
import csv
from bs4 import BeautifulSoup

# Read the keywords from a file
with open("keywords.txt", "r") as file:
    keywords = file.read().splitlines()

# Define the User-Agent header
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"}

# Create a new CSV file and write the headers
with open("results.csv", "w", newline="") as file:
    writer = csv.writer(file)
    writer.writerow(["Keyword", "Total Results"])

    # Perform the search for each keyword and write the total number of results to the CSV file
    for keyword in keywords:
        response = requests.get(f"https://www.google.com/search?q={keyword}", headers=headers)
        soup = BeautifulSoup(response.content, "html.parser")
        result_stats = soup.find("div", {"id": "result-stats"})
        if result_stats:
            total_results = result_stats.get_text().split()[1].replace(",", "")
            writer.writerow([keyword, total_results])
            print(f"Keyword: {keyword}, Total Results: {total_results}")
        else:
            writer.writerow([keyword, "Not found"])
            print(f"Keyword: {keyword}, Total Results: Not found")