Scraping Question & Answers

scrape_questions.py

from bs4 import BeautifulSoup
import requests
import csv

def main():
    print("Scraping Begin")
    print("--------------------------------")
    qna = {}

    for i in range (46):
        url = "https://www.gkseries.com/mass-communication-and-journalism-questions-answers/" + str(i) + "-mass-communication-and-journalism-multiple-choice-questions-and-answers"
        print(url)
        page = requests.get(url=url)
        soup = BeautifulSoup(page.text, 'html.parser')
        for mcq in soup.find_all("div", class_="mcq"):
            question = mcq.find("div", class_="question-content")
            stripped_question = "\n".join(question.text.strip().splitlines()[2:])

            print(stripped_question)
            ans = mcq.find("blockquote", class_="blockquote")
            print(ans.text)
            qna[stripped_question] = ans.text.strip()

    print("Scraping End")
    print("--------------------------------")      
    
    print("Save to CSV")
    with open('dict.csv', 'w') as csv_file:  
        writer = csv.writer(csv_file)
        for key, value in qna.items():
            writer.writerow([key, value])
main()