request_articles_async.py

import asyncio
import requests
from bs4 import BeautifulSoup as bs
from requests.adapters import HTTPAdapter
from concurrent.futures import ThreadPoolExecutor


def fetch(session, url):
    with session.get(url) as response:
        src_html = response.text
        soup = bs(src_html, "html.parser")
        main_content = soup.select("#main-content")
        for d in main_content:
            main_c = d.get_text()
            _segment(main_c)

async def get_data_asynchronous(urls):
    with ThreadPoolExecutor(max_workers=20) as executor:
        with requests.Session() as session:
            # Set any session parameters here before calling `fetch`
            session.mount('http://', HTTPAdapter(max_retries=3))
            session.mount('https://', HTTPAdapter(max_retries=3))
            loop = asyncio.get_event_loop()
            tasks = [
                loop.run_in_executor(
                    executor,
                    fetch,
                    *(session, url)
                )
                for url in urls
            ]
            for response in await asyncio.gather(*tasks):
                pass