Fast AsyncIO HTTP requests

requests.py

import http.cookies
http.cookies._is_legal_key = lambda _: True
import asyncio
import aiohttp
import nest_asyncio
import time
nest_asyncio.apply()

async def gather_with_concurrency(n, *tasks):
    semaphore = asyncio.Semaphore(n)

    async def sem_task(task):
        async with semaphore:
            return await task

    return await asyncio.gather(*(sem_task(task) for task in tasks))
conn = aiohttp.TCPConnector(limit=None, ttl_dns_cache=300)
session = aiohttp.ClientSession(connector=conn)

async def get_async(url):
    async with session.get(url, ssl=False) as response:
        obj = await response.read()
        results[url] = obj
urls = [f"https://jsonplaceholder.typicode.com/todos/{i}" for i in range(4000)]

conc_req = 60
s = time.perf_counter()
await gather_with_concurrency(conc_req, *[get_async(i) for i in urls])
elapsed = time.perf_counter() - s

Outside Jupyter context, this worked for me (on v3.9.5) when

1. declaring results = {}
2. putting gather_with_concurrency() into an asyncio event loop:

loop = asyncio.get_event_loop()
loop.run_until_complete(gather_with_concurrency(conc_req, *[get_async(i) for i in urls]))
loop.close()

3. changing get_async() like so:

async def get_async(url):
    conn = aiohttp.TCPConnector(limit=None, ttl_dns_cache=300)
    session = aiohttp.ClientSession(connector=conn)

    async with session.get(url, ssl=False) as response:
        obj = await response.read()
        results[url] = obj

        await session.close()

.. which makes this a great implementation, crazy fast 🦊