Asyncio + aiohttp DevCru sample.

async.py

"""
IO-bound concurrency with asyncio and aiohttp.

Rewritten from the previous approach that was using gevent, due to 
http://www.gevent.org/api/gevent.monkey.html which conflicted with
`google.cloud` libs.

`gevent` allows to create pools to manage greenlets.

What are greenlets?

Greenlets are lightweight thread-like structures that are scheduled and
managed inside the process.

Reference https://learn-gevent-socketio.readthedocs.io/en/latest/greenlets.html
"""


import asyncio
import os
from pathlib import Path
from pprint import pprint
from urllib.parse import urljoin

import aiohttp
from bulb.platform.common.logging_config import config_logging

from .compare_bills.core import helpers

url = urljoin(
    "https://account.platform.stage.bulb.co.uk/",
    "/v1/members?member_ids=M-01-0000-00%d"
)


async def query_member(session, url):
    cookies = helpers.load_cookies(
        Path(__file__).parent / "../../cookies-stage.txt"
    )

    async with session.get(url, cookies=cookies) as resp:
        # Returning the resp object would close the connection, so
        # any other operations on the response need to be inside this context.
        json = await resp.json()
        pprint(url)
        pprint(json)
        print(os.linesep * 2)


async def query_members(url):
  async with aiohttp.ClientSession() as session:
    awaitables = []  # coroutines in our case. Or coros.
    for i in range(90, 95):
        awaitables.append(query_member(session, url % i))

    # Is there a limit to how long this list can be?
    # 
    # ClientSession can have 100 open connections by default. Can be configured.
    #
    # Don't go overboard. There's an OS-level limit to the number of
    # open connections.
    #
    # Options:
    # Limit the number of concurrent requests to a reasonable amount using:
    # 1. semaphore (asyncio way)
    # 2. aiostream: https://aiostream.readthedocs.io/en/latest/operators.html#aiostream.stream.map
    # 3. the current way
    await asyncio.gather(*awaitables)
        

def main():
    asyncio.run(query_members(url))


if __name__ == "__main__":
    main()