asyncworker_streaming_example.py

import asyncio
from http import HTTPStatus
from typing import AsyncIterator

from aiohttp import ClientSession

WORDLIST_URL = "https://s3.amazonaws.com/diogo.martins/public/portuguese-brazil.txt"


async def stream_wordlist(limit: int=None) -> AsyncIterator[str]:
    async with ClientSession() as session:
        async with session.get(WORDLIST_URL) as response:
            i = 0
            async for row in response.content:
                yield row.rstrip().decode('utf-8')
                i += 1
                if i == limit:
                    break


async def produce_messages():
    async for word in stream_wordlist(limit=10000):
        print(word)

asyncio.get_event_loop().run_until_complete(produce_messages())