bulker.py

import asyncio
import aiohttp
import async_timeout



headers = {}
links = []  # TODO: Populate this somehow with urls
done = asyncio.Queue()


async def handle_link(url):
    async with aiohttp.ClientSession() as session:
        with async_timeout.timeout(10):
            async with session.get(url, headers=headers) as response:
                if response.status == 200:
                    html = await response.text()
                    return {'error': '', 'html': html}
                else:
                    return {'error': response.status, 'html': ''}
                done.put_nowait((url, html))


tasks = [handle_link(l) for l in links]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))