Asyncio Python -- Download in parallel

asyncdownload.py

import aiohttp
import asyncio

BATCH_SIZE = 20
POOL_SIZE = 5


async def get_content(client, url):
  async with client.get(url) as response:
    return await response.text()


async def download(client, url, callback):
  content = await get_content(client, url)
  callback(url, content)
  return


def schedule_downloads(args):
  loop = asyncio.get_event_loop()
  conn = aiohttp.TCPConnector(limit=POOL_SIZE)
  with aiohttp.ClientSession(loop=loop, connector=conn) as client:
    for i in range((len(args) // BATCH_SIZE) + 1):
      tasks = []
      batch = args[(i * BATCH_SIZE):((i + 1) * BATCH_SIZE)]
      for arg in batch:
        tasks.append(asyncio.ensure_future(download(client, *arg)))
      loop.run_until_complete(asyncio.wait(tasks))