Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Comparison of asynchronous HTTP requests (aiohttp/asyncio) and traditional sequential requests (with Requests library)
"""Comparison of fetching web pages sequentially vs. asynchronously
Requirements: Python 3.5+, Requests, aiohttp, cchardet
Inspiration comes from this blog post:
http://mahugh.com/2017/05/23/http-requests-asyncio-aiohttp-vs-requests/
"""
import asyncio
from timeit import default_timer
from aiohttp import ClientSession
import requests
def demo_sequential(urls):
"""Fetch list of web pages sequentially."""
start_time = default_timer()
for url in urls:
start_time_url = default_timer()
_ = requests.get(url)
elapsed = default_timer() - start_time_url
# print the time taken to receive response
print(f"{url!s:30} {elapsed} {asterisks(elapsed)}")
# Total time taken by all requests to complete sequentially
tot_elapsed = default_timer() - start_time
print(f"TOTAL SECONDS - {tot_elapsed}" + '\n')
def demo_async(urls):
"""Make all the calls asynchronously."""
start_time = default_timer()
loop = asyncio.get_event_loop() # event loop
future = asyncio.ensure_future(fetch_all(urls)) # tasks to do
loop.run_until_complete(future) # loop until done
tot_elapsed = default_timer() - start_time
print(' WITH ASYNCIO: ' + f'{tot_elapsed} {asterisks(tot_elapsed)}')
async def fetch_all(urls):
"""Launch requests for all APIs."""
tasks = []
fetch.start_time = dict() # dictionary of start times for each url
async with ClientSession() as session:
for url in urls:
task = asyncio.ensure_future(fetch(url, session))
tasks.append(task) # create list of tasks
_ = await asyncio.gather(*tasks) # gather task responses
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
fetch.start_time[url] = default_timer()
async with session.get(url) as response:
resp = await response.read()
elapsed = default_timer() - fetch.start_time[url]
print(f'{url!s:30} {elapsed} {asterisks(elapsed)}')
return resp
def asterisks(num):
"""Returns a string of asterisks reflecting the magnitude of a number."""
return int(num * 10) * '*'
if __name__ == '__main__':
# A dummy rest API
URL_ROOT = "https://jsonplaceholder.typicode.com"
URL_LIST = [f'{URL_ROOT}/posts',
f'{URL_ROOT}/comments',
f'{URL_ROOT}/albums',
f'{URL_ROOT}/photos',
f'{URL_ROOT}/todos',
f'{URL_ROOT}/users']
demo_sequential(URL_LIST)
demo_async(URL_LIST)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment