Skip to content

Instantly share code, notes, and snippets.

@dmahugh
Created May 23, 2017 17:21
Show Gist options
  • Save dmahugh/b043ecbc4c61920aa685e0febbabb959 to your computer and use it in GitHub Desktop.
Save dmahugh/b043ecbc4c61920aa685e0febbabb959 to your computer and use it in GitHub Desktop.
comparison of asynchronous HTTP requests (aiohttp/asyncio) and traditional sequential requests (with Requests library)
"""Comparison of fetching web pages sequentially vs. asynchronously
Requirements: Python 3.5+, Requests, aiohttp, cchardet
For a walkthrough see this blog post:
http://mahugh.com/2017/05/23/http-requests-asyncio-aiohttp-vs-requests/
"""
import asyncio
from timeit import default_timer
from aiohttp import ClientSession
import requests
def demo_sequential(urls):
"""Fetch list of web pages sequentially."""
start_time = default_timer()
for url in urls:
start_time_url = default_timer()
_ = requests.get(url)
elapsed = default_timer() - start_time_url
print('{0:30}{1:5.2f} {2}'.format(url, elapsed, asterisks(elapsed)))
tot_elapsed = default_timer() - start_time
print(' TOTAL SECONDS: '.rjust(30, '-') + '{0:5.2f} {1}'. \
format(tot_elapsed, asterisks(tot_elapsed)) + '\n')
def demo_async(urls):
"""Fetch list of web pages asynchronously."""
start_time = default_timer()
loop = asyncio.get_event_loop() # event loop
future = asyncio.ensure_future(fetch_all(urls)) # tasks to do
loop.run_until_complete(future) # loop until done
tot_elapsed = default_timer() - start_time
print(' WITH ASYNCIO: '.rjust(30, '-') + '{0:5.2f} {1}'. \
format(tot_elapsed, asterisks(tot_elapsed)))
async def fetch_all(urls):
"""Launch requests for all web pages."""
tasks = []
fetch.start_time = dict() # dictionary of start times for each url
async with ClientSession() as session:
for url in urls:
task = asyncio.ensure_future(fetch(url, session))
tasks.append(task) # create list of tasks
_ = await asyncio.gather(*tasks) # gather task responses
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
fetch.start_time[url] = default_timer()
async with session.get(url) as response:
resp = await response.read()
elapsed = default_timer() - fetch.start_time[url]
print('{0:30}{1:5.2f} {2}'.format(url, elapsed, asterisks(elapsed)))
return resp
def asterisks(num):
"""Returns a string of asterisks reflecting the magnitude of a number."""
return int(num*10)*'*'
if __name__ == '__main__':
URL_LIST = ['https://facebook.com',
'https://github.com',
'https://google.com',
'https://microsoft.com',
'https://yahoo.com']
demo_sequential(URL_LIST)
demo_async(URL_LIST)
@rs2
Copy link

rs2 commented Jul 4, 2018

Apparently, ensure_future is the older way of creating tasks...

@lalamax3d
Copy link

sir you are genius and i am grateful for sharing such knowledgeable piece of software

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment