Created
May 23, 2017 17:21
-
-
Save dmahugh/b043ecbc4c61920aa685e0febbabb959 to your computer and use it in GitHub Desktop.
comparison of asynchronous HTTP requests (aiohttp/asyncio) and traditional sequential requests (with Requests library)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"""Comparison of fetching web pages sequentially vs. asynchronously | |
Requirements: Python 3.5+, Requests, aiohttp, cchardet | |
For a walkthrough see this blog post: | |
http://mahugh.com/2017/05/23/http-requests-asyncio-aiohttp-vs-requests/ | |
""" | |
import asyncio | |
from timeit import default_timer | |
from aiohttp import ClientSession | |
import requests | |
def demo_sequential(urls): | |
"""Fetch list of web pages sequentially.""" | |
start_time = default_timer() | |
for url in urls: | |
start_time_url = default_timer() | |
_ = requests.get(url) | |
elapsed = default_timer() - start_time_url | |
print('{0:30}{1:5.2f} {2}'.format(url, elapsed, asterisks(elapsed))) | |
tot_elapsed = default_timer() - start_time | |
print(' TOTAL SECONDS: '.rjust(30, '-') + '{0:5.2f} {1}'. \ | |
format(tot_elapsed, asterisks(tot_elapsed)) + '\n') | |
def demo_async(urls): | |
"""Fetch list of web pages asynchronously.""" | |
start_time = default_timer() | |
loop = asyncio.get_event_loop() # event loop | |
future = asyncio.ensure_future(fetch_all(urls)) # tasks to do | |
loop.run_until_complete(future) # loop until done | |
tot_elapsed = default_timer() - start_time | |
print(' WITH ASYNCIO: '.rjust(30, '-') + '{0:5.2f} {1}'. \ | |
format(tot_elapsed, asterisks(tot_elapsed))) | |
async def fetch_all(urls): | |
"""Launch requests for all web pages.""" | |
tasks = [] | |
fetch.start_time = dict() # dictionary of start times for each url | |
async with ClientSession() as session: | |
for url in urls: | |
task = asyncio.ensure_future(fetch(url, session)) | |
tasks.append(task) # create list of tasks | |
_ = await asyncio.gather(*tasks) # gather task responses | |
async def fetch(url, session): | |
"""Fetch a url, using specified ClientSession.""" | |
fetch.start_time[url] = default_timer() | |
async with session.get(url) as response: | |
resp = await response.read() | |
elapsed = default_timer() - fetch.start_time[url] | |
print('{0:30}{1:5.2f} {2}'.format(url, elapsed, asterisks(elapsed))) | |
return resp | |
def asterisks(num): | |
"""Returns a string of asterisks reflecting the magnitude of a number.""" | |
return int(num*10)*'*' | |
if __name__ == '__main__': | |
URL_LIST = ['https://facebook.com', | |
'https://github.com', | |
'https://google.com', | |
'https://microsoft.com', | |
'https://yahoo.com'] | |
demo_sequential(URL_LIST) | |
demo_async(URL_LIST) |
sir you are genius and i am grateful for sharing such knowledgeable piece of software
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Apparently,
ensure_future
is the older way of creating tasks...