Created
October 23, 2020 05:52
-
-
Save velotiotech/3bd9314613b1079d1dd2a69048676457 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import asyncio | |
import time | |
from aiohttp import ClientSession, ClientResponseError | |
async def fetch_url_data(session, url): | |
try: | |
async with session.get(url, timeout=60) as response: | |
resp = await response.read() | |
except Exception as e: | |
print(e) | |
else: | |
return resp | |
return | |
async def fetch_async(loop, r): | |
url = "https://www.velotio.com/careers" | |
tasks = [] | |
async with ClientSession() as session: | |
for i in range(r): | |
task = asyncio.ensure_future(fetch_url_data(session, url)) | |
tasks.append(task) | |
responses = await asyncio.gather(*tasks) | |
return responses | |
if __name__ == '__main__': | |
for ntimes in [1, 10, 50, 100, 500]: | |
start_time = time.time() | |
loop = asyncio.get_event_loop() | |
future = asyncio.ensure_future(fetch_async(loop, ntimes)) | |
loop.run_until_complete(future) #will run until it finish or get any error | |
responses = future.result() | |
print(f'Fetch total {ntimes} urls and process takes {time.time() - start_time} seconds') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Accepting event loop object in fetch_async function and not using it. Can we avoid passing loop argument in fetch_async or is it needed to pass argument while calling function?