Skip to content

Instantly share code, notes, and snippets.

@norbinsh
Created September 28, 2018 19:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save norbinsh/02c398e9e1d6d0ce1a9031aefc0e6050 to your computer and use it in GitHub Desktop.
Save norbinsh/02c398e9e1d6d0ce1a9031aefc0e6050 to your computer and use it in GitHub Desktop.
import aiohttp
import asyncio
import os
import time
def timeit(method):
def timed(*args, **kw):
ts = time.time()
result = method(*args, **kw)
te = time.time()
if 'log_time' in kw:
name = kw.get('log_name', method.__name__.upper())
kw['log_time'][name] = int((te - ts) * 1000)
else:
print('%r finished in %2.2fs' % (method.__name__, (te - ts)))
return result
return timed
@timeit
def main() -> None:
loop = asyncio.get_event_loop()
loop.run_until_complete(orchestrator())
async def retrieve_url(url: str) -> str:
print(f"Retrieving URL: {url}")
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
resp.raise_for_status()
return await resp.text()
async def html_that(response) -> str:
print(f"Wrapping with <html>")
html_content = f"""
<html>
<head>
<title>
Blocking can be bad for you.
</title>
</head>
<body>
<h1>
Blocking can be bad for you.
</h1>
<p>
{response}
</p>
</body>
</html>
"""
return html_content
async def io_save(input_file: str, html_page: str) -> None:
print(f"Opening file: {input_file}")
with open(input_file, "w") as write_file:
write_file.write(html_page)
async def clean_up(input_file: str) -> None:
if os.path.exists(input_file):
print(f"Deleting file: {input_file}")
os.remove(input_file)
else:
print(f"Did not find file: {input_file}")
async def orchestrator():
url = 'https://www.google.com/robots.txt'
ifile = "robots.html"
tasks = []
for _ in range(1, 50 + 1):
tasks.append(asyncio.create_task(retrieve_url(url)))
for task in tasks:
resp = await task
html_content = await html_that(resp)
await io_save(ifile, html_content)
await clean_up(ifile)
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment