Skip to content

Instantly share code, notes, and snippets.

@norbinsh
Created September 28, 2018 16:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save norbinsh/47c8ec36ea48648cf8bee3c56e3cad16 to your computer and use it in GitHub Desktop.
Save norbinsh/47c8ec36ea48648cf8bee3c56e3cad16 to your computer and use it in GitHub Desktop.
import requests
import os
import time
def timeit(method):
def timed(*args, **kw):
ts = time.time()
result = method(*args, **kw)
te = time.time()
if 'log_time' in kw:
name = kw.get('log_name', method.__name__.upper())
kw['log_time'][name] = int((te - ts) * 1000)
else:
print('%r finished in %2.2fs' % (method.__name__, (te - ts)))
return result
return timed
@timeit
def main() -> None:
url = 'https://www.google.com/robots.txt'
ifile = "robots.html"
for i in range(1, 50 + 1):
print(f"Iteration number {i}")
raw_data = retrieve_url(url)
html_data = html_that(raw_data)
io_save(ifile, html_data)
clean_up(ifile)
def retrieve_url(url: str) -> requests.models.Response:
print(f"Retrieving URL: {url}")
response = requests.get(url, timeout=5)
response.raise_for_status()
return response
def html_that(response: requests.models.Response) -> str:
print(f"Wrapping with <html>")
html_content = f"""
<html>
<head>
<title>
Blocking can be bad for you.
</title>
</head>
<body>
<h1>
Blocking can be bad for you.
</h1>
<p>
{response.text}
</p>
</body>
</html>
"""
return html_content
def io_save(input_file: str, html_page: str) -> None:
print(f"Opening file: {input_file}")
with open(input_file, "w") as write_file:
write_file.write(html_page)
def clean_up(input_file: str) -> None:
if os.path.exists(input_file):
print(f"Deleting file: {input_file}")
os.remove(input_file)
else:
print(f"Did not find file: {input_file}")
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment