Created
September 28, 2018 16:18
-
-
Save norbinsh/47c8ec36ea48648cf8bee3c56e3cad16 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests | |
import os | |
import time | |
def timeit(method): | |
def timed(*args, **kw): | |
ts = time.time() | |
result = method(*args, **kw) | |
te = time.time() | |
if 'log_time' in kw: | |
name = kw.get('log_name', method.__name__.upper()) | |
kw['log_time'][name] = int((te - ts) * 1000) | |
else: | |
print('%r finished in %2.2fs' % (method.__name__, (te - ts))) | |
return result | |
return timed | |
@timeit | |
def main() -> None: | |
url = 'https://www.google.com/robots.txt' | |
ifile = "robots.html" | |
for i in range(1, 50 + 1): | |
print(f"Iteration number {i}") | |
raw_data = retrieve_url(url) | |
html_data = html_that(raw_data) | |
io_save(ifile, html_data) | |
clean_up(ifile) | |
def retrieve_url(url: str) -> requests.models.Response: | |
print(f"Retrieving URL: {url}") | |
response = requests.get(url, timeout=5) | |
response.raise_for_status() | |
return response | |
def html_that(response: requests.models.Response) -> str: | |
print(f"Wrapping with <html>") | |
html_content = f""" | |
<html> | |
<head> | |
<title> | |
Blocking can be bad for you. | |
</title> | |
</head> | |
<body> | |
<h1> | |
Blocking can be bad for you. | |
</h1> | |
<p> | |
{response.text} | |
</p> | |
</body> | |
</html> | |
""" | |
return html_content | |
def io_save(input_file: str, html_page: str) -> None: | |
print(f"Opening file: {input_file}") | |
with open(input_file, "w") as write_file: | |
write_file.write(html_page) | |
def clean_up(input_file: str) -> None: | |
if os.path.exists(input_file): | |
print(f"Deleting file: {input_file}") | |
os.remove(input_file) | |
else: | |
print(f"Did not find file: {input_file}") | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment