Last active
July 2, 2019 07:07
-
-
Save lokesh1729/db6739a872e95ebd8df70d9b2f6191a3 to your computer and use it in GitHub Desktop.
#threading #python
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import concurrent.futures | |
import json | |
import time | |
import requests | |
URLS = [ | |
"https://reqres.in/api/users", | |
"http://dummy.restapiexample.com/api/v1/employees", | |
# "https://jsonplaceholder.typicode.com/todos", | |
# "https://jsonplaceholder.typicode.com/posts", | |
] | |
# Retrieve a single page and report the URL and contents | |
def load_url(url, timeout): | |
with requests.Session() as s: | |
return s.get(url, timeout=timeout).content | |
# We can use a with statement to ensure threads are cleaned up promptly | |
result = [] | |
start_time = time.time() | |
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: | |
# Start the load operations and mark each future with its URL | |
future_to_url = {executor.submit(load_url, url, 60): url for url in URLS} | |
for future in concurrent.futures.as_completed(future_to_url): | |
url = future_to_url[future] | |
try: | |
data = json.loads(future.result().decode("utf-8")) | |
result.append(data) | |
except Exception as exc: | |
print("%r generated an exception: %s" % (url, exc)) | |
else: | |
print("%r page is %d bytes" % (url, len(data))) | |
print(result) | |
print("total time took is %s" % (time.time() - start_time)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment