Skip to content

Instantly share code, notes, and snippets.

@yakirk
Created August 26, 2024 18:38
Show Gist options
  • Save yakirk/1c72aad0cdd8a30f25d04ac59af7b7d8 to your computer and use it in GitHub Desktop.
Save yakirk/1c72aad0cdd8a30f25d04ac59af7b7d8 to your computer and use it in GitHub Desktop.
import concurrent.futures
import requests
from time import time
# List of URLs to request
urls = []
for i in range(10000):
urls.append(f"http://<Prometheus-URL>:9090/debug/pprof/heap?seconds={i}")
def fetch_url(url):
try:
print(url)
response = requests.get(url)
return (
url, response.status_code, response.text[:100]) # Return URL, status code, and first 100 chars of content
except requests.RequestException as e:
return (url, None, str(e)) # In case of error, return the exception
def fetch_all_urls(urls):
results = []
with concurrent.futures.ThreadPoolExecutor(max_workers=100000) as executor:
future_to_url = {executor.submit(fetch_url, url): url for url in urls}
for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
try:
result = future.result()
results.append(result)
except Exception as exc:
results.append((url, None, str(exc)))
return results
if __name__ == "__main__":
start_time = time()
results = fetch_all_urls(urls)
end_time = time()
for url, status, content in results:
print(f"URL: {url} \nStatus: {status}\nContent: {content}\n")
print(f"Time taken: {end_time - start_time} seconds")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment