Skip to content

Instantly share code, notes, and snippets.

@OlegKorn
Created February 5, 2024 01:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save OlegKorn/5f63eba9e912704a7d7e9997d35cdd7b to your computer and use it in GitHub Desktop.
Save OlegKorn/5f63eba9e912704a7d7e9997d35cdd7b to your computer and use it in GitHub Desktop.
something with semaphor asyncio requests
urls = open_file(0, 500) #array of urls
async def fetch_url(i, url, semaphore):
async with semaphore:
async with httpx.AsyncClient() as client:
reply = await client.get(url)
print(f'got {url}')
return i, reply
async def main():
semaphore = asyncio.Semaphore(3)
tasks = [fetch_url(i, url, semaphore) for i, url in enumerate(urls)]
responses = await asyncio.gather(*tasks)
for i, r in responses:
print(i, r.status)
if __name__ == "__main__":
asyncio.run(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment