Created
February 5, 2024 01:06
-
-
Save OlegKorn/5f63eba9e912704a7d7e9997d35cdd7b to your computer and use it in GitHub Desktop.
something with semaphor asyncio requests
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
urls = open_file(0, 500) #array of urls | |
async def fetch_url(i, url, semaphore): | |
async with semaphore: | |
async with httpx.AsyncClient() as client: | |
reply = await client.get(url) | |
print(f'got {url}') | |
return i, reply | |
async def main(): | |
semaphore = asyncio.Semaphore(3) | |
tasks = [fetch_url(i, url, semaphore) for i, url in enumerate(urls)] | |
responses = await asyncio.gather(*tasks) | |
for i, r in responses: | |
print(i, r.status) | |
if __name__ == "__main__": | |
asyncio.run(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment