Skip to content

Instantly share code, notes, and snippets.

@master-of-zen
Created October 6, 2019 20:30
Show Gist options
  • Save master-of-zen/e24fad6a0f121d89b7ee73416caa91eb to your computer and use it in GitHub Desktop.
Save master-of-zen/e24fad6a0f121d89b7ee73416caa91eb to your computer and use it in GitHub Desktop.
import asyncio
from aiohttp import ClientSession
from urllib.parse import urlencode
async def fetch(url, session):
async with session.get(url) as response:
delay = response.headers.get("DELAY")
date = response.headers.get("DATE")
print("{}:{} with delay {}".format(date, response.url, delay))
return await response.read()
async def bound_fetch(sem, url, session):
# Getter function with semaphore.
async with sem:
await fetch(url, session)
async def run(urls):
tasks = []
# Fetch all responses within one Client session,
# keep connection alive for all requests.
async with ClientSession() as session:
for i in urls:
task = asyncio.ensure_future(fetch(i, session))
tasks.append(task)
responses = asyncio.gather(*tasks)
# you now have all response bodies in this variable
await responses
def main():
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run([f'http://libgen.is/search.php?&{urlencode({"req": "python", "res": 100, "page": page})}'
for page in range(1, 10)]))
loop.run_until_complete(future)
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment