Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save taizan-hokuto/5b9075f01b6a87bc0110c88cb7eac6c0 to your computer and use it in GitHub Desktop.
Save taizan-hokuto/5b9075f01b6a87bc0110c88cb7eac6c0 to your computer and use it in GitHub Desktop.
import asyncio
import time
import requests
import aiohttp
import ramdom
async def download(url, session):
print("download start")
s = session.get(url)
print("download {url} completed")
await asyncio.sleep(0)
sentence = s.text
return sentence
async def text_preprocess(sentence):
a = random.randint(0,6)
print(f"processing for {a} sec")
# await asyncio.sleep(a)
return a
async def download_and_preprocess(url, session):
sentence = await download(url, session)
return await text_preprocess(sentence)
async def do_tasks(urls):
session = requests.Session()
tasks = [asyncio.create_task(download_and_preprocess(url, session)) for url in urls]
results = await asyncio.gather(*tasks)
session.close()
return results
def main():
urls = ["https://www.example.com/domains/reserved","https://www.example.org/domains/reserved","https://www.example.edu/domains/reserved"]
loop = asyncio.get_event_loop()
results = loop.run_until_complete(do_tasks(urls))
for r in results:
print(f"result:{r}")
start = time.time()
main()
process_time = time.time() - start
print(process_time)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment