Skip to content

Instantly share code, notes, and snippets.

@gabicavalcante
Created November 17, 2020 20:11
Show Gist options
  • Save gabicavalcante/b3d92dc6c60cace8a30fe7e621876bbb to your computer and use it in GitHub Desktop.
Save gabicavalcante/b3d92dc6c60cace8a30fe7e621876bbb to your computer and use it in GitHub Desktop.
import asyncio
import time
import queue
async def load_manifest(value, q):
await asyncio.sleep(1)
q.put(f"Processing Task {value}\n")
async def crawler_github_files(item, q):
await asyncio.gather(
*[
load_manifest((item, i), q)
for i in range(0, item+1)
]
)
async def run_parser():
q = queue.Queue()
await asyncio.wait([crawler_github_files(i, q) for i in range(5)], timeout=5)
with open('log.txt', 'w') as fp:
while not q.empty():
item = q.get(block=False)
if item is None:
break
fp.write(str(item))
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(run_parser())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment