Skip to content

Instantly share code, notes, and snippets.

@michaelBenin
Created February 24, 2018 04:22
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save michaelBenin/f4b2eb5c0bc387e2af4febc3b54587ac to your computer and use it in GitHub Desktop.
Save michaelBenin/f4b2eb5c0bc387e2af4febc3b54587ac to your computer and use it in GitHub Desktop.
Bluebird's Promise.map in python
'''Debugging: https://docs.python.org/3/library/asyncio-dev.html#asyncio-dev'''
def fetch_urls_concurrent(urls):
loop = asyncio.new_event_loop()
resp_list = []
async def async_req_urls():
'''Change max workers for concurrency'''
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
futures = []
for url in urls:
futures.append(loop.run_in_executor(
executor,
fetch_url,
url
)
)
for response in await asyncio.gather( * futures):
resp_list.append(response)
loop.run_until_complete(async_req_urls())
loop.close()
return resp_list
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment