Skip to content

Instantly share code, notes, and snippets.

@naiquevin
Last active August 29, 2015 14:01
Show Gist options
  • Save naiquevin/18131dcb67db5f92cfa1 to your computer and use it in GitHub Desktop.
Save naiquevin/18131dcb67db5f92cfa1 to your computer and use it in GitHub Desktop.
Concurrently run list of functions and get results as a list
from operator import itemgetter
from threading import Thread
try:
from Queue import Queue
except ImportError:
from queue import Queue
def concurrently(fns):
enqueue_result = lambda q, idx, fn, *args: q.put((idx, fn(*args)))
q = Queue()
threads = [Thread(target=enqueue_result, args=(q, idx, fn)+args)
for (idx, (fn, args)) in enumerate(fns)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
results = []
while not q.empty():
results.append(q.get())
return map(itemgetter(1), sorted(results, key=itemgetter(0)))
def cmap(f, *args):
return concurrently([(f, a) for a in zip(*args)])
#!/usr/bin/python3
from concurrent.futures import ThreadPoolExecutor
from test_cmap import get_status_code, urls
with ThreadPoolExecutor(max_workers=len(urls)) as executor:
print(list(executor.map(get_status_code, urls)))
import requests
from concurrently import cmap
urls = ['http://github.com',
'https://github.com/new',
'http://google.com']
def get_status_code(url):
r = requests.get(url)
return r.status_code
def test_cmap_sequentially():
for url in urls:
print(get_status_code(url))
def test_cmap_concurrently():
print(list(cmap(get_status_code, urls)))
if __name__ == '__main__':
# test_cmap_sequentially()
test_cmap_concurrently()
import time
from concurrently import concurrently
def a(x):
time.sleep(3)
return x * 2
def b(x, y):
time.sleep(5)
return [x]*y
def test_sequentially():
print([a(10), b(3, 4)])
def test_concurrently():
print(concurrently([(a, (10,)), (b, (3, 4))]))
if __name__ == '__main__':
# test_sequentially()
test_concurrently()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment