Created
May 19, 2018 00:02
-
-
Save benhg/906caf862c50012af82a643d736bd005 to your computer and use it in GitHub Desktop.
parallel_function decorator for parallelizing python functions which take iterables as input
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def parallel_function(f): | |
def easy_parallize(f, sequence): | |
""" assumes f takes sequence as input, easy w/ Python's scope """ | |
from multiprocessing import Pool | |
pool = Pool(processes=4) # depends on available cores | |
result = pool.map(f, sequence) # for i in sequence: result[i] = f(i) | |
cleaned = [x for x in result if not x is None] # getting results | |
cleaned = asarray(cleaned) | |
pool.close() # not optimal! but easy | |
pool.join() | |
return cleaned | |
from functools import partial | |
return partial(easy_parallize, f) | |
function.parallel = parallel_function(test_primes) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment