Created
September 8, 2023 17:33
-
-
Save hmahadik/2a5c93f0034452b0e6160d132711429a to your computer and use it in GitHub Desktop.
Parallelize any function using multi-processing in python
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" Based on https://stackoverflow.com/questions/67132064/python-multiprocessing-pool-as-decorator """ | |
import functools | |
import time | |
from multiprocessing import Pool | |
def parallelize(func=None, **options): | |
if func is None: | |
return functools.partial(parallel, **options) | |
def wrapper(iterable, *args, **kwargs): | |
processes = options["processes"] | |
with Pool(processes) as pool: | |
result = pool.map(func, iterable) | |
return result | |
return wrapper | |
def test(kwargs): | |
foo = kwargs['foo'] | |
pid = kwargs['pid'] | |
print(f"Process {pid}: foo = {foo}") | |
def main(): | |
parallelize(test, processes=6)([dict(pid=i, foo="bar") for i in range(10)]) | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Output should look like: