Skip to content

Instantly share code, notes, and snippets.

@EdwinChan
Last active May 8, 2024 20:58
Show Gist options
  • Save EdwinChan/3c13d3a746bb3ec5082f to your computer and use it in GitHub Desktop.
Save EdwinChan/3c13d3a746bb3ec5082f to your computer and use it in GitHub Desktop.
Trick for using multiprocessing with nested functions and lambda expressions
import concurrent.futures
import multiprocessing
import sys
import uuid
def globalize(func):
def result(*args, **kwargs):
return func(*args, **kwargs)
result.__name__ = result.__qualname__ = uuid.uuid4().hex
setattr(sys.modules[result.__module__], result.__name__, result)
return result
def main():
@globalize
def func1(x):
return x
func2 = globalize(lambda x: x)
with multiprocessing.Pool() as pool:
print(pool.map(func1, range(10)))
print(pool.map(func2, range(10)))
with concurrent.futures.ThreadPoolExecutor() as executor:
print(list(executor.map(func1, range(10))))
print(list(executor.map(func2, range(10))))
if __name__ == '__main__':
main()
@pk1234dva
Copy link

@EdwinChan Thank you.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment