Skip to content

Instantly share code, notes, and snippets.

@CMCDragonkai
Created October 27, 2016 12:46
Show Gist options
  • Save CMCDragonkai/1f367f38187f25aff0488096193c5809 to your computer and use it in GitHub Desktop.
Save CMCDragonkai/1f367f38187f25aff0488096193c5809 to your computer and use it in GitHub Desktop.
Python: Multiprocessing Async Apply
#!/usr/bin/env python3
# the explaination for this behaviour is: http://stackoverflow.com/q/40283772/582917
import functools
from multiprocessing import Pool
def processing_function(unprocessed_data):
return unprocessed_data
def callback_function(processed_data):
print("FUNCTION: " + str(processed_data))
def create_processing_closure(initial_data):
def processing_function(unprocessed_data):
return initial_data + unprocessed_data
return processing_function
def create_callback_closure():
def callback(processed_data):
print("CLOSURE: " + str(processed_data))
return callback
def create_processing_lambda(initial_data):
return lambda unprocessed_data: initial_data + unprocessed_data
def create_callback_lambda():
return lambda processed_data: print("LAMBDA: " + str(processed_data))
def processing_partial(unprocessed_data1, unprocessed_data2):
return (unprocessed_data1 + unprocessed_data2)
def callback_partial(initial_data, processed_data):
print("PARTIAL: " + str(processed_data))
pool = Pool(processes=1)
print("Testing if they work normally...")
f1 = processing_function
f2 = callback_function
f2(f1(1))
f3 = create_processing_closure(1)
f4 = create_callback_closure()
f4(f3(1))
f5 = create_processing_lambda(1)
f6 = create_callback_lambda()
f6(f5(1))
f7 = functools.partial(processing_partial, 1)
f8 = functools.partial(callback_partial, 1)
f8(f7(1))
# bonus round!
x = 1
f9 = lambda unprocessed_data: unprocessed_data + x
f10 = lambda processed_data: print("GLOBAL LAMBDA: " + str(processed_data))
f10(f9(1))
print("Testing if they work in apply_async...")
# works
pool.apply_async(f1, args=(1,), callback=f2)
# doesn't work
pool.apply_async(f3, args=(1,), callback=f4)
# doesn't work
pool.apply_async(f5, args=(1,), callback=f6)
# works
pool.apply_async(f7, args=(1,), callback=f8)
# doesn't work
pool.apply_async(f9, args=(1,), callback=f10)
pool.close()
pool.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment