Created
October 10, 2014 20:02
-
-
Save tianhuil/b6e95d00fa5d89505e4a to your computer and use it in GitHub Desktop.
When you encounter pickle errors in multiprocessing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# from http://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-pythons-multiprocessing-pool-ma | |
from multiprocessing import Pool, cpu_count | |
from multiprocessing.pool import ApplyResult | |
# --------- see Stenven's solution above ------------- | |
from copy_reg import pickle | |
from types import MethodType | |
def _pickle_method(method): | |
func_name = method.im_func.__name__ | |
obj = method.im_self | |
cls = method.im_class | |
return _unpickle_method, (func_name, obj, cls) | |
def _unpickle_method(func_name, obj, cls): | |
for cls in cls.mro(): | |
try: | |
func = cls.__dict__[func_name] | |
except KeyError: | |
pass | |
else: | |
break | |
return func.__get__(obj, cls) | |
class Myclass(object): | |
def __init__(self, nobj, workers=cpu_count()): | |
print "Constructor ..." | |
# multi-processing | |
pool = Pool(processes=workers) | |
async_results = [ pool.apply_async(self.process_obj, (i,)) for i in range(nobj) ] | |
pool.close() | |
# waiting for all results | |
map(ApplyResult.wait, async_results) | |
lst_results=[r.get() for r in async_results] | |
print lst_results | |
def __del__(self): | |
print "... Destructor" | |
def process_obj(self, index): | |
print "object %d" % index | |
return "results" | |
pickle(MethodType, _pickle_method, _unpickle_method) | |
Myclass(nobj=8, workers=3) | |
# problem !!! the destructor is called nobj times (instead of once) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment