Skip to content

Instantly share code, notes, and snippets.

@bnyeggen
Created July 16, 2011 14:17
Show Gist options
  • Save bnyeggen/1086393 to your computer and use it in GitHub Desktop.
Save bnyeggen/1086393 to your computer and use it in GitHub Desktop.
Example showing how to use instance methods with the multiprocessing module
from multiprocessing import Pool
from functools import partial
def _pickle_method(method):
func_name = method.im_func.__name__
obj = method.im_self
cls = method.im_class
if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names
cls_name = cls.__name__.lstrip('_')
func_name = '_' + cls_name + func_name
return _unpickle_method, (func_name, obj, cls)
def _unpickle_method(func_name, obj, cls):
for cls in cls.__mro__:
try:
func = cls.__dict__[func_name]
except KeyError:
pass
else:
break
return func.__get__(obj, cls)
import copy_reg
import types
copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
class someClass(object):
def __init__(self):
pass
def f(self, x=None):
#can put something expensive here to verify CPU utilization
if x is None: return 99
return x*x
def go(self):
pool = Pool()
print pool.map(self.f, range(10))
if __name__=='__main__':
sc = someClass()
sc.go()
x=[someClass(),someClass(),someClass()]
p=Pool()
filled_f=partial(someClass.f,x=9)
print p.map(filled_f,x)
print p.map(someClass.f,x)
@DomHudson
Copy link

Either store the state in the local scope and return it out, or use multiprocessing.Value.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment