Multiprocessing pool: How to call an arbitrary list of methods on a list of class objects

run_routines_on_objects_in_parallel_and_return.py

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# LIBRARY PART
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

from functools import partial
import multiprocessing

NUM_CORES = multiprocessing.cpu_count()


def run_routines_on_objects_in_parallel_and_return(in_object_list, method_list):
	"""
	Arguments:
		in_object_list: list of arbritrary objects
		method_list: list of method names (strings)
		(All objects in the list must expose every method listed in method_list.)
	Purpose:
		Distributes the objects in the list over the available CPU cores.
		Calles every method in method_list in the order of appearance on every object.
	Returns:
		list of modified objects (order is not maintained)
	"""

	# Generate a function handle containing the list of methods as a default parameter
	func_handle = partial(__run_routines_on_object_and_return__, method_list)

	# Fire up a multiprocessing pool, one process per available CPU core
	with multiprocessing.Pool(processes = NUM_CORES) as p:
		out_object_list = list(p.imap_unordered( # if list order matters, use 'p.imap' instead
		func_handle, # partial function handler
		(in_object for in_object in in_object_list) # generator yielding objects from the list
		))

	# Return the list of modified objects
	return out_object_list


def __run_routines_on_object_and_return__(method_list, in_object):
	"""
	Helper routine for run_routines_on_objects_in_parallel_and_return
	"""

	# Iterate over list of method names
	for method_name in method_list:
		# Run method on object
		getattr(in_object, method_name)()

	# Return modified object
	return in_object


# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# DEMO PART
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

from pprint import pprint as pp
from pprint import pformat as pf
import random


class some_class:

	def __init__(self):
		self.some_dict = {'some_string': None, 'some_float': None}

	def some_routine(self):
		self.some_dict.update({'some_string': 'some_value %d' % int(10 * random.random())})

	def some_other_routine(self):
		self.some_dict.update({'some_float': random.random()})

	def __repr__(self):
		return pf(self.some_dict)


if __name__ == '__main__':

	# Generate a list of class instances
	object_list = [some_class() for item in range(20)]
	# Print list of class instances
	pp([item for item in object_list])

	# Run a sequence of class methods on every object in the list - in parallel
	new_object_list = run_routines_on_objects_in_parallel_and_return(
		object_list,
		['some_routine', 'some_other_routine']
		)
	# Print new list of class instances
	pp([item for item in new_object_list])