Python多进程库multiprocessing的封装

multiprocess.py

#!/usr/bin/python
#coding: utf-8

"""
Usage:

import multiprocess

slices = multiprocess.split_list(filelist, 8) #分成8份
processes = map(lambda slice: multiprocess.spawn(file_processor, slice), slices)
sys.exit(multiprocess.start_and_join(processes))

"""


import sys
from multiprocessing import Process

def split_list(data, n_slice, hash_func=lambda i, d: i): #default: sequential
    slices = []
    for i in range(n_slice):
        slices.append([])

    for i, d in enumerate(data):
        slices[hash_func(i, d) % n_slice].append(d)

    return slices

def spawn(target, *args, **kwargs):
    return Process(target=target, args=args, kwargs=kwargs)

def start_and_join(processes, killall_if_fail=True):
    for p in processes:
        p.start()

    exitcode = 0
    for p in processes:
        p.join()
        if p.exitcode != 0:
            exitcode = p.exitcode
            break

    if exitcode != 0:
        for p in processes:
            if killall_if_fail and p.is_alive():
                p.terminate()

    return exitcode