Skip to content

Instantly share code, notes, and snippets.

@felix021 felix021/multiprocess.py
Last active Apr 12, 2016

Embed
What would you like to do?
Python多进程库multiprocessing的封装
#!/usr/bin/python
#coding: utf-8
"""
Usage:
import multiprocess
slices = multiprocess.split_list(filelist, 8) #分成8份
processes = map(lambda slice: multiprocess.spawn(file_processor, slice), slices)
sys.exit(multiprocess.start_and_join(processes))
"""
import sys
from multiprocessing import Process
def split_list(data, n_slice, hash_func=lambda i, d: i): #default: sequential
slices = []
for i in range(n_slice):
slices.append([])
for i, d in enumerate(data):
slices[hash_func(i, d) % n_slice].append(d)
return slices
def spawn(target, *args, **kwargs):
return Process(target=target, args=args, kwargs=kwargs)
def start_and_join(processes, killall_if_fail=True):
for p in processes:
p.start()
exitcode = 0
for p in processes:
p.join()
if p.exitcode != 0:
exitcode = p.exitcode
break
if exitcode != 0:
for p in processes:
if killall_if_fail and p.is_alive():
p.terminate()
return exitcode
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.