multiprocess.py

# -*- coding: utf-8 -*-
import multiprocessing
import subprocess
import sys
import time


class Job:
    done = False

    def __init__(self, command):
        self.command = command


class Worker:
    process = None

    def work(self, job):
        self.job = job
        self.process = subprocess.Popen(job.command, shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)

    @property
    def status(self):
        if self.process:
            returncode = self.process.poll()
            if returncode is None:
                return 'WORKING'
            elif returncode == 0:
                return 'SUCCESSFUL'
            else:
                return 'FAILED'
        else:
            return 'FREE'

    def free(self):
        if self.job:
            self.job.done = True
        self.process = None

    def stop(self):
        if self.job:
            self.job.done = True
        if self.process and self.process.poll():
            self.process.kill()

if __name__ == '__main__':
    jobs = [Job(x) for x in sys.argv[1:]]
    workers = [Worker() for x in range(multiprocessing.cpu_count())]
    queue = [(i, x) for i, x in enumerate(jobs)]
    while not all(x.done for x in jobs):
        for w in workers:
            if w.status == 'WORKING':  # 実行中
                pass
            elif w.status in ['SUCCESSFUL', 'FAILED']:   # 終了
                outs, errs = w.process.communicate()  # 同期
                if outs:
                    print(outs)
                if errs:
                    print(errs)
                if w.status == 'SUCCESSFUL':  # 成功
                    w.free()
                elif w.status == 'FAILED':  # 失敗
                    [x.stop() for x in workers]
                    exit(1)
            elif w.status == 'FREE':  # 未割当
                try:
                    i, j = queue.pop(0)
                    w.work(j)
                except IndexError:  # キューが空
                    pass
        time.sleep(0.01)