stream_procs_select.py

import subprocess
import time
import select
import sys


def stream_procs(procs, poll_interval_ms=100, timeout_ms=None):
    """
    yields output for all processes until they're all finished

    note that the processes themselves must not buffer their output
    (for example, you'll want to run child python processes with the
    "python -u" flag)
    """
    start_time = time.time()

    while any(p.poll() is None for p in procs):
        if timeout_ms is not None:
            elapsed_time = time.time() - start_time
            if elapsed_time > (timeout_ms / 1000.0):
                for proc in procs:
                    is_still_running = proc.poll() is None
                    if is_still_running:
                        proc.kill()
                break

        outputs = [p.stdout for p in procs if p.poll() is None]
        readable, _, _ = select.select(outputs, [], [],
                                       poll_interval_ms / 1000.0)
        for buf in readable:
            line = buf.readline()
            if line:
                yield line

    # fully read stdout out of all procs
    for proc in procs:
        line = proc.stdout.readline()
        while line:
            yield line
            line = proc.stdout.readline()


# Example Usage
cmds = [
    'python -u slow_program.py',
    'python -u fast_program.py',
]
procs = [subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE)
         for cmd in cmds]
for line in stream_procs(procs):
    processed_line = '>>> {}'.format(line.decode('utf-8'))
    sys.stdout.write(processed_line)