How to manage a pool of subprocesses in Python

mybunch.py

#!/usr/bin/python

import os
from pipes import quote
from subprocess import Popen, PIPE
import sys
from time import sleep


N_PROCS = 4  # adjust this or make this an option

def main():
    commands = [
        ['sleep', '6'],
        ['sleep', '5'],
        ['sleep', '4'],
        ['sleep', '3'],
        ['sleep', '2'],
        ['sleep', '1'],
    ]  # Populate this with real commands, read from arguments, or else
    procs = {}
    ret_code = 0
    while commands or procs:
        # Give processes time to do stuff, adjust to suit
        if procs:
            sleep(0.1)
        # Poll for completed processes
        # Handle their results
        for key, (command, proc) in procs.copy().items():
            if proc.poll() is None:
                continue
            out, err = proc.communicate()
            sys.stdout.write(out)
            sys.stderr.write(err)
            command_ret_code = proc.wait()
            sys.stderr.write('%s  # return %d\n' % (
                ' '.join([quote(item) for item in command]), command_ret_code))
            # Set overall exit code to return error on any error
            if command_ret_code:
                ret_code = 1
            del procs[key]
        # Run more commands, if we have space
        while commands and len(procs) < N_PROCS:
            command = commands.pop(0)
            try:
                proc = Popen(
                    command, stdout=PIPE, stderr=PIPE, stdin=open(os.devnull))
            except OSError as exc:
                # OSError does not set the executable name!
                if not exc.filename:
                    exc.filename = command[0]
                ret_code = 1
                sys.stderr.write('%s\n' % exc)
                continue
            procs[id(proc)] = (command, proc)
    sys.exit(ret_code)


if __name__ == '__main__':
    main()