Batch execution Given an list of shell commands, this script batches them out according to user-defined batch size or default CPU count.

batchjobs.py

import threading, os, sys
from multiprocessing import cpu_count

NUM_CPUS = cpu_count()

def batch_process(command_list, batch_size=NUM_CPUS):
    iteratorlock = threading.Lock()
    exceptions = []
    cmd = command_list.__iter__()
    
    def runall():
        while True:
            iteratorlock.acquire()

            try:
                try:
                    if exceptions: return
                    next_job = cmd.next()
                finally: iteratorlock.release()
            except StopIteration: return
            
            try: os.system(next_job)
            except Exception(next_job):
                failure = sys.exc_info()
                iteratorlock.acquire()
                try: exceptions.append(failure)
                finally: iteratorlock.release()
                    
    jobs = [threading.Thread(target=runall) for j in xrange(batch_size)]
    for job in jobs: job.start()
    for job in jobs: job.join()
    
    if exceptions:
        alfa, beta, dlta = exceptions[0]
        raise alfa, beta, dlta