parallel for

gistfile1.py

import os
import sys

class ParallelFor(object):

    def __init__(self, iterable, num_children=2):
        self.iterable = iterable
        self.num_children = num_children
        self._is_child = False
        self._working = None

    def __iter__(self):
        return self._working or iter([])

    def __enter__(self):
        iterable = self.iterable
        if not hasattr(iterable, '__len__'):
            iterable = list(iterable)
            
        chunksize, extra = divmod(len(iterable), self.num_children)
        if extra:
            chunksize += 1
        if len(iterable) == 0:
            chunksize = 0

        for i in range(self.num_children-1):
            pid = os.fork()
            if pid == 0:
                self._is_child = True
                self._working = iter(iterable[chunksize*(i+1):chunksize*(i+2)])
                return self._working
        self._working = iter(iterable[:chunksize])
        return self._working

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def close(self, status_code=0):
         if self._is_child:
             sys.exit(status_code)
         else:
             os.wait()
        
parallel_for = ParallelFor

if __name__ == "__main__":
    import time
    
    l = range(10)
    with parallel_for(l, 4) as p:
        for i in p:
            print "pid %s: %s" % (os.getpid(), i)
            time.sleep(0.01)