multiprocessing pool test

gistfile1.txt

#!/usr/bin/env python

import multiprocessing, sys, random, time

def process_file(filename, foo, bar, baz='biz'):
    print "FOO"
    if random.random() < 0.5:
        raise Exception("blah")
    return 123

if __name__ == '__main__':
    # setup code setting parameters foo, bar, and biz

    psize = 4 # multiprocessing.cpu_count()*2
    pool = multiprocessing.Pool(processes=psize)

    a = map(lambda x: pool.apply_async(process_file, (x, 'foo', 'bar'), dict(baz='biz')), sys.argv[1:])
    pool.close()
    pool.join()