Test program for http://stackoverflow.com/q/24952918/35070

test_mp.py

import multiprocessing as mp

def worker(filex):
    """ reading lots of data """

    raw = filter(lambda x: len(x.split())>3,
        open(filex).readlines())
    data = {}
    # putting all the data in the data dictionary
    return data

# multiprocessing options
nproc = mp.cpu_count()
pool = mp.Pool(processes=nproc)
files_to_parse = ['/etc/passwd', '/etc/hosts']
traj = pool.map(worker, tuple(files_to_parse))
pool.close()