Test program for http://stackoverflow.com/questions/27809586/unexpected-memory-footprint-differences-when-spawning-python-multiprocessing-poo

pool_memory_test.py

import multiprocessing as mp
import sys
import resource


def memory():
    self = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
    return self + children


def debug(prefix):
    msg = '{}: {}'.format(prefix, memory())
    sys.stderr.write(msg + '\n')


def process(data):
    return data.upper()


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--pool_first', action='store_true')
    parser.add_argument('--size', type=int, default=1)
    args = parser.parse_args()
    
    create_data = lambda size: '.' * args.size
    # one worker process for faster and cleaner results
    create_pool = lambda: mp.Pool(1)

    if args.pool_first:
        debug('start')
        p = create_pool()
        debug('pool created')
        data = create_data()
        debug('data created')
    else:
        debug('start')
        data = create_data()
        debug('data created')
        p = create_pool()
        debug('pool created')
    p.map(process, data)
    debug('processed')
    # save references to avoid garbage collector affect results
    _ = p
    _ = data