multiprocessing shared array

mp.py

# -*- coding: utf-8 -*-
"""
Created on Thu Jun 26 13:00:02 2014

@author: pawel
"""

import multiprocessing as mp
import numpy as np
import logging
import os

os.system("taskset -p 0xff %d" % os.getpid())

logger = mp.log_to_stderr(logging.WARNING)

def free_memory():
    total = 0
    with open('/proc/meminfo', 'r') as f:
        for line in f:
            line = line.strip()
            if any(line.startswith(field) for field in ('MemFree', 'Buffers', 'Cached')):
                if len(line.split())<3:
                    continue
                field, amount, unit = line.split()
                amount = int(amount)
                if unit != 'kB':
                    raise ValueError(
                        'Unknown unit {u!r} in /proc/meminfo'.format(u = unit))
                total += amount
    return total

def worker(data,i):
    for _ in range(1000000):
        x = data[i,:].sum()    # Exercise access to data
    logger.warn('Free memory: {m}'.format(m = free_memory()))

def main():
    N = 15000
    data = np.ones((N,N))

    procs = [mp.Process(target = worker, args = (data, i)) for i in range(32)]
    for proc in procs:
        proc.start()
    for proc in procs:
        proc.join()

logger.warn('Initial free: {m}'.format(m = free_memory()))
logger.warn('After allocating data: {m}'.format(m = free_memory()))

if __name__ == '__main__':
    main()