moved to https://github.com/zonca/numpy_memory_allocation

build_cython.sh

python setup.py build_ext -i

cython_malloc.pyx

import random
from libc.stdlib cimport malloc, free

def create_array_malloc(long number=1):
    # allocate number * sizeof(double) bytes of memory
    cdef double *my_array = <double *>malloc(number * sizeof(double))
    cdef long i
    for i in range(number):
        my_array[i] = 0.
    if not my_array:
        raise MemoryError()

poolsimple.py

import gc

import sys
import traceback

import time
import numpy as np

def create_arrays(div, ndata):
    dlist = []
    for d in range(div):
        dlist.append(np.ones(ndata, dtype=np.float64))
    print("  done with allocation", flush=True)
    time.sleep(5)

def main():
    datasize = 5000000000 # ~40GB
    mindiv = 4096 # 1 array of ~40GB
    maxdiv = 4096*4 # 65536 arrays of ~610KB

    div = mindiv
    while div <= maxdiv:
        ndata = datasize // div
        asize = ndata * 8
        asize_mb = asize / 1000000.0

        print("Testing {} arrays size of {:2.2f} MB".format(div, asize_mb),
            flush=True)

        create_arrays(div, ndata)
        print("  Arrays freed", flush=True)

        time.sleep(5)

        div *= 2


if __name__ == "__main__":
    try:
        main()
    except:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
        print("".join(lines), flush=True)

poolsimple_cmalloc.py

import gc

import sys
import traceback

import time
import numpy as np

def create_arrays(div, ndata):
    dlist = []
    for d in range(div):
        dlist.append(np.ones(ndata, dtype=np.float64))
    print("  done with allocation", flush=True)
    time.sleep(5)

def main():
    datasize = 5000000000 # ~40GB
    mindiv = 4096 # 1 array of ~40GB
    maxdiv = 4096*4 # 65536 arrays of ~610KB

    div = mindiv
    while div <= maxdiv:
        ndata = datasize // div
        asize = ndata * 8
        asize_mb = asize / 1000000.0

        print("Testing {} arrays size of {:2.2f} MB".format(div, asize_mb),
            flush=True)

        create_arrays(div, ndata)
        print("  Arrays freed", flush=True)

        time.sleep(5)

        div *= 2

    print("Allocate with malloc now", flush=True)
    from cython_malloc import create_array_malloc
    create_array_malloc(datasize)

    time.sleep(10)

if __name__ == "__main__":
    try:
        main()
    except:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
        print("".join(lines), flush=True)

pypool.py

from mpi4py import MPI

import sys
import traceback

import time
import numpy as np


def main():
    datasize = 5000000000 # ~40GB
    mindiv = 1024 # 1 array of ~40GB
    maxdiv = 1024*16 # 65536 arrays of ~610KB

    rank = MPI.COMM_WORLD.rank
    nproc = MPI.COMM_WORLD.size

    div = mindiv
    while div < maxdiv:
        ndata = datasize // div
        asize = ndata * 8
        asize_mb = asize / 1000000.0

        if rank == 0:
            print("Testing {} arrays size of {:2.2f} MB".format(div, asize_mb),
                flush=True)

        for p in range(nproc):
            # Take turns allocating and freeing
            if rank == p:
                dlist = []
                for d in range(div):
                    dlist.append(np.ones(ndata, dtype=np.float64))
                print("  Proc {} done with allocation".format(p), flush=True)
                del dlist
                print("  Proc {} freed".format(p), flush=True)
            MPI.COMM_WORLD.barrier()

        time.sleep(10)
        MPI.COMM_WORLD.barrier()
        if rank == 0:
            print("  All procs done",
                flush=True)

        div *= 2

    return


if __name__ == "__main__":
    try:
        main()
    except:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
        lines = [ "Proc {}: {}".format(MPI.COMM_WORLD.rank, x) for x in lines ]
        print("".join(lines), flush=True)
        MPI.COMM_WORLD.Abort()

setup.py

from distutils.core import setup
from Cython.Build import cythonize

setup(
    name = "My hello app",
    ext_modules = cythonize('cython_malloc.pyx'),
    )