Memory Allocation Problem on JURON

.sync.toml

# This file steers synchronization of the current folder with different target machines
# See https://github.com/AndiH/simpler-rsync for more information

[juron]
	hostname = "juron"
	target_folder = "~/scratch/py-memory-test"

README.md

Files for a minimal error thingy

Makefile

PYVERSION = python3
NPROC := 20

CC = mpicc

.PHONY: all run
all: run
run: 
	mpirun -n ${NPROC} ${PYVERSION} test_memory.py

test_memory: test_memory.c
	${CC} -O0 -o $@ $<

mpi_init_ctypes.py

#!python3

import ctypes

lib = ctypes.CDLL("libmpi.so", ctypes.RTLD_GLOBAL)
#print(lib.MPI_Init(None,None))
ierr = lib.MPI_Init(None,None)
assert ierr==0

r = ctypes.create_string_buffer(4096)
n = ctypes.c_int()
ierr = lib.MPI_Get_library_version(r, ctypes.byref(n))
assert ierr==0
print(r[0:n.value])

ierr = lib.MPI_Finalize()
assert ierr==0

test_memory.c

#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <unistd.h>

#define UPPERLIMIT 250*1024/8


unsigned long long getTotalSystemMemory()
{
    long pages = sysconf(_SC_PHYS_PAGES);
    long page_size = sysconf(_SC_PAGE_SIZE);
    return pages * page_size;
}

unsigned long long getFreeSystemMemory() {
    long free_pages = sysconf(_SC_AVPHYS_PAGES);
    long page_size = sysconf(_SC_PAGE_SIZE);
    return free_pages * page_size;
}

int main() {
    MPI_Init(NULL, NULL);
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    if (rank == 0) {

        double* a[UPPERLIMIT];
        int oneMeg = 1024 * 1024;

        for(int i = 0; i < UPPERLIMIT; i++){
            a[i] = (double*) malloc(oneMeg * sizeof(double));

            for (int j = 0; j < oneMeg; j++)
                a[i][j] = 23.42;

            if (i%100 == 0){
                printf(
                    "Iteration %3d\t Size of a: %5.2f GB\t Mem free: %3.2f%\n", 
                    i, 
                    ((float)i * oneMeg * sizeof(a[i][0]))/1024/1024/1024, 
                    ((double) getFreeSystemMemory()) / ((double) getTotalSystemMemory()) * 100.
                );
                if(a[i] == NULL) {
                    printf("is null\n");
                }
            }
        }

        sleep(5);
        for (int i = 0; i < UPPERLIMIT; i++)
            free(a[i]);
    }

    MPI_Finalize();
    return 0;
}

test_memory.py

from sys import getsizeof
import os
# import mpi4py
# mpi4py.rc.initialize = False
# from mpi4py import MPI
import numpy as np
# ^--- this will break things!
import ctypes

# !pip[3] install --user psutil
import psutil
# !pip[3] install --user hurry.filesize
from hurry.filesize import size

# def size(a):
#     return a

# MPI.Init()

libmpi = ctypes.CDLL("libmpi.so", ctypes.RTLD_GLOBAL)
libmpi.MPI_Init(None,None)
 
rank = int(os.environ['OMPI_COMM_WORLD_RANK'])
# rank = MPI.COMM_WORLD.Get_rank() 
if rank is not None:
    if rank == 0:
        data = []
        # generate 190 GB of data: 3040 numpy 1024x1024x8 float64 arrays (64MB each)
        for i in range(int(490*1024/(1024*1024*8*64/8/1024/1024))):
            # data.append(np.ones((1024,1024,8), dtype=np.float64))
            data += [1.] * (64 * 1024*1024 // 8)
            if i % 100 == 0:
                print('Rank: {:2}\t Data size: {:9.2f} MB\t Mem used {} ({:4.1f} %)'.format(rank, (len(data) * getsizeof(data[0])/1024/1024), size(psutil.virtual_memory().used), psutil.virtual_memory().percent))


libmpi.MPI_Finalize()