make_mmap.py

import numpy as np

# make memmap file and fill it with some random data

# create a memmap file
fp = np.memmap('data.memmap', dtype='float32', mode='w+', shape=(100000, 40000))

for chunk in range(100):
    # fill the memmap file with some random data
    fp[chunk*1000:(chunk+1)*1000] = np.random.rand(1000, 40000)

# (venv) elephantum@LAPTOP-FQGTSCNO:~/Epoch8/tmp$ ls -lh
# total 15G
# -rw-r--r-- 1 elephantum elephantum  15G Nov 10 01:06 data.memmap
# -rw-r--r-- 1 elephantum elephantum 1.3K Nov 10 01:06 make_mmap.ipynb
# -rw-r--r-- 1 elephantum elephantum 1.7K Nov 10 01:08 use_mmap.ipynb

use_mmap.py

import numpy as np

fp = np.memmap('data.memmap', dtype='float32', mode='r', shape=(100000, 40000))

fp.shape
# (100000, 40000)

x = fp[np.random.randint(0, 100000, 100)].reshape(-1)
# (4000000,)