Sharing memory between processes with various methods

benchmark.ipynb

buffer.py

import time
import ctypes
from multiprocessing import Process, Event, Array
from argparse import ArgumentParser
import numpy as np


def producer(buf, shape, event):
    t0 = time.perf_counter()
    buf[:] = np.random.random(shape)
    event.set()
    print("producer", time.perf_counter() - t0)


def consumer(buf, shape, event):
    event.wait()
    t0 = time.perf_counter()
    a = np.frombuffer(buf.get_obj())
    _ = a**2
    print("consumer", time.perf_counter() - t0)


if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument('size', type=int)
    args = parser.parse_args()

    buf = Array(ctypes.c_double, args.size)
    event = Event()
    shape = args.size

    procs = [Process(target=t, args=(buf, shape, event))
             for t in (producer, consumer)]
    list(map(lambda p: p.start(), procs))
    list(map(lambda p: p.join(), procs))

hdf5.py

import time
from multiprocessing import Process, Event
from argparse import ArgumentParser
import numpy as np
import h5py


def producer(shape, event):
    t0 = time.perf_counter()
    with h5py.File("shared.h5", "w") as hfile:
        hfile.create_dataset('/data', data=np.random.random(shape), chunks=True)
    event.set()
    print("producer", time.perf_counter() - t0)


def consumer(_, event):
    event.wait()
    t0 = time.perf_counter()
    with h5py.File("shared.h5", 'r') as hfile:
        _ = hfile['/data'][:]**2
    print("consumer", time.perf_counter() - t0)


if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument('size', type=int)
    args = parser.parse_args()

    event = Event()
    shape = args.size

    procs = [Process(target=t, args=(shape, event))
             for t in (producer, consumer)]
    list(map(lambda p: p.start(), procs))
    list(map(lambda p: p.join(), procs))

manager.py

import time
from multiprocessing import Manager, Process
from argparse import ArgumentParser
import numpy as np


def producer(namespace, event):
    t0 = time.perf_counter()
    namespace.array = np.random.random(namespace.shape)
    event.set()
    print("producer", time.perf_counter() - t0)


def consumer(namespace, event):
    event.wait()
    t0 = time.perf_counter()
    _ = namespace.array**2
    print("consumer", time.perf_counter() - t0)


if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument('size', type=int)
    args = parser.parse_args()

    with Manager() as mgr:
        namespace = mgr.Namespace()
        event = mgr.Event()
        namespace.shape = args.size

        procs = [Process(target=t, args=(namespace, event))
                 for t in (producer, consumer)]
        list(map(lambda p: p.start(), procs))
        list(map(lambda p: p.join(), procs))