Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Sharing memory between processes with various methods
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
import time
import ctypes
from multiprocessing import Process, Event, Array
from argparse import ArgumentParser
import numpy as np
def producer(buf, shape, event):
t0 = time.perf_counter()
buf[:] = np.random.random(shape)
event.set()
print("producer", time.perf_counter() - t0)
def consumer(buf, shape, event):
event.wait()
t0 = time.perf_counter()
a = np.frombuffer(buf.get_obj())
_ = a**2
print("consumer", time.perf_counter() - t0)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('size', type=int)
args = parser.parse_args()
buf = Array(ctypes.c_double, args.size)
event = Event()
shape = args.size
procs = [Process(target=t, args=(buf, shape, event))
for t in (producer, consumer)]
list(map(lambda p: p.start(), procs))
list(map(lambda p: p.join(), procs))
import time
from multiprocessing import Process, Event
from argparse import ArgumentParser
import numpy as np
import h5py
def producer(shape, event):
t0 = time.perf_counter()
with h5py.File("shared.h5", "w") as hfile:
hfile.create_dataset('/data', data=np.random.random(shape), chunks=True)
event.set()
print("producer", time.perf_counter() - t0)
def consumer(_, event):
event.wait()
t0 = time.perf_counter()
with h5py.File("shared.h5", 'r') as hfile:
_ = hfile['/data'][:]**2
print("consumer", time.perf_counter() - t0)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('size', type=int)
args = parser.parse_args()
event = Event()
shape = args.size
procs = [Process(target=t, args=(shape, event))
for t in (producer, consumer)]
list(map(lambda p: p.start(), procs))
list(map(lambda p: p.join(), procs))
import time
from multiprocessing import Manager, Process
from argparse import ArgumentParser
import numpy as np
def producer(namespace, event):
t0 = time.perf_counter()
namespace.array = np.random.random(namespace.shape)
event.set()
print("producer", time.perf_counter() - t0)
def consumer(namespace, event):
event.wait()
t0 = time.perf_counter()
_ = namespace.array**2
print("consumer", time.perf_counter() - t0)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('size', type=int)
args = parser.parse_args()
with Manager() as mgr:
namespace = mgr.Namespace()
event = mgr.Event()
namespace.shape = args.size
procs = [Process(target=t, args=(namespace, event))
for t in (producer, consumer)]
list(map(lambda p: p.start(), procs))
list(map(lambda p: p.join(), procs))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment