Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Array access benchmark of Python multi-process.
"""
共有メモリのオーバヘッドを計測する。
"""
import time
import numpy as np
import multiprocessing
import ctypes
import random
np.random.seed(1)
# src=np.random.rand(30*1024*1024).tolist()
src=[0]*100*1024*1024
#通常のローカルメモリ
def singleList(s,l=1):
print("single+List")
#srcはダブル配列
#for加算
start = time.time()
for c in range(l):
ret=0
for i in s:
ret=ret+i
return time.time()-start
def singleNumPy(s,l=1):
print("single+NumPy")
#srcはダブル配列
s=np.asarray(s)
#for加算
start = time.time()
for c in range(l):
ret=0
for i in s:
ret=ret+i
return time.time()-start
#シェアドメモリ(ロックあり)
def singleSharedArray(s,l=1):
print("single+sharedArray")
#シェアドメモリに変換
sa= multiprocessing.Array(ctypes.c_double,len(s),lock=True)
for i in range(len(s)):
sa[i]=s[i]
#for加算
start = time.time()
for c in range(l):
ret=0
for i in sa:
ret=ret+i
return time.time()-start
#シェアドメモリ(ロックなし)
def singleSharedRawArray(s,l=1):
print("single+sharedRawArray")
import multiprocessing
import ctypes
#シェアドメモリに変換
sa= multiprocessing.RawArray(ctypes.c_double,len(s))
for i in range(len(s)):
sa[i]=s[i]
#for加算
start = time.time()
for c in range(l):
ret=0
for i in sa:
ret=ret+i
return time.time()-start
def func(args):
sa=args
ret=0
for i in sa:
ret=ret+i
return ret
def mpSharedRawArray(s,l=1):
print("MP+sharedRawArray")
import multiprocessing
import ctypes
global sa
#シェアドメモリに変換
sa= multiprocessing.RawArray(ctypes.c_double,len(s))
for i in range(len(s)):
sa[i]=s[i]
p=[multiprocessing.Process(target=func, args=(sa,)) for i in range(l)]
start = time.time()
[i.start() for i in p]
[i.join() for i in p]
return time.time()-start
def func2(args):
sa=np.asarray(args).tolist()
ret=0
for i in sa:
ret=ret+i
return ret
def mpList(s,l=1):
print("MP+sharedRawArray+List")
import multiprocessing
import ctypes
global sa
#シェアドメモリに変換
sa= multiprocessing.RawArray(ctypes.c_double,len(s))
for i in range(len(s)):
sa[i]=s[i]
p=[multiprocessing.Process(target=func2, args=(sa,)) for i in range(l)]
start = time.time()
[i.start() for i in p]
[i.join() for i in p]
return time.time()-start
def func3(args):
sa=np.asarray(args)
ret=0
for i in sa:
ret=ret+i
return ret
def mpNumPy(s,l=1):
print("MP+sharedRawArray+NumPy")
import multiprocessing
import ctypes
global sa
#シェアドメモリに変換
sa= multiprocessing.RawArray(ctypes.c_double,len(s))
for i in range(len(s)):
sa[i]=s[i]
p=[multiprocessing.Process(target=func3, args=(sa,)) for i in range(l)]
start = time.time()
[i.start() for i in p]
[i.join() for i in p]
return time.time()-start
NOL=4
print("%5.2fsec"%(singleList(src,NOL)))
print("%5.2fsec"%(singleNumPy(src,NOL)))
print("%5.2fsec"%(singleSharedRawArray(src,NOL)))
print("%5.2fsec"%(mpList(src,NOL)))
print("%5.2fsec"%(mpNumPy(src,NOL)))
print("%5.2fsec"%(mpSharedRawArray(src,NOL)))
#print("%5.2fsec"%(singleSharedArray(src,NOL)))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.