Last active
October 21, 2017 01:17
-
-
Save rvprasad/9994608818a7a3cfa8b6a40431d4f16a to your computer and use it in GitHub Desktop.
Illustrates performance degradation when large data chunks are used with multiprocesing module in Python.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Python -- v3.6 | |
import begin | |
import multiprocessing | |
import time | |
def worker(varying_data, aux_data): | |
t = 0 | |
for j in range(1, 10000): | |
t += varying_data | |
return t | |
aux_data = None | |
def initializer(init_data): | |
global aux_data | |
aux_data = init_data | |
def with_initializer_worker_wrapper(varying_data): | |
return worker(varying_data, aux_data) | |
@begin.subcommand | |
def with_initializer(): | |
iterations = 10 | |
for i in range(3, 8): | |
start_time = time.time() | |
aux_data = [i] * pow(10, i) | |
pool = multiprocessing.Pool(4, initializer, (aux_data,)) | |
data = [1 for x in range(1, 1001)] | |
tmp = 0 | |
for i in range(1, iterations): | |
tmp = sum(pool.map(with_initializer_worker_wrapper, data)) | |
pool.close() | |
pool.join() | |
pool.terminate() | |
end_time = time.time() | |
secs_per_iteration = (end_time - start_time) / iterations | |
print("aux_data {0:>10,} ints : {1:>6.6f} secs per iteration {2}" | |
.format(len(aux_data), secs_per_iteration, tmp)) | |
def without_initializer_worker_wrapper(data): | |
return worker(*data) | |
@begin.subcommand | |
def without_initializer(): | |
iterations = 10 | |
for i in range(3, 8): | |
start_time = time.time() | |
aux_data = [i] * pow(10, i) | |
pool = multiprocessing.Pool(4) | |
data = [(1, aux_data) for x in range(1, 1001)] | |
tmp = 0 | |
for i in range(1, iterations): | |
tmp = sum(pool.map(without_initializer_worker_wrapper, data)) | |
pool.close() | |
pool.join() | |
pool.terminate() | |
end_time = time.time() | |
secs_per_iteration = (end_time - start_time) / iterations | |
print("aux_data {0:>10,} ints : {1:>6.6f} secs per iteration {2}" | |
.format(len(aux_data), secs_per_iteration, tmp)) | |
@begin.start | |
def entry(): | |
pass |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Related gist: test_pool_map_graph.gp