joblib_batch_mechanism.py

import time
from math import sqrt
from joblib import Parallel, delayed

# Results obtained on my dual code Thinkpad laptop by using this modification:
# https://github.com/cjauvin/joblib/compare/parallel_job_batch

start = time.time()
Parallel(n_jobs=1)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 4.2 secs

start = time.time()
Parallel(n_jobs=2)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 16.8 secs

start = time.time()
Parallel(n_jobs=2, batch_size=2)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 9.6 secs

start = time.time()
Parallel(n_jobs=2, batch_size=10)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 3.4 secs

start = time.time()
Parallel(n_jobs=2, batch_size=100)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 1.9 secs

start = time.time()
Parallel(n_jobs=2, batch_size=1000)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 1.8 secs

start = time.time()
Parallel(n_jobs=2, batch_size=25000)(delayed(sqrt)(i**2) for i in xrange(50000))
print time.time() - start # 1.8 secs