Example Schwimmbad usage

0_schwimmbad_demo.py

"""
A tutorial introduction to Schwimmbad

https://github.com/adrn/schwimmbad

To install the requirements:

    > cat requirements.txt | xargs pip install

"""

import numpy as np
import time

data = np.arange(1e6)

def do_the_processing(x):
    # Do something with each value in the data set!
    # For example, here we just square each value
    return x**4


print("using a loop...\n------------------")
start = time.time()

values = []
for x in data:
    values.append(do_the_processing(x))

end = time.time()
print(values[:10])
print("Time elapsed: %.4f\n"%(end - start))


print("using map...\n------------------")
start = time.time()
values = list(map(do_the_processing, data))
end = time.time()

print(values[:10])
print("Time elapsed: %.4f\n"%(end - start))


print("Using schwimmbad to do this (in serial)\n------------------")
from schwimmbad import SerialPool

start = time.time()

pool = SerialPool()

values = list(pool.map(do_the_processing, data))

end = time.time()

print(values[:10])
print("Time elapsed: %.4f\n"%(end - start))

pool.close()


print("Using schwimmbad with multiprocessing (2 threads)\n------------------")
from schwimmbad import MultiPool

N = 2
start = time.time()

pool = MultiPool()

values = list(pool.map(do_the_processing, data))

# with MultiPool(processes=N) as pool:
#     """
#     The context manager (`with`) ensures the file gets closed once we're finished
#     """
#     values = list(pool.map(do_the_processing, data))

end = time.time()
print(values[:10])
print("Time elapsed: %.4f\n"%(end - start))
pool.close()




print("Using schwimmbad with multiprocessing in a context manager (8 threads)\n------------------")
from schwimmbad import MultiPool

N = 8
start = time.time()

with MultiPool(processes=N) as pool:
    """
    The context manager (`with`) ensures the file gets closed once we're finished
    """
    values = list(pool.map(do_the_processing, data))

end = time.time()
print(values[:10])
print("Time elapsed: %.4f\n"%(end - start))

1_schwimmbad_mpi.py

"""
Run with:

    > mpiexec -n 2 python mpi-demo.py

"""

import sys
import time
import numpy as np
from schwimmbad import MPIPool


def do_the_processing(x):
    # Do something with each value in the data set!
    # For example, here we just square each value
    return x**2


start = time.time()

pool = MPIPool()

if not pool.is_master():
    pool.wait()
    sys.exit(0)



data = np.arange(10000)

values = list(pool.map(do_the_processing,data))

end = time.time()
print("Time elapsed: %.4f\n"%(end - start))

print(values[:10])

pool.close()

requirements.txt

six
numpy
ipython
mpi4py
schwimmbad