Python multiprocessing example

python_multiprocessing_example.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
A basic example using multiple processes

We spawn 4 processes.  Each process is made to draw N/4 random
integers from 1 to 10, for some fixed large N.

Then the processes adds up its list of numbers, passes it into a queue,
and the main program adds up the subtotals to get a grand total.

This is basically copied from Ryan W. Smith's blog,
https://www.praetorian.com/blog/multi-core-and-distributed-programming-in-python

But I simplified the example and made it work for Python 3.

"""

import random
import time
import sys
from multiprocessing import Process, Queue, cpu_count
import numpy as np
random.seed()

def do_work(q, N):
    # Create a random list of N integers
    myList = np.random.randint(0, 10, N)
    finalSum = sum(myList)

    # Put the result in the Queue to return the the calling process
    q.put(finalSum)

if __name__ == '__main__':
    N = 33333
    startTime = time.time()

    # Not used but this might be nice to know when deciding how
    # many processes to create.  In this example we just hardcode
    # to 4 processes, however.
    num_cores = cpu_count()

    # Create a Queue to share results
    q = Queue()
    # Create 4 sub-processes to do the work
    processes = []
    for i in range(4):
        processes.append(Process(target=do_work, args=(q, N/4)))

    for i in range(4):
        processes[i].start()

    results = []
    # Grab 4 values from the queue, one for each process
    for i in range(4):
       # set block=True to block until we get a result
       results.append(q.get(True))

    # Sum the partial results to get the final result
    finalSum = sum(results)

    for i in range(4):
        processes[i].join()

    endTime = time.time()
    workTime =  endTime - startTime

    print("The job took " + str(workTime) + " seconds to complete")
    print("The final sum was: " + str(finalSum))