Behaviors of Python threading, multiprocessing and futures.

README.md

Behaviors of Python threading, multiprocessing and concurrent.futures

I saw the behaviors as follows:

1. threading.Thread and multiprocessing.Process do not raise to the main thread.
2. multiprocessing.Pool and multiprocessing.pool.ThreadPool (this is not documented at all) leave zombie threads.

The results show that concurrent.futures is the best choice for threading and multiprocessing in Python!

I did this experiment on Python 2.7.9, and Python 2.x does not have concurrent.futures, so I installed it by pip command pip install futures.

output.log

Exception in thread Thread-1:
Traceback (most recent call last):
  File "/Users/takuya/anaconda/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/Users/takuya/anaconda/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "example_thread_process.py", line 10, in bar
    raise ValueError("bar raise")
ValueError: bar raise

main_Thread [NOT raised]
Zombie thread: 0
main_ThreadPool [RAISED]
Zombie thread: 7
main_ThreadPoolExecutorMap [RAISED]
Zombie thread: 0
main_ThreadPoolExecutorSubmit [RAISED]
Zombie thread: 0
Process Process-1:
Traceback (most recent call last):
  File "/Users/takuya/anaconda/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/Users/takuya/anaconda/lib/python2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "example_thread_process.py", line 10, in bar
    raise ValueError("bar raise")
ValueError: bar raise
main_Process [NOT raised]
Zombie thread: 0
main_ProcessPool [RAISED]
Zombie thread: 3
main_ProcessPoolExecutorMap [RAISED]
Zombie thread: 0
main_ProcessPoolExecutorSubmit [RAISED]
Zombie thread: 0

python_thread_process.py

from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
import multiprocessing
from multiprocessing.pool import ThreadPool
import threading
import time


def bar(i=0):
    if i == 0:
        raise ValueError("bar raise")
    return i ** 2


def main_Thread():
    thread = threading.Thread(target=bar)
    thread.start()
    thread.join()
    raise RuntimeError("Exception not caught")


def main_ThreadPool():
    p = ThreadPool(4)
    for i in p.map(bar, xrange(4)):
        print i
    raise RuntimeError("Exception not caught")


def main_ThreadPoolExecutorMap():
    with ThreadPoolExecutor(4) as ex:
        for i in ex.map(bar, xrange(4)):
            print i
    raise RuntimeError("Exception not caught")


def main_ThreadPoolExecutorSubmit():
    with ThreadPoolExecutor(4) as ex:
        s = ex.submit(bar)
        print s.result()
    raise RuntimeError("Exception not caught")


def main_Process():
    thread = multiprocessing.Process(target=bar)
    thread.start()
    thread.join()
    raise RuntimeError("Exception not caught")


def main_ProcessPool():
    p = multiprocessing.Pool(4)
    for i in p.map(bar, xrange(4)):
        print i
    raise RuntimeError("Exception not caught")


def main_ProcessPoolExecutorMap():
    with ProcessPoolExecutor(4) as ex:
        for i in ex.map(bar, xrange(4)):
            print i
    raise RuntimeError("Exception not caught")


def main_ProcessPoolExecutorSubmit():
    with ProcessPoolExecutor(4) as ex:
        s = ex.submit(bar, 0)
        print s.result()
    raise RuntimeError("Exception not caught")


def run(fun):
    ac = threading.active_count()
    try:
        fun()
    except RuntimeError:
        print fun.__name__, "[NOT raised]"
    except ValueError:
        print fun.__name__, "[RAISED]"
    time.sleep(1)
    print "Zombie thread:", threading.active_count() - ac

if __name__ == '__main__':
    run(main_Thread)
    run(main_ThreadPool)
    run(main_ThreadPoolExecutorMap)
    run(main_ThreadPoolExecutorSubmit)
    run(main_Process)
    run(main_ProcessPool)
    run(main_ProcessPoolExecutorMap)
    run(main_ProcessPoolExecutorSubmit)