Paralleling tests
$ nosetests --processes 5 test_debug_parallel.py | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 SecondTest | |
INFO - heavy operation is beginning... 1 ThirdTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47620 140735328031504 1396607234.95 setUpClass 1 FirstTest | |
INFO - heavy operation has ended 1 SecondTest | |
DEBUG - 47621 140735328031504 1396607234.96 setUpClass 1 SecondTest | |
INFO - heavy operation has ended 1 ThirdTest | |
DEBUG - 47622 140735328031504 1396607234.98 setUpClass 1 ThirdTest | |
DEBUG - 47620 140735328031504 1396607235.13 5 FirstTest | |
DEBUG - 47621 140735328031504 1396607235.15 5 SecondTest | |
DEBUG - 47622 140735328031504 1396607235.17 5 ThirdTest | |
DEBUG - 47620 140735328031504 1396607235.25 4 FirstTest | |
DEBUG - 47621 140735328031504 1396607235.27 4 SecondTest | |
DEBUG - 47622 140735328031504 1396607235.29 4 ThirdTest | |
DEBUG - 47620 140735328031504 1396607235.34 1 FirstTest | |
DEBUG - 47621 140735328031504 1396607235.36 1 SecondTest | |
DEBUG - 47622 140735328031504 1396607235.38 1 ThirdTest | |
DEBUG - 47622 140735328031504 1396607235.44 _3rd ThirdTest | |
DEBUG - 47621 140735328031504 1396607235.57 _2nd SecondTest | |
DEBUG - 47620 140735328031504 1396607235.58 3 FirstTest | |
DEBUG - 47622 140735328031504 1396607235.68 3 ThirdTest | |
DEBUG - 47620 140735328031504 1396607235.73 2 FirstTest | |
DEBUG - 47620 140735328031504 1396607235.73 tearDownClass FirstTest | |
.....DEBUG - 47621 140735328031504 1396607235.81 3 SecondTest | |
DEBUG - 47622 140735328031504 1396607235.83 2 ThirdTest | |
DEBUG - 47622 140735328031504 1396607235.83 tearDownClass ThirdTest | |
......DEBUG - 47621 140735328031504 1396607235.96 2 SecondTest | |
DEBUG - 47621 140735328031504 1396607235.96 tearDownClass SecondTest | |
...... | |
---------------------------------------------------------------------- | |
Ran 17 tests in 4.101s | |
OK | |
$ SHARE=True nosetests --processes 5 test_debug_parallel.py | |
INFO - Shared: True | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47651 140735328031504 1396607263.19 setUpClass 1 FirstTest | |
DEBUG - 47660 140735328031504 1396607263.37 1 FirstTest | |
DEBUG - 47659 140735328031504 1396607263.37 4 FirstTest | |
..DEBUG - 47658 140735328031504 1396607263.41 5 FirstTest | |
.DEBUG - 47662 140735328031504 1396607263.46 2 FirstTest | |
.DEBUG - 47659 140735328031504 1396607263.49 4 SecondTest | |
.DEBUG - 47658 140735328031504 1396607263.5 1 SecondTest | |
.DEBUG - 47661 140735328031504 1396607263.54 3 FirstTest | |
.DEBUG - 47660 140735328031504 1396607263.55 5 SecondTest | |
.DEBUG - 47658 140735328031504 1396607263.66 2 SecondTest | |
DEBUG - 47660 140735328031504 1396607263.67 4 ThirdTest | |
DEBUG - 47662 140735328031504 1396607263.68 _2nd SecondTest | |
...DEBUG - 47661 140735328031504 1396607263.72 5 ThirdTest | |
DEBUG - 47660 140735328031504 1396607263.73 _3rd ThirdTest | |
DEBUG - 47659 140735328031504 1396607263.74 3 SecondTest | |
...DEBUG - 47658 140735328031504 1396607263.75 1 ThirdTest | |
.DEBUG - 47661 140735328031504 1396607263.87 2 ThirdTest | |
.DEBUG - 47662 140735328031504 1396607263.92 3 ThirdTest | |
.DEBUG - 47651 140735328031504 1396607263.92 tearDownClass FirstTest | |
---------------------------------------------------------------------- | |
Ran 17 tests in 3.737s | |
OK | |
$ SPLIT=True nosetests --processes 5 test_debug_parallel.py | |
INFO - Can Split: True | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47678 140735328031504 1396607281.78 setUpClass 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47679 140735328031504 1396607281.8 setUpClass 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47680 140735328031504 1396607281.82 setUpClass 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47681 140735328031504 1396607281.84 setUpClass 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47682 140735328031504 1396607281.85 setUpClass 1 FirstTest | |
DEBUG - 47680 140735328031504 1396607281.91 1 FirstTest | |
DEBUG - 47680 140735328031504 1396607281.91 tearDownClass FirstTest | |
INFO - heavy operation is beginning... 2 SecondTest | |
DEBUG - 47679 140735328031504 1396607281.93 4 FirstTest | |
DEBUG - 47679 140735328031504 1396607281.93 tearDownClass FirstTest | |
INFO - heavy operation is beginning... 2 SecondTest | |
..DEBUG - 47678 140735328031504 1396607281.97 5 FirstTest | |
DEBUG - 47678 140735328031504 1396607281.97 tearDownClass FirstTest | |
.INFO - heavy operation is beginning... 2 SecondTest | |
DEBUG - 47682 140735328031504 1396607282.01 2 FirstTest | |
DEBUG - 47682 140735328031504 1396607282.01 tearDownClass FirstTest | |
INFO - heavy operation is beginning... 2 SecondTest | |
.DEBUG - 47681 140735328031504 1396607282.08 3 FirstTest | |
DEBUG - 47681 140735328031504 1396607282.08 tearDownClass FirstTest | |
INFO - heavy operation is beginning... 2 SecondTest | |
.INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47680 140735328031504 1396607284.91 setUpClass 2 SecondTest | |
INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47679 140735328031504 1396607284.93 setUpClass 2 SecondTest | |
INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47678 140735328031504 1396607284.97 setUpClass 2 SecondTest | |
INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47682 140735328031504 1396607285.01 setUpClass 2 SecondTest | |
DEBUG - 47679 140735328031504 1396607285.05 4 SecondTest | |
DEBUG - 47679 140735328031504 1396607285.05 tearDownClass SecondTest | |
DEBUG - 47678 140735328031504 1396607285.06 1 SecondTest | |
DEBUG - 47678 140735328031504 1396607285.06 tearDownClass SecondTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
..INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47681 140735328031504 1396607285.08 setUpClass 2 SecondTest | |
DEBUG - 47680 140735328031504 1396607285.1 5 SecondTest | |
DEBUG - 47680 140735328031504 1396607285.1 tearDownClass SecondTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
.DEBUG - 47679 140735328031504 1396607285.2 2 SecondTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
.DEBUG - 47682 140735328031504 1396607285.22 _2nd SecondTest | |
DEBUG - 47682 140735328031504 1396607285.22 tearDownClass SecondTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
.DEBUG - 47681 140735328031504 1396607285.32 3 SecondTest | |
DEBUG - 47681 140735328031504 1396607285.32 tearDownClass SecondTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
.INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47678 140735328031504 1396607288.06 setUpClass 3 ThirdTest | |
INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47680 140735328031504 1396607288.1 setUpClass 3 ThirdTest | |
INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47679 140735328031504 1396607288.2 setUpClass 3 ThirdTest | |
DEBUG - 47680 140735328031504 1396607288.22 4 ThirdTest | |
DEBUG - 47680 140735328031504 1396607288.22 tearDownClass ThirdTest | |
INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47682 140735328031504 1396607288.22 setUpClass 3 ThirdTest | |
DEBUG - 47678 140735328031504 1396607288.24 5 ThirdTest | |
DEBUG - 47678 140735328031504 1396607288.24 tearDownClass ThirdTest | |
..DEBUG - 47682 140735328031504 1396607288.28 _3rd ThirdTest | |
DEBUG - 47682 140735328031504 1396607288.28 tearDownClass ThirdTest | |
DEBUG - 47679 140735328031504 1396607288.3 1 ThirdTest | |
DEBUG - 47679 140735328031504 1396607288.3 tearDownClass ThirdTest | |
..INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47681 140735328031504 1396607288.33 setUpClass 3 ThirdTest | |
DEBUG - 47680 140735328031504 1396607288.37 2 ThirdTest | |
.DEBUG - 47681 140735328031504 1396607288.57 3 ThirdTest | |
DEBUG - 47681 140735328031504 1396607288.57 tearDownClass ThirdTest | |
. | |
---------------------------------------------------------------------- | |
Ran 17 tests in 9.858s | |
OK | |
$ py.test -s -n 5 test_debug_parallel.py | |
================================================== test session starts ================================================== | |
platform darwin -- Python 2.7.5 -- py-1.4.20 -- pytest-2.5.2 | |
plugins: xdist | |
gw0 [17] / gw1 [17] / gw2 [17] / gw3 [17] / gw4 [17] | |
scheduling tests via LoadScheduling | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation is beginning... 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
INFO - heavy operation has ended 1 FirstTest | |
DEBUG - 47869 140735328031504 1396607478.69 setUpClass 1 FirstTest | |
DEBUG - 47870 140735328031504 1396607478.69 setUpClass 1 FirstTest | |
DEBUG - 47868 140735328031504 1396607478.69 setUpClass 1 FirstTest | |
DEBUG - 47872 140735328031504 1396607478.69 setUpClass 1 FirstTest | |
DEBUG - 47871 140735328031504 1396607478.69 setUpClass 1 FirstTest | |
DEBUG - 47869 140735328031504 1396607478.78 1 FirstTest | |
DEBUG - 47869 140735328031504 1396607478.78 tearDownClass FirstTest | |
.INFO - heavy operation is beginning... 2 SecondTest | |
DEBUG - 47872 140735328031504 1396607478.81 4 FirstTest | |
DEBUG - 47872 140735328031504 1396607478.81 tearDownClass FirstTest | |
.INFO - heavy operation is beginning... 2 SecondTest | |
DEBUG - 47870 140735328031504 1396607478.84 2 FirstTest | |
DEBUG - 47870 140735328031504 1396607478.84 tearDownClass FirstTest | |
.INFO - heavy operation is beginning... 2 SecondTest | |
DEBUG - 47868 140735328031504 1396607478.87 5 FirstTest | |
DEBUG - 47868 140735328031504 1396607478.87 tearDownClass FirstTest | |
.INFO - heavy operation is beginning... 2 ThirdTest | |
DEBUG - 47871 140735328031504 1396607478.93 3 FirstTest | |
DEBUG - 47871 140735328031504 1396607478.93 tearDownClass FirstTest | |
.INFO - heavy operation is beginning... 2 ThirdTest | |
INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47869 140735328031504 1396607481.79 setUpClass 2 SecondTest | |
INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47872 140735328031504 1396607481.83 setUpClass 2 SecondTest | |
INFO - heavy operation has ended 2 SecondTest | |
DEBUG - 47870 140735328031504 1396607481.84 setUpClass 2 SecondTest | |
INFO - heavy operation has ended 2 ThirdTest | |
DEBUG - 47868 140735328031504 1396607481.88 setUpClass 2 ThirdTest | |
DEBUG - 47872 140735328031504 1396607481.92 1 SecondTest | |
.INFO - heavy operation has ended 2 ThirdTest | |
DEBUG - 47871 140735328031504 1396607481.95 setUpClass 2 ThirdTest | |
DEBUG - 47869 140735328031504 1396607481.97 5 SecondTest | |
.DEBUG - 47871 140735328031504 1396607482.04 1 ThirdTest | |
.DEBUG - 47868 140735328031504 1396607482.06 5 ThirdTest | |
.DEBUG - 47870 140735328031504 1396607482.09 3 SecondTest | |
DEBUG - 47869 140735328031504 1396607482.1 4 SecondTest | |
DEBUG - 47869 140735328031504 1396607482.1 tearDownClass SecondTest | |
..DEBUG - 47871 140735328031504 1396607482.1 _3rd ThirdTest | |
DEBUG - 47871 140735328031504 1396607482.1 tearDownClass ThirdTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
.DEBUG - 47872 140735328031504 1396607482.13 _2nd SecondTest | |
DEBUG - 47872 140735328031504 1396607482.13 tearDownClass SecondTest | |
INFO - heavy operation is beginning... 3 ThirdTest | |
.DEBUG - 47868 140735328031504 1396607482.18 4 ThirdTest | |
DEBUG - 47868 140735328031504 1396607482.18 tearDownClass ThirdTest | |
.DEBUG - 47870 140735328031504 1396607482.24 2 SecondTest | |
DEBUG - 47870 140735328031504 1396607482.24 tearDownClass SecondTest | |
.INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47871 140735328031504 1396607485.1 setUpClass 3 ThirdTest | |
INFO - heavy operation has ended 3 ThirdTest | |
DEBUG - 47872 140735328031504 1396607485.13 setUpClass 3 ThirdTest | |
DEBUG - 47871 140735328031504 1396607485.25 2 ThirdTest | |
DEBUG - 47871 140735328031504 1396607485.26 tearDownClass ThirdTest | |
.DEBUG - 47872 140735328031504 1396607485.38 3 ThirdTest | |
DEBUG - 47872 140735328031504 1396607485.38 tearDownClass ThirdTest | |
. | |
============================================== 17 passed in 10.19 seconds =============================================== |
# Try with: | |
# nosetests --processes 5 test_debug_parallel.py -- setUpClass called once per suite, | |
# same behaviour as SHARE=False or SPLIT=False | |
# SHARE=True nosetests --processes 5 test_debug_parallel.py -- setUpClass called once for all inheritors | |
# SPLIT=True nosetests --processes 5 test_debug_parallel.py -- many-many setUpClass | |
# py.test -s -n 5 test_parallel.py -- same as nose _multiprocess_can_split_=True | |
# Requirements: | |
# python_2.7 | |
# py.test | |
# xdist | |
# nose | |
import os | |
import time | |
import logging | |
from threading import current_thread | |
from multiprocessing.process import current_process | |
import testtools | |
log = logging.getLogger() | |
log.setLevel(logging.DEBUG) | |
handler = logging.StreamHandler() | |
handler.setLevel(logging.DEBUG) | |
formatter = logging.Formatter("%(levelname)s - %(message)s") | |
handler.setFormatter(formatter) | |
log.addHandler(handler) | |
def d(s): | |
log.debug("{0} {1} {2} {3}".format(current_process().ident, current_thread().ident, time.time(), s)) | |
i = 0 # per process counter | |
quant = 0.03 # increase if too fast | |
class FirstTest(testtools.TestCase): | |
if 'SPLIT' in os.environ: | |
can_spilt = os.getenv('SPLIT').lower() in ['true'] | |
_multiprocess_can_split_ = can_spilt | |
log.info("Can Split: "+str(can_spilt)) | |
if 'SHARE' in os.environ: | |
shared = os.getenv('SHARE').lower() in ['true'] | |
_multiprocess_shared_ = shared | |
log.info("Shared: "+str(shared)) | |
@classmethod | |
def setUpClass(cls): | |
global i | |
i += 1 | |
log.info("heavy operation is beginning... {0} {1}".format(i, cls.__name__)) | |
time.sleep(100 * quant) | |
log.info("heavy operation has ended {0} {1}".format(i, cls.__name__)) | |
d("setUpClass {0} {1}".format(i, cls.__name__)) | |
@classmethod | |
def tearDownClass(cls): | |
d("tearDownClass " + cls.__name__) | |
def test_one(self): | |
time.sleep(3 * quant) | |
d("1 " + self.__class__.__name__) | |
def test_two(self): | |
time.sleep(5 * quant) | |
d("2 " + self.__class__.__name__) | |
def test_three(self): | |
time.sleep(8 * quant) | |
d("3 " + self.__class__.__name__) | |
def test_four(self): | |
time.sleep(4 * quant) | |
d("4 " + self.__class__.__name__) | |
def test_five(self): | |
time.sleep(6 * quant) | |
d("5 " + self.__class__.__name__) | |
class SecondTest(FirstTest): | |
def test_this(self): | |
time.sleep(7 * quant) | |
d("_2nd " + self.__class__.__name__) | |
class ThirdTest(FirstTest): | |
def test_that(self): | |
time.sleep(2 * quant) | |
d("_3rd " + self.__class__.__name__) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment