/00-intro_errorreporting.txt
Created Aug 18, 2012
| These are snippets of py.test in action, used in a talk given at | |
| PyCon AU 2012 in Hobart, Tasmania. They are all relevant for | |
| py.test 2.2 except where specified. Where taken from open source | |
| projects I have listed a URL, some examples are from the py.test | |
| documentation, some are from my workplace. | |
| Apart from things called test_*, these functions should probably | |
| be in your conftest.py, although they can generally start life in | |
| your test files. | |
| Talk info: http://2012.pycon-au.org/schedule/52/view_talk | |
| Slides: http://www.slideshare.net/pfctdayelise/funcargs-other-fun-with-pytest | |
| http://pytest.org/ | |
| http://stackoverflow.com/questions/tagged/py.test | |
| http://codespeak.net/mailman/listinfo/py-dev | |
| http://lists.idyll.org/listinfo/testing-in-python | |
| ############## | |
| # informative error reporting | |
| # tb==native | |
| Traceback (most recent call last): | |
| File "/workspace/Review/GFESuite/tests/unit/formatters/test_Precis.py", | |
| line 882, in test_lowDetail12hourWxBrackets | |
| assert 'morningZ' in words | |
| AssertionError: assert 'morningZ' in 'morning shower or two' | |
| # tb==short | |
| tests/unit/formatters/test_Precis.py:882: in test_lowDetail12hourWxBrackets | |
| > assert 'morningZ' in words | |
| E assert 'morningZ' in 'morning shower or two' | |
| # tb==long | |
| mockAccessor = | |
| def test_lowDetail12hourWxBrackets(mockAccessor): | |
| """ | |
| Initial 6 hours of wx is correctly being washed out to the first 12 hours. | |
| """ | |
| mockAccessor.mockGrids([ | |
| Flat("Sky", 0, 24, 0), | |
| Flat("Wx", 0, 6, "Sct:SH:m::"), | |
| Flat("Wx", 6, 24, "NoWx"), | |
| ]) | |
| _icon, words = precisIconWords(mockAccessor, period=6, detail='low') | |
| assert 'early' not in words | |
| > assert 'morningZ' in words | |
| E assert 'morningZ' in 'morning shower or two' | |
| tests/unit/formatters/test_Precis.py:882: AssertionError |
| # add back in unittest assert statements | |
| # apparently cribbed from something in nose | |
| def pytest_namespace(): | |
| """Make unittest assert methods available. | |
| This is useful for things such floating point checks with assertAlmostEqual. | |
| """ | |
| import unittest | |
| class Dummy(unittest.TestCase): | |
| def dummy(self): | |
| pass | |
| obj = Dummy('dummy') | |
| names = {name: member | |
| for name, member in inspect.getmembers(obj) | |
| if name.startswith('assert') and '_' not in name} | |
| return names | |
| # test file | |
| def test_kn2kmh(): | |
| py.test.assertAlmostEqual(UnitConvertor.kn2kmh(10), 18.52, places=4) | |
| ########################################################### | |
| # add a hook for winpdb | |
| def pytest_addoption(parser): | |
| """pytest hook that adds a GFE specific option. | |
| """ | |
| # Add options. | |
| group = parser.getgroup('graphical forecast editor options') | |
| group.addoption('--winpdb', dest='usewinpdb', action='store_true', default=False, | |
| help=('start the WinPDB Python debugger before calling each test function. ' | |
| 'Suggest only using this with a single test at a time (i.e. -k .')) | |
| def pytest_configure(config): | |
| # Only do these if this process is the master. | |
| if not hasattr(config, 'slaveinput'): | |
| # Activate winpdb plugin if appropriate. | |
| if config.getvalue("usewinpdb"): | |
| config.pluginmanager.register(WinPdbInvoke(), 'winpdb') | |
| class WinPdbInvoke: | |
| def __init__(self): | |
| print "initialising winpdb invoker" | |
| def pytest_pyfunc_call(self, pyfuncitem): | |
| import rpdb2 | |
| rpdb2.start_embedded_debugger('0') | |
| # then run: py.test -k test_some_specific_thing --winpdb |
| # SKIP | |
| # inside a test, if you need to check something after the environment | |
| # has been loaded | |
| if not config.get('ifpServer.allowOfficalWrites'): | |
| py.test.skip('Official DB writes are not allowed.') | |
| # decorators: | |
| import sys | |
| win32only = pytest.mark.skipif("sys.platform != 'win32'") | |
| @win32only | |
| def test_foo(): | |
| .... | |
| @py.test.mark.skipif('True') | |
| def test_foo1(): | |
| print "foo1" | |
| @py.test.mark.skipif('False') | |
| def test_foo2(): | |
| print "foo2" | |
| def test_foo3(): | |
| py.test.skip('inside skip') | |
| print "foo3" | |
| # XFAIL | |
| @py.test.mark.xfail | |
| def test_foo4(): | |
| assert False | |
| @py.test.mark.xfail(reason='This is a bad idea') | |
| def test_foo5(): | |
| assert False | |
| @py.test.mark.xfail(reason='Maybe this was a bad idea once') | |
| def test_foo6(): | |
| assert True | |
| def test_foo7(): | |
| # force test to be recorded as an xfail, | |
| # even if it would otherwise pass | |
| py.test.xfail() | |
| assert True | |
| # output: | |
| test_foo4 xfail | |
| test_foo5 xfail | |
| test_foo6 XPASS | |
| test_foo7 xfail | |
| # with --runxfail: | |
| test_foo4 FAILED | |
| test_foo5 FAILED | |
| test_foo6 PASSED | |
| test_foo7 FAILED | |
| # plus tracebacks | |
| # example custom marks | |
| @py.test.mark.slow | |
| @py.test.mark.dstAffected | |
| @py.test.mark.mantis1543 | |
| @py.test.mark.flaky | |
| @py.test.mark.unicode | |
| @py.test.mark.regression | |
| # in 2.2 - parametrize | |
| @pytest.mark.parametrize(("input", "expected"), [ | |
| ("3+5", 8), | |
| ("2+4", 6), | |
| ("6*9", 42), | |
| ]) | |
| def test_eval(input, expected): | |
| assert eval(input) == expected |
| # code | |
| def isSquare(n): | |
| n = n ** 0.5 | |
| return int(n) == n | |
| # test file | |
| def pytest_generate_tests(metafunc): | |
| squares = [1, 4, 9, 16, 25, 36, 49] | |
| for n in range(1, 50): | |
| expected = n in squares | |
| if metafunc.function.__name__ == 'test_isSquare': | |
| metafunc.addcall(id=n, funcargs=dict(n=n, expected=expected)) | |
| def test_isSquare(n, expected): | |
| assert isSquare(n) == expected | |
| # then: | |
| # conftest.py | |
| def pytest_generate_tests(__multicall__, metafunc): | |
| """Supports parametrised tests using generate_ fns. | |
| Use multicall to call any other pytest_generate_tests hooks first. | |
| If the test_ fn has a generate_ fn then call it with the metafunc | |
| to let it parametrise the test. | |
| """ | |
| __multicall__.execute() | |
| name = metafunc.function.__name__.replace('test_', 'generate_') | |
| fn = getattr(metafunc.module, name, None) | |
| if fn: | |
| fn(metafunc) | |
| # generate function is simplified, no boilerplate! | |
| # and we can have one per test function with multiple pairs in a single module, woot! | |
| def generate_isSquare(metafunc): | |
| squares = [1, 4, 9, 16, 25, 36, 49] | |
| for n in range(1, 50): | |
| expected = n in squares | |
| metafunc.addcall(id=n, funcargs=dict(n=n, expected=expected)) | |
| import os.path | |
| def getssh(): # pseudo application code | |
| return os.path.join(os.path.expanduser("~admin"), '.ssh') | |
| def test_getssh(monkeypatch): | |
| def mockreturn(path): | |
| return '/abc' | |
| monkeypatch.setattr(os.path, 'expanduser', mockreturn) | |
| x = getssh() | |
| assert x == '/abc/.ssh' | |
| ###################### | |
| # a funcarg to hide/abstract away some monkeypatching | |
| def pytest_funcarg__noPreviousWarnings(request): | |
| """pytest funcarg to avoid retrieving REAL previously issued warnings""" | |
| _ = request.getfuncargvalue("textImporter") | |
| def setup(): | |
| import RecognisedWarnings as RW | |
| monkeypatch = request.getfuncargvalue("monkeypatch") | |
| noPreviousWarnings = lambda _x, _y, _z: None | |
| monkeypatch.setattr(RW, '_getFDBViewerXML', noPreviousWarnings) | |
| def teardown(obj): | |
| pass | |
| return request.cached_setup(setup, teardown, scope='function') |
| # http://anders.conbere.org/blog/2011/05/03/setup_and_teardown_methods_with_py.test/ | |
| # conftest.py | |
| def setup_fixtures(): | |
| db.insert('...') | |
| return db | |
| def teardown_fixtures(db): | |
| db.destroy('..') | |
| def py_test_funcarg__db(request): | |
| return request.cached_setup( | |
| setup = setup_fixtures, | |
| teardown = teardown_fixtures, | |
| scope = "module") | |
| # test_db.py | |
| def test_db(db): | |
| assert(len(db.query(x=y)) >= 1) | |
| ##################### | |
| # a real DB example | |
| # still far from a good example for most use cases I suspect, what with the lack of ORM and all | |
| def setupTestDb(): | |
| """Setup test gfedb ensuring we only do this once.""" | |
| import gfeDB | |
| if gfeDB.DB_NAME == 'gfedb': | |
| from NeoConfig import config as neoConfig | |
| name = 'gfedbtest{}'.format(neoConfig['instance.port']) | |
| cmd = """\ | |
| mysql -u root -e "DROP DATABASE IF EXISTS {name}; | |
| CREATE DATABASE {name}; | |
| USE {name}; | |
| GRANT ALL ON * TO gfe; | |
| GRANT ALL ON * TO gfe@localhost; | |
| FLUSH PRIVILEGES;"; | |
| mysqldump -u root --no-data gfedb | mysql -u root {name} | |
| """.format(name=name) | |
| subprocess.check_output(cmd, shell=True) | |
| gfeDB.DB_NAME = name | |
| def pytest_funcarg__testDb(request): | |
| """pytest funcarg for a test gfedb.""" | |
| return request.cached_setup(setup=setupTestDb, | |
| scope='session') | |
| def pytest_funcarg__emptyDb(request): | |
| """pytest funcarg to truncate all tables in the test gfedb.""" | |
| _ = request.getfuncargvalue('testDb') | |
| def setup(): | |
| from NeoConfig import config as neoConfig | |
| name = 'gfedbtest{}'.format(neoConfig['instance.port']) | |
| cmd = """\ | |
| mysql -u root -e "USE {name}; | |
| DELETE FROM fire_event_tb; | |
| DELETE FROM forecast_tb; | |
| DELETE FROM issuance_tb; | |
| DELETE FROM precis_pop_areas_tb; | |
| DELETE FROM warning_tb;" | |
| """.format(name=name) | |
| subprocess.check_output(cmd, shell=True) | |
| return request.cached_setup(setup=setup, | |
| scope='function') | |
| ########################### | |
| # funcarg to express a prerequisite | |
| # https://github.com/lunaryorn/pyudev/blob/develop/tests/test_libudev.py | |
| def pytest_funcarg__libudev(request): | |
| try: | |
| return _libudev.load_udev_library() | |
| except ImportError: | |
| pytest.skip('udev not available') |
| # django example | |
| # http://pytest-django.readthedocs.org/en/latest/helpers.html | |
| from myapp.views import my_view | |
| def test_details(rf): | |
| request = rf.get('/customer/details') | |
| response = my_view(request) | |
| assert response.status_code == 200 | |
| def test_an_admin_view(admin_client): | |
| response = admin_client.get('/admin/') | |
| assert response.status_code == 200 | |
| ############################## | |
| # Google App Engine | |
| # I wonder if these examples should be using monkeypatch to do os.environ.update?? | |
| # http://pypi.python.org/pypi/pytest_gae/0.2.1 | |
| import os | |
| from webtest import TestApp | |
| from main import make_application | |
| def pytest_funcarg__anon_app(request): | |
| os.environ.update({'USER_EMAIL': '', | |
| 'USER_ID': '', | |
| 'AUTH_DOMAIN': 'google', | |
| 'USER_IS_ADMIN': '0'}) | |
| return TestApp(make_application()) | |
| def pytest_funcarg__user_app(request): | |
| os.environ.update({'USER_EMAIL': 'simple@google.com', | |
| 'USER_ID': '1', | |
| 'AUTH_DOMAIN': 'google', | |
| 'USER_IS_ADMIN': '0'}) | |
| return TestApp(make_application()) | |
| def pytest_funcarg__admin_app(request): | |
| os.environ.update({'USER_EMAIL': 'admin@google.com', | |
| 'USER_ID': '2', | |
| 'AUTH_DOMAIN': 'google', | |
| 'USER_IS_ADMIN': '1'}) | |
| return TestApp(make_application()) | |
| def test_index(anon_app): | |
| assert "Index" in anon_app.get('/index') | |
| def test_user_with_user(user_app): | |
| assert "User" in user_app.get('/users') | |
| def test_user_with_anon(anon_app): | |
| assert '302 Moved Temporarily' == anon_app.get('/users').status | |
| def test_user_with_admin(admin_app): | |
| assert "Admin" in admin_app.get('/users') |
| # https://bitbucket.org/hpk42/py/src/0fca612a4bbd/conftest.py: | |
| def pytest_generate_tests(metafunc): | |
| multi = getattr(metafunc.function, 'multi', None) | |
| if multi is not None: | |
| assert len(multi.kwargs) == 1 | |
| for name, l in multi.kwargs.items(): | |
| for val in l: | |
| metafunc.addcall(funcargs={name: val}) | |
| elif 'anypython' in metafunc.funcargnames: | |
| for name in ('python2.4', 'python2.5', 'python2.6', | |
| 'python2.7', 'python3.1', 'pypy-c', 'jython'): | |
| metafunc.addcall(id=name, param=name) | |
| def pytest_funcarg__anypython(request): | |
| name = request.param | |
| executable = getexecutable(name) | |
| if executable is None: | |
| if sys.platform == "win32": | |
| executable = winpymap.get(name, None) | |
| if executable: | |
| executable = py.path.local(executable) | |
| if executable.check(): | |
| return executable | |
| py.test.skip("no %s found" % (name,)) | |
| return executable |
| #probable in 2.3 (not yet released) | |
| # content of conftest.py | |
| import pytest | |
| import smtplib | |
| @pytest.factory(scope="session", | |
| params=["merlinux.eu", "mail.python.org"]) | |
| def smtp(testcontext): | |
| smtp = smtplib.SMTP(testcontext.param) | |
| def fin(): | |
| print ("finalizing %s" % smtp) | |
| smtp.close() | |
| testcontext.addfinalizer(fin) | |
| return smtp | |
| # content of test_module.py | |
| def test_ehlo(smtp): | |
| response = smtp.ehlo() | |
| assert response[0] == 250 | |
| assert "merlinux" in response[1] | |
| assert 0 # for demo purposes | |
| def test_noop(smtp): | |
| response = smtp.noop() | |
| assert response[0] == 250 | |
| assert 0 # for demo purposes | |
| $ py.test --collectonly | |
| =========================== test session starts ============================ | |
| platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8 | |
| plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov | |
| collecting ... collected 4 items | |
| <Module 'test_module.py'> | |
| <Function 'test_ehlo[merlinux.eu]'> | |
| <Function 'test_noop[merlinux.eu]'> | |
| <Function 'test_ehlo[mail.python.org]'> | |
| <Function 'test_noop[mail.python.org]'> | |
| ============================= in 0.02 seconds ============================= | |
| # v0 | |
| # Feb 2010 | |
| # data/test/dbconfig/TEXT/Misc/District_TestScript_2.py | |
| { | |
| "name": "Thunderstorms with heavy rain are not dry", | |
| "commentary": "Mantis 01530", | |
| "productType": "District", | |
| "createGrids": [ | |
| ("Fcst", "Wx", "WEATHER", 0, 24, "Chc:TS:!::r", "all"), | |
| ], | |
| "notCheckStrings": [ | |
| "dry" | |
| ], | |
| "fileChanges": [ | |
| ("District_NSWRO_Definition", "TextUtility", "add", defaultEditAreas, "undo"), | |
| ], | |
| "cmdLineVars": str({ | |
| ('Generate Days', 'productIssuance'): 'All Periods', | |
| ('Issuance Type', 'issuanceType'): 'Morning', | |
| ('Issued By', 'issuedBy'): None, | |
| ('CompleteUpdate', 'completeUpdate'): 'yes'}), | |
| }, | |
| # v1 | |
| # early 2011 - move to pytest, directory restructure | |
| def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester): | |
| formatterTester.run({ | |
| "commentary": "Mantis 01530", | |
| "productType": "District", | |
| "createGrids": [ | |
| Flat("Wx", 0, 24, "Chc:TS:!::r"), | |
| ], | |
| "notCheckStrings": [ | |
| "dry" | |
| ], | |
| "cmdLineVars": cmdLineVarsIssuanceMorning | |
| }, defaults()) | |
| # v2 | |
| # feb 2011 - introduction of gridCreator | |
| def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester, gridCreator): | |
| """Mantis 01530 | |
| """ | |
| gridCreator.createGrids([ | |
| Flat("Wx", 0, 24, "Chc:TS:!::r"), | |
| ]) | |
| formatterTester.run({ | |
| "productType": "District", | |
| "notCheckStrings": [ | |
| "dry" | |
| ], | |
| "cmdLineVars": cmdLineVarsIssuanceMorning | |
| }, defaults()) | |
| # lots of specifying areas like this: | |
| siteID = siteConfig.GFESUITE_SITEID | |
| DistrictEditAreaDictionary = { | |
| "NSWRO" : ['NSW_CW013'], | |
| "VICRO" : ['VIC_CW010', 'VIC_CW011'], | |
| "QLDRO" : [], | |
| "SARO" : [], | |
| "TASRO" : [], | |
| "WARO" : [], | |
| "NTRO" : [], | |
| } | |
| area = DistrictEditAreaDictionary[siteID] | |
| # v3 | |
| # March 2011 | |
| def pytest_funcarg__districtArea(request): | |
| def start(): | |
| areas = { | |
| "VICRO": DefaultEditArea("VIC_PW007", "Central"), | |
| "NSWRO": DefaultEditArea("NSW_PW014", "Riverrina"), | |
| "TASRO": DefaultEditArea("TAS_PW002", "North East"), | |
| "SARO": DefaultEditArea("SA_PW012", "North West Pastoral"), | |
| } | |
| return __areaStart(areas) | |
| return request.cached_setup(setup=start, scope='session') | |
| def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester, gridCreator, districtArea): | |
| """Mantis 01530 | |
| """ | |
| gridCreator.createGrids([ | |
| Flat("Wx", 0, 24, "Chc:TS:!::r"), | |
| ]) | |
| formatterTester.run({ | |
| "productType": "District", | |
| "notCheckStrings": [ | |
| "dry" | |
| ], | |
| "cmdLineVars": cmdLineVarsIssuanceMorning(districtArea.aac) | |
| }, defaults(districtArea.aac)) | |
| # v4 | |
| # Nov 2011 | |
| # tests/system/formatters/wxPhrase/test_attributes.py | |
| @py.test.mark.mantis1530 | |
| def test_thunderstormsWithHeavyRainAreNotDry(mockAccessor): | |
| """'heavy rain' and 'dry' attributes were getting mixed up. | |
| """ | |
| mockAccessor.mockGrids([ | |
| Flat("Wx", 0, 24, "Chc:TS:!::r"), | |
| ]) | |
| words = weatherWordsLandAreal(mockAccessor) | |
| assert "dry" not in words |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment