public

pfctdayelise / 00-intro_errorreporting.txt

Last active 2014-04-20

These are snippets of py.test in action, used in a talk given at 
PyCon AU 2012 in Hobart, Tasmania. They are all relevant for 
py.test 2.2 except where specified. Where taken from open source 
projects I have listed a URL, some examples are from the py.test 
documentation, some are from my workplace.
 
Apart from things called test_*, these functions should probably
be in your conftest.py, although they can generally start life in
your test files.
 
Talk info: http://2012.pycon-au.org/schedule/52/view_talk
Slides: http://www.slideshare.net/pfctdayelise/funcargs-other-fun-with-pytest
 
http://pytest.org/
 
http://stackoverflow.com/questions/tagged/py.test
http://codespeak.net/mailman/listinfo/py-dev
http://lists.idyll.org/listinfo/testing-in-python
 
##############
# informative error reporting
 
# tb==native
 
Traceback (most recent call last):
  File "/workspace/Review/GFESuite/tests/unit/formatters/test_Precis.py", 
line 882, in test_lowDetail12hourWxBrackets
    assert 'morningZ' in words
AssertionError: assert 'morningZ' in 'morning shower or two'
 
 
# tb==short
 
tests/unit/formatters/test_Precis.py:882: in test_lowDetail12hourWxBrackets
>       assert 'morningZ' in words
E       assert 'morningZ' in 'morning shower or two'
 
 
# tb==long
 
mockAccessor = 
 
    def test_lowDetail12hourWxBrackets(mockAccessor):
        """
        Initial 6 hours of wx is correctly being washed out to the first 12 hours.
        """
        mockAccessor.mockGrids([
                Flat("Sky", 0, 24, 0),
                Flat("Wx", 0, 6, "Sct:SH:m::"),
                Flat("Wx", 6, 24, "NoWx"),
                ])
        _icon, words = precisIconWords(mockAccessor, period=6, detail='low')
        assert 'early' not in words
>       assert 'morningZ' in words
E       assert 'morningZ' in 'morning shower or two'
 
 
tests/unit/formatters/test_Precis.py:882: AssertionError

# add back in unittest assert statements
# apparently cribbed from something in nose
 
def pytest_namespace():
    """Make unittest assert methods available.
    This is useful for things such floating point checks with assertAlmostEqual.
    """
 
    import unittest
 
    class Dummy(unittest.TestCase):
        def dummy(self):
            pass
    obj = Dummy('dummy')
 
    names = {name: member
             for name, member in inspect.getmembers(obj)
             if name.startswith('assert') and '_' not in name}
 
    return names
 
 
# test file
def test_kn2kmh():
    py.test.assertAlmostEqual(UnitConvertor.kn2kmh(10), 18.52, places=4)
 
 
###########################################################
# add a hook for winpdb
 
def pytest_addoption(parser):
    """pytest hook that adds a GFE specific option.
    """
 
    # Add options.
    group = parser.getgroup('graphical forecast editor options')
    group.addoption('--winpdb', dest='usewinpdb', action='store_true', default=False,
                    help=('start the WinPDB Python debugger before calling each test function. '
                          'Suggest only using this with a single test at a time (i.e. -k .'))
 
def pytest_configure(config):
    # Only do these if this process is the master.
    if not hasattr(config, 'slaveinput'):
        # Activate winpdb plugin if appropriate.
        if config.getvalue("usewinpdb"):
            config.pluginmanager.register(WinPdbInvoke(), 'winpdb')
 
class WinPdbInvoke:
 
    def __init__(self):
        print "initialising winpdb invoker"
 
    def pytest_pyfunc_call(self, pyfuncitem):
        import rpdb2
        rpdb2.start_embedded_debugger('0')
 
 
# then run: py.test -k test_some_specific_thing --winpdb

# SKIP
# inside a test, if you need to check something after the environment
# has been loaded
 
    if not config.get('ifpServer.allowOfficalWrites'):
        py.test.skip('Official DB writes are not allowed.')
 
 
# decorators:
 
import sys
 
win32only = pytest.mark.skipif("sys.platform != 'win32'")
 
@win32only
def test_foo():
    ....
 
@py.test.mark.skipif('True')
def test_foo1():
    print "foo1"
 
@py.test.mark.skipif('False')
def test_foo2():
    print "foo2"
 
def test_foo3():
    py.test.skip('inside skip')
    print "foo3"
 
 
# XFAIL
@py.test.mark.xfail
def test_foo4():
    assert False
 
@py.test.mark.xfail(reason='This is a bad idea')
def test_foo5():
    assert False
 
@py.test.mark.xfail(reason='Maybe this was a bad idea once')
def test_foo6():
    assert True
 
def test_foo7():
    # force test to be recorded as an xfail,
    # even if it would otherwise pass
    py.test.xfail()
    assert True
 
# output:
 
test_foo4 xfail
test_foo5 xfail
test_foo6 XPASS
test_foo7 xfail
 
# with --runxfail:
 
test_foo4 FAILED
test_foo5 FAILED
test_foo6 PASSED
test_foo7 FAILED
# plus tracebacks
 
 
# example custom marks
 
@py.test.mark.slow
 
@py.test.mark.dstAffected
 
@py.test.mark.mantis1543
 
@py.test.mark.flaky
 
@py.test.mark.unicode
 
@py.test.mark.regression
 
 
# in 2.2 -  parametrize
 
@pytest.mark.parametrize(("input", "expected"), [
    ("3+5", 8),
    ("2+4", 6),
    ("6*9", 42),
])
def test_eval(input, expected):
    assert eval(input) == expected

# code
def isSquare(n):
    n = n ** 0.5
    return int(n) == n
 
# test file
def pytest_generate_tests(metafunc):
    squares = [1, 4, 9, 16, 25, 36, 49]
    for n in range(1, 50):
        expected = n in squares
        if metafunc.function.__name__ == 'test_isSquare':
            metafunc.addcall(id=n, funcargs=dict(n=n, expected=expected))
 
 
def test_isSquare(n, expected):
    assert isSquare(n) == expected
 
 
# then:
# conftest.py
def pytest_generate_tests(__multicall__, metafunc):
    """Supports parametrised tests using generate_ fns.
    Use multicall to call any other pytest_generate_tests hooks first.
    If the test_ fn has a generate_ fn then call it with the metafunc
to let it parametrise the test.
    """
    __multicall__.execute()
    name = metafunc.function.__name__.replace('test_', 'generate_')
    fn = getattr(metafunc.module, name, None)
    if fn:
        fn(metafunc)
 
 
# generate function is simplified, no boilerplate!
# and we can have one per test function with multiple pairs in a single module, woot!
def generate_isSquare(metafunc):
    squares = [1, 4, 9, 16, 25, 36, 49]
    for n in range(1, 50):
        expected = n in squares
        metafunc.addcall(id=n, funcargs=dict(n=n, expected=expected))

import os.path
 
def getssh(): # pseudo application code
    return os.path.join(os.path.expanduser("~admin"), '.ssh')
 
 
def test_getssh(monkeypatch):
 
    def mockreturn(path):
        return '/abc'
    monkeypatch.setattr(os.path, 'expanduser', mockreturn)
    
    x = getssh()
    assert x == '/abc/.ssh'
 
 
######################
# a funcarg to hide/abstract away some monkeypatching
 
def pytest_funcarg__noPreviousWarnings(request):
    """pytest funcarg to avoid retrieving REAL previously issued warnings"""
    _ = request.getfuncargvalue("textImporter")
 
    def setup():
        import RecognisedWarnings as RW
        monkeypatch = request.getfuncargvalue("monkeypatch")
        noPreviousWarnings = lambda _x, _y, _z: None
        monkeypatch.setattr(RW, '_getFDBViewerXML', noPreviousWarnings)
 
    def teardown(obj):
        pass
 
    return request.cached_setup(setup, teardown, scope='function')

# http://anders.conbere.org/blog/2011/05/03/setup_and_teardown_methods_with_py.test/
 
# conftest.py
def setup_fixtures():
    db.insert('...')
    return db
 
def teardown_fixtures(db):
    db.destroy('..')
 
def py_test_funcarg__db(request):
    return request.cached_setup(
        setup = setup_fixtures,
        teardown = teardown_fixtures,
        scope = "module")
 
# test_db.py
def test_db(db):
    assert(len(db.query(x=y)) >= 1)
 
 
#####################
# a real DB example
# still far from a good example for most use cases I suspect, what with the lack of ORM and all
 
def setupTestDb():
    """Setup test gfedb ensuring we only do this once."""
 
    import gfeDB
    if gfeDB.DB_NAME == 'gfedb':
        from NeoConfig import config as neoConfig
        name = 'gfedbtest{}'.format(neoConfig['instance.port'])
        cmd = """\
            mysql -u root -e "DROP DATABASE IF EXISTS {name};
                              CREATE DATABASE {name};
                              USE {name};
                              GRANT ALL ON * TO gfe;
                              GRANT ALL ON * TO gfe@localhost;
                              FLUSH PRIVILEGES;";
            mysqldump -u root --no-data gfedb | mysql -u root {name}
            """.format(name=name)
        subprocess.check_output(cmd, shell=True)
        gfeDB.DB_NAME = name
 
 
def pytest_funcarg__testDb(request):
    """pytest funcarg for a test gfedb."""
 
    return request.cached_setup(setup=setupTestDb,
                                scope='session')
 
 
def pytest_funcarg__emptyDb(request):
    """pytest funcarg to truncate all tables in the test gfedb."""
 
    _ = request.getfuncargvalue('testDb')
 
    def setup():
        from NeoConfig import config as neoConfig
        name = 'gfedbtest{}'.format(neoConfig['instance.port'])
        cmd = """\
            mysql -u root -e "USE {name};
                              DELETE FROM fire_event_tb;
                              DELETE FROM forecast_tb;
                              DELETE FROM issuance_tb;
                              DELETE FROM precis_pop_areas_tb;
                              DELETE FROM warning_tb;"
            """.format(name=name)
        subprocess.check_output(cmd, shell=True)
 
    return request.cached_setup(setup=setup,
                                scope='function')
 
###########################
# funcarg to express a prerequisite
 
 
# https://github.com/lunaryorn/pyudev/blob/develop/tests/test_libudev.py
 
def pytest_funcarg__libudev(request):
    try:
        return _libudev.load_udev_library()
    except ImportError:
        pytest.skip('udev not available')

# django example
# http://pytest-django.readthedocs.org/en/latest/helpers.html
 
from myapp.views import my_view
 
def test_details(rf):
    request = rf.get('/customer/details')
    response = my_view(request)
    assert response.status_code == 200
 
def test_an_admin_view(admin_client):
    response = admin_client.get('/admin/')
    assert response.status_code == 200
 
 
##############################
# Google App Engine
 
# I wonder if these examples should be using monkeypatch to do os.environ.update??
 
# http://pypi.python.org/pypi/pytest_gae/0.2.1
 
 
import os
 
from webtest import TestApp
from main import make_application
 
 
def pytest_funcarg__anon_app(request):
    os.environ.update({'USER_EMAIL': '',
                        'USER_ID': '',
                        'AUTH_DOMAIN': 'google',
                        'USER_IS_ADMIN': '0'})
    return TestApp(make_application())
 
 
def pytest_funcarg__user_app(request):
    os.environ.update({'USER_EMAIL': 'simple@google.com',
                       'USER_ID': '1',
                       'AUTH_DOMAIN': 'google',
                       'USER_IS_ADMIN': '0'})
    return TestApp(make_application())
 
 
def pytest_funcarg__admin_app(request):
    os.environ.update({'USER_EMAIL': 'admin@google.com',
                       'USER_ID': '2',
                       'AUTH_DOMAIN': 'google',
                       'USER_IS_ADMIN': '1'})
    return TestApp(make_application())
 
 
def test_index(anon_app):
    assert "Index" in anon_app.get('/index')
 
 
def test_user_with_user(user_app):
    assert "User" in user_app.get('/users')
 
 
def test_user_with_anon(anon_app):
    assert '302 Moved Temporarily' == anon_app.get('/users').status
 
 
def test_user_with_admin(admin_app):
    assert "Admin" in admin_app.get('/users')

# https://bitbucket.org/hpk42/py/src/0fca612a4bbd/conftest.py:
 
def pytest_generate_tests(metafunc):
    multi = getattr(metafunc.function, 'multi', None)
    if multi is not None:
        assert len(multi.kwargs) == 1
        for name, l in multi.kwargs.items():
            for val in l:
                metafunc.addcall(funcargs={name: val})
    elif 'anypython' in metafunc.funcargnames:
        for name in ('python2.4', 'python2.5', 'python2.6',
                     'python2.7', 'python3.1', 'pypy-c', 'jython'):
            metafunc.addcall(id=name, param=name)
 
 
def pytest_funcarg__anypython(request):
    name = request.param
    executable = getexecutable(name)
    if executable is None:
        if sys.platform == "win32":
            executable = winpymap.get(name, None)
            if executable:
                executable = py.path.local(executable)
                if executable.check():
                    return executable
        py.test.skip("no %s found" % (name,))
    return executable

#probable in 2.3 (not yet released)
 
# content of conftest.py
import pytest
import smtplib
 
@pytest.factory(scope="session",
                params=["merlinux.eu", "mail.python.org"])
def smtp(testcontext):
    smtp = smtplib.SMTP(testcontext.param)
    def fin():
        print ("finalizing %s" % smtp)
        smtp.close()
    testcontext.addfinalizer(fin)
    return smtp
 
 
# content of test_module.py
def test_ehlo(smtp):
    response = smtp.ehlo()
    assert response[0] == 250
    assert "merlinux" in response[1]
    assert 0  # for demo purposes
 
def test_noop(smtp):
    response = smtp.noop()
    assert response[0] == 250
    assert 0  # for demo purposes
 
 
$ py.test --collectonly
=========================== test session starts ============================
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
collecting ... collected 4 items
<Module 'test_module.py'>
  <Function 'test_ehlo[merlinux.eu]'>
  <Function 'test_noop[merlinux.eu]'>
  <Function 'test_ehlo[mail.python.org]'>
  <Function 'test_noop[mail.python.org]'>
 
=============================  in 0.02 seconds =============================

# v0
# Feb 2010
# data/test/dbconfig/TEXT/Misc/District_TestScript_2.py
    
    {
    "name": "Thunderstorms with heavy rain are not dry",
    "commentary": "Mantis 01530",
    "productType": "District",  
    "createGrids": [
    ("Fcst", "Wx", "WEATHER",   0,   24, "Chc:TS:!::r", "all"),
    ],
    "notCheckStrings": [
       "dry"
       ],
    "fileChanges": [
       ("District_NSWRO_Definition", "TextUtility", "add", defaultEditAreas, "undo"),
       ],
    "cmdLineVars": str({
      ('Generate Days', 'productIssuance'): 'All Periods',
      ('Issuance Type', 'issuanceType'): 'Morning',
      ('Issued By', 'issuedBy'): None,
      ('CompleteUpdate', 'completeUpdate'): 'yes'}),
    },
 
 
# v1
# early 2011 - move to pytest, directory restructure
 
def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester):
    formatterTester.run({
        "commentary": "Mantis 01530",
        "productType": "District",
        "createGrids": [
            Flat("Wx",    0,   24, "Chc:TS:!::r"),
            ],
        "notCheckStrings": [
            "dry"
            ],
        "cmdLineVars": cmdLineVarsIssuanceMorning
        }, defaults())
 
# v2
# feb 2011 - introduction of gridCreator
 
def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester, gridCreator):
    """Mantis 01530
    """
    gridCreator.createGrids([
            Flat("Wx",    0,   24, "Chc:TS:!::r"),
            ])
    formatterTester.run({
        "productType": "District",
        "notCheckStrings": [
            "dry"
            ],
        "cmdLineVars": cmdLineVarsIssuanceMorning
        }, defaults())
 
 
# lots of specifying areas like this:
 
siteID = siteConfig.GFESUITE_SITEID
DistrictEditAreaDictionary = {
       "NSWRO" : ['NSW_CW013'],
       "VICRO" : ['VIC_CW010', 'VIC_CW011'],
       "QLDRO" : [],
       "SARO" : [],
       "TASRO" : [],
       "WARO" : [],
       "NTRO" : [],
}
area =  DistrictEditAreaDictionary[siteID]
 
 
# v3
# March 2011
 
def pytest_funcarg__districtArea(request):
 
    def start():
 
        areas = {
             "VICRO": DefaultEditArea("VIC_PW007", "Central"),
             "NSWRO": DefaultEditArea("NSW_PW014", "Riverrina"),
             "TASRO": DefaultEditArea("TAS_PW002", "North East"),
             "SARO": DefaultEditArea("SA_PW012", "North West Pastoral"),
        }
 
        return __areaStart(areas)
 
    return request.cached_setup(setup=start, scope='session')
 
 
def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester, gridCreator, districtArea):
    """Mantis 01530
    """
    gridCreator.createGrids([
            Flat("Wx",    0,   24, "Chc:TS:!::r"),
            ])
    formatterTester.run({
        "productType": "District",
        "notCheckStrings": [
            "dry"
            ],
        "cmdLineVars": cmdLineVarsIssuanceMorning(districtArea.aac)
        }, defaults(districtArea.aac))
 
 
 
# v4
# Nov 2011
# tests/system/formatters/wxPhrase/test_attributes.py
 
@py.test.mark.mantis1530
def test_thunderstormsWithHeavyRainAreNotDry(mockAccessor):
    """'heavy rain' and 'dry' attributes were getting mixed up.
    """
    mockAccessor.mockGrids([
            Flat("Wx", 0, 24, "Chc:TS:!::r"),
            ])
    words = weatherWordsLandAreal(mockAccessor)
    assert "dry" not in words