00-intro_errorreporting.txt

## 00-intro_errorreporting.txt
These are snippets of py.test in action, used in a talk given at
PyCon AU 2012 in Hobart, Tasmania. They are all relevant for
py.test 2.2 except where specified. Where taken from open source
projects I have listed a URL, some examples are from the py.test
documentation, some are from my workplace.

Apart from things called test_*, these functions should probably
be in your conftest.py, although they can generally start life in
your test files.

Talk info: http://2012.pycon-au.org/schedule/52/view_talk
Slides: http://www.slideshare.net/pfctdayelise/funcargs-other-fun-with-pytest

http://pytest.org/

http://stackoverflow.com/questions/tagged/py.test
http://codespeak.net/mailman/listinfo/py-dev
http://lists.idyll.org/listinfo/testing-in-python

##############
# informative error reporting

# tb==native

Traceback (most recent call last):
  File "/workspace/Review/GFESuite/tests/unit/formatters/test_Precis.py",
line 882, in test_lowDetail12hourWxBrackets
    assert 'morningZ' in words
AssertionError: assert 'morningZ' in 'morning shower or two'


# tb==short

tests/unit/formatters/test_Precis.py:882: in test_lowDetail12hourWxBrackets
>       assert 'morningZ' in words
E       assert 'morningZ' in 'morning shower or two'


# tb==long

mockAccessor =

    def test_lowDetail12hourWxBrackets(mockAccessor):
        """
        Initial 6 hours of wx is correctly being washed out to the first 12 hours.
        """
        mockAccessor.mockGrids([
                Flat("Sky", 0, 24, 0),
                Flat("Wx", 0, 6, "Sct:SH:m::"),
                Flat("Wx", 6, 24, "NoWx"),
                ])
        _icon, words = precisIconWords(mockAccessor, period=6, detail='low')
        assert 'early' not in words
>       assert 'morningZ' in words
E       assert 'morningZ' in 'morning shower or two'


tests/unit/formatters/test_Precis.py:882: AssertionError

## 01-extensible.py
# add back in unittest assert statements
# apparently cribbed from something in nose

def pytest_namespace():
    """Make unittest assert methods available.
    This is useful for things such floating point checks with assertAlmostEqual.
    """

    import unittest

    class Dummy(unittest.TestCase):
        def dummy(self):
            pass
    obj = Dummy('dummy')

    names = {name: member
             for name, member in inspect.getmembers(obj)
             if name.startswith('assert') and '_' not in name}

    return names


# test file
def test_kn2kmh():
    py.test.assertAlmostEqual(UnitConvertor.kn2kmh(10), 18.52, places=4)


###########################################################
# add a hook for winpdb

def pytest_addoption(parser):
    """pytest hook that adds a GFE specific option.
    """

    # Add options.
    group = parser.getgroup('graphical forecast editor options')
    group.addoption('--winpdb', dest='usewinpdb', action='store_true', default=False,
                    help=('start the WinPDB Python debugger before calling each test function. '
                          'Suggest only using this with a single test at a time (i.e. -k .'))

def pytest_configure(config):
    # Only do these if this process is the master.
    if not hasattr(config, 'slaveinput'):
        # Activate winpdb plugin if appropriate.
        if config.getvalue("usewinpdb"):
            config.pluginmanager.register(WinPdbInvoke(), 'winpdb')

class WinPdbInvoke:

    def __init__(self):
        print "initialising winpdb invoker"

    def pytest_pyfunc_call(self, pyfuncitem):
        import rpdb2
        rpdb2.start_embedded_debugger('0')


# then run: py.test -k test_some_specific_thing --winpdb

## 02-marks.py
# SKIP
# inside a test, if you need to check something after the environment
# has been loaded

    if not config.get('ifpServer.allowOfficalWrites'):
        py.test.skip('Official DB writes are not allowed.')


# decorators:

import sys

win32only = pytest.mark.skipif("sys.platform != 'win32'")

@win32only
def test_foo():
    ....

@py.test.mark.skipif('True')
def test_foo1():
    print "foo1"

@py.test.mark.skipif('False')
def test_foo2():
    print "foo2"

def test_foo3():
    py.test.skip('inside skip')
    print "foo3"


# XFAIL
@py.test.mark.xfail
def test_foo4():
    assert False

@py.test.mark.xfail(reason='This is a bad idea')
def test_foo5():
    assert False

@py.test.mark.xfail(reason='Maybe this was a bad idea once')
def test_foo6():
    assert True

def test_foo7():
    # force test to be recorded as an xfail,
    # even if it would otherwise pass
    py.test.xfail()
    assert True

# output:

test_foo4 xfail
test_foo5 xfail
test_foo6 XPASS
test_foo7 xfail

# with --runxfail:

test_foo4 FAILED
test_foo5 FAILED
test_foo6 PASSED
test_foo7 FAILED
# plus tracebacks


# example custom marks

@py.test.mark.slow

@py.test.mark.dstAffected

@py.test.mark.mantis1543

@py.test.mark.flaky

@py.test.mark.unicode

@py.test.mark.regression


# in 2.2 -  parametrize

@pytest.mark.parametrize(("input", "expected"), [
    ("3+5", 8),
    ("2+4", 6),
    ("6*9", 42),
])
def test_eval(input, expected):
    assert eval(input) == expected

## 03-generatetests.py
# code
def isSquare(n):
    n = n ** 0.5
    return int(n) == n

# test file
def pytest_generate_tests(metafunc):
    squares = [1, 4, 9, 16, 25, 36, 49]
    for n in range(1, 50):
        expected = n in squares
        if metafunc.function.__name__ == 'test_isSquare':
            metafunc.addcall(id=n, funcargs=dict(n=n, expected=expected))


def test_isSquare(n, expected):
    assert isSquare(n) == expected


# then:
# conftest.py
def pytest_generate_tests(__multicall__, metafunc):
    """Supports parametrised tests using generate_ fns.
    Use multicall to call any other pytest_generate_tests hooks first.
    If the test_ fn has a generate_ fn then call it with the metafunc
to let it parametrise the test.
    """
    __multicall__.execute()
    name = metafunc.function.__name__.replace('test_', 'generate_')
    fn = getattr(metafunc.module, name, None)
    if fn:
        fn(metafunc)


# generate function is simplified, no boilerplate!
# and we can have one per test function with multiple pairs in a single module, woot!
def generate_isSquare(metafunc):
    squares = [1, 4, 9, 16, 25, 36, 49]
    for n in range(1, 50):
        expected = n in squares
        metafunc.addcall(id=n, funcargs=dict(n=n, expected=expected))


## 04-monkeypatch.py
import os.path

def getssh(): # pseudo application code
    return os.path.join(os.path.expanduser("~admin"), '.ssh')


def test_getssh(monkeypatch):

    def mockreturn(path):
        return '/abc'
    monkeypatch.setattr(os.path, 'expanduser', mockreturn)

    x = getssh()
    assert x == '/abc/.ssh'


######################
# a funcarg to hide/abstract away some monkeypatching

def pytest_funcarg__noPreviousWarnings(request):
    """pytest funcarg to avoid retrieving REAL previously issued warnings"""
    _ = request.getfuncargvalue("textImporter")

    def setup():
        import RecognisedWarnings as RW
        monkeypatch = request.getfuncargvalue("monkeypatch")
        noPreviousWarnings = lambda _x, _y, _z: None
        monkeypatch.setattr(RW, '_getFDBViewerXML', noPreviousWarnings)

    def teardown(obj):
        pass

    return request.cached_setup(setup, teardown, scope='function')

## 05-funcargs.py
# http://anders.conbere.org/blog/2011/05/03/setup_and_teardown_methods_with_py.test/

# conftest.py
def setup_fixtures():
    db.insert('...')
    return db

def teardown_fixtures(db):
    db.destroy('..')

def py_test_funcarg__db(request):
    return request.cached_setup(
        setup = setup_fixtures,
        teardown = teardown_fixtures,
        scope = "module")

# test_db.py
def test_db(db):
    assert(len(db.query(x=y)) >= 1)


#####################
# a real DB example
# still far from a good example for most use cases I suspect, what with the lack of ORM and all

def setupTestDb():
    """Setup test gfedb ensuring we only do this once."""

    import gfeDB
    if gfeDB.DB_NAME == 'gfedb':
        from NeoConfig import config as neoConfig
        name = 'gfedbtest{}'.format(neoConfig['instance.port'])
        cmd = """\
            mysql -u root -e "DROP DATABASE IF EXISTS {name};
                              CREATE DATABASE {name};
                              USE {name};
                              GRANT ALL ON * TO gfe;
                              GRANT ALL ON * TO gfe@localhost;
                              FLUSH PRIVILEGES;";
            mysqldump -u root --no-data gfedb | mysql -u root {name}
            """.format(name=name)
        subprocess.check_output(cmd, shell=True)
        gfeDB.DB_NAME = name


def pytest_funcarg__testDb(request):
    """pytest funcarg for a test gfedb."""

    return request.cached_setup(setup=setupTestDb,
                                scope='session')


def pytest_funcarg__emptyDb(request):
    """pytest funcarg to truncate all tables in the test gfedb."""

    _ = request.getfuncargvalue('testDb')

    def setup():
        from NeoConfig import config as neoConfig
        name = 'gfedbtest{}'.format(neoConfig['instance.port'])
        cmd = """\
            mysql -u root -e "USE {name};
                              DELETE FROM fire_event_tb;
                              DELETE FROM forecast_tb;
                              DELETE FROM issuance_tb;
                              DELETE FROM precis_pop_areas_tb;
                              DELETE FROM warning_tb;"
            """.format(name=name)
        subprocess.check_output(cmd, shell=True)

    return request.cached_setup(setup=setup,
                                scope='function')

###########################
# funcarg to express a prerequisite


# https://github.com/lunaryorn/pyudev/blob/develop/tests/test_libudev.py

def pytest_funcarg__libudev(request):
    try:
        return _libudev.load_udev_library()
    except ImportError:
        pytest.skip('udev not available')

## 06-frameworks.py
# django example
# http://pytest-django.readthedocs.org/en/latest/helpers.html

from myapp.views import my_view

def test_details(rf):
    request = rf.get('/customer/details')
    response = my_view(request)
    assert response.status_code == 200

def test_an_admin_view(admin_client):
    response = admin_client.get('/admin/')
    assert response.status_code == 200


##############################
# Google App Engine

# I wonder if these examples should be using monkeypatch to do os.environ.update??

# http://pypi.python.org/pypi/pytest_gae/0.2.1


import os

from webtest import TestApp
from main import make_application


def pytest_funcarg__anon_app(request):
    os.environ.update({'USER_EMAIL': '',
                        'USER_ID': '',
                        'AUTH_DOMAIN': 'google',
                        'USER_IS_ADMIN': '0'})
    return TestApp(make_application())


def pytest_funcarg__user_app(request):
    os.environ.update({'USER_EMAIL': 'simple@google.com',
                       'USER_ID': '1',
                       'AUTH_DOMAIN': 'google',
                       'USER_IS_ADMIN': '0'})
    return TestApp(make_application())


def pytest_funcarg__admin_app(request):
    os.environ.update({'USER_EMAIL': 'admin@google.com',
                       'USER_ID': '2',
                       'AUTH_DOMAIN': 'google',
                       'USER_IS_ADMIN': '1'})
    return TestApp(make_application())


def test_index(anon_app):
    assert "Index" in anon_app.get('/index')


def test_user_with_user(user_app):
    assert "User" in user_app.get('/users')


def test_user_with_anon(anon_app):
    assert '302 Moved Temporarily' == anon_app.get('/users').status


def test_user_with_admin(admin_app):
    assert "Admin" in admin_app.get('/users')

## 07-generatefuncargs.py
# https://bitbucket.org/hpk42/py/src/0fca612a4bbd/conftest.py:

def pytest_generate_tests(metafunc):
    multi = getattr(metafunc.function, 'multi', None)
    if multi is not None:
        assert len(multi.kwargs) == 1
        for name, l in multi.kwargs.items():
            for val in l:
                metafunc.addcall(funcargs={name: val})
    elif 'anypython' in metafunc.funcargnames:
        for name in ('python2.4', 'python2.5', 'python2.6',
                     'python2.7', 'python3.1', 'pypy-c', 'jython'):
            metafunc.addcall(id=name, param=name)


def pytest_funcarg__anypython(request):
    name = request.param
    executable = getexecutable(name)
    if executable is None:
        if sys.platform == "win32":
            executable = winpymap.get(name, None)
            if executable:
                executable = py.path.local(executable)
                if executable.check():
                    return executable
        py.test.skip("no %s found" % (name,))
    return executable

## 08-funcargsfuture.py
#probable in 2.3 (not yet released)

# content of conftest.py
import pytest
import smtplib

@pytest.factory(scope="session",
                params=["merlinux.eu", "mail.python.org"])
def smtp(testcontext):
    smtp = smtplib.SMTP(testcontext.param)
    def fin():
        print ("finalizing %s" % smtp)
        smtp.close()
    testcontext.addfinalizer(fin)
    return smtp


# content of test_module.py
def test_ehlo(smtp):
    response = smtp.ehlo()
    assert response[0] == 250
    assert "merlinux" in response[1]
    assert 0  # for demo purposes

def test_noop(smtp):
    response = smtp.noop()
    assert response[0] == 250
    assert 0  # for demo purposes


$ py.test --collectonly
=========================== test session starts ============================
platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
collecting ... collected 4 items
<Module 'test_module.py'>
  <Function 'test_ehlo[merlinux.eu]'>
  <Function 'test_noop[merlinux.eu]'>
  <Function 'test_ehlo[mail.python.org]'>
  <Function 'test_noop[mail.python.org]'>

=============================  in 0.02 seconds =============================


## 09-testevolution.py
# v0
# Feb 2010
# data/test/dbconfig/TEXT/Misc/District_TestScript_2.py

    {
    "name": "Thunderstorms with heavy rain are not dry",
    "commentary": "Mantis 01530",
    "productType": "District",
    "createGrids": [
    ("Fcst", "Wx", "WEATHER",   0,   24, "Chc:TS:!::r", "all"),
    ],
    "notCheckStrings": [
       "dry"
       ],
    "fileChanges": [
       ("District_NSWRO_Definition", "TextUtility", "add", defaultEditAreas, "undo"),
       ],
    "cmdLineVars": str({
      ('Generate Days', 'productIssuance'): 'All Periods',
      ('Issuance Type', 'issuanceType'): 'Morning',
      ('Issued By', 'issuedBy'): None,
      ('CompleteUpdate', 'completeUpdate'): 'yes'}),
    },


# v1
# early 2011 - move to pytest, directory restructure

def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester):
    formatterTester.run({
        "commentary": "Mantis 01530",
        "productType": "District",
        "createGrids": [
            Flat("Wx",    0,   24, "Chc:TS:!::r"),
            ],
        "notCheckStrings": [
            "dry"
            ],
        "cmdLineVars": cmdLineVarsIssuanceMorning
        }, defaults())

# v2
# feb 2011 - introduction of gridCreator

def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester, gridCreator):
    """Mantis 01530
    """
    gridCreator.createGrids([
            Flat("Wx",    0,   24, "Chc:TS:!::r"),
            ])
    formatterTester.run({
        "productType": "District",
        "notCheckStrings": [
            "dry"
            ],
        "cmdLineVars": cmdLineVarsIssuanceMorning
        }, defaults())


# lots of specifying areas like this:

siteID = siteConfig.GFESUITE_SITEID
DistrictEditAreaDictionary = {
       "NSWRO" : ['NSW_CW013'],
       "VICRO" : ['VIC_CW010', 'VIC_CW011'],
       "QLDRO" : [],
       "SARO" : [],
       "TASRO" : [],
       "WARO" : [],
       "NTRO" : [],
}
area =  DistrictEditAreaDictionary[siteID]


# v3
# March 2011

def pytest_funcarg__districtArea(request):

    def start():

        areas = {
             "VICRO": DefaultEditArea("VIC_PW007", "Central"),
             "NSWRO": DefaultEditArea("NSW_PW014", "Riverrina"),
             "TASRO": DefaultEditArea("TAS_PW002", "North East"),
             "SARO": DefaultEditArea("SA_PW012", "North West Pastoral"),
        }

        return __areaStart(areas)

    return request.cached_setup(setup=start, scope='session')


def test_Thunderstorms_with_heavy_rain_are_not_dry(formatterTester, gridCreator, districtArea):
    """Mantis 01530
    """
    gridCreator.createGrids([
            Flat("Wx",    0,   24, "Chc:TS:!::r"),
            ])
    formatterTester.run({
        "productType": "District",
        "notCheckStrings": [
            "dry"
            ],
        "cmdLineVars": cmdLineVarsIssuanceMorning(districtArea.aac)
        }, defaults(districtArea.aac))


# v4
# Nov 2011
# tests/system/formatters/wxPhrase/test_attributes.py

@py.test.mark.mantis1530
def test_thunderstormsWithHeavyRainAreNotDry(mockAccessor):
    """'heavy rain' and 'dry' attributes were getting mixed up.
    """
    mockAccessor.mockGrids([
            Flat("Wx", 0, 24, "Chc:TS:!::r"),
            ])
    words = weatherWordsLandAreal(mockAccessor)
    assert "dry" not in words