Testing out pytest.

apply_sabotage.py

import pytest

######### CONSTANTS #########
TIMEOUT = 10


######### TEST #########
@pytest.mark.timeout(TIMEOUT)
def test_func(assert_stmt, func_to_test, apply_penalties):
    # Track global vars
    d = {}
    # !!!Unsafe!!!
    # func_to_test & assert_stmt are both strings
    exec(func_to_test[1], d, d)
    exec(assert_stmt[1], d, d)


######### MAIN #########
#pytest.main('-s test_module.py')

conftest.py

""" 
Apply list of tests to list of python programs. 

The tests and programs are retrieved from a server as `string`s. 

"""
import pytest
import requests


@pytest.mark.tryfirst
def pytest_runtest_makereport(item, call, __multicall__):
    # execute all other hooks to obtain the report object
    rep = __multicall__.execute()

    # set an report attribute for each phase of a call, which can
    # be "setup", "call", "teardown"
    setattr(item, "rep_" + rep.when, rep)
    return rep


def pytest_addoption(parser):
    parser.addoption("--challenge", action="store", default="1",
        help="list of challenge ids to pass to test functions")


######### HELPERS #########
def get_x_json(challenge_id, x):
    "Retrieve (Solutions|Sabotages) for given challenge_id from site"
    DOMAIN = 'http://localhost:8000'
    API_CALL = '{0}/api/challenge/{1}/{2}/'.format(DOMAIN, challenge_id, x)

    get_xs = requests.get(API_CALL, auth=('thebest', 'wafful'))
    s_json = get_xs.json()
    x_json = s_json[x]
    return x_json


def get_sabotages(challenge_id):
    "Retrieve Sabotages for given challenge_id from site"
    sabotage_json = get_x_json(challenge_id, 'sabotages')
    sabotages = [(sa['resource_id'], sa['sabotage']) for sa in sabotage_json]
    return sabotages


def get_solutions(challenge_id):
    "Retrieve Solutions for given challenge_id from site"
    solutions_json = get_x_json(challenge_id, 'solutions')
    solutions = []
    for s in solutions_json:
        solutions.append((s['resource_id'], s['solution']))

    return solutions


def penalize_solution(solution_id):
    pass


def reward_sabotage(sabotage_id):
    pass


######### FIXTURES #########
#@pytest.fixture(scope="module", params=get_solutions(1))
#def func_to_test(request):
    #return request.param


#@pytest.fixture(scope="function", params=get_sabotages(1))
#def assert_stmt(request):
    #challenge_id = request.config.option.challenge
    #return request.param


def pytest_generate_tests(metafunc):
    challenge_id = metafunc.config.option.challenge
    metafunc.parametrize("assert_stmt", get_sabotages(challenge_id))
    metafunc.parametrize("func_to_test", get_solutions(challenge_id))


@pytest.fixture
def apply_penalties(request):
    """
    Penalize Solutions for failing Sabotages.
    Reward successful Sabotages.
    """
    def fin():
        if request.node.rep_setup.passed:
            if request.node.rep_call.failed:
                sabotage = request.getfuncargvalue('assert_stmt')
                solution = request.getfuncargvalue('func_to_test')

                solution_id = solution[0]
                penalize_solution(solution_id)

                sabotage_id = sabotage[0]
                reward_sabotage(sabotage_id)
        elif request.node.rep_setup.failed:
            # Unsure what to do on setup failure
            pass
    request.addfinalizer(fin)