mutation testing with redbaron and pytest

mod.py

def inc(n):
    """increments n by one"""
    return n + 1


def square(n):
    """squares n"""
    return n ** 2

mutant_tester.py

from contextlib import redirect_stdout
from importlib import import_module
import io
import sys
import types

import pytest
from redbaron import RedBaron


def run_pytest_quietly():
    with redirect_stdout(io.StringIO()) as _:
        result = pytest.main([])
    return result


module_name = sys.argv[1]

module = import_module(module_name)

with open(module.__file__) as fp:
    source_code = fp.read()
red_ast = RedBaron(source_code)  # AST means Abstract Syntax Tree

initial_runner_state = run_pytest_quietly()

# keep track of mutations that pass all tests
passing_mutants = list()
# iterate over all nodes that match the query
for node in red_ast("binary_operator"):
    # keep track of initial value
    initial_node_value = node.value
    # iterate over some alternative values for this node type
    for alternative_value in ("-", "*", "/", "**", "%"):

        # mutate the node and compile the source code
        node.value = alternative_value
        mutant_source_code = red_ast.dumps()
        compiled_mutant_code = compile(mutant_source_code, "None", "exec")

        # instanciate a new module and execute the compiled code
        # in its local context
        mutant_module = types.ModuleType(module_name)
        exec(compiled_mutant_code, mutant_module.__dict__)
        # put it back where the test runner can find it
        sys.modules[module_name] = mutant_module

        runner_state = run_pytest_quietly()

        # check if the runner results changed
        # different results means the tests detect the mutation
        # and that is good
        if runner_state == initial_runner_state:
            passing_mutants.append(node)
            break

    # reset node to initial value
    node.value = initial_node_value

# present the nodes that need better testing
source_code_lines = source_code.split("\n")
for mutant_node in passing_mutants:
    bbox = mutant_node.absolute_bounding_box
    print(
        f"line {bbox.top_left.line}, column {mutant_node.get_absolute_bounding_box_of_attribute('value').top_left.column}"
    )
    print("\n".join(source_code_lines[bbox.top_left.line - 1 : bbox.bottom_right.line]))

test_mod.py

import mod


def test_inc_returns_int():
    assert isinstance(mod.inc(1), int)
    assert isinstance(mod.inc(2), int)


def test_inc_returns_larger_than_input():
    assert 1 < mod.inc(1)