A proof of concept that shows how to run checks and tests on Python code samples in reStructuredText documents.

sample_code_tester.py

import re
import argparse
import tempfile
import textwrap
from flake8.api import legacy as flake8


def parse_cmdline_args():
    parser = argparse.ArgumentParser(description="Code Lint tool")
    parser.add_argument('--file', '-f', type=str, help='File to run checks on.',
                        required=True)
    parser.add_argument('--ignore', '-i', type=str, nargs='+', default='',
                        help='Flake8 rules to ignore')
    arguments = parser.parse_args()
    return arguments


def rst_stripper(document):
    """
    Extract Python code from reStructuredText document

    :param document: Filename of the .rst document that contains Python
                     code samples
    """
    pattern = r"(\.\. code-block:: python\s+$)((\n +.*|\s)+)"
    python_code = ""
    with open(document) as f:
        f_content = f.read()
        code_match_obj = re.finditer(pattern, f_content, re.M)
        for match in code_match_obj:
            for group_text in match.groups():
                group_text = re.sub(r".. code-block:: python", '', group_text)
                python_code += textwrap.dedent(text=group_text)
    
    print(python_code)   
    return python_code


def run_flake8_checks(python_code, document, ignore_list):
    """
    Run flake8 checks on document

    :param python_code: Python code sample extracted from document
    :param document: Filename of the .rst document that contains Python
                     code samples
    :param ignore_list: List of errors and warnings to ignore                     
    """
    
    with tempfile.NamedTemporaryFile() as tmpfilename:
        tmpfilename.write(bytes(python_code, 'UTF-8'))
        style_guide = flake8.get_style_guide(ignore=ignore_list)
        tmpfilename.seek(0)

        # Lint file and generate flake8 report
        report = style_guide.input_file(tmpfilename.name)
        if report.total_errors > 0:
            print(f"Found {report.total_errors} errors in {document}")
            print()


def main():
    args = parse_cmdline_args()
    document = args.file
    ignore_errors = args.ignore
    python_code = rst_stripper(args.file)
    run_flake8_checks(python_code, document, ignore_errors)


if __name__ == '__main__':
    main()

This gist extracts python code from a README document and runs flake8 checks on it. Modifying it to run pytest or tox should not be difficult. The extracted code is saved in a temporary file.

To run this script, do:

$ python sample_code_tester.py --file README.rst

Sample output:

from selenium import webdriver
from axe_selenium_python import Axe

def test_google():
    driver = webdriver.Firefox()
    driver.get("http://www.google.com")
    axe = Axe(driver)
    # Inject axe-core javascript into page.
    axe.inject()
    # Run axe accessibility checks.
    results = axe.execute()
    # Write results to file
    axe.write_results(results, 'a11y.json')
    driver.close()
    # Assert no violations are found
    assert len(results["violations"]) == 0, axe.report(results["violations"])



/tmp/tmp2t4o9cz6:6:1: E302 expected 2 blank lines, found 1
/tmp/tmp2t4o9cz6:20:1: W391 blank line at end of file
Found 2 errors in README.rst

It is also possible to ignore specific errors:

$ python sample_code_tester.py --file README.rst --ignore E302

from selenium import webdriver
from axe_selenium_python import Axe

def test_google():
    driver = webdriver.Firefox()
    driver.get("http://www.google.com")
    axe = Axe(driver)
    # Inject axe-core javascript into page.
    axe.inject()
    # Run axe accessibility checks.
    results = axe.execute()
    # Write results to file
    axe.write_results(results, 'a11y.json')
    driver.close()
    # Assert no violations are found
    assert len(results["violations"]) == 0, axe.report(results["violations"])



/tmp/tmpp0vt66mc:20:1: W391 blank line at end of file
Found 1 errors in README2.rst