test_analysis.py

#!/usr/bin/env python3.6
#
# For the full story: http://nedbatchelder.com/blog/201612/finding_test_coupling.html
#

import re

def read_output(filename):
    failed = False
    test_names = []

    with open(filename) as f:
        for line in f:
            if not line.startswith('#'):
                continue
            if 'ERROR' in line:
                failed = True
            _, test_name, test_class, _ = line.split(None, 3)
            # Tests can have randomized components in them, strip those out.
            test_name = re.sub(r"__\d+", "__#", test_name)
            # Four underscores means crazy stuff, strip that out.
            test_name = re.sub(r"____.*$", "", test_name)
            # One pesky test has different numbers at the end.
            if test_name.startswith("test_post_with_invalid_timestamp_1_"):
                test_name = "test_post_with_invalid_timestamp_1_XXXXXXXXXX"
            test_names.append((test_name, test_class))

    return failed, test_names

FAIL_TEST = "test_provider_login_can_handle_unicode_email_inactive_account"

def check_set_difference(s1, s2):
    s1 = set(s1)
    s2 = set(s2)
    if s1 != s2:
        print("Difference!")
        for d1, d2 in zip(sorted(s1-s2), sorted(s2-s1)):
            print(f"    {d1}")
            print(f"    {d2}")

def main():
    all_tests = None

    not_bad_tests = set()
    maybe_bad_tests = set()
    for i in range(1, 999):
        try:
            failed, test_names = read_output(f"test{i}.txt")
        except FileNotFoundError:
            break
        if all_tests is None:
            all_tests = set(test_names)
        elif all_tests != set(test_names):
            check_set_difference(all_tests, test_names)
            break
        fail_index = next(i for i,name in enumerate(test_names) if FAIL_TEST in name)
        before_fail = test_names[:fail_index]
        if failed:
            maybe_bad_tests.update(before_fail)
        else:
            not_bad_tests.update(before_fail)
        maybe_bad_tests -= not_bad_tests
        print(
            f"Test run {i:3}: "
            f"{'FAILED' if failed else 'passed'}, "
            f"{len(test_names)} tests, "
            f"fail_test was {fail_index:4} "
            f"({len(not_bad_tests):4} not-bad, {len(maybe_bad_tests):4} maybe-bad)"
        )

    print(f"{len(not_bad_tests)} not-bad, {len(maybe_bad_tests)} maybe-bad")
    print("\n".join(sorted(f"{t} {c}" for t, c in maybe_bad_tests)))

    print("\n\nClasses:")
    print("\n".join(sorted(set(c for t, c in maybe_bad_tests))))

main()