StopwatchTestRunner

testrunner.py

import time
import statistics
import unittest
from unittest import TextTestRunner
from django.test.runner import DiscoverRunner


class StopwatchTestResult(unittest.TextTestResult):
    """
    Times test runs and formats the result
    """

    # Collection shared between all result instaces to calculate statistics
    timings = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.start = 0
        self.stop = 0
        self.elapsed = 0

    def startTest(self, test):
        self.start = time.time()
        super().startTest(test)

    def stopTest(self, test):
        super().stopTest(test)
        self.stop = time.time()
        self.elapsed = self.stop - self.start
        self.timings[test] = self.elapsed

    def getDescription(self, test):
        """
        Format test result with timing info
        e.g. `test_add [0.1s]`
        """
        description = super().getDescription(test)
        return f'{description} [{self.elapsed:0.4f}s]'

    @classmethod
    def print_stats(cls):
        """
        Calculate and print timings

        These data are likely skewed, as is normal for reaction time data,
        therefore mean and standard deviation are difficult to interpret. Thus,
        the IQR is used to identify outliers.
        """
        timings = StopwatchTestResult.timings.values()
        count = len(timings)
        mean = statistics.mean(timings)
        stdev = statistics.stdev(timings)
        slowest = max(timings)
        q1, median, q3 = statistics.quantiles(timings)
        fastest = min(timings)
        total = sum(timings)

        print()
        print("Statistics")
        print("==========")
        print("")
        print(f"count: {count:.0f}")
        print(f" mean: {mean:.4f}s")
        print(f"  std: {stdev:.4f}s")
        print(f"  min: {fastest:.4f}s")
        print(f"  25%: {q1:.4f}s")
        print(f"  50%: {median:.4f}s")
        print(f"  75%: {q3:.4f}s")
        print(f"  max: {slowest:.4f}s")
        print(f"total: {total:.4f}s")

        # https://en.wikipedia.org/wiki/Interquartile_range
        iqr = q3 - q1
        fast = q1 - 1.5 * iqr
        slow_threshold = q3 + 1.5 * iqr

        slow_tests = [
            (test, elapsed)
            for test, elapsed
            in StopwatchTestResult.timings.items()
            if elapsed >= slow_threshold
        ]

        if not slow_tests: return
        print()
        print("Outliers")
        print("========")
        print("These were particularly slow:")
        print()
        for test, elapsed in slow_tests:
            print(' ', test, f"[{elapsed:0.4f}s]")


class StopwatchTestRunner(DiscoverRunner):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._stats = kwargs['stats']

    @classmethod
    def add_arguments(cls, parser):
        DiscoverRunner.add_arguments(parser)
        parser.add_argument(
            "--stats",
            action="store_true",
            help="Print timing statistics",
        )

    def get_resultclass(self):
        # super().get_resultclass() or
        return StopwatchTestResult

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        super().run_tests(test_labels, extra_tests, **kwargs)
        if self._stats:
            StopwatchTestResult.print_stats()