COMS 4113: Script to run tests automatically...

sir_testalot.py

#!/usr/bin/env python3
import argparse
import sys
import logging
import subprocess


log = logging.getLogger(__name__)


def parse_args():
    prog = "Run individual go unit tests with a timeout for many times."
    parser = argparse.ArgumentParser(prog)
    parser.add_argument("-t", "--tests", default=None,
                        help="Test names, comma-separated. Defaults to all"
                             " tests.")
    parser.add_argument("-n", "--iterations", default=50, type=int,
                        help="How many times to run each test.")
    parser.add_argument("--shutdown", default=False, action="store_true",
                        help="Shutdown the machine after testing. Useful for"
                             " testing on VMs and then shutting them down to"
                             " reduce usage.")
    return parser.parse_args()


def list_go_tests(path="."):
    res = []
    out = subprocess.check_output(["go", "test", "-list", path])
    for line in out.splitlines():
        line = line.strip().decode("utf-8")
        if line.startswith("Test"):
            res.append(line)
    return res


def main():
    logging.basicConfig(level=logging.INFO)
    args = parse_args()
    if args.tests:
        tests = args.tests.split(",")
    else:
        tests = list_go_tests()
    results = []
    for test in tests:
        successes, fails = 0, 0
        for i in range(1, args.iterations + 1):
            log.info("test '%s': Started iteration %d", test, i)
            try:
                out = subprocess.check_output(
                    ["go", "test", "-v", "-count", "1", "-timeout", "2m",
                     "-run", "^%s$" % test])
            except subprocess.CalledProcessError as e:
                log.info("Iteration %d failed", i)
                fails += 1
                out = e.output
                out_file = "test_%s_try_%d.txt" % (test.lower(), i)
                with open(out_file, "w") as f:
                    f.write(out.decode("utf-8"))
            else:
                log.info("Iteration %d succeeded", i)
                successes += 1
        log.info("Test '%s': Successes = %d, Fails = %d", test, successes,
                 fails)
        results.append({"test": test, "successes": successes, "fails": fails})
    log.info("Summarizing results...")
    for res in results:
        log.info("Test '%s': Successes = %d, Fails = %d", res["test"],
                 res["successes"], res["fails"])
    if args.shutdown:
        subprocess.run(["sudo", "/usr/sbin/shutdown", "now"])


if __name__ == "__main__":
    sys.exit(main())