Python unit test bisecter
#!/usr/bin/python3 | |
""" | |
Find which test in the test list is causing the failure of a known-failing | |
test. That is - Given a test list which dictates a specific test order, | |
under which a test X (which passes when run in isolation) is failing, find | |
out which of the tests that, on the list, run before X, are causing it to | |
fail. | |
Many of our test runs use parallelization to run faster. Sometimes we see test | |
failures which we can't reproduce locally, because locally we usually run | |
sequentially; and even then, the test ordering seems to be somewhat | |
unpredictable so it's hard to reproduce the exact test ordering seen in our | |
test runner. | |
Most of the time these failures are due to unidentified test interdependencies: | |
either test A causes test B to pass (where running test B in isolation would | |
fail), or test A causes B to fail (where running B in isolation would pass). | |
And we have seen more complex scenarios where C passes, A-B-C passes, but A-C | |
fails (because A sets C up for failure, while B would set C up for success). | |
We added some diagnostic output to our test runner so it would show exactly the | |
list of tests each process runs. This way we can copy the list and run it | |
locally, which usually reproduces the failure. | |
But we needed a tool to then determine exactly which of the tests preceding the | |
failing one was setting up the failure conditions. So I wrote this simple | |
bisecter script, which expects a list of test names, which must contain the | |
faily test "A", and of course, the name of the faily test "A". It looks for "A" | |
in the list and will use bisection to determine which of the tests preceding | |
"A" is causing the failure. | |
Note it's not very tunable, it will run "make test" with | |
ARGS='--failfast $LIST_OF_TESTS' | |
And interpret any non-zero exit code as "a test failed". | |
""" | |
import argparse | |
import math | |
import subprocess | |
import sys | |
def bisect_run(f_list, f_test): | |
# Always called with a f_list that causes f_test to fail. | |
if len(f_list) == 1: | |
return("The test that causes the failure is {}".format(f_list[0])) | |
if len(f_list) == 0: | |
return("No test causes the failure? what?") | |
first_half = f_list[:len(f_list)/2] | |
second_half = f_list[len(f_list)/2:] | |
print("{} elements in the list, about {} iterations left".format( | |
len(f_list), int(math.log(len(f_list), 2)))) | |
try: | |
list_of_tests = first_half[:] | |
list_of_tests.append(f_test) | |
test_plan = " ".join(list_of_tests) | |
subprocess.check_output( | |
"make test ARGS='--failfast {}'".format(test_plan), | |
shell=True, stderr=subprocess.PIPE) | |
except: | |
print("Test causing failure is in first half of given list") | |
return bisect_run(first_half, f_test) | |
else: | |
print("Test causing failure is in second half of given list") | |
return bisect_run(second_half, f_test) | |
def main(): | |
parser = argparse.ArgumentParser(description=""" | |
Find which test in the test list is causing the failure of a known-failing | |
test. That is - Given a test list which dictates a specific test order, | |
under which a test X (which passes when run in isolation) is failing, find | |
out which of the tests that, on the list, run before X, are causing it to | |
fail. | |
""") | |
parser.add_argument("test_list", help="File containing a list of " | |
"test names, one per line.") | |
parser.add_argument("failing_test", help="Name of the test that fails. " | |
"It must exist in the test_list file.") | |
args = parser.parse_args() | |
with open(args.test_list, "r") as test_list_file: | |
test_list = [s.strip() for s in test_list_file.readlines()] | |
# We don't need to bother with tests before failing_test | |
f_index = test_list.index(args.failing_test) | |
test_list = test_list[:f_index-1] | |
print(bisect_run(test_list, args.failing_test)) | |
sys.exit(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment