test.py

import numpy as np
import json
import subprocess

test_binaries = [
    "a.exe",
    "b.exe"]
test_args_path = 'test_args.json'


def create_tests(path):
    f = open(path)
    args = json.load(f)
    f.close()
    # print(args)
    tests_by_args = {}

    for arg in args:
        start = arg['interval'][0]
        stop = arg['interval'][1]
        step = (stop - start)/arg['amount']
        tests_by_args[arg['name']] = [format(x, '.6f')
                                      for x in np.arange(start, stop, step)]

    tests = []

    for arg in args:
        total = len(tests)
        if total == 0:
            for test_input in tests_by_args[arg['name']] + (arg.get('special_tests') or []):
                tests.append(str(test_input))
        else:
            temp = []
            for test_input_prev in tests:
                for test_input in tests_by_args[arg['name']] + (arg.get('special_tests') or []):
                    temp.append(test_input_prev + '\n' + str(test_input))
            tests = temp

    return tests


def run_once(path, input):
    result = subprocess.run([path],
                            capture_output=True, text=True, input=input)
    return result.stdout


tests = create_tests(test_args_path)

for test_input in tests:
    test_output1 = run_once(test_binaries[0], test_input).strip()
    test_output2 = run_once(test_binaries[1], test_input).strip()
    if test_output1 != test_output2:
        print("\nNot equal with input:\n" + test_input + "\nOutput " +
              test_binaries[0] + ":\n" + test_output1 + "\nOutput " + test_binaries[1] + ":\n" + test_output2)

test_args.json

[
  {
    "name": "x",
    "type": "double",
    "interval": [-10, 10],
    "amount": 50
  },
  {
    "name": "eps",
    "type": "double",
    "special_tests": [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1],
    "interval": [0.000001, 0.1],
    "amount": 10
  }
]