Test Oracle

Test Oracle.py

import random, os

# define constants
correct_program = "correct.exe"
incorrect_program = "incorrect.exe"
input_file = "input.txt"
correct_output = "correct_output.txt"
incorrect_output = "incorrect_output.txt"
num_testcases = 10

# generate test cases and write to input file
# create small, random test cases, consistent with the problem description
testcases = [random.random() for i in xrange(num_testcases)]
with open(input_file, 'w') as input_f:
  for testcase in testcases:
    # write test case to input file, consistent with the format described in the problem
    input_f.write(str(testcase) + '\n')
        
# execute the two programs, resulting in them generating the output files
os.system(correct_program)
os.system(incorrect_program)

# compare the two output files and find a point of difference
with open(correct_output, 'r') as correct_f:
  with open(incorrect_output, 'r') as incorrect_f:
    for i in xrange(num_testcases):
      # assuming that we output 1 line for each test case
      # If multiple lines are output per test case, then all those should be read here
      correct, incorrect = correct_f.readline(), incorrect_f.readline()
      if correct != incorrect:
        print testcases[i], "\ncorrect = ", correct, "\nincorrect = ", incorrect