00_Evaluator.md

Hunter's Debug Evaluator

This is a simple little module that I use to figure out why programs are giving different output with the same input. Not a common problem, but one I've been asked enough for it to matter.

How to Use

Here's a simple file that might fail unexpectedly:

import numpy as np
from eval import save_dict, dump_saves

# if we're expecting consistent output we should be using np.zeros instead
arr = np.empty(shape=(6, 1))
save_dict({'arr': arr})

dump_saves()

Run the file twice:

$ python main.py
$ python main.py

This will create a file for each run named variablesX.pk where X is replaced with the run number. You can then evaluate the output by running the eval file:

$ python eval.py

Variables 1: 1
Variables 2: 2

===================================================

Non-matching values for variable arr at 5:

Variables 1:
[[2.02949914e-316]
 [0.00000000e+000]
 [0.00000000e+000]
 [0.00000000e+000]
 [0.00000000e+000]
 [0.00000000e+000]]

===================================================

Variables 2:
[[7.8067827e-317]
 [0.0000000e+000]
 [0.0000000e+000]
 [0.0000000e+000]
 [0.0000000e+000]
 [0.0000000e+000]]

===================================================



Failed matching. See above issues.

eval.py

from pickle import load, dump
from os import listdir, getcwd
from os.path import isfile, join
from inspect import currentframe
import numpy as np
import logging

saved_vars = []
count = 0

def save_class(obj):
    global saved_vars, count
    cf = currentframe()
    saved_vars.append({'state_nr': count, 'line_nr': cf.f_back.f_lineno, **vars(obj)})
    count += 1

def save_dict(obj):
    global saved_vars, count
    cf = currentframe()
    saved_vars.append({'state_nr': count, 'line_nr': cf.f_back.f_lineno, **obj})
    count += 1

def dump_saves():
    files = max([0, *[int(f[len("variables"):-len(".pk")]) for f in listdir(getcwd()) if isfile(join(getcwd(), f)) and f.find("variables") != -1 and f.find(".pk") != -1]])
    with open(f"variables{files + 1}.pk", "wb") as f:
        dump(saved_vars, f)

def eval():
    a = int(input("Variables 1: "))
    b = int(input("Variables 2: "))

    with open(f"variables{a}.pk", "rb") as f:
        variables1 = load(f)

    with open(f"variables{b}.pk", "rb") as f:
        variables2 = load(f)

    if (lv1 := len(variables1)) != (lv2 := len(variables2)):
        print(f"Incompatible state counts {lv1} and {lv2}")
        exit()

    for state in range(len(variables1)):
        v1state = variables1[state]
        v2state = variables2[state]

        if not isinstance(v1state, dict):
            print("Non-dict history entry")
            exit()
        if not isinstance(v2state, dict):
            print("Non-dict history entry")
            exit()

        ln1 = v1state['line_nr']
        ln2 = v2state['line_nr']

        if ln1 != ln2 or ln1 == None:
            print(f"Invalid line numbers {ln1} and {ln2}")
            exit()

        v1k = set(v1state.keys())
        v2k = set(v2state.keys())

        if len((diff := v1k.difference(v2k))) != 0:
            print(f"Non-matching keys: {diff} at line {ln1}")
            exit()

        for key in v1k:
            failed = False
            v1v = v1state[key]
            v2v = v2state[key]
            if (t1 := type(v1v)) != (t2 := type(v2v)):
                failed = True
                print(f"Non-matching values for variable {key} at {ln1}: {t1} and {t2}")
            elif isinstance(v1v, list) or isinstance(v1v, np.ndarray):
                l1 = np.array(v1v)
                l2 = np.array(v2v)
                if not (l1 == l2).all():
                    failed = True
                    print("\n===================================================\n")
                    print(f"Non-matching values for variable {key} at {ln1}:\n")
                    print(f"Variables {a}:")
                    print(v1v)
                    print("\n===================================================\n")
                    print(f"Variables {b}:")
                    print(v2v)
                    print("\n===================================================\n")
            elif v1v != v2v:
                failed = True
                print("\n===================================================\n")
                print(f"Non-matching values for variable {key} at {ln1}:\n")
                print(f"Variables {a}:")
                print(v1v)
                print("\n===================================================\n")
                print(f"Variables {b}:")
                print(v2v)
                print("\n===================================================\n")
            else:
                logging.debug(f"Pass {key}")
            if failed:
                print("\n\nFailed matching. See above issues.")
                exit()
            

if __name__ == "__main__":
    eval()