compare execution results

compare.py

import csv
import sys

# These tests are failing when gas costs are radically changed.

SKIPPED = [
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_session_to_stored_contract_",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_session_to_stored_versioned_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_versioned_session_to_stored_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_versioned_session_to_stored_versioned_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_hash_to_stored_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_hash_to_stored_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_hash_to_stored_versioned_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_p ayment_by_hash_to_stored_versioned_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_name_to_stored_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_name_to_stored_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_name_to_stored_versioned_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_name_to_stored_versioned_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_hash_to_stored_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_hash_to_stored_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_hash_to_stored_versioned_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_hash_to_stored_versioned_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_name_to_stored_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_name_to_stored_session",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_name_to_stored_versioned_contract",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_versioned_payment_by_name_to_stored_versioned_session",
    ),
    ("casper_engine_tests::test::explorer::faucet", "faucet_costs"),
    (
        "casper_engine_tests::test::gas_counter",
        "should_correctly_measure_gas_for_opcodes",
    ),
    (
        "casper_engine_tests::test::regression::host_function_metrics_size_and_gas_cost",
        "host_function_metrics_has_acceptable_gas_cost",
    ),
    (
        "casper_engine_tests::test::regression::regression_20210924",
        "should_execute_do_minimum_session",
    ),
    (
        "casper_engine_tests::test::regression::slow_input",
        "should_charge_extra_per_amount_of_br_table_elements",
    ),
    # Different outcome (error does not match)
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_contract_to_stored_session_should_fail",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_contract_to_stored_versioned_session_should_fail",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_versioned_contract_to_stored_session_should_fail",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "payment_bytes_to_stored_versioned_contract_to_stored_versioned_session_should_fail",
    ),
    (
        "casper_engine_tests::test::contract_api::get_call_stack::payment",
        "stored_payment_by_hash_to_stored_versioned_session",
    ),
]


def get_data(reader):
    next(reader)
    for (module_name, file, line, function, duration_us, gas, error, *data) in reader:
        if (module_name, function) in SKIPPED:
            continue
        yield (
            module_name,
            file,
            line,
            function,
            int(duration_us),
            int(gas),
            mask_error(error),
            *data,
        )


def mask_error(error):
    # Some tests assert for forged urefs, but the uref addr is non-deterministic across runs, so strip it.
    if "Forged reference: URef" in error:
        return "Forged reference: URef(xxx)"
    else:
        return error


_, a, b = sys.argv

a_reader = csv.reader(open(a))
b_reader = csv.reader(open(b))

total_gas_a = 0
total_gas_b = 0
total_time_a = 0
total_time_b = 0
matching_entries = 0

for lhs, rhs in zip(get_data(a_reader), get_data(b_reader)):
    (
        module_name_a,
        file_a,
        line_a,
        function_a,
        duration_us_a,
        gas_a,
        error_a,
        *data_a,
    ) = lhs
    (
        module_name_b,
        file_b,
        line_b,
        function_b,
        duration_us_b,
        gas_b,
        error_b,
        *data_b,
    ) = rhs
    if (module_name_a, file_a, line_a, function_a, error_a, *data_a) != (
        module_name_b,
        file_b,
        line_b,
        function_b,
        error_b,
        *data_b,
    ):
        print(f"\n{lhs!r}\n{rhs!r}")
        break
    else:
        matching_entries += 1
        gas_a = gas_a
        gas_b = gas_b
        total_gas_a += gas_a
        total_gas_b += gas_b
        total_time_a += duration_us_a
        total_time_b = duration_us_b

print(f"total gas a = {total_gas_a}")
print(f"total time a = {total_time_a}")
print(f"total gas b = {total_gas_b}")
print(f"total time b = {total_time_b}")
print(f"matching entries {matching_entries}")