Death testing

example_use.cpp

#include <stddef.h>
#include <string_view>

#include <pmretvals.h>
#include <test_death.hpp>

using namespace std;

int test_case_operator_dereference_value_initalized_iterator() {
    string_view::iterator it; // note: for IDL to work correctly, default init and value init are equivalent
    (void)*it; // cannot dereference value-initialized string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_dereference_value_initalized_iterator_end() {
    string_view sv("text");
    string_view::iterator it = sv.end();
    (void)*it; // cannot dereference end string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_arrow_value_initalized_iterator() {
    string_view::iterator it;
    (void)it.operator->(); // cannot dereference value-initialized string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_arrow_value_initalized_iterator_end() {
    string_view sv("text");
    string_view::iterator it = sv.end();
    (void)it.operator->(); // cannot dereference end string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_preincrement_value_initialized_iterator() {
    string_view::iterator it;
    ++it; // cannot increment value-initialized string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_preincrement_off_end() {
    string_view sv("text");
    string_view::iterator it = sv.begin();
    for (size_t idx = 0; idx < 5; ++idx) {
        ++it; // cannot increment string_view iterator past end
    }

    return PM_TEST_CASCADE;
}

int test_case_operator_predecrement_value_initialized_iterator() {
    string_view::iterator it;
    --it; // cannot decrement value-initialized string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_predecrement_before_begin() {
    string_view sv("text");
    string_view::iterator it = sv.begin();
    --it; // cannot decrement string_view iterator before begin
    return PM_TEST_CASCADE;
}

int test_case_operator_advance_value_initialized_iterator() {
    string_view::iterator it;
    it += 1; // cannot seek value-initialized string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_advance_value_initialized_iterator_zero() {
    string_view::iterator it;
    it += 0; // OK
    return PM_TEST_PASS;
}

int test_case_operator_advance_before_begin() {
    string_view sv("text");
    string_view::iterator it = sv.begin();
    it += -1; // cannot seek string_view iterator before begin
    return PM_TEST_CASCADE;
}

int test_case_operator_advance_after_end() {
    string_view sv("text");
    string_view::iterator it = sv.begin();
    it += 5; // cannot seek string_view iterator after end
    return PM_TEST_CASCADE;
}

int test_case_operator_retreat_value_initialized_iterator() {
    string_view::iterator it;
    it -= 1; // cannot seek value-initialized string_view iterator
    return PM_TEST_CASCADE;
}

int test_case_operator_retreat_value_initialized_iterator_zero() {
    string_view::iterator it;
    it -= 0; // OK
    return PM_TEST_PASS;
}

int test_case_operator_retreat_before_begin() {
    string_view sv("text");
    string_view::iterator it = sv.begin();
    it -= 1; // cannot seek string_view iterator before begin
    return PM_TEST_CASCADE;
}

int test_case_operator_retreat_after_end() {
    string_view sv("text");
    string_view::iterator it = sv.begin();
    it -= -5; // cannot seek string_view iterator after end
    return PM_TEST_CASCADE;
}

int test_case_operator_subtract_incompatible_different_views() {
    string_view sv1("text");
    string_view sv2("text2");
    (void)(sv1.begin() - sv2.begin()); // cannot subtract incompatible string_view iterators
    return PM_TEST_CASCADE;
}

int test_case_operator_subtract_incompatible_value_initialized() {
    string_view sv1("text");
    (void)(sv1.begin() - string_view::iterator{}); // cannot subtract incompatible string_view iterators
    return PM_TEST_CASCADE;
}

int test_case_operator_equal_incompatible_different_views() {
    string_view sv1("text");
    string_view sv2("text2");
    (void)(sv1.begin() == sv2.begin()); // cannot compare incompatible string_view iterators for equality
    return PM_TEST_CASCADE;
}

int test_case_operator_equal_incompatible_value_initialized() {
    string_view sv1("text");
    (void)(sv1.begin() == string_view::iterator{}); // cannot compare incompatible string_view iterators for equality
    return PM_TEST_CASCADE;
}

int test_case_operator_less_incompatible_different_views() {
    string_view sv1("text");
    string_view sv2("text2");
    (void)(sv1.begin() < sv2.begin()); // cannot compare incompatible string_view iterators
    return PM_TEST_CASCADE;
}

int test_case_operator_less_incompatible_value_initialized() {
    string_view sv1("text");
    (void)(sv1.begin() < string_view::iterator{}); // cannot compare incompatible string_view iterators
    return PM_TEST_CASCADE;
}

int test_case_operator_subscript_out_of_range() {
    string_view sv("text");
    (void)sv[5]; // string_view subscript out of range
    return PM_TEST_CASCADE;
}

int test_case_front_empty() {
    string_view sv;
    (void)sv.front(); // cannot call front on empty string_view
    return PM_TEST_CASCADE;
}

int test_case_back_empty() {
    string_view sv;
    (void)sv.back(); // cannot call back on empty string_view
    return PM_TEST_CASCADE;
}

int test_case_remove_prefix_too_large() {
    string_view sv("text");
    sv.remove_prefix(5); // cannot remove prefix longer than total size
    return PM_TEST_CASCADE;
}

int test_case_remove_prefix_zero() {
    string_view sv;
    sv.remove_prefix(0); // OK
    return PM_TEST_PASS;
}

int test_case_remove_suffix_too_large() {
    string_view sv("text");
    sv.remove_suffix(5); // cannot remove suffix longer than total size
    return PM_TEST_CASCADE;
}

int test_case_remove_suffix_zero() {
    string_view sv;
    sv.remove_suffix(0); // OK
    return PM_TEST_PASS;
}

int test_case_remove_prefix_incompatible() {
    string_view sv("text");
    auto old_range = sv.begin();
    sv.remove_prefix(1);
    auto new_range = sv.begin();
    (void)(old_range == new_range); // cannot compare incompatible string_view iterators for equality
    return PM_TEST_CASCADE;
}

int test_case_remove_suffix_incompatible() {
    string_view sv("text");
    auto old_range = sv.begin();
    sv.remove_suffix(1);
    auto new_range = sv.begin();
    (void)(old_range == new_range); // cannot compare incompatible string_view iterators for equality
    return PM_TEST_CASCADE;
}

int test_case_Copy_s() {
    string_view sv("text");
    char buffer[2];
#pragma warning(suppress: 28020) // yay PREfast catches this mistake at compile time!
    sv._Copy_s(buffer, 2, 4); // CRT invalid parameter handler (memcpy_s failed)
    return PM_TEST_CASCADE;
}

int test_case_null_constructor() {
#pragma warning(suppress: 6387) // yay PREfast catches this mistake at compile time!
    string_view sv(nullptr, 1); // non-zero size null string_view
    return PM_TEST_CASCADE;
}

int test_case_null_constructor_zero() {
    string_view sv(nullptr, 0); // OK
    (void)sv;
    return PM_TEST_PASS;
}

#if _ITERATOR_DEBUG_LEVEL == 0
int main() {
    return PM_TEST_PASS;
}
#else /* ^^^ _ITERATOR_DEBUG_LEVEL == 0 ^^^ // vvv _ITERATOR_DEBUG_LEVEL != 0 vvv */
int main(int argc, char* argv[]) {
    std_testing::death_test_executive exec([] {
        test_case_operator_advance_value_initialized_iterator_zero();
        test_case_operator_retreat_value_initialized_iterator_zero();
        test_case_remove_prefix_zero();
        test_case_remove_suffix_zero();
        test_case_null_constructor_zero();
        return PM_TEST_PASS;
        });
    exec.add_death_test(test_case_operator_dereference_value_initalized_iterator);
    exec.add_death_test(test_case_operator_dereference_value_initalized_iterator_end);
    exec.add_death_test(test_case_operator_arrow_value_initalized_iterator);
    exec.add_death_test(test_case_operator_arrow_value_initalized_iterator_end);
    exec.add_death_test(test_case_operator_preincrement_value_initialized_iterator);
    exec.add_death_test(test_case_operator_preincrement_off_end);
    exec.add_death_test(test_case_operator_predecrement_value_initialized_iterator);
    exec.add_death_test(test_case_operator_predecrement_before_begin);
    exec.add_death_test(test_case_operator_advance_value_initialized_iterator);
    exec.add_death_test(test_case_operator_advance_before_begin);
    exec.add_death_test(test_case_operator_advance_after_end);
    exec.add_death_test(test_case_operator_retreat_value_initialized_iterator);
    exec.add_death_test(test_case_operator_retreat_before_begin);
    exec.add_death_test(test_case_operator_retreat_after_end);
    exec.add_death_test(test_case_operator_subtract_incompatible_different_views);
    exec.add_death_test(test_case_operator_subtract_incompatible_value_initialized);
    exec.add_death_test(test_case_operator_equal_incompatible_different_views);
    exec.add_death_test(test_case_operator_equal_incompatible_value_initialized);
    exec.add_death_test(test_case_operator_less_incompatible_different_views);
    exec.add_death_test(test_case_operator_less_incompatible_value_initialized);
    exec.add_death_test(test_case_operator_subscript_out_of_range);
    exec.add_death_test(test_case_front_empty);
    exec.add_death_test(test_case_back_empty);
    exec.add_death_test(test_case_remove_prefix_too_large);
    exec.add_death_test(test_case_remove_suffix_too_large);
    exec.add_death_test(test_case_remove_prefix_incompatible);
    exec.add_death_test(test_case_remove_suffix_incompatible);
    exec.add_death_test(test_case_Copy_s);
    exec.add_death_test(test_case_null_constructor);
    return exec.run(argc, argv);
}
#endif /* _ITERATOR_DEBUG_LEVEL == 0 */

test_death.hpp

#pragma once
#include <crtdbg.h>
#include <locale.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <exception>
#include <limits>
#include <string>
#include <system_error>
#include <vector>
#ifdef _MSC_EXTENSIONS
#include <test_windows.h>
#endif
#include <pmretvals.h>

namespace std_testing {

constexpr int internal_failure = 103;
using test_function_t = int (*)();

#ifdef _MSC_EXTENSIONS
[[noreturn]] inline void api_bogus(const char * const api_name) {
    const DWORD last_error = ::GetLastError();
    const std::string msg = std::system_category().message(last_error);
    printf("%s failed; LastError: 0x%08lX\n %s",
        api_name, last_error, msg.c_str());
    exit(PM_TEST_CASCADE);
}


class death_test_executive {
    const std::wstring this_program;
    const test_function_t run_normal_tests;
    std::vector<test_function_t> death_tests;

    int execute_death_test(const char * const test_id) const {
        _locale_t loc = _create_locale(LC_NUMERIC, "C");
        int testId = _atoi_l(test_id, loc);
        errno_t err = errno;
        _free_locale(loc);

        if (err == 0) {
            _set_abort_behavior(0, _WRITE_ABORT_MSG);
            _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE);
            _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDOUT);

            death_tests[testId]();
            return 0;
        } else {
            puts("failed to parse test_id");
            return internal_failure;
        }
    }

    DWORD dispatch_death_test(const size_t test_id) const {
        STARTUPINFOW si{};
        si.cb = sizeof(si);

        PROCESS_INFORMATION pi{};

        std::wstring test_id_str(1, L' ');
        test_id_str.append(std::to_wstring(test_id));
        if (::CreateProcessW(this_program.c_str(),
            &test_id_str[0],
            0,
            0,
            FALSE,
            0,
            0,
            0,
            &si,
            &pi) == 0) {
            api_bogus("CreateProcessW");
        }

        if (::WaitForSingleObject(pi.hProcess, INFINITE) != WAIT_OBJECT_0) {
            api_bogus("WaitForSingleObject");
        }

        DWORD exit_code = 0;
        if (::GetExitCodeProcess(pi.hProcess, &exit_code) == 0) {
            api_bogus("GetExitCodeProcess");
        }

        ::CloseHandle(pi.hThread);
        ::CloseHandle(pi.hProcess);

        return exit_code;
    }

    static std::wstring get_current_process_path() {
        std::wstring result(MAX_PATH, L'\0');
        for (;;) {
            const DWORD result_size = ::GetModuleFileNameW(0, &result[0],
                static_cast<DWORD>(result.size()));
            const size_t str_size = result.size();
            if (result_size == str_size) {
                // buffer was not big enough
                const size_t str_max_size = result.max_size();
                const size_t result_max_size = str_max_size - str_max_size / 2;
                if (result_size >= result_max_size) {
                    api_bogus("GetModuleFileNameW");
                }

                result.resize(result_size + result_size / 2);
            } else if (result_size == 0) {
                api_bogus("GetModuleFileNameW");
            } else {
                result.resize(result_size);
                break;
            }
        }

        return result;
    }
public:
    explicit death_test_executive(const test_function_t normal_tests_function)
        : this_program(get_current_process_path())
        , run_normal_tests(normal_tests_function) {
    }

    void add_death_test(const test_function_t test) {
        death_tests.push_back(test);
    }

    int run(int argc, char* argv[]) const {
        if (argc == 1) {
            // first pass, run normal tests and sub-process loop
            printf("running normal tests...");
            const int normal_tests_result = run_normal_tests();
            if (normal_tests_result == PM_TEST_PASS) {
                puts(" passed!");
            } else {
                puts("normal tests failed, skipping death tests");
                return normal_tests_result;
            }

            ::SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
            const size_t death_tests_size = death_tests.size();
            for (size_t idx = 0; idx < death_tests_size; ++idx) {
                printf("running death test %zu... ", idx);
                const DWORD death_test_result = dispatch_death_test(idx);
                if (death_test_result <= 1000U) {
                    printf("returned %lu", death_test_result);
                } else {
                    printf("returned 0x%lX", death_test_result);
                }

                if (death_test_result == 0 || death_test_result == 100) {
                    puts(", a success code (this is bad)");
                    puts("Terminate!");
                    return PM_TEST_FAIL;
                } else if (death_test_result == internal_failure) {
                    puts(", an internal test harness failure");
                    puts("Terminate!");
                    return PM_TEST_CASCADE;
                } else {
                    puts(", a failure code (this is good)");
                }
            }

            return PM_TEST_PASS;
        } else if (argc == 2) {
            return execute_death_test(argv[1]);
        } else {
            return PM_TEST_CASCADE;
        }
    }
};

#else /* ^^^ _MSC_EXTENSIONS ^^ // vvv !_MSC_EXTENSIONS vvv */

class death_test_executive {
    const test_function_t run_normal_tests;
public:
    explicit death_test_executive(test_function_t normal_tests_function)
        : run_normal_tests(normal_tests_function) {
    }

    void add_death_test(test_function_t) {
        // skipped under /Za
    }

    int run(int, char*[]) const {
        puts("skipping death tests (/Za prevents including windows.h), running normal tests...");
        return run_normal_tests();
    }
};

#endif // _MSC_EXTENSIONS

} // namespace std_testing