diff --git a/README.md b/README.md index e854ecde..6116653a 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t - [Using _snitch_ as a regular library](#using-snitch-as-a-regular-library) - [Using _snitch_ as a header-only library](#using-snitch-as-a-header-only-library) - [Example build configuration with meson](#example-build-configuration-with-meson) -- [Example build configuration with vcpkg](https://github.com/snitch-org/snitch/tree/main/doc/vcpkg-example/README.md) +- [Example build configuration with vcpkg](#example-build-configuration-with-vcpkg) - [Benchmark](#benchmark) - [Documentation](#documentation) - [Detailed comparison with _Catch2_](#detailed-comparison-with-catch2) @@ -29,6 +29,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t - [Run-time and compile-time](#run-time-and-compile-time) - [Exception checks](#exception-checks) - [Miscellaneous](#miscellaneous) + - [Advanced API](#advanced-api) - [Tags](#tags) - [Matchers](#matchers) - [Sections](#sections) @@ -214,6 +215,11 @@ then you can configure with: And this disables the build step that generates the single-header file "`snitch_all.hpp`". +## Example build configuration with vcpkg + +See [the dedicated page in the docs folder](doc/vcpkg-example/README.md). + + ## Benchmark The following benchmarks were done using real-world tests from another library ([_observable_unique_ptr_](https://github.com/cschreib/observable_unique_ptr)), which generates about 4000 test cases and 25000 checks. This library uses "typed" tests almost exclusively, where each test case is instantiated several times, each time with a different tested type (here, 25 types). Building and running the tests was done without parallelism to simplify the comparison. The benchmarks were run on a desktop with the following specs: @@ -476,6 +482,13 @@ This reports the current test case as "skipped". Any previously reported status This is similar to `SKIP`, except that the test case continues. Further failure will not be reported. This is only recommended as an alternative to `SKIP()` when exceptions cannot be used. +### Advanced API + +`snitch::notify_exception_handled();` + +If handling exceptions explicitly with a `try/catch` block in a test case, this should be called at the end of the `catch` block. This clears up internal state that would have been used to report that exception, had it not been handled. Calling this is not strictly necessary in most cases, but omitting it can lead to confusing contextual data (incorrect section/capture/info) if another exception is thrown afterwards and not handled. + + ### Tags Tags are assigned to each test case using the [Test case macros](#test-case-macros), as a single string. Within this string, individual tags must be surrounded by square brackets, with no white-space between tags (although white space within a tag is allowed). For example: diff --git a/include/snitch/snitch_macros_exceptions.hpp b/include/snitch/snitch_macros_exceptions.hpp index 83400b66..883e20e9 100644 --- a/include/snitch/snitch_macros_exceptions.hpp +++ b/include/snitch/snitch_macros_exceptions.hpp @@ -17,6 +17,7 @@ SNITCH_NO_EXCEPTION_THROWN = true; \ } catch (const __VA_ARGS__&) { \ snitch::registry::report_assertion(true, #__VA_ARGS__ " was thrown as expected"); \ + snitch::notify_exception_handled(); \ } catch (...) { \ try { \ throw; \ @@ -25,12 +26,12 @@ false, \ #__VA_ARGS__ " expected but other std::exception thrown; message: ", \ e.what()); \ - MAYBE_ABORT; \ } catch (...) { \ snitch::registry::report_assertion( \ false, #__VA_ARGS__ " expected but other unknown exception thrown"); \ - MAYBE_ABORT; \ } \ + snitch::notify_exception_handled(); \ + MAYBE_ABORT; \ } \ if (SNITCH_NO_EXCEPTION_THROWN) { \ snitch::registry::report_assertion( \ @@ -58,12 +59,14 @@ false, "could not match caught " #EXCEPTION " with expected content: ", \ SNITCH_TEMP_MATCHER.describe_match( \ e, snitch::matchers::match_status::failed)); \ + snitch::notify_exception_handled(); \ MAYBE_ABORT; \ } else { \ snitch::registry::report_assertion( \ true, "caught " #EXCEPTION " matched expected content: ", \ SNITCH_TEMP_MATCHER.describe_match( \ e, snitch::matchers::match_status::matched)); \ + snitch::notify_exception_handled(); \ } \ } catch (...) { \ try { \ @@ -72,12 +75,12 @@ snitch::registry::report_assertion( \ false, #EXCEPTION " expected but other std::exception thrown; message: ", \ e.what()); \ - MAYBE_ABORT; \ } catch (...) { \ snitch::registry::report_assertion( \ false, #EXCEPTION " expected but other unknown exception thrown"); \ - MAYBE_ABORT; \ } \ + snitch::notify_exception_handled(); \ + MAYBE_ABORT; \ } \ if (SNITCH_NO_EXCEPTION_THROWN) { \ snitch::registry::report_assertion( \ @@ -111,6 +114,7 @@ false, "expected " #__VA_ARGS__ \ " not to throw but it threw an unknown exception"); \ } \ + snitch::notify_exception_handled(); \ MAYBE_ABORT; \ } \ } while (0) diff --git a/include/snitch/snitch_test_data.hpp b/include/snitch/snitch_test_data.hpp index 733216be..e58d7e2d 100644 --- a/include/snitch/snitch_test_data.hpp +++ b/include/snitch/snitch_test_data.hpp @@ -374,4 +374,18 @@ struct scoped_test_check { }; } // namespace snitch::impl +namespace snitch { +#if SNITCH_WITH_EXCEPTIONS +/*! \brief Notify the testing framework that an exception was manually handled. + * \details If handling exceptions explicitly with a `try/catch` block in a test case, + * this should be called at the end of the `catch` block. This clears up internal state + * that would have been used to report that exception, had it not been handled. Calling + * this is not strictly necessary in most cases, but omitting it can lead to confusing + * contextual data (incorrect section/capture/info) if another exception is thrown afterwards + * and not handled. + */ +SNITCH_EXPORT void notify_exception_handled() noexcept; +#endif +} // namespace snitch + #endif diff --git a/src/snitch_capture.cpp b/src/snitch_capture.cpp index 0c406cc8..c98498ec 100644 --- a/src/snitch_capture.cpp +++ b/src/snitch_capture.cpp @@ -92,7 +92,9 @@ small_string& add_capture(test_state& state) { } #if SNITCH_WITH_EXCEPTIONS - state.held_info.reset(); + if (std::uncaught_exceptions() == 0) { + notify_exception_handled(); + } #endif state.info.captures.grow(1); diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index 63ac7cac..36a74562 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -456,7 +456,8 @@ void report_assertion_impl( register_assertion(success, state); #if SNITCH_WITH_EXCEPTIONS - const bool use_held_info = state.unhandled_exception && state.held_info.has_value(); + const bool use_held_info = (state.unhandled_exception || std::uncaught_exceptions() > 0) && + state.held_info.has_value(); const auto captures_buffer = impl::make_capture_buffer( use_held_info ? state.held_info.value().captures : state.info.captures); @@ -655,15 +656,7 @@ impl::test_state registry::run(impl::test_case& test) noexcept { } if (state.unhandled_exception) { - auto& current_section = state.held_info.has_value() - ? state.held_info.value().sections.current_section - : state.info.sections.current_section; - - // Close all sections that were left open by the exception. - while (!current_section.empty()) { - registry::report_section_ended(current_section.back()); - current_section.pop_back(); - } + notify_exception_handled(); } state.unhandled_exception = false; diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index 0f4ccd8b..8cc72db8 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -71,7 +71,9 @@ section_entry_checker::~section_entry_checker() { section_entry_checker::operator bool() { #if SNITCH_WITH_EXCEPTIONS - state.held_info.reset(); + if (std::uncaught_exceptions() == 0) { + notify_exception_handled(); + } #endif auto& sections = state.info.sections; diff --git a/src/snitch_test_data.cpp b/src/snitch_test_data.cpp index c1d90f5c..e37665a2 100644 --- a/src/snitch_test_data.cpp +++ b/src/snitch_test_data.cpp @@ -1,5 +1,7 @@ #include "snitch/snitch_test_data.hpp" +#include "snitch/snitch_registry.hpp" + #if SNITCH_WITH_EXCEPTIONS # include #endif @@ -36,8 +38,11 @@ void pop_location(test_state& test) noexcept { scoped_test_check::scoped_test_check(const source_location& location) noexcept : test(get_current_test()) { + #if SNITCH_WITH_EXCEPTIONS - test.held_info.reset(); + if (std::uncaught_exceptions() == 0) { + notify_exception_handled(); + } #endif push_location(test, {location.file, location.line, location_type::in_check}); @@ -59,3 +64,24 @@ scoped_test_check::~scoped_test_check() noexcept { pop_location(test); } } // namespace snitch::impl + +namespace snitch { +#if SNITCH_WITH_EXCEPTIONS +void notify_exception_handled() noexcept { + auto& state = impl::get_current_test(); + if (!state.held_info.has_value()) { + return; + } + + // Close all sections that were left open by the exception. + auto& current_held_section = state.held_info.value().sections.current_section; + const auto& current_section = state.info.sections.current_section; + while (current_held_section.size() > current_section.size()) { + registry::report_section_ended(current_held_section.back()); + current_held_section.pop_back(); + } + + state.held_info.reset(); +} +#endif +} // namespace snitch diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index 4c619f9f..a842aaf5 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -250,6 +250,15 @@ + +
+ + unexpected std::exception caught; message: unexpected error + + +
+ + not interesting @@ -301,6 +310,15 @@ + +
+ + failure + + +
+ +
@@ -451,6 +469,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index b804a355..ca44b684 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -376,6 +376,15 @@
+ +
+ + unexpected std::exception caught; message: unexpected error + + +
+ + not interesting @@ -436,6 +445,18 @@ + +
+ + failure + + +
+ + no exception caught + + +
@@ -595,6 +616,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests index 50acafc6..c8af5592 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests @@ -324,6 +324,15 @@ * + + test unexpected throw in check & section mayfail + + [!mayfail] + + *testing_reporters.cpp + * + + test SKIP @@ -360,6 +369,15 @@ * + + test SECTION mayfail + + [!mayfail] + + *testing_reporters.cpp + * + + test multiple SECTION diff --git a/tests/approval_tests/data/expected/reporter_console_default b/tests/approval_tests/data/expected/reporter_console_default index e7ab4200..9b26116f 100644 --- a/tests/approval_tests/data/expected/reporter_console_default +++ b/tests/approval_tests/data/expected/reporter_console_default @@ -80,6 +80,10 @@ failed: running test case "test unexpected throw in check & section fail" in section "section 1" somewhere inside check at *testing_reporters.cpp:* unexpected std::exception caught; message: unexpected error +allowed failure: running test case "test unexpected throw in check & section mayfail" + in section "section 1" + somewhere inside check at *testing_reporters.cpp:* + unexpected std::exception caught; message: unexpected error skipped: running test case "test SKIP" at *testing_reporters.cpp:* not interesting @@ -107,6 +111,10 @@ failed: running test case "test SECTION" in section "section" at *testing_reporters.cpp:* failure +allowed failure: running test case "test SECTION mayfail" + in section "section" + at *testing_reporters.cpp:* + failure failed: running test case "test multiple SECTION" in section "section 1" at *testing_reporters.cpp:* @@ -195,4 +203,4 @@ skipped: running test case "test SKIP in SECTION" at *testing_reporters.cpp:* stopping here ========================================== -error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 46 test cases, 94 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_full b/tests/approval_tests/data/expected/reporter_console_full index d96914f7..50a6e87d 100644 --- a/tests/approval_tests/data/expected/reporter_console_full +++ b/tests/approval_tests/data/expected/reporter_console_full @@ -277,6 +277,14 @@ failed: running test case "test unexpected throw in check & section fail" unexpected std::exception caught; message: unexpected error leaving section: section 1 finished: test unexpected throw in check & section fail (*) +starting: test unexpected throw in check & section mayfail at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* +allowed failure: running test case "test unexpected throw in check & section mayfail" + in section "section 1" + somewhere inside check at *testing_reporters.cpp:* + unexpected std::exception caught; message: unexpected error +leaving section: section 1 +finished: test unexpected throw in check & section mayfail (*) starting: test SKIP at *testing_reporters.cpp:* skipped: running test case "test SKIP" at *testing_reporters.cpp:* @@ -323,6 +331,17 @@ passed: running test case "test SECTION" at *testing_reporters.cpp:* no exception caught finished: test SECTION (*) +starting: test SECTION mayfail at *testing_reporters.cpp:* +entering section: section at *testing_reporters.cpp:* +allowed failure: running test case "test SECTION mayfail" + in section "section" + at *testing_reporters.cpp:* + failure +leaving section: section +passed: running test case "test SECTION mayfail" + at *testing_reporters.cpp:* + no exception caught +finished: test SECTION mayfail (*) starting: test multiple SECTION at *testing_reporters.cpp:* entering section: section 1 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" @@ -450,4 +469,4 @@ leaving section: section 2 leaving section: section 1 finished: test SKIP in SECTION (*) ========================================== -error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 46 test cases, 94 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_list_tests b/tests/approval_tests/data/expected/reporter_console_list_tests index 685f23d4..98ed91d1 100644 --- a/tests/approval_tests/data/expected/reporter_console_list_tests +++ b/tests/approval_tests/data/expected/reporter_console_list_tests @@ -47,12 +47,16 @@ Matching test cases: test unexpected throw in section fail test unexpected throw in check fail test unexpected throw in check & section fail + test unexpected throw in check & section mayfail + [!mayfail] test SKIP test INFO test multiple INFO test SECTION + test SECTION mayfail + [!mayfail] test multiple SECTION test SECTION & INFO test SECTION & CAPTURE test SKIP in SECTION -44 matching test cases +46 matching test cases diff --git a/tests/approval_tests/data/expected/reporter_console_withcolor b/tests/approval_tests/data/expected/reporter_console_withcolor index 55178903..e0a969d2 100644 --- a/tests/approval_tests/data/expected/reporter_console_withcolor +++ b/tests/approval_tests/data/expected/reporter_console_withcolor @@ -80,6 +80,10 @@ in section "section 1" somewhere inside check at *testing_reporters.cpp:* unexpected std::exception caught; message: unexpected error +allowed failure: running test case "test unexpected throw in check & section mayfail" + in section "section 1" + somewhere inside check at *testing_reporters.cpp:* + unexpected std::exception caught; message: unexpected error skipped: running test case "test SKIP" at *testing_reporters.cpp:* not interesting @@ -107,6 +111,10 @@ in section "section" at *testing_reporters.cpp:* failure +allowed failure: running test case "test SECTION mayfail" + in section "section" + at *testing_reporters.cpp:* + failure failed: running test case "test multiple SECTION" in section "section 1" at *testing_reporters.cpp:* @@ -195,4 +203,4 @@ at *testing_reporters.cpp:* stopping here ========================================== -error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 46 test cases, 94 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_teamcity_default b/tests/approval_tests/data/expected/reporter_teamcity_default index 8a3168f2..e56401f4 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_default +++ b/tests/approval_tests/data/expected/reporter_teamcity_default @@ -101,6 +101,11 @@ ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] +##teamCity[testStarted name='test unexpected throw in check & section mayfail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[testStdOut name='test unexpected throw in check & section mayfail' out='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] +##teamCity[testFinished name='test unexpected throw in check & section mayfail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] ##teamCity[testFinished name='test SKIP' duration='*'] @@ -118,6 +123,11 @@ ##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|n failure'] ##teamCity[blockClosed name='section'] ##teamCity[testFinished name='test SECTION' duration='*'] +##teamCity[testStarted name='test SECTION mayfail'] +##teamCity[blockOpened name='section' description=''] +##teamCity[testStdOut name='test SECTION mayfail' out='*testing_reporters.cpp:*|n failure'] +##teamCity[blockClosed name='section'] +##teamCity[testFinished name='test SECTION mayfail' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 1'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_full b/tests/approval_tests/data/expected/reporter_teamcity_full index cd4999da..a88e33cd 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_full +++ b/tests/approval_tests/data/expected/reporter_teamcity_full @@ -138,6 +138,11 @@ ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] +##teamCity[testStarted name='test unexpected throw in check & section mayfail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[testStdOut name='test unexpected throw in check & section mayfail' out='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] +##teamCity[testFinished name='test unexpected throw in check & section mayfail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] ##teamCity[testFinished name='test SKIP' duration='*'] @@ -158,6 +163,12 @@ ##teamCity[blockClosed name='section'] ##teamCity[testStdOut name='test SECTION' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION' duration='*'] +##teamCity[testStarted name='test SECTION mayfail'] +##teamCity[blockOpened name='section' description=''] +##teamCity[testStdOut name='test SECTION mayfail' out='*testing_reporters.cpp:*|n failure'] +##teamCity[blockClosed name='section'] +##teamCity[testStdOut name='test SECTION mayfail' out='*testing_reporters.cpp:*|n no exception caught'] +##teamCity[testFinished name='test SECTION mayfail' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 1'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_list_tests b/tests/approval_tests/data/expected/reporter_teamcity_list_tests index c9000a5f..d14c4853 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_list_tests +++ b/tests/approval_tests/data/expected/reporter_teamcity_list_tests @@ -34,10 +34,12 @@ test unexpected throw fail test unexpected throw in section fail test unexpected throw in check fail test unexpected throw in check & section fail +test unexpected throw in check & section mayfail test SKIP test INFO test multiple INFO test SECTION +test SECTION mayfail test multiple SECTION test SECTION & INFO test SECTION & CAPTURE diff --git a/tests/runtime_tests/capture.cpp b/tests/runtime_tests/capture.cpp index 15ab2bf2..8bceaf06 100644 --- a/tests/runtime_tests/capture.cpp +++ b/tests/runtime_tests/capture.cpp @@ -274,7 +274,7 @@ TEST_CASE("capture", "[test macros]") { CHECK_CAPTURES("j := 2"); } - SECTION("with handled exception then unhandled no capture") { + SECTION("with handled exception then unhandled no capture missing notify") { framework.test_case.func = []() { try { int i = 1; @@ -288,10 +288,26 @@ TEST_CASE("capture", "[test macros]") { framework.run_test(); REQUIRE(framework.get_num_failures() == 1u); - // FIXME: expected nothing - // https://github.com/snitch-org/snitch/issues/179 CHECK_CAPTURES("i := 1"); } + + SECTION("with handled exception then unhandled no capture") { + framework.test_case.func = []() { + try { + int i = 1; + SNITCH_CAPTURE(i); + throw std::runtime_error("bad"); + } catch (...) { + snitch::notify_exception_handled(); + } + + throw std::runtime_error("bad"); + }; + + framework.run_test(); + REQUIRE(framework.get_num_failures() == 1u); + CHECK_NO_CAPTURE; + } #endif } diff --git a/tests/runtime_tests/section.cpp b/tests/runtime_tests/section.cpp index fa545143..dfd57479 100644 --- a/tests/runtime_tests/section.cpp +++ b/tests/runtime_tests/section.cpp @@ -8,6 +8,16 @@ using namespace std::literals; +#if SNITCH_WITH_EXCEPTIONS +struct destructor_asserter { + bool pass = true; + + ~destructor_asserter() { + SNITCH_CHECK(pass); + } +}; +#endif + SNITCH_WARNING_PUSH SNITCH_WARNING_DISABLE_UNREACHABLE @@ -149,6 +159,49 @@ TEST_CASE("section", "[test macros]") { CHECK_SECTIONS("section 1"); CHECK_CASE(snitch::test_case_state::failed, 1u, 1u); } + + SECTION("unexpected throw with destructor assert ok") { + framework.test_case.func = []() { + SNITCH_SECTION("section 1") { + destructor_asserter a{.pass = true}; + SNITCH_SECTION("section 2") { + throw std::runtime_error("no can do"); + } + } + }; + + framework.run_test(); + + REQUIRE(framework.get_num_failures() == 1u); + + CHECK_SECTIONS("section 1", "section 2"); + CHECK_CASE(snitch::test_case_state::failed, 2u, 1u); + } + + SECTION("unexpected throw with destructor assert nok") { + framework.test_case.func = []() { + SNITCH_SECTION("section 1") { + destructor_asserter a{.pass = false}; + SNITCH_SECTION("section 2") { + throw std::runtime_error("no can do"); + } + } + }; + + framework.run_test(); + + // This is what we want: + // REQUIRE(framework.get_num_failures() == 2u); + // CHECK_SECTIONS_FOR_FAILURE(0u, "section 1", "section 2"); // exception + // CHECK_SECTIONS_FOR_FAILURE(1u, "section 1"); // destructor + // CHECK_CASE(snitch::test_case_state::failed, 2u, 2u); + + // This is what we get: + REQUIRE(framework.get_num_failures() == 2u); + CHECK_SECTIONS_FOR_FAILURE(0u, "section 1", "section 2"); // destructor + CHECK_SECTIONS_FOR_FAILURE(1u, "section 1", "section 2"); // exception + CHECK_CASE(snitch::test_case_state::failed, 2u, 2u); + } #endif SECTION("nested sections varying depth") { @@ -418,7 +471,7 @@ TEST_CASE("section", "[test macros]") { CHECK_SECTIONS("section 2"); } - SECTION("with handled exception then unhandled no section") { + SECTION("with handled exception then unhandled no section missing notify") { framework.test_case.func = []() { try { SNITCH_SECTION("section 1") { @@ -432,10 +485,26 @@ TEST_CASE("section", "[test macros]") { framework.run_test(); REQUIRE(framework.get_num_failures() == 1u); - // FIXME: expected nothing - // https://github.com/snitch-org/snitch/issues/179 CHECK_SECTIONS("section 1"); } + + SECTION("with handled exception then unhandled no section") { + framework.test_case.func = []() { + try { + SNITCH_SECTION("section 1") { + throw std::runtime_error("bad"); + } + } catch (...) { + snitch::notify_exception_handled(); + } + + throw std::runtime_error("bad"); + }; + + framework.run_test(); + REQUIRE(framework.get_num_failures() == 1u); + CHECK_NO_SECTION; + } #endif } diff --git a/tests/testing_event.cpp b/tests/testing_event.cpp index 55498e48..8a4bfab6 100644 --- a/tests/testing_event.cpp +++ b/tests/testing_event.cpp @@ -68,7 +68,7 @@ void copy_assertion_data(snitch::small_string_span pool, U& c, const T& e) { } template -std::optional +std::pair, std::size_t> get_nth_event(snitch::small_vector_span events, std::size_t id) { auto iter = events.cbegin(); bool first = true; @@ -87,9 +87,9 @@ get_nth_event(snitch::small_vector_span events, std::s } while (id != 0); if (iter == events.cend()) { - return {}; + return {std::nullopt, events.size()}; } else { - return std::get(*iter); + return {std::get(*iter), iter - events.cbegin()}; } } @@ -211,12 +211,12 @@ owning_event::data deep_copy(snitch::small_string_span pool, const snitch::event std::optional get_failure_event(snitch::small_vector_span events, std::size_t id) { - return get_nth_event(events, id); + return get_nth_event(events, id).first; } std::optional get_success_event(snitch::small_vector_span events, std::size_t id) { - return get_nth_event(events, id); + return get_nth_event(events, id).first; } std::optional get_test_id(const owning_event::data& e) noexcept { @@ -278,12 +278,76 @@ void mock_framework::run_test() { std::optional mock_framework::get_failure_event(std::size_t id) const { - return get_nth_event(events, id); + return get_nth_event(events, id).first; } std::optional mock_framework::get_success_event(std::size_t id) const { - return get_nth_event(events, id); + return get_nth_event(events, id).first; +} + +bool mock_framework::check_balanced_section_events() const { + bool test_case_started = false; + snitch::small_vector sections; + for (const auto& e : events) { + bool good = std::visit( + snitch::overload{ + [&](const owning_event::section_started& s) { + if (!test_case_started) { + return false; + } + sections.push_back(s.id); + return true; + }, + [&](const owning_event::section_ended& s) { + if (!test_case_started) { + return false; + } + if (sections.empty()) { + return false; + } + if (sections.back().name != s.id.name || + sections.back().description != s.id.description) { + return false; + } + + sections.pop_back(); + return true; + }, + [&](const owning_event::test_case_started&) { + test_case_started = true; + return sections.empty(); + }, + [&](const owning_event::test_case_ended&) { + test_case_started = false; + return sections.empty(); + }, + [](const auto&) { return true; }}, + e); + + if (!good) { + return false; + } + } + + return sections.empty(); +} + +snitch::small_vector +mock_framework::get_sections_for_failure_event(std::size_t id) const { + auto [event, pos] = get_nth_event(events, id); + + snitch::small_vector sections; + for (std::size_t i = 0; i < pos; ++i) { + std::visit( + snitch::overload{ + [&](const owning_event::section_started& s) { sections.push_back(s.id.name); }, + [&](const owning_event::section_ended&) { sections.pop_back(); }, + [](const auto&) {}}, + events[i]); + } + + return sections; } std::size_t mock_framework::get_num_registered_tests() const { diff --git a/tests/testing_event.hpp b/tests/testing_event.hpp index 4012803f..358fb869 100644 --- a/tests/testing_event.hpp +++ b/tests/testing_event.hpp @@ -59,12 +59,12 @@ struct section_started { }; struct section_ended { - snitch::section_id id = {}; - snitch::source_location location = {}; - bool skipped = false; - std::size_t assertion_count = 0; - std::size_t assertion_failure_count = 0; - std::size_t allowed_assertion_failure_count = 0; + snitch::section_id id = {}; + snitch::source_location location = {}; + bool skipped = false; + std::size_t assertion_count = 0; + std::size_t assertion_failure_count = 0; + std::size_t allowed_assertion_failure_count = 0; #if SNITCH_WITH_TIMINGS float duration = 0.0f; #endif @@ -217,6 +217,11 @@ struct mock_framework { std::optional get_failure_event(std::size_t id = 0) const; std::optional get_success_event(std::size_t id = 0) const; + bool check_balanced_section_events() const; + + snitch::small_vector + get_sections_for_failure_event(std::size_t id = 0) const; + std::size_t get_num_registered_tests() const; std::size_t get_num_runs() const; std::size_t get_num_failures() const; @@ -386,16 +391,21 @@ struct has_expr_data { #define CHECK_SECTIONS_FOR_FAILURE(FAILURE_ID, ...) \ do { \ + INFO("failure ID: ", FAILURE_ID); \ + REQUIRE(framework.check_balanced_section_events()); \ auto failure = framework.get_failure_event(FAILURE_ID); \ REQUIRE(failure.has_value()); \ const char* EXPECTED_SECTIONS[] = {__VA_ARGS__}; \ REQUIRE( \ failure.value().sections.size() == sizeof(EXPECTED_SECTIONS) / sizeof(const char*)); \ + const auto section_from_events = framework.get_sections_for_failure_event(FAILURE_ID); \ + REQUIRE(section_from_events.size() == sizeof(EXPECTED_SECTIONS) / sizeof(const char*)); \ std::size_t SECTION_INDEX = 0; \ for (std::string_view SECTION_NAME : EXPECTED_SECTIONS) { \ CHECK( \ failure.value().sections[SECTION_INDEX].id.name == \ std::string_view{SECTION_NAME}); \ + CHECK(section_from_events[SECTION_INDEX] == std::string_view{SECTION_NAME}); \ ++SECTION_INDEX; \ } \ } while (0) @@ -404,9 +414,12 @@ struct has_expr_data { #define CHECK_NO_SECTION_FOR_FAILURE(FAILURE_ID) \ do { \ + REQUIRE(framework.check_balanced_section_events()); \ auto failure = framework.get_failure_event(FAILURE_ID); \ REQUIRE(failure.has_value()); \ CHECK(failure.value().sections.empty()); \ + const auto section_from_events = framework.get_sections_for_failure_event(FAILURE_ID); \ + CHECK(section_from_events.empty()); \ } while (0) #define CHECK_NO_SECTION CHECK_NO_SECTION_FOR_FAILURE(0u) diff --git a/tests/testing_reporters.cpp b/tests/testing_reporters.cpp index 467e3bb2..afce1e78 100644 --- a/tests/testing_reporters.cpp +++ b/tests/testing_reporters.cpp @@ -136,6 +136,13 @@ void register_tests_for_reporters(snitch::registry& r) { SNITCH_CHECK(throw_unexpectedly() == 0); } }); + r.add( + {"test unexpected throw in check & section mayfail", "[!mayfail]"}, SNITCH_CURRENT_LOCATION, + []() { + SNITCH_SECTION("section 1") { + SNITCH_CHECK(throw_unexpectedly() == 0); + } + }); #endif r.add({"test SKIP"}, SNITCH_CURRENT_LOCATION, []() { SNITCH_SKIP("not interesting"); }); @@ -162,6 +169,12 @@ void register_tests_for_reporters(snitch::registry& r) { } }); + r.add({"test SECTION mayfail", "[!mayfail]"}, SNITCH_CURRENT_LOCATION, []() { + SNITCH_SECTION("section") { + SNITCH_FAIL_CHECK("failure"); + } + }); + r.add({"test multiple SECTION"}, SNITCH_CURRENT_LOCATION, []() { SNITCH_SECTION("section 1") { SNITCH_FAIL_CHECK("failure 1");