Fix color codes used even when --color never is specified

README.md

@@ -37,6 +37,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t
         - [Built-in reporters](#built-in-reporters)
         - [Overriding the default reporter](#overriding-the-default-reporter)
         - [Registering a new reporter](#registering-a-new-reporter)
+    - [Output colors](#output-colors)
     - [Command-line API](#command-line-api)
     - [Selecting which tests to run](#selecting-which-tests-to-run)
     - [Using your own main function](#using-your-own-main-function)
@@ -863,6 +864,14 @@ All callback functions are optional except `REPORT`. If a callback is unused, si
 An example can be found in [`include/snitch_reporter_teamcity.hpp`](include/snitch_reporter_teamcity.hpp) / [`src/snitch_reporter_teamcity.cpp`](src/snitch_reporter_teamcity.cpp).
 
 
+### Output colors
+
+_snitch_ is able to use color codes when outputting text to the console. These help with readability, but only when the output is printed directly into a terminal that supports color codes. If the chosen output target does not support color codes (which includes in particular the Windows command prompt, outputting to a file, or some CI frameworks), the output will contain gibberish symbols, e.g., `error: missing ...`, hence color codes should be disabled. There are two ways to do this:
+
+ 1. At build-time using `-DSNITCH_DEFAULT_WITH_COLOR=on/off` (CMake) or `-Dsnitch:default_with_color=true/false` (meson). This selects whether color codes are used or not when no specific command-line option is provided to the test executable. This is enabled by default, but you can turn it off if your typical output targets do not support color codes.
+ 2. At run-time using the `--color` (or `--colour-mode`) command-line option (see [the command-line API](#command-line-api) for more information). This allows enabling and disabling color codes for each test run, without rebuilding the tests. This is more useful if your workflow involves some targets which support color codes, and others that do not.
+
+
 ### Command-line API
 
 _snitch_ offers the following command-line API:

include/snitch/snitch_cli.hpp

@@ -40,7 +40,13 @@ void print(Args&&... args) noexcept {
     console_print(message);
 }
 
-SNITCH_EXPORT void print_help(std::string_view program_name) noexcept;
+struct print_help_settings {
+    bool with_color = true;
+};
+
+SNITCH_EXPORT void print_help(
+    std::string_view           program_name,
+    const print_help_settings& settings = print_help_settings{}) noexcept;
 
 SNITCH_EXPORT std::optional<input> parse_arguments(int argc, const char* const argv[]) noexcept;
 

src/snitch_registry.cpp

@@ -694,7 +694,7 @@ bool registry::run_tests(std::string_view run_name) noexcept {
 namespace {
 bool run_tests_impl(registry& r, const cli::input& args) noexcept {
     if (get_option(args, "--help")) {
-        cli::print_help(args.executable);
+        cli::print_help(args.executable, {.with_color = r.with_color});
         return true;
     }
 

tests/runtime_tests/cli.cpp

@@ -208,6 +208,28 @@ TEST_CASE("parse arguments multiple positional", "[cli]") {
     CHECK(console.messages.empty());
 }
 
+TEST_CASE("parse arguments error no color", "[cli]") {
+    console_output_catcher console;
+
+    SECTION("duplicate arg") {
+        const arg_vector args = {"test", "--color", "never", "--color", "always"};
+        snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        CHECK(!contains_color_codes(console.messages));
+    }
+
+    SECTION("missing value") {
+        const arg_vector args = {"test", "--color", "never", "--verbosity"};
+        snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        CHECK(!contains_color_codes(console.messages));
+    }
+
+    SECTION("unknown arg") {
+        const arg_vector args = {"test", "--color", "never", "--foobar"};
+        snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        CHECK(!contains_color_codes(console.messages));
+    }
+}
+
 TEST_CASE("get option", "[cli]") {
     const arg_vector args = {"test", "--help", "--verbosity", "high"};
     auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());

tests/runtime_tests/registry.cpp

@@ -874,6 +874,7 @@ TEST_CASE("run tests cli", "[registry][cli]") {
     SECTION("no argument") {
         const arg_vector args = {"test"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -882,16 +883,34 @@ TEST_CASE("run tests cli", "[registry][cli]") {
         CHECK_RUN(false, 5u, 3u, 0u, 1u, 3u, 3u, 0u);
 #endif
     }
+}
+
+TEST_CASE("print help cli", "[registry][cli]") {
+    mock_framework framework;
+    framework.setup_reporter();
+    register_tests(framework);
+    console_output_catcher console;
 
     SECTION("--help") {
         const arg_vector args = {"test", "--help"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
         CHECK(framework.events.empty());
         CHECK(framework.get_num_runs() == 0u);
         CHECK(console.messages == contains_substring("test [options...]"));
     }
+
+    SECTION("--help no color") {
+        const arg_vector args = {"test", "--help", "--color", "never"};
+        auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
+        framework.registry.run_tests(*input);
+
+        snitch::impl::stdout_print(console.messages);
+        CHECK(!contains_color_codes(console.messages));
+    }
 }
 
 TEST_CASE("list stuff cli", "[registry][cli]") {
@@ -903,6 +922,7 @@ TEST_CASE("list stuff cli", "[registry][cli]") {
     SECTION("--list-tests") {
         const arg_vector args = {"test", "--list-tests"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
         REQUIRE(framework.events.size() == 15u);
@@ -915,6 +935,7 @@ TEST_CASE("list stuff cli", "[registry][cli]") {
     SECTION("--list-tests filtered") {
         const arg_vector args = {"test", "--list-tests", "how*"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
         REQUIRE(framework.events.size() == 6u);
@@ -931,6 +952,7 @@ TEST_CASE("list stuff cli", "[registry][cli]") {
     SECTION("--list-tags") {
         const arg_vector args = {"test", "--list-tags"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
         CHECK(framework.events.empty());
@@ -944,6 +966,7 @@ TEST_CASE("list stuff cli", "[registry][cli]") {
     SECTION("--list-tests-with-tag") {
         const arg_vector args = {"test", "--list-tests-with-tag", "[other_tag]"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
         REQUIRE(framework.events.size() == 4u);
@@ -956,6 +979,7 @@ TEST_CASE("list stuff cli", "[registry][cli]") {
     SECTION("--list-reporters") {
         const arg_vector args = {"test", "--list-reporters"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
 
         SECTION("default") {
             framework.registry.run_tests(*input);
@@ -990,6 +1014,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter") {
         const arg_vector args = {"test", "how many*"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1002,6 +1027,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter multiple AND") {
         const arg_vector args = {"test", "how many*", "*templated*"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1014,6 +1040,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter multiple OR") {
         const arg_vector args = {"test", "how many*,*are you"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1026,6 +1053,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter exclusion") {
         const arg_vector args = {"test", "~*fail"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1038,6 +1066,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter hidden") {
         const arg_vector args = {"test", "hidden test*"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1050,6 +1079,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter tag") {
         const arg_vector args = {"test", "[skipped]"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
         CHECK_RUN(true, 1u, 0u, 0u, 1u, 0u, 0u, 0u);
@@ -1058,6 +1088,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter multiple tags") {
         const arg_vector args = {"test", "[other_tag][tag]"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1070,6 +1101,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter tag AND name") {
         const arg_vector args = {"test", "[tag]", "*many lights"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS
@@ -1082,6 +1114,7 @@ TEST_CASE("run tests filtered cli", "[registry][cli]") {
     SECTION("test filter tag OR name") {
         const arg_vector args = {"test", "[other_tag],how are*"};
         auto input = snitch::cli::parse_arguments(static_cast<int>(args.size()), args.data());
+        framework.registry.configure(*input);
         framework.registry.run_tests(*input);
 
 #if SNITCH_WITH_EXCEPTIONS

tests/testing.cpp

@@ -12,3 +12,19 @@
 #    define SNITCH_EXPORT
 #    include "snitch_main.cpp"
 #endif
+
+bool contains_color_codes(std::string_view msg) noexcept {
+    constexpr std::array codes = {snitch::impl::color::error,      snitch::impl::color::warning,
+                                  snitch::impl::color::status,     snitch::impl::color::fail,
+                                  snitch::impl::color::skipped,    snitch::impl::color::pass,
+                                  snitch::impl::color::highlight1, snitch::impl::color::highlight2,
+                                  snitch::impl::color::reset};
+
+    for (const auto c : codes) {
+        if (msg.find(c) != std::string_view::npos) {
+            return true;
+        }
+    }
+
+    return false;
+}

tests/testing.hpp

@@ -101,3 +101,5 @@ struct filldata<T*> {
 #    define SNITCH_WARNING_DISABLE_PRECEDENCE
 #    define SNITCH_WARNING_DISABLE_ASSIGNMENT
 #endif
+
+bool contains_color_codes(std::string_view msg) noexcept;