diff --git a/CMakeLists.txt b/CMakeLists.txt index 125db05..d1f28b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,10 @@ set_target_properties(clar PROPERTIES C_EXTENSIONS OFF ) +if(NOT WIN32) + set(CLAR_LIBRARIES m) +endif() + if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) include(CTest) if(BUILD_TESTING) diff --git a/README.md b/README.md index 4159598..d2d681e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,13 @@ Come out and Clar ================= -In Catalan, "clar" means clear, easy to perceive. Using clar will make it -easy to test and make clear the quality of your code. +Clar is a minimal C unit testing and benchmarking framework. It +provides a simple mechanism for writing tests and asserting +postconditions, while providing support for TAP and JUnit style +outputs. -> _Historical note_ -> -> Originally the clar project was named "clay" because the word "test" has its -> roots in the latin word *"testum"*, meaning "earthen pot", and *"testa"*, -> meaning "piece of burned clay"? -> -> This is because historically, testing implied melting metal in a pot to -> check its quality. Clay is what tests are made of. +In Catalan, "clar" means clear, easy to perceive. Using Clar will make it +easy to test and make clear the quality of your code. ## Quick Usage Overview @@ -19,9 +15,7 @@ Clar is a minimal C unit testing framework. It's been written to replace the old framework in [libgit2][libgit2], but it's both very versatile and straightforward to use. -Can you count to funk? - -- **Zero: Initialize test directory** +1. **Initialize test directory** ~~~~ sh $ mkdir tests @@ -29,7 +23,7 @@ Can you count to funk? $ cp $CLAR_ROOT/example/*.c tests ~~~~ -- **One: Write some tests** +2. **Write some tests** File: tests/adding.c: @@ -59,7 +53,7 @@ Can you count to funk? } ~~~~~ -- **Two: Build the test executable** +3. **Build the test executable** ~~~~ sh $ cd tests @@ -68,7 +62,7 @@ Can you count to funk? $ gcc -I. clar.c main.c adding.c -o testit ~~~~ -- **Funk: Funk it.** +4. **Run the tests** ~~~~ sh $ ./testit @@ -319,6 +313,92 @@ void test_example__a_test_with_auxiliary_methods(void) } ~~~~ +## Benchmarks + +The clar mixer (`generate.py`) and runner can also be used to support +simple benchmark capabilities. When running in benchmark mode, Clar +will run each test multiple times in succession, using a high-resolution +platform timer to measure the elapsed time of each run. + +By default, Clar will run each test repeatedly for 3 seconds (with +a minimum of 10 runs), but you can define the explicit number of +runs for each test in the definition. + +By default, Clar will run the initialization and cleanup functions +before _each_ test run. This allows for consistent setup and teardown +behavior, and predictability with existing test setups. However, you +can avoid this additional overhead by defining a _reset_ function. +This will be called between test runs instead of the cleanup and +re-initialization; in this case, initialization will occur only +before all test runs, and cleanup will be performed only when all +test runs are complete. + +To configure a benchmark application instead of a test application: + +1. **Set clar into benchmark mode in your main function** + + ~~~~ c + int main(int argc, char *argv[]) + { + clar_test_set_mode(CL_TEST_BENCHMARK); + clar_test_init(argc, argv); + res = clar_test_run(); + clar_test_shutdown(); + return res; + } + ~~~~ + +2. **Optionally, set up your initialization, cleanup, and reset + functions** + + ~~~~ c + void test_foo__initialize(void) + { + global_data = malloc(1024 * 1024 * 1024); + memset(global_data, 0, 1024 * 1024 * 1024); + } + + void test_foo__reset(void) + { + memset(global_data, 0, 1024 * 1024 * 1024); + } + + void test_foo__cleanup(void) + { + global_data = malloc(1024 * 1024 * 1024); + } + ~~~~ + +3. **Optionally, configure tests with a specific run number** + + ~~~~ c + /* Run this test 500 times */ + void test_foo__bar(void) + /* [clar]:runs=500 */ + { + bar(); + } + ~~~~ + +3. **Run the benchmarks** + + When running in benchmark mode, you'll see timings output; if you + write a summary file, it will be a JSON file that contains the + time information. + + ~~~~ sh + $ ./benchmarks -r/path/to/results.json + Started benchmarks (mean time ± stddev / min time … max time): + + foo::bar: 24.75 ms ± 1.214 ms / range: 24.41 ms … 38.06 ms (500 runs) + foo::baz: 24.67 ms ± 248.2 μs / range: 24.41 ms … 25.41 ms (478 runs) + foo::qux: 25.98 ms ± 333.0 μs / range: 25.64 ms … 26.82 ms (112 runs) + ~~~~ + +Note: you can change the prefix of the test function names from `test_` +to something of your choice by using the `--prefix=...` option for +the `generate.py` mixer script. + About Clar ========== diff --git a/clar.c b/clar.c index bf3d7f0..35166e4 100644 --- a/clar.c +++ b/clar.c @@ -87,6 +87,7 @@ typedef struct stat STAT_T; #endif +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #include "clar.h" @@ -100,11 +101,11 @@ fixture_path(const char *base, const char *fixture_name); #endif struct clar_error { - const char *file; + const char *message; + char *description; const char *function; + const char *file; uintmax_t line_number; - const char *error_msg; - char *description; struct clar_error *next; }; @@ -117,13 +118,22 @@ struct clar_explicit { }; struct clar_report { + const char *suite; const char *test; + const char *description; int test_number; - const char *suite; + + int runs; enum cl_test_status status; time_t start; - double elapsed; + + double *times; + double time_min; + double time_max; + double time_mean; + double time_stddev; + double time_total; struct clar_error *errors; struct clar_error *last_error; @@ -137,10 +147,12 @@ struct clar_summary { }; static struct { + enum cl_test_mode test_mode; enum cl_test_status test_status; - const char *active_test; const char *active_suite; + const char *active_test; + const char *active_description; int total_skipped; int total_errors; @@ -149,6 +161,7 @@ static struct { int suites_ran; enum cl_output_format output_format; + enum cl_summary_format summary_format; int report_errors_only; int exit_on_error; @@ -181,28 +194,32 @@ static struct { struct clar_func { const char *name; + const char *description; + int runs; void (*ptr)(void); }; struct clar_suite { const char *name; struct clar_func initialize; + struct clar_func reset; struct clar_func cleanup; const struct clar_func *tests; size_t test_count; int enabled; }; -/* From clar_print_*.c */ +/* From print.h */ static void clar_print_init(int test_count, int suite_count, const char *suite_names); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed); -static void clar_print_onsuite(const char *suite_name, int suite_index); +static void clar_print_suite_start(const char *suite_name, int suite_index); +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number); +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); -/* From clar_sandbox.c */ +/* From sandbox.c */ static void clar_tempdir_init(void); static void clar_tempdir_shutdown(void); static int clar_sandbox_create(const char *suite_name, const char *test_name); @@ -212,6 +229,8 @@ static int clar_sandbox_cleanup(void); static struct clar_summary *clar_summary_init(const char *filename); static int clar_summary_shutdown(struct clar_summary *fp); +#include "clar/counter.h" + /* Load the declarations for the test suite */ #include "clar.suite" @@ -268,70 +287,121 @@ clar_report_all(void) } } -#ifdef WIN32 -# define clar_time DWORD - -static void clar_time_now(clar_time *out) +static void +compute_times(void) { - *out = GetTickCount(); -} + double total_squares = 0; + int i; -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)*end - (double)*start) / 1000; -} -#else -# include + _clar.last_report->time_min = _clar.last_report->times[0]; + _clar.last_report->time_max = _clar.last_report->times[0]; + _clar.last_report->time_total = _clar.last_report->times[0]; -# define clar_time struct timeval + for (i = 1; i < _clar.last_report->runs; i++) { + if (_clar.last_report->times[i] < _clar.last_report->time_min) + _clar.last_report->time_min = _clar.last_report->times[i]; -static void clar_time_now(clar_time *out) -{ - gettimeofday(out, NULL); -} + if (_clar.last_report->times[i] > _clar.last_report->time_max) + _clar.last_report->time_max = _clar.last_report->times[i]; -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) - - ((double)start->tv_sec + (double)start->tv_usec / 1.0E6); + _clar.last_report->time_total += _clar.last_report->times[i]; + } + + if (_clar.last_report->runs <= 1) { + _clar.last_report->time_stddev = 0; + } else { + _clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs; + + for (i = 0; i < _clar.last_report->runs; i++) { + double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ? + _clar.last_report->times[i] - _clar.last_report->time_mean : + _clar.last_report->time_mean - _clar.last_report->times[i]; + + total_squares += (dev * dev); + } + + _clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs); + } } -#endif static void clar_run_test( const struct clar_suite *suite, const struct clar_func *test, const struct clar_func *initialize, + const struct clar_func *reset, const struct clar_func *cleanup) { - clar_time start, end; + int runs = test->runs, i = 0; - _clar.trampoline_enabled = 1; + _clar.last_report->start = time(NULL); + _clar.last_report->times = &_clar.last_report->time_mean; CL_TRACE(CL_TRACE__TEST__BEGIN); clar_sandbox_create(suite->name, test->name); - _clar.last_report->start = time(NULL); - clar_time_now(&start); + clar_print_test_start(suite->name, test->name, _clar.tests_ran); + + _clar.trampoline_enabled = 1; if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) initialize->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - test->ptr(); + + do { + struct clar_counter start, end; + double elapsed; + + if (i > 0 && reset->ptr != NULL) { + reset->ptr(); + } else if (i > 0) { + if (_clar.local_cleanup != NULL) + _clar.local_cleanup(_clar.local_cleanup_payload); + if (cleanup->ptr != NULL) + cleanup->ptr(); + if (initialize->ptr != NULL) + initialize->ptr(); + } + + clar_counter_now(&start); + test->ptr(); + clar_counter_now(&end); + + elapsed = clar_counter_diff(&start, &end); + + /* + * unless the number of runs was explicitly given + * in benchmark mode, use the first run as a sample + * to determine how many runs we should attempt + */ + if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) { + runs = MAX(CLAR_BENCHMARK_RUN_MIN, (CLAR_BENCHMARK_RUN_TIME / elapsed)); + runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs); + } + + if (i == 0 && runs > 1) { + _clar.last_report->times = calloc(runs, sizeof(double)); + + if (_clar.last_report->times == NULL) + clar_abort("Failed to allocate report times.\n"); + } + + _clar.last_report->runs++; + _clar.last_report->times[i] = elapsed; + } while(++i < runs); + CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_time_now(&end); - _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_time_diff(&start, &end); + compute_times(); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -354,7 +424,7 @@ clar_run_test( if (_clar.report_errors_only) { clar_report_errors(_clar.last_report); } else { - clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status); + clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report); } } @@ -373,10 +443,11 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) return; if (!_clar.report_errors_only) - clar_print_onsuite(suite->name, ++_clar.suites_ran); + clar_print_suite_start(suite->name, ++_clar.suites_ran); _clar.active_suite = suite->name; _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_BEGIN); if (filter) { @@ -405,11 +476,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) continue; _clar.active_test = test[i].name; + _clar.active_description = test[i].description; if ((report = calloc(1, sizeof(*report))) == NULL) clar_abort("Failed to allocate report.\n"); report->suite = _clar.active_suite; report->test = _clar.active_test; + report->description = _clar.active_description; report->test_number = _clar.tests_ran; report->status = CL_TEST_NOTRUN; @@ -421,13 +494,14 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.last_report = report; - clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup); + clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) return; } _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_END); } @@ -593,6 +667,14 @@ clar_test_init(int argc, char **argv) { const char *summary_env; + if (_clar.test_mode == CL_TEST_BENCHMARK) { + _clar.output_format = CL_OUTPUT_TIMING; + _clar.summary_format = CL_SUMMARY_JSON; + } else { + _clar.output_format = CL_OUTPUT_CLAP; + _clar.summary_format = CL_SUMMARY_JUNIT; + } + if (argc > 1) clar_parse_args(argc, argv); @@ -619,6 +701,12 @@ clar_test_init(int argc, char **argv) clar_tempdir_init(); } +void +clar_test_set_mode(enum cl_test_mode mode) +{ + _clar.test_mode = mode; +} + int clar_test_run(void) { @@ -668,6 +756,9 @@ clar_test_shutdown(void) free(error); } + if (report->times != &report->time_mean) + free(report->times); + report_next = report->next; free(report); } @@ -711,8 +802,8 @@ void clar__fail( const char *file, const char *function, size_t line, - const char *error_msg, - const char *description, + const char *error_message, + const char *error_description, int should_abort) { struct clar_error *error; @@ -731,10 +822,10 @@ void clar__fail( error->file = _clar.invoke_file ? _clar.invoke_file : file; error->function = _clar.invoke_func ? _clar.invoke_func : function; error->line_number = _clar.invoke_line ? _clar.invoke_line : line; - error->error_msg = error_msg; + error->message = error_message; - if (description != NULL && - (error->description = strdup(description)) == NULL) + if (error_description != NULL && + (error->description = strdup(error_description)) == NULL) clar_abort("Failed to allocate description.\n"); _clar.total_errors++; @@ -749,14 +840,14 @@ void clar__assert( const char *file, const char *function, size_t line, - const char *error_msg, - const char *description, + const char *error_message, + const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, description, should_abort); + clar__fail(file, function, line, error_message, error_description, should_abort); } void clar__assert_equal( diff --git a/clar.h b/clar.h index ca72292..b6e5069 100644 --- a/clar.h +++ b/clar.h @@ -18,6 +18,16 @@ # define CLAR_MAX_PATH PATH_MAX #endif +/* + * In benchmark mode, by default, clar will run the test repeatedly for + * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least + * `CLAR_BENCHMARK_RUN_MIN` iterations. + */ + +#define CLAR_BENCHMARK_RUN_TIME 3.0 +#define CLAR_BENCHMARK_RUN_MIN 10 +#define CLAR_BENCHMARK_RUN_MAX 30000000 + #ifndef CLAR_SELFTEST # define CLAR_CURRENT_FILE __FILE__ # define CLAR_CURRENT_LINE __LINE__ @@ -28,6 +38,11 @@ # define CLAR_CURRENT_FUNC "func" #endif +enum cl_test_mode { + CL_TEST_STANDARD, + CL_TEST_BENCHMARK, +}; + enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, @@ -38,10 +53,17 @@ enum cl_test_status { enum cl_output_format { CL_OUTPUT_CLAP, CL_OUTPUT_TAP, + CL_OUTPUT_TIMING, +}; + +enum cl_summary_format { + CL_SUMMARY_JUNIT, + CL_SUMMARY_JSON, }; /** Setup clar environment */ void clar_test_init(int argc, char *argv[]); +void clar_test_set_mode(enum cl_test_mode mode); int clar_test_run(void); void clar_test_shutdown(void); diff --git a/clar/counter.h b/clar/counter.h new file mode 100644 index 0000000..89b43d8 --- /dev/null +++ b/clar/counter.h @@ -0,0 +1,167 @@ +#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \ + if (start_usec > end_usec) { \ + out_sec = (end_sec - 1) - start_sec; \ + out_usec = (end_usec + 1000000) - start_usec; \ + } else { \ + out_sec = end_sec - start_sec; \ + out_usec = end_usec - start_usec; \ + } + +#ifdef _WIN32 + +struct clar_counter { + LARGE_INTEGER value; +}; + +void clar_counter_now(struct clar_counter *out) +{ + QueryPerformanceCounter(&out->value); +} + +double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + LARGE_INTEGER freq; + + QueryPerformanceFrequency(&freq); + return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart; +} + +#elif __APPLE__ + +#include +#include + +static double clar_counter_scaling_factor = -1; + +struct clar_counter { + union { + uint64_t absolute_time; + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + if (clar_counter_scaling_factor == 0) { + mach_timebase_info_data_t info; + + clar_counter_scaling_factor = + mach_timebase_info(&info) == KERN_SUCCESS ? + ((double)info.numer / (double)info.denom) / 1.0E6 : + -1; + } + + /* mach_timebase_info failed; fall back to gettimeofday */ + if (clar_counter_scaling_factor < 0) + gettimeofday(&out->val.tv, NULL); + else + out->val.absolute_time = mach_absolute_time(); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + if (clar_counter_scaling_factor < 0) { + time_t sec; + suseconds_t usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); + } else { + return (double)(end->val.absolute_time - start->val.absolute_time) * + clar_counter_scaling_factor; + } +} + +#elif defined(__amigaos4__) + +#include + +struct clar_counter { + struct TimeVal tv; +} + +static void clar_counter_now(struct clar_counter *out) +{ + ITimer->GetUpTime(&out->tv); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + uint32_t sec, usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->tv.Seconds, start->tv.Microseconds, + end->tv.Seconds, end->tv.Microseconds); + + return (double)sec + ((double)usec / 1000000.0); +} + +#else + +#include + +struct clar_counter { + int type; + union { +#ifdef CLOCK_MONOTONIC + struct timespec tp; +#endif + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ +#ifdef CLOCK_MONOTONIC + if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) { + out->type = 0; + return; + } +#endif + + /* Fall back to using gettimeofday */ + out->type = 1; + gettimeofday(&out->val.tv, NULL); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + time_t sec; + suseconds_t usec; + +#ifdef CLOCK_MONOTONIC + if (start->type == 0) { + time_t sec; + long nsec; + + if (start->val.tp.tv_sec > end->val.tp.tv_sec) { + sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec; + nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec; + } else { + sec = end->val.tp.tv_sec - start->val.tp.tv_sec; + nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec; + } + + return (double)sec + ((double)nsec / 1000000000.0); + } +#endif + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); +} + +#endif diff --git a/clar/print.h b/clar/print.h index e3e8ef8..311bd9c 100644 --- a/clar/print.h +++ b/clar/print.h @@ -43,7 +43,7 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con error->file, error->line_number); - clar_print_indented(error->error_msg, 2); + clar_print_indented(error->message, 2); if (error->description != NULL) clar_print_indented(error->description, 2); @@ -52,22 +52,31 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con fflush(stdout); } -static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number) { - (void)test_name; (void)test_number; if (_clar.verbosity > 1) { printf("%s::%s: ", suite_name, test_name); + fflush(stdout); + } +} - switch (status) { +static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + (void)suite_name; + (void)test_name; + (void)test_number; + + if (_clar.verbosity > 1) { + switch (report->status) { case CL_TEST_OK: printf("ok\n"); break; case CL_TEST_FAILURE: printf("fail\n"); break; case CL_TEST_SKIP: printf("skipped\n"); break; case CL_TEST_NOTRUN: printf("notrun\n"); break; } } else { - switch (status) { + switch (report->status) { case CL_TEST_OK: printf("."); break; case CL_TEST_FAILURE: printf("F"); break; case CL_TEST_SKIP: printf("S"); break; @@ -78,7 +87,7 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name } } -static void clar_print_clap_onsuite(const char *suite_name, int suite_index) +static void clar_print_clap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity == 1) printf("\n%s", suite_name); @@ -129,14 +138,21 @@ static void print_escaped(const char *str) printf("%s", str); } -static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)suite_name; + (void)test_name; + (void)test_number; +} + +static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { const struct clar_error *error = _clar.last_report->errors; (void)test_name; (void)test_number; - switch(status) { + switch(report->status) { case CL_TEST_OK: printf("ok %d - %s::%s\n", test_number, suite_name, test_name); break; @@ -145,7 +161,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, printf(" ---\n"); printf(" reason: |\n"); - clar_print_indented(error->error_msg, 6); + clar_print_indented(error->message, 6); if (error->description) clar_print_indented(error->description, 6); @@ -166,7 +182,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, fflush(stdout); } -static void clar_print_tap_onsuite(const char *suite_name, int suite_index) +static void clar_print_tap_suite_start(const char *suite_name, int suite_index) { printf("# start of suite %d: %s\n", suite_index, suite_name); } @@ -178,6 +194,115 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) fflush(stdout); } +/* timings format: useful for benchmarks */ + +static void clar_print_timing_init(int test_count, int suite_count, const char *suite_names) +{ + (void)test_count; + (void)suite_count; + (void)suite_names; + + printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n"); +} + +static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count) +{ + (void)test_count; + (void)suite_count; + (void)error_count; +} + +static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error) +{ + (void)num; + (void)report; + (void)error; +} + +static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)test_number; + + printf("%s::%s: ", suite_name, test_name); + fflush(stdout); +} + +static void clar_print_timing_time(double t) +{ + static const char *units[] = { "sec", "ms", "μs", "ns" }; + static const int units_len = sizeof(units) / sizeof(units[0]); + int unit = 0, exponent = 0, digits; + + while (t < 1.0 && unit < units_len - 1) { + t *= 1000.0; + unit++; + } + + while (t > 0.0 && t < 1.0 && exponent < 10) { + t *= 10.0; + exponent++; + } + + digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1); + + printf("%.*f", digits, t); + + if (exponent > 0) + printf("e-%d", exponent); + + printf(" %s", units[unit]); +} + +static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + const struct clar_error *error = _clar.last_report->errors; + + (void)suite_name; + (void)test_name; + (void)test_number; + + switch(report->status) { + case CL_TEST_OK: + clar_print_timing_time(report->time_mean); + + if (report->runs > 1) { + printf(" ± "); + clar_print_timing_time(report->time_stddev); + + printf(" / range: "); + clar_print_timing_time(report->time_min); + printf(" … "); + clar_print_timing_time(report->time_max); + printf(" (%d runs)", report->runs); + } + + printf("\n"); + break; + case CL_TEST_FAILURE: + printf("failed: %s\n", error->message); + break; + case CL_TEST_SKIP: + case CL_TEST_NOTRUN: + printf("skipped\n"); + break; + } + + fflush(stdout); +} + +static void clar_print_timing_suite_start(const char *suite_name, int suite_index) +{ + if (_clar.verbosity == 1) + printf("\n%s", suite_name); + + (void)suite_index; +} + +static void clar_print_timing_onabort(const char *fmt, va_list arg) +{ + vfprintf(stderr, fmt, arg); +} + /* indirection between protocol output selection */ #define PRINT(FN, ...) do { \ @@ -188,6 +313,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) case CL_OUTPUT_TAP: \ clar_print_tap_##FN (__VA_ARGS__); \ break; \ + case CL_OUTPUT_TIMING: \ + clar_print_timing_##FN (__VA_ARGS__); \ + break; \ default: \ abort(); \ } \ @@ -208,14 +336,19 @@ static void clar_print_error(int num, const struct clar_report *report, const st PRINT(error, num, report, error); } -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number) +{ + PRINT(test_start, suite_name, test_name, test_number); +} + +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { - PRINT(ontest, suite_name, test_name, test_number, status); + PRINT(test_finish, suite_name, test_name, test_number, report); } -static void clar_print_onsuite(const char *suite_name, int suite_index) +static void clar_print_suite_start(const char *suite_name, int suite_index) { - PRINT(onsuite, suite_name, suite_index); + PRINT(suite_start, suite_name, suite_index); } static void clar_print_onabortv(const char *msg, va_list argp) diff --git a/clar/summary.h b/clar/summary.h index 0d0b646..d3f5657 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -1,8 +1,24 @@ - #include #include -static int clar_summary_close_tag( +static int clar_summary_time_digits(double t) +{ + int digits = 3; + + if (t >= 100.0) + return 1; + else if (t >= 10.0) + return 2; + + while (t > 0.0 && t < 1.0 && digits < 10) { + t *= 10.0; + digits++; + } + + return digits; +} + +static int clar_summary_junit_close_tag( struct clar_summary *summary, const char *tag, int indent) { const char *indt; @@ -14,12 +30,12 @@ static int clar_summary_close_tag( return fprintf(summary->fp, "%s\n", indt, tag); } -static int clar_summary_testsuites(struct clar_summary *summary) +static int clar_summary_junit_testsuites(struct clar_summary *summary) { return fprintf(summary->fp, "\n"); } -static int clar_summary_testsuite(struct clar_summary *summary, +static int clar_summary_junit_testsuite(struct clar_summary *summary, int idn, const char *name, time_t timestamp, int test_count, int fail_count, int error_count) { @@ -40,15 +56,15 @@ static int clar_summary_testsuite(struct clar_summary *summary, idn, name, iso_dt, test_count, fail_count, error_count); } -static int clar_summary_testcase(struct clar_summary *summary, +static int clar_summary_junit_testcase(struct clar_summary *summary, const char *name, const char *classname, double elapsed) { return fprintf(summary->fp, - "\t\t\n", - name, classname, elapsed); + "\t\t\n", + name, classname, clar_summary_time_digits(elapsed), elapsed); } -static int clar_summary_failure(struct clar_summary *summary, +static int clar_summary_junit_failure(struct clar_summary *summary, const char *type, const char *message, const char *desc) { return fprintf(summary->fp, @@ -56,22 +72,26 @@ static int clar_summary_failure(struct clar_summary *summary, type, message, desc); } -static int clar_summary_skipped(struct clar_summary *summary) +static int clar_summary_junit_skipped(struct clar_summary *summary) { return fprintf(summary->fp, "\t\t\t\n"); } -struct clar_summary *clar_summary_init(const char *filename) +struct clar_summary *clar_summary_junit_init(const char *filename) { struct clar_summary *summary; FILE *fp; - if ((fp = fopen(filename, "w")) == NULL) - clar_abort("Failed to open the summary file '%s': %s.\n", - filename, strerror(errno)); + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } - if ((summary = malloc(sizeof(struct clar_summary))) == NULL) - clar_abort("Failed to allocate summary.\n"); + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } summary->filename = filename; summary->fp = fp; @@ -79,12 +99,12 @@ struct clar_summary *clar_summary_init(const char *filename) return summary; } -int clar_summary_shutdown(struct clar_summary *summary) +int clar_summary_junit_shutdown(struct clar_summary *summary) { struct clar_report *report; const char *last_suite = NULL; - if (clar_summary_testsuites(summary) < 0) + if (clar_summary_junit_testsuites(summary) < 0) goto on_error; report = _clar.reports; @@ -92,38 +112,38 @@ int clar_summary_shutdown(struct clar_summary *summary) struct clar_error *error = report->errors; if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_testsuite(summary, 0, report->suite, + if (clar_summary_junit_testsuite(summary, 0, report->suite, report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) goto on_error; } last_suite = report->suite; - clar_summary_testcase(summary, report->test, report->suite, report->elapsed); + clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total); while (error != NULL) { - if (clar_summary_failure(summary, "assert", - error->error_msg, error->description) < 0) + if (clar_summary_junit_failure(summary, "assert", + error->message, error->description) < 0) goto on_error; error = error->next; } if (report->status == CL_TEST_SKIP) - clar_summary_skipped(summary); + clar_summary_junit_skipped(summary); - if (clar_summary_close_tag(summary, "testcase", 2) < 0) + if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0) goto on_error; report = report->next; if (!report || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_close_tag(summary, "testsuite", 1) < 0) + if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0) goto on_error; } } - if (clar_summary_close_tag(summary, "testsuites", 0) < 0 || + if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 || fclose(summary->fp) != 0) goto on_error; @@ -137,3 +157,154 @@ int clar_summary_shutdown(struct clar_summary *summary) free(summary); return -1; } + +struct clar_summary *clar_summary_json_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +int clar_summary_json_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + int i; + + fprintf(summary->fp, "{\n"); + fprintf(summary->fp, " \"tests\": [\n"); + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (report != _clar.reports) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"name\": \"%s::%s\",\n", report->suite, report->test); + + if (report->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", report->description); + + fprintf(summary->fp, " \"results\": {\n"); + + fprintf(summary->fp, " \"status\": "); + if (report->status == CL_TEST_OK) + fprintf(summary->fp, "\"ok\",\n"); + else if (report->status == CL_TEST_FAILURE) + fprintf(summary->fp, "\"failed\",\n"); + else if (report->status == CL_TEST_SKIP) + fprintf(summary->fp, "\"skipped\"\n"); + else + clar_abort("unknown test status %d", report->status); + + if (report->status == CL_TEST_OK) { + fprintf(summary->fp, " \"mean\": %.*f,\n", + clar_summary_time_digits(report->time_mean), report->time_mean); + fprintf(summary->fp, " \"stddev\": %.*f,\n", + clar_summary_time_digits(report->time_stddev), report->time_stddev); + fprintf(summary->fp, " \"min\": %.*f,\n", + clar_summary_time_digits(report->time_min), report->time_min); + fprintf(summary->fp, " \"max\": %.*f,\n", + clar_summary_time_digits(report->time_max), report->time_max); + fprintf(summary->fp, " \"times\": [\n"); + + for (i = 0; i < report->runs; i++) { + if (i > 0) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " %.*f", + clar_summary_time_digits(report->times[i]), report->times[i]); + } + + fprintf(summary->fp, "\n ]\n"); + } + + if (report->status == CL_TEST_FAILURE) { + fprintf(summary->fp, " \"errors\": [\n"); + + while (error != NULL) { + if (error != report->errors) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->message); + + if (error->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); + + fprintf(summary->fp, " \"function\": \"%s\",\n", error->function); + fprintf(summary->fp, " \"file\": \"%s\",\n", error->file); + fprintf(summary->fp, " \"line\": %" PRIuMAX "\n", error->line_number); + fprintf(summary->fp, " }"); + + error = error->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + } + + fprintf(summary->fp, " }\n"); + fprintf(summary->fp, " }"); + + report = report->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + fprintf(summary->fp, "}\n"); + + if (fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} + +/* indirection between protocol output selection */ + +#define SUMMARY(FN, ...) do { \ + switch (_clar.summary_format) { \ + case CL_SUMMARY_JUNIT: \ + return clar_summary_junit_##FN (__VA_ARGS__); \ + break; \ + case CL_SUMMARY_JSON: \ + return clar_summary_json_##FN (__VA_ARGS__); \ + break; \ + default: \ + abort(); \ + } \ + } while(0) + +struct clar_summary *clar_summary_init(const char *filename) +{ + SUMMARY(init, filename); +} + +int clar_summary_shutdown(struct clar_summary *summary) +{ + SUMMARY(shutdown, summary); +} diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b72f187..4f81df3 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -25,4 +25,4 @@ target_include_directories(example PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(example clar) +target_link_libraries(example clar ${CLAR_LIBRARIES}) diff --git a/generate.py b/generate.py index 80996ac..7843beb 100755 --- a/generate.py +++ b/generate.py @@ -17,8 +17,13 @@ def __init__(self, module): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL }' - return ' { "%s", &%s }' % (cb['short_name'], cb['symbol']) + return ' { NULL, NULL, 0, NULL }' + + return ' { "%s", %s, %d, &%s }' % \ + (cb['short_name'], \ + '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['runs'], \ + cb['symbol']) class DeclarationTemplate(Template): def render(self): @@ -27,6 +32,9 @@ def render(self): for initializer in self.module.initializers: out += "extern %s;\n" % initializer['declaration'] + if self.module.reset: + out += "extern %s;\n" % self.module.reset['declaration'] + if self.module.cleanup: out += "extern %s;\n" % self.module.cleanup['declaration'] @@ -34,7 +42,7 @@ def render(self): class CallbacksTemplate(Template): def render(self): - out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name + out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name) out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks) out += "\n};\n" return out @@ -58,14 +66,16 @@ def render(self): { "${clean_name}", ${initialize}, + ${reset}, ${cleanup}, ${cb_ptr}, ${cb_count}, ${enabled} }""" ).substitute( clean_name = name, initialize = self._render_callback(initializer), + reset = self._render_callback(self.module.reset), cleanup = self._render_callback(self.module.cleanup), - cb_ptr = "_clar_cb_%s" % self.module.name, + cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), enabled = int(self.module.enabled) ) @@ -73,8 +83,10 @@ def render(self): return ','.join(templates) - def __init__(self, name): + def __init__(self, name, app_name, prefix): self.name = name + self.app_name = app_name + self.prefix = prefix self.mtime = None self.enabled = True @@ -85,7 +97,7 @@ def clean_name(self): def _skip_comments(self, text): SKIP_COMMENTS_REGEX = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) def _replacer(match): @@ -95,24 +107,63 @@ def _replacer(match): return re.sub(SKIP_COMMENTS_REGEX, _replacer, text) def parse(self, contents): - TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{" + TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{" contents = self._skip_comments(contents) - regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE) + regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE) self.callbacks = [] self.initializers = [] + self.reset = None self.cleanup = None - for (declaration, symbol, short_name) in regex.findall(contents): + for (declaration, symbol, short_name, options) in regex.findall(contents): + runs = 0 + description = None + + while options != '': + match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options) + + if match == None: + print("Invalid options: '%s' for '%s'" % (options, symbol)) + sys.exit(1) + + key = match.group(1) + value = match.group(2) + options = match.group(3) + + match = re.search(r'^\"(.*)\"$', value) + if match != None: + value = match.group(1) + + match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value) + if match != None: + print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol)) + sys.exit(1) + + if key == "description": + description = value + elif key == "runs": + if not value.isnumeric(): + print("Invalid option: '%s' in runs for '%s'" % (option, symbol)) + sys.exit(1) + runs = int(value) + else: + print("Invalid option: '%s' for '%s'" % (key, symbol)) + sys.exit(1) + data = { "short_name" : short_name, "declaration" : declaration, - "symbol" : symbol + "symbol" : symbol, + "description" : description, + "runs" : runs } if short_name.startswith('initialize'): self.initializers.append(data) + elif short_name == 'reset': + self.reset = data elif short_name == 'cleanup': self.cleanup = data else: @@ -172,8 +223,8 @@ def find_modules(self): return modules - def load_cache(self): - path = os.path.join(self.output, '.clarcache') + def load_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) cache = {} try: @@ -185,18 +236,18 @@ def load_cache(self): return cache - def save_cache(self): - path = os.path.join(self.output, '.clarcache') + def save_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) with open(path, 'wb') as cache: pickle.dump(self.modules, cache) - def load(self, force = False): + def load(self, app_name, prefix, force = False): module_data = self.find_modules() - self.modules = {} if force else self.load_cache() + self.modules = {} if force else self.load_cache(app_name) for path, name in module_data: if name not in self.modules: - self.modules[name] = Module(name) + self.modules[name] = Module(name, app_name, prefix) if not self.modules[name].refresh(path): del self.modules[name] @@ -215,8 +266,8 @@ def suite_count(self): def callback_count(self): return sum(len(module.callbacks) for module in self.modules.values()) - def write(self): - output = os.path.join(self.output, 'clar.suite') + def write(self, name): + output = os.path.join(self.output, "%s.suite" % name) if not self.should_generate(output): return False @@ -232,16 +283,17 @@ def write(self): t = Module.CallbacksTemplate(module) data.write(t.render()) - suites = "static struct clar_suite _clar_suites[] = {" + ','.join( + suites = "static struct %s_suite _%s_suites[] = {" % (name, name) + suites += ','.join( Module.InfoTemplate(module).render() for module in modules ) + "\n};\n" data.write(suites) - data.write("static const size_t _clar_suite_count = %d;\n" % self.suite_count()) - data.write("static const size_t _clar_callback_count = %d;\n" % self.callback_count()) + data.write("static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count())) + data.write("static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count())) - self.save_cache() + self.save_cache(name) return True if __name__ == '__main__': @@ -251,6 +303,8 @@ def write(self): parser.add_option('-f', '--force', action="store_true", dest='force', default=False) parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[]) parser.add_option('-o', '--output', dest='output') + parser.add_option('-n', '--name', dest='name', default='clar') + parser.add_option('-p', '--prefix', dest='prefix', default='test') options, args = parser.parse_args() if len(args) > 1: @@ -260,7 +314,7 @@ def write(self): path = args.pop() if args else '.' output = options.output or path suite = TestSuite(path, output) - suite.load(options.force) + suite.load(options.name, options.prefix, options.force) suite.disable(options.excluded) - if suite.write(): - print("Written `clar.suite` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count())) + if suite.write(options.name): + print("Written `%s.suite` (%d tests in %d suites)" % (options.name, suite.callback_count(), suite.suite_count())) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 96abd6e..f0b5a32 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,7 +38,7 @@ target_include_directories(selftest PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(selftest clar) +target_link_libraries(selftest clar ${CLAR_LIBRARIES}) add_test(NAME build_selftest_suite COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --config "$" --target selftest_suite diff --git a/test/selftest_suite/CMakeLists.txt b/test/selftest_suite/CMakeLists.txt index 9597d67..d96f18c 100644 --- a/test/selftest_suite/CMakeLists.txt +++ b/test/selftest_suite/CMakeLists.txt @@ -37,4 +37,4 @@ target_include_directories(selftest_suite PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(selftest_suite clar) +target_link_libraries(selftest_suite clar ${CLAR_LIBRARIES})