From c23b5f36c02d64964f5eecbc0446cfb23a412cd9 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 17 Jan 2025 12:53:04 +0000 Subject: [PATCH 01/10] allow customization for app and test prefixes Allow the resulting application names to be configured by the user, instead of hardcoding `clar_suite.h` and `clar.suite`. This configuration will also customize the struct names (`clar_func`, etc.) Also allow the test names to be configured by the user, instead of hardcoding `test_` as a prefix. This allows users to generate test functions with uniquely prefixed names, for example, if they were generating benchmark code instead of unit tests. --- generate.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/generate.py b/generate.py index 80996ac..e135f13 100755 --- a/generate.py +++ b/generate.py @@ -34,7 +34,7 @@ def render(self): class CallbacksTemplate(Template): def render(self): - out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name + out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name) out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks) out += "\n};\n" return out @@ -65,7 +65,7 @@ def render(self): clean_name = name, initialize = self._render_callback(initializer), cleanup = self._render_callback(self.module.cleanup), - cb_ptr = "_clar_cb_%s" % self.module.name, + cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), enabled = int(self.module.enabled) ) @@ -73,8 +73,10 @@ def render(self): return ','.join(templates) - def __init__(self, name): + def __init__(self, name, app_name, prefix): self.name = name + self.app_name = app_name + self.prefix = prefix self.mtime = None self.enabled = True @@ -172,8 +174,8 @@ def find_modules(self): return modules - def load_cache(self): - path = os.path.join(self.output, '.clarcache') + def load_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) cache = {} try: @@ -185,18 +187,18 @@ def load_cache(self): return cache - def save_cache(self): - path = os.path.join(self.output, '.clarcache') + def save_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) with open(path, 'wb') as cache: pickle.dump(self.modules, cache) - def load(self, force = False): + def load(self, app_name, prefix, force = False): module_data = self.find_modules() - self.modules = {} if force else self.load_cache() + self.modules = {} if force else self.load_cache(app_name) for path, name in module_data: if name not in self.modules: - self.modules[name] = Module(name) + self.modules[name] = Module(name, app_name, prefix) if not self.modules[name].refresh(path): del self.modules[name] @@ -215,8 +217,8 @@ def suite_count(self): def callback_count(self): return sum(len(module.callbacks) for module in self.modules.values()) - def write(self): - output = os.path.join(self.output, 'clar.suite') + def write(self, name): + output = os.path.join(self.output, "%s.suite" % name) if not self.should_generate(output): return False @@ -232,16 +234,17 @@ def write(self): t = Module.CallbacksTemplate(module) data.write(t.render()) - suites = "static struct clar_suite _clar_suites[] = {" + ','.join( + suites = "static struct %s_suite _%s_suites[] = {" % (name, name) + suites += ','.join( Module.InfoTemplate(module).render() for module in modules ) + "\n};\n" data.write(suites) - data.write("static const size_t _clar_suite_count = %d;\n" % self.suite_count()) - data.write("static const size_t _clar_callback_count = %d;\n" % self.callback_count()) + data.write("static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count())) + data.write("static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count())) - self.save_cache() + self.save_cache(name) return True if __name__ == '__main__': @@ -251,6 +254,8 @@ def write(self): parser.add_option('-f', '--force', action="store_true", dest='force', default=False) parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[]) parser.add_option('-o', '--output', dest='output') + parser.add_option('-n', '--name', dest='name', default='clar') + parser.add_option('-p', '--prefix', dest='prefix', default='test') options, args = parser.parse_args() if len(args) > 1: @@ -260,7 +265,7 @@ def write(self): path = args.pop() if args else '.' output = options.output or path suite = TestSuite(path, output) - suite.load(options.force) + suite.load(options.name, options.prefix, options.force) suite.disable(options.excluded) - if suite.write(): - print("Written `clar.suite` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count())) + if suite.write(options.name): + print("Written `%s.suite` (%d tests in %d suites)" % (options.name, suite.callback_count(), suite.suite_count())) From c328be26a5b932a048d5cbb8ce66c9831f6b1571 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 17 Jan 2025 13:15:07 +0000 Subject: [PATCH 02/10] tests have optional metadata (eg descriptions) Tests can now have optional metadata, provided as comments in the test definition. For example: ``` void test_spline__reticulation(void) /*[clar]: description="ensure that splines are reticulated" */ { ... } ``` This description is preserved and produced as part of the summary XML. --- clar.c | 21 ++++++++++++++------- clar/summary.h | 19 +++++++++++++++---- generate.py | 47 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/clar.c b/clar.c index bf3d7f0..d6e4c63 100644 --- a/clar.c +++ b/clar.c @@ -117,9 +117,10 @@ struct clar_explicit { }; struct clar_report { + const char *suite; const char *test; + const char *description; int test_number; - const char *suite; enum cl_test_status status; time_t start; @@ -139,8 +140,9 @@ struct clar_summary { static struct { enum cl_test_status test_status; - const char *active_test; const char *active_suite; + const char *active_test; + const char *active_description; int total_skipped; int total_errors; @@ -181,6 +183,7 @@ static struct { struct clar_func { const char *name; + const char *description; void (*ptr)(void); }; @@ -377,6 +380,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.active_suite = suite->name; _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_BEGIN); if (filter) { @@ -405,11 +409,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) continue; _clar.active_test = test[i].name; + _clar.active_description = test[i].description; if ((report = calloc(1, sizeof(*report))) == NULL) clar_abort("Failed to allocate report.\n"); report->suite = _clar.active_suite; report->test = _clar.active_test; + report->description = _clar.active_description; report->test_number = _clar.tests_ran; report->status = CL_TEST_NOTRUN; @@ -428,6 +434,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) } _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_END); } @@ -712,7 +719,7 @@ void clar__fail( const char *function, size_t line, const char *error_msg, - const char *description, + const char *error_description, int should_abort) { struct clar_error *error; @@ -733,8 +740,8 @@ void clar__fail( error->line_number = _clar.invoke_line ? _clar.invoke_line : line; error->error_msg = error_msg; - if (description != NULL && - (error->description = strdup(description)) == NULL) + if (error_description != NULL && + (error->description = strdup(error_description)) == NULL) clar_abort("Failed to allocate description.\n"); _clar.total_errors++; @@ -750,13 +757,13 @@ void clar__assert( const char *function, size_t line, const char *error_msg, - const char *description, + const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, description, should_abort); + clar__fail(file, function, line, error_msg, error_description, should_abort); } void clar__assert_equal( diff --git a/clar/summary.h b/clar/summary.h index 0d0b646..3ba0982 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -41,11 +41,22 @@ static int clar_summary_testsuite(struct clar_summary *summary, } static int clar_summary_testcase(struct clar_summary *summary, - const char *name, const char *classname, double elapsed) + const struct clar_report *report) { - return fprintf(summary->fp, + if (fprintf(summary->fp, "\t\t\n", - name, classname, elapsed); + report->test, report->suite, report->elapsed) < 0) + return -1; + + if (report->description) { + fprintf(summary->fp, "\t\t\t\n"); + fprintf(summary->fp, "\t\t\t\t\n"); + fprintf(summary->fp, "\t\t\t\t\t%s\n", report->description); + fprintf(summary->fp, "\t\t\t\t\n"); + fprintf(summary->fp, "\t\t\t\n"); + } + + return 0; } static int clar_summary_failure(struct clar_summary *summary, @@ -99,7 +110,7 @@ int clar_summary_shutdown(struct clar_summary *summary) last_suite = report->suite; - clar_summary_testcase(summary, report->test, report->suite, report->elapsed); + clar_summary_testcase(summary, report); while (error != NULL) { if (clar_summary_failure(summary, "assert", diff --git a/generate.py b/generate.py index e135f13..321acf7 100755 --- a/generate.py +++ b/generate.py @@ -17,8 +17,12 @@ def __init__(self, module): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL }' - return ' { "%s", &%s }' % (cb['short_name'], cb['symbol']) + return ' { NULL, NULL, NULL }' + + return ' { "%s", %s, &%s }' % \ + (cb['short_name'], \ + '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['symbol']) class DeclarationTemplate(Template): def render(self): @@ -87,7 +91,7 @@ def clean_name(self): def _skip_comments(self, text): SKIP_COMMENTS_REGEX = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) def _replacer(match): @@ -97,20 +101,49 @@ def _replacer(match): return re.sub(SKIP_COMMENTS_REGEX, _replacer, text) def parse(self, contents): - TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{" + TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{" contents = self._skip_comments(contents) - regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE) + regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE) self.callbacks = [] self.initializers = [] self.cleanup = None - for (declaration, symbol, short_name) in regex.findall(contents): + for (declaration, symbol, short_name, options) in regex.findall(contents): + description = None + + while options != '': + match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options) + + if match == None: + print("Invalid options: '%s' for '%s'" % (options, symbol)) + sys.exit(1) + + key = match.group(1) + value = match.group(2) + options = match.group(3) + + match = re.search(r'^\"(.*)\"$', value) + if match != None: + value = match.group(1) + + match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value) + if match != None: + print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol)) + sys.exit(1) + + if key == "description": + description = value + else: + print("Invalid option: '%s' for '%s'" % (key, symbol)) + sys.exit(1) + data = { "short_name" : short_name, "declaration" : declaration, - "symbol" : symbol + "symbol" : symbol, + "description" : description } if short_name.startswith('initialize'): From affe43764fd89416ad5fb07c88a54ad3cbd86cc3 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 17 Jan 2025 23:57:27 +0000 Subject: [PATCH 03/10] use monotonic performance counter for elapsed time Move the elapsed time calculation to `counter.h`, and use high-resolution monotonic performance counters on all platforms. --- clar.c | 44 +++---------- clar/counter.h | 167 +++++++++++++++++++++++++++++++++++++++++++++++++ clar/summary.h | 2 +- 3 files changed, 177 insertions(+), 36 deletions(-) create mode 100644 clar/counter.h diff --git a/clar.c b/clar.c index d6e4c63..3a4191a 100644 --- a/clar.c +++ b/clar.c @@ -196,7 +196,7 @@ struct clar_suite { int enabled; }; -/* From clar_print_*.c */ +/* From print.h */ static void clar_print_init(int test_count, int suite_count, const char *suite_names); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); @@ -205,7 +205,7 @@ static void clar_print_onsuite(const char *suite_name, int suite_index); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); -/* From clar_sandbox.c */ +/* From sandbox.c */ static void clar_tempdir_init(void); static void clar_tempdir_shutdown(void); static int clar_sandbox_create(const char *suite_name, const char *test_name); @@ -215,6 +215,8 @@ static int clar_sandbox_cleanup(void); static struct clar_summary *clar_summary_init(const char *filename); static int clar_summary_shutdown(struct clar_summary *fp); +#include "clar/counter.h" + /* Load the declarations for the test suite */ #include "clar.suite" @@ -271,35 +273,6 @@ clar_report_all(void) } } -#ifdef WIN32 -# define clar_time DWORD - -static void clar_time_now(clar_time *out) -{ - *out = GetTickCount(); -} - -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)*end - (double)*start) / 1000; -} -#else -# include - -# define clar_time struct timeval - -static void clar_time_now(clar_time *out) -{ - gettimeofday(out, NULL); -} - -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) - - ((double)start->tv_sec + (double)start->tv_usec / 1.0E6); -} -#endif - static void clar_run_test( const struct clar_suite *suite, @@ -307,16 +280,17 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - clar_time start, end; + struct clar_counter start, end; _clar.trampoline_enabled = 1; + _clar.last_report->start = time(NULL); CL_TRACE(CL_TRACE__TEST__BEGIN); clar_sandbox_create(suite->name, test->name); _clar.last_report->start = time(NULL); - clar_time_now(&start); + clar_counter_now(&start); if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) @@ -327,14 +301,14 @@ clar_run_test( CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_time_now(&end); + clar_counter_now(&end); _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_time_diff(&start, &end); + _clar.last_report->elapsed = clar_counter_diff(&start, &end); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); diff --git a/clar/counter.h b/clar/counter.h new file mode 100644 index 0000000..89b43d8 --- /dev/null +++ b/clar/counter.h @@ -0,0 +1,167 @@ +#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \ + if (start_usec > end_usec) { \ + out_sec = (end_sec - 1) - start_sec; \ + out_usec = (end_usec + 1000000) - start_usec; \ + } else { \ + out_sec = end_sec - start_sec; \ + out_usec = end_usec - start_usec; \ + } + +#ifdef _WIN32 + +struct clar_counter { + LARGE_INTEGER value; +}; + +void clar_counter_now(struct clar_counter *out) +{ + QueryPerformanceCounter(&out->value); +} + +double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + LARGE_INTEGER freq; + + QueryPerformanceFrequency(&freq); + return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart; +} + +#elif __APPLE__ + +#include +#include + +static double clar_counter_scaling_factor = -1; + +struct clar_counter { + union { + uint64_t absolute_time; + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + if (clar_counter_scaling_factor == 0) { + mach_timebase_info_data_t info; + + clar_counter_scaling_factor = + mach_timebase_info(&info) == KERN_SUCCESS ? + ((double)info.numer / (double)info.denom) / 1.0E6 : + -1; + } + + /* mach_timebase_info failed; fall back to gettimeofday */ + if (clar_counter_scaling_factor < 0) + gettimeofday(&out->val.tv, NULL); + else + out->val.absolute_time = mach_absolute_time(); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + if (clar_counter_scaling_factor < 0) { + time_t sec; + suseconds_t usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); + } else { + return (double)(end->val.absolute_time - start->val.absolute_time) * + clar_counter_scaling_factor; + } +} + +#elif defined(__amigaos4__) + +#include + +struct clar_counter { + struct TimeVal tv; +} + +static void clar_counter_now(struct clar_counter *out) +{ + ITimer->GetUpTime(&out->tv); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + uint32_t sec, usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->tv.Seconds, start->tv.Microseconds, + end->tv.Seconds, end->tv.Microseconds); + + return (double)sec + ((double)usec / 1000000.0); +} + +#else + +#include + +struct clar_counter { + int type; + union { +#ifdef CLOCK_MONOTONIC + struct timespec tp; +#endif + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ +#ifdef CLOCK_MONOTONIC + if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) { + out->type = 0; + return; + } +#endif + + /* Fall back to using gettimeofday */ + out->type = 1; + gettimeofday(&out->val.tv, NULL); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + time_t sec; + suseconds_t usec; + +#ifdef CLOCK_MONOTONIC + if (start->type == 0) { + time_t sec; + long nsec; + + if (start->val.tp.tv_sec > end->val.tp.tv_sec) { + sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec; + nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec; + } else { + sec = end->val.tp.tv_sec - start->val.tp.tv_sec; + nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec; + } + + return (double)sec + ((double)nsec / 1000000000.0); + } +#endif + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); +} + +#endif diff --git a/clar/summary.h b/clar/summary.h index 3ba0982..b069430 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -44,7 +44,7 @@ static int clar_summary_testcase(struct clar_summary *summary, const struct clar_report *report) { if (fprintf(summary->fp, - "\t\t\n", + "\t\t\n", report->test, report->suite, report->elapsed) < 0) return -1; From cffcc7a605cc60b74458c6d34afc934958456644 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 12:16:25 +0000 Subject: [PATCH 04/10] add test_start print callback Refactor the `ontest` callback (which is implicitly test _finished_) into a test started and test finished callback. This allows printers to show the test name (at start) and its conclusion in two steps, which is advantageous for users to see the current test during long-running test executions. In addition, rename `onsuite` to `suite_start` for consistency. --- clar.c | 11 ++++++----- clar/print.h | 45 +++++++++++++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/clar.c b/clar.c index 3a4191a..f6a3c86 100644 --- a/clar.c +++ b/clar.c @@ -200,8 +200,9 @@ struct clar_suite { static void clar_print_init(int test_count, int suite_count, const char *suite_names); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed); -static void clar_print_onsuite(const char *suite_name, int suite_index); +static void clar_print_suite_start(const char *suite_name, int suite_index); +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number); +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); @@ -289,7 +290,7 @@ clar_run_test( clar_sandbox_create(suite->name, test->name); - _clar.last_report->start = time(NULL); + clar_print_test_start(suite->name, test->name, _clar.tests_ran); clar_counter_now(&start); if (setjmp(_clar.trampoline) == 0) { @@ -331,7 +332,7 @@ clar_run_test( if (_clar.report_errors_only) { clar_report_errors(_clar.last_report); } else { - clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status); + clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report); } } @@ -350,7 +351,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) return; if (!_clar.report_errors_only) - clar_print_onsuite(suite->name, ++_clar.suites_ran); + clar_print_suite_start(suite->name, ++_clar.suites_ran); _clar.active_suite = suite->name; _clar.active_test = NULL; diff --git a/clar/print.h b/clar/print.h index e3e8ef8..f139f2d 100644 --- a/clar/print.h +++ b/clar/print.h @@ -52,22 +52,31 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con fflush(stdout); } -static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number) { - (void)test_name; (void)test_number; if (_clar.verbosity > 1) { printf("%s::%s: ", suite_name, test_name); + fflush(stdout); + } +} - switch (status) { +static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + (void)suite_name; + (void)test_name; + (void)test_number; + + if (_clar.verbosity > 1) { + switch (report->status) { case CL_TEST_OK: printf("ok\n"); break; case CL_TEST_FAILURE: printf("fail\n"); break; case CL_TEST_SKIP: printf("skipped\n"); break; case CL_TEST_NOTRUN: printf("notrun\n"); break; } } else { - switch (status) { + switch (report->status) { case CL_TEST_OK: printf("."); break; case CL_TEST_FAILURE: printf("F"); break; case CL_TEST_SKIP: printf("S"); break; @@ -78,7 +87,7 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name } } -static void clar_print_clap_onsuite(const char *suite_name, int suite_index) +static void clar_print_clap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity == 1) printf("\n%s", suite_name); @@ -129,14 +138,21 @@ static void print_escaped(const char *str) printf("%s", str); } -static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)suite_name; + (void)test_name; + (void)test_number; +} + +static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { const struct clar_error *error = _clar.last_report->errors; (void)test_name; (void)test_number; - switch(status) { + switch(report->status) { case CL_TEST_OK: printf("ok %d - %s::%s\n", test_number, suite_name, test_name); break; @@ -166,7 +182,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, fflush(stdout); } -static void clar_print_tap_onsuite(const char *suite_name, int suite_index) +static void clar_print_tap_suite_start(const char *suite_name, int suite_index) { printf("# start of suite %d: %s\n", suite_index, suite_name); } @@ -208,14 +224,19 @@ static void clar_print_error(int num, const struct clar_report *report, const st PRINT(error, num, report, error); } -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number) +{ + PRINT(test_start, suite_name, test_name, test_number); +} + +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { - PRINT(ontest, suite_name, test_name, test_number, status); + PRINT(test_finish, suite_name, test_name, test_number, report); } -static void clar_print_onsuite(const char *suite_name, int suite_index) +static void clar_print_suite_start(const char *suite_name, int suite_index) { - PRINT(onsuite, suite_name, suite_index); + PRINT(suite_start, suite_name, suite_index); } static void clar_print_onabortv(const char *msg, va_list argp) From d75edb6b9eb0c8224099bb7e7a711e2b872e8a36 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 12:50:42 +0000 Subject: [PATCH 05/10] multiple runs Allow tests to specify that they should have multiple runs. These runs all occur within a single initialization and cleanup phase, and is useful for repeatedly testing the same thing as quickly as possible. The time for each run is recorded, which may be useful for benchmarking that test run. --- CMakeLists.txt | 4 ++ clar.c | 83 +++++++++++++++++++++++++++--- clar/summary.h | 2 +- example/CMakeLists.txt | 2 +- generate.py | 14 +++-- test/CMakeLists.txt | 2 +- test/selftest_suite/CMakeLists.txt | 2 +- 7 files changed, 94 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 125db05..d1f28b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,10 @@ set_target_properties(clar PROPERTIES C_EXTENSIONS OFF ) +if(NOT WIN32) + set(CLAR_LIBRARIES m) +endif() + if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) include(CTest) if(BUILD_TESTING) diff --git a/clar.c b/clar.c index f6a3c86..5b7e7da 100644 --- a/clar.c +++ b/clar.c @@ -122,9 +122,17 @@ struct clar_report { const char *description; int test_number; + int runs; + enum cl_test_status status; time_t start; - double elapsed; + + double *times; + double time_min; + double time_max; + double time_mean; + double time_stddev; + double time_total; struct clar_error *errors; struct clar_error *last_error; @@ -184,6 +192,7 @@ static struct { struct clar_func { const char *name; const char *description; + int runs; void (*ptr)(void); }; @@ -274,6 +283,43 @@ clar_report_all(void) } } +static void +compute_times(void) +{ + double total_squares = 0; + int i; + + _clar.last_report->time_min = _clar.last_report->times[0]; + _clar.last_report->time_max = _clar.last_report->times[0]; + _clar.last_report->time_total = _clar.last_report->times[0]; + + for (i = 1; i < _clar.last_report->runs; i++) { + if (_clar.last_report->times[i] < _clar.last_report->time_min) + _clar.last_report->time_min = _clar.last_report->times[i]; + + if (_clar.last_report->times[i] > _clar.last_report->time_max) + _clar.last_report->time_max = _clar.last_report->times[i]; + + _clar.last_report->time_total += _clar.last_report->times[i]; + } + + if (_clar.last_report->runs <= 1) { + _clar.last_report->time_stddev = 0; + } else { + _clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs; + + for (i = 0; i < _clar.last_report->runs; i++) { + double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ? + _clar.last_report->times[i] - _clar.last_report->time_mean : + _clar.last_report->time_mean - _clar.last_report->times[i]; + + total_squares += (dev * dev); + } + + _clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs); + } +} + static void clar_run_test( const struct clar_suite *suite, @@ -281,9 +327,17 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - struct clar_counter start, end; + int runs, i; + + runs = (test->runs > 0) ? test->runs : 1; + + _clar.last_report->times = (runs > 1) ? + calloc(runs, sizeof(double)) : + &_clar.last_report->time_mean; + + if (!_clar.last_report->times) + clar_abort("Failed to allocate report times.\n"); - _clar.trampoline_enabled = 1; _clar.last_report->start = time(NULL); CL_TRACE(CL_TRACE__TEST__BEGIN); @@ -291,25 +345,35 @@ clar_run_test( clar_sandbox_create(suite->name, test->name); clar_print_test_start(suite->name, test->name, _clar.tests_ran); - clar_counter_now(&start); + + _clar.trampoline_enabled = 1; if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) initialize->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - test->ptr(); + + for (i = 0; i < runs; i++) { + struct clar_counter start, end; + + clar_counter_now(&start); + test->ptr(); + clar_counter_now(&end); + + _clar.last_report->runs++; + _clar.last_report->times[i] = clar_counter_diff(&start, &end); + } + CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_counter_now(&end); - _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_counter_diff(&start, &end); + compute_times(); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -650,6 +714,9 @@ clar_test_shutdown(void) free(error); } + if (report->times != &report->time_mean) + free(report->times); + report_next = report->next; free(report); } diff --git a/clar/summary.h b/clar/summary.h index b069430..c9930a5 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -45,7 +45,7 @@ static int clar_summary_testcase(struct clar_summary *summary, { if (fprintf(summary->fp, "\t\t\n", - report->test, report->suite, report->elapsed) < 0) + report->test, report->suite, report->time_total) < 0) return -1; if (report->description) { diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b72f187..4f81df3 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -25,4 +25,4 @@ target_include_directories(example PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(example clar) +target_link_libraries(example clar ${CLAR_LIBRARIES}) diff --git a/generate.py b/generate.py index 321acf7..7775b07 100755 --- a/generate.py +++ b/generate.py @@ -17,11 +17,12 @@ def __init__(self, module): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL, NULL }' + return ' { NULL, NULL, 0, NULL }' - return ' { "%s", %s, &%s }' % \ + return ' { "%s", %s, %d, &%s }' % \ (cb['short_name'], \ '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['runs'], \ cb['symbol']) class DeclarationTemplate(Template): @@ -111,6 +112,7 @@ def parse(self, contents): self.cleanup = None for (declaration, symbol, short_name, options) in regex.findall(contents): + runs = 0 description = None while options != '': @@ -135,6 +137,11 @@ def parse(self, contents): if key == "description": description = value + elif key == "runs": + if not value.isnumeric(): + print("Invalid option: '%s' in runs for '%s'" % (option, symbol)) + sys.exit(1) + runs = int(value) else: print("Invalid option: '%s' for '%s'" % (key, symbol)) sys.exit(1) @@ -143,7 +150,8 @@ def parse(self, contents): "short_name" : short_name, "declaration" : declaration, "symbol" : symbol, - "description" : description + "description" : description, + "runs" : runs } if short_name.startswith('initialize'): diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 96abd6e..f0b5a32 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,7 +38,7 @@ target_include_directories(selftest PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(selftest clar) +target_link_libraries(selftest clar ${CLAR_LIBRARIES}) add_test(NAME build_selftest_suite COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --config "$" --target selftest_suite diff --git a/test/selftest_suite/CMakeLists.txt b/test/selftest_suite/CMakeLists.txt index 9597d67..d96f18c 100644 --- a/test/selftest_suite/CMakeLists.txt +++ b/test/selftest_suite/CMakeLists.txt @@ -37,4 +37,4 @@ target_include_directories(selftest_suite PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(selftest_suite clar) +target_link_libraries(selftest_suite clar ${CLAR_LIBRARIES}) From d0aaa9d054ddf60da2ea27fa8f29b21de30cea6f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 17:08:36 +0000 Subject: [PATCH 06/10] add benchmark mode An application can provide _benchmarks_ instead of _tests_. Benchmarks can run multiple times, will calculate the times of each run, and some simple additional data (mean, min, max, etc). This information will be displayed and will optionally be emitted in the summary output. Test hosts can indicate that they're benchmarks (not tests) by setting the mode before parsing the arguments. This will switch the output and summary format types. --- clar.c | 16 ++++ clar.h | 12 +++ clar/print.h | 112 +++++++++++++++++++++++ clar/summary.h | 234 +++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 337 insertions(+), 37 deletions(-) diff --git a/clar.c b/clar.c index 5b7e7da..ee46720 100644 --- a/clar.c +++ b/clar.c @@ -146,6 +146,7 @@ struct clar_summary { }; static struct { + enum cl_test_mode test_mode; enum cl_test_status test_status; const char *active_suite; @@ -159,6 +160,7 @@ static struct { int suites_ran; enum cl_output_format output_format; + enum cl_summary_format summary_format; int report_errors_only; int exit_on_error; @@ -639,6 +641,14 @@ clar_test_init(int argc, char **argv) { const char *summary_env; + if (_clar.test_mode == CL_TEST_BENCHMARK) { + _clar.output_format = CL_OUTPUT_TIMING; + _clar.summary_format = CL_SUMMARY_JSON; + } else { + _clar.output_format = CL_OUTPUT_CLAP; + _clar.summary_format = CL_SUMMARY_JUNIT; + } + if (argc > 1) clar_parse_args(argc, argv); @@ -665,6 +675,12 @@ clar_test_init(int argc, char **argv) clar_tempdir_init(); } +void +clar_test_set_mode(enum cl_test_mode mode) +{ + _clar.test_mode = mode; +} + int clar_test_run(void) { diff --git a/clar.h b/clar.h index ca72292..0c19755 100644 --- a/clar.h +++ b/clar.h @@ -28,6 +28,11 @@ # define CLAR_CURRENT_FUNC "func" #endif +enum cl_test_mode { + CL_TEST_STANDARD, + CL_TEST_BENCHMARK, +}; + enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, @@ -38,10 +43,17 @@ enum cl_test_status { enum cl_output_format { CL_OUTPUT_CLAP, CL_OUTPUT_TAP, + CL_OUTPUT_TIMING, +}; + +enum cl_summary_format { + CL_SUMMARY_JUNIT, + CL_SUMMARY_JSON, }; /** Setup clar environment */ void clar_test_init(int argc, char *argv[]); +void clar_test_set_mode(enum cl_test_mode mode); int clar_test_run(void); void clar_test_shutdown(void); diff --git a/clar/print.h b/clar/print.h index f139f2d..faac971 100644 --- a/clar/print.h +++ b/clar/print.h @@ -194,6 +194,115 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) fflush(stdout); } +/* timings format: useful for benchmarks */ + +static void clar_print_timing_init(int test_count, int suite_count, const char *suite_names) +{ + (void)test_count; + (void)suite_count; + (void)suite_names; + + printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n"); +} + +static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count) +{ + (void)test_count; + (void)suite_count; + (void)error_count; +} + +static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error) +{ + (void)num; + (void)report; + (void)error; +} + +static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)test_number; + + printf("%s::%s: ", suite_name, test_name); + fflush(stdout); +} + +static void clar_print_timing_time(double t) +{ + static const char *units[] = { "sec", "ms", "μs", "ns" }; + static const int units_len = sizeof(units) / sizeof(units[0]); + int unit = 0, exponent = 0, digits; + + while (t < 1.0 && unit < units_len - 1) { + t *= 1000.0; + unit++; + } + + while (t > 0.0 && t < 1.0 && exponent < 10) { + t *= 10.0; + exponent++; + } + + digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1); + + printf("%.*f", digits, t); + + if (exponent > 0) + printf("e-%d", exponent); + + printf(" %s", units[unit]); +} + +static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + const struct clar_error *error = _clar.last_report->errors; + + (void)suite_name; + (void)test_name; + (void)test_number; + + switch(report->status) { + case CL_TEST_OK: + clar_print_timing_time(report->time_mean); + + if (report->runs > 1) { + printf(" ± "); + clar_print_timing_time(report->time_stddev); + + printf(" / range: "); + clar_print_timing_time(report->time_min); + printf(" … "); + clar_print_timing_time(report->time_max); + printf(" (%d runs)", report->runs); + } + + printf("\n"); + break; + case CL_TEST_FAILURE: + printf("failed: %s\n", error->error_msg); + break; + case CL_TEST_SKIP: + case CL_TEST_NOTRUN: + printf("skipped\n"); + break; + } + + fflush(stdout); +} + +static void clar_print_timing_suite_start(const char *suite_name, int suite_index) +{ + if (_clar.verbosity == 1) + printf("\n%s", suite_name); + + (void)suite_index; +} + +static void clar_print_timing_onabort(const char *fmt, va_list arg) +{ + vfprintf(stderr, fmt, arg); +} + /* indirection between protocol output selection */ #define PRINT(FN, ...) do { \ @@ -204,6 +313,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) case CL_OUTPUT_TAP: \ clar_print_tap_##FN (__VA_ARGS__); \ break; \ + case CL_OUTPUT_TIMING: \ + clar_print_timing_##FN (__VA_ARGS__); \ + break; \ default: \ abort(); \ } \ diff --git a/clar/summary.h b/clar/summary.h index c9930a5..0cd2b8f 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -1,8 +1,24 @@ - #include #include -static int clar_summary_close_tag( +static int clar_summary_time_digits(double t) +{ + int digits = 3; + + if (t >= 100.0) + return 1; + else if (t >= 10.0) + return 2; + + while (t > 0.0 && t < 1.0 && digits < 10) { + t *= 10.0; + digits++; + } + + return digits; +} + +static int clar_summary_junit_close_tag( struct clar_summary *summary, const char *tag, int indent) { const char *indt; @@ -14,12 +30,12 @@ static int clar_summary_close_tag( return fprintf(summary->fp, "%s\n", indt, tag); } -static int clar_summary_testsuites(struct clar_summary *summary) +static int clar_summary_junit_testsuites(struct clar_summary *summary) { return fprintf(summary->fp, "\n"); } -static int clar_summary_testsuite(struct clar_summary *summary, +static int clar_summary_junit_testsuite(struct clar_summary *summary, int idn, const char *name, time_t timestamp, int test_count, int fail_count, int error_count) { @@ -40,26 +56,15 @@ static int clar_summary_testsuite(struct clar_summary *summary, idn, name, iso_dt, test_count, fail_count, error_count); } -static int clar_summary_testcase(struct clar_summary *summary, - const struct clar_report *report) +static int clar_summary_junit_testcase(struct clar_summary *summary, + const char *name, const char *classname, double elapsed) { - if (fprintf(summary->fp, - "\t\t\n", - report->test, report->suite, report->time_total) < 0) - return -1; - - if (report->description) { - fprintf(summary->fp, "\t\t\t\n"); - fprintf(summary->fp, "\t\t\t\t\n"); - fprintf(summary->fp, "\t\t\t\t\t%s\n", report->description); - fprintf(summary->fp, "\t\t\t\t\n"); - fprintf(summary->fp, "\t\t\t\n"); - } - - return 0; + return fprintf(summary->fp, + "\t\t\n", + name, classname, clar_summary_time_digits(elapsed), elapsed); } -static int clar_summary_failure(struct clar_summary *summary, +static int clar_summary_junit_failure(struct clar_summary *summary, const char *type, const char *message, const char *desc) { return fprintf(summary->fp, @@ -67,22 +72,26 @@ static int clar_summary_failure(struct clar_summary *summary, type, message, desc); } -static int clar_summary_skipped(struct clar_summary *summary) +static int clar_summary_junit_skipped(struct clar_summary *summary) { return fprintf(summary->fp, "\t\t\t\n"); } -struct clar_summary *clar_summary_init(const char *filename) +struct clar_summary *clar_summary_junit_init(const char *filename) { struct clar_summary *summary; FILE *fp; - if ((fp = fopen(filename, "w")) == NULL) - clar_abort("Failed to open the summary file '%s': %s.\n", - filename, strerror(errno)); + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } - if ((summary = malloc(sizeof(struct clar_summary))) == NULL) - clar_abort("Failed to allocate summary.\n"); + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } summary->filename = filename; summary->fp = fp; @@ -90,12 +99,12 @@ struct clar_summary *clar_summary_init(const char *filename) return summary; } -int clar_summary_shutdown(struct clar_summary *summary) +int clar_summary_junit_shutdown(struct clar_summary *summary) { struct clar_report *report; const char *last_suite = NULL; - if (clar_summary_testsuites(summary) < 0) + if (clar_summary_junit_testsuites(summary) < 0) goto on_error; report = _clar.reports; @@ -103,17 +112,17 @@ int clar_summary_shutdown(struct clar_summary *summary) struct clar_error *error = report->errors; if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_testsuite(summary, 0, report->suite, + if (clar_summary_junit_testsuite(summary, 0, report->suite, report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) goto on_error; } last_suite = report->suite; - clar_summary_testcase(summary, report); + clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total); while (error != NULL) { - if (clar_summary_failure(summary, "assert", + if (clar_summary_junit_failure(summary, "assert", error->error_msg, error->description) < 0) goto on_error; @@ -121,20 +130,20 @@ int clar_summary_shutdown(struct clar_summary *summary) } if (report->status == CL_TEST_SKIP) - clar_summary_skipped(summary); + clar_summary_junit_skipped(summary); - if (clar_summary_close_tag(summary, "testcase", 2) < 0) + if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0) goto on_error; report = report->next; if (!report || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_close_tag(summary, "testsuite", 1) < 0) + if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0) goto on_error; } } - if (clar_summary_close_tag(summary, "testsuites", 0) < 0 || + if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 || fclose(summary->fp) != 0) goto on_error; @@ -148,3 +157,154 @@ int clar_summary_shutdown(struct clar_summary *summary) free(summary); return -1; } + +struct clar_summary *clar_summary_json_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +int clar_summary_json_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + int i; + + fprintf(summary->fp, "{\n"); + fprintf(summary->fp, " \"tests\": [\n"); + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (report != _clar.reports) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"name\": \"%s::%s\",\n", report->suite, report->test); + + if (report->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", report->description); + + fprintf(summary->fp, " \"results\": {\n"); + + fprintf(summary->fp, " \"status\": "); + if (report->status == CL_TEST_OK) + fprintf(summary->fp, "\"ok\",\n"); + else if (report->status == CL_TEST_FAILURE) + fprintf(summary->fp, "\"failed\",\n"); + else if (report->status == CL_TEST_SKIP) + fprintf(summary->fp, "\"skipped\"\n"); + else + clar_abort("unknown test status %d", report->status); + + if (report->status == CL_TEST_OK) { + fprintf(summary->fp, " \"mean\": %.*f,\n", + clar_summary_time_digits(report->time_mean), report->time_mean); + fprintf(summary->fp, " \"stddev\": %.*f,\n", + clar_summary_time_digits(report->time_stddev), report->time_stddev); + fprintf(summary->fp, " \"min\": %.*f,\n", + clar_summary_time_digits(report->time_min), report->time_min); + fprintf(summary->fp, " \"max\": %.*f,\n", + clar_summary_time_digits(report->time_max), report->time_max); + fprintf(summary->fp, " \"times\": [\n"); + + for (i = 0; i < report->runs; i++) { + if (i > 0) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " %.*f", + clar_summary_time_digits(report->times[i]), report->times[i]); + } + + fprintf(summary->fp, "\n ]\n"); + } + + if (report->status == CL_TEST_FAILURE) { + fprintf(summary->fp, " \"errors\": [\n"); + + while (error != NULL) { + if (error != report->errors) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->error_msg); + + if (error->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); + + fprintf(summary->fp, " \"function\": \"%s\",\n", error->function); + fprintf(summary->fp, " \"file\": \"%s\",\n", error->file); + fprintf(summary->fp, " \"line\": %" PRIuMAX "\n", error->line_number); + fprintf(summary->fp, " }"); + + error = error->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + } + + fprintf(summary->fp, " }\n"); + fprintf(summary->fp, " }"); + + report = report->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + fprintf(summary->fp, "}\n"); + + if (fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} + +/* indirection between protocol output selection */ + +#define SUMMARY(FN, ...) do { \ + switch (_clar.summary_format) { \ + case CL_SUMMARY_JUNIT: \ + return clar_summary_junit_##FN (__VA_ARGS__); \ + break; \ + case CL_SUMMARY_JSON: \ + return clar_summary_json_##FN (__VA_ARGS__); \ + break; \ + default: \ + abort(); \ + } \ + } while(0) + +struct clar_summary *clar_summary_init(const char *filename) +{ + SUMMARY(init, filename); +} + +int clar_summary_shutdown(struct clar_summary *summary) +{ + SUMMARY(shutdown, summary); +} From 54751ad9af7ddc0623960ab7bd1c6be3b81cdc0f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 17:13:44 +0000 Subject: [PATCH 07/10] refactor error struct --- clar.c | 14 +++++++------- clar/print.h | 6 +++--- clar/summary.h | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/clar.c b/clar.c index ee46720..cf63f2a 100644 --- a/clar.c +++ b/clar.c @@ -100,11 +100,11 @@ fixture_path(const char *base, const char *fixture_name); #endif struct clar_error { - const char *file; + const char *message; + char *description; const char *function; + const char *file; uintmax_t line_number; - const char *error_msg; - char *description; struct clar_error *next; }; @@ -776,7 +776,7 @@ void clar__fail( const char *file, const char *function, size_t line, - const char *error_msg, + const char *error_message, const char *error_description, int should_abort) { @@ -796,7 +796,7 @@ void clar__fail( error->file = _clar.invoke_file ? _clar.invoke_file : file; error->function = _clar.invoke_func ? _clar.invoke_func : function; error->line_number = _clar.invoke_line ? _clar.invoke_line : line; - error->error_msg = error_msg; + error->message = error_message; if (error_description != NULL && (error->description = strdup(error_description)) == NULL) @@ -814,14 +814,14 @@ void clar__assert( const char *file, const char *function, size_t line, - const char *error_msg, + const char *error_message, const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, error_description, should_abort); + clar__fail(file, function, line, error_message, error_description, should_abort); } void clar__assert_equal( diff --git a/clar/print.h b/clar/print.h index faac971..311bd9c 100644 --- a/clar/print.h +++ b/clar/print.h @@ -43,7 +43,7 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con error->file, error->line_number); - clar_print_indented(error->error_msg, 2); + clar_print_indented(error->message, 2); if (error->description != NULL) clar_print_indented(error->description, 2); @@ -161,7 +161,7 @@ static void clar_print_tap_test_finish(const char *suite_name, const char *test_ printf(" ---\n"); printf(" reason: |\n"); - clar_print_indented(error->error_msg, 6); + clar_print_indented(error->message, 6); if (error->description) clar_print_indented(error->description, 6); @@ -279,7 +279,7 @@ static void clar_print_timing_test_finish(const char *suite_name, const char *te printf("\n"); break; case CL_TEST_FAILURE: - printf("failed: %s\n", error->error_msg); + printf("failed: %s\n", error->message); break; case CL_TEST_SKIP: case CL_TEST_NOTRUN: diff --git a/clar/summary.h b/clar/summary.h index 0cd2b8f..d3f5657 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -123,7 +123,7 @@ int clar_summary_junit_shutdown(struct clar_summary *summary) while (error != NULL) { if (clar_summary_junit_failure(summary, "assert", - error->error_msg, error->description) < 0) + error->message, error->description) < 0) goto on_error; error = error->next; @@ -243,7 +243,7 @@ int clar_summary_json_shutdown(struct clar_summary *summary) fprintf(summary->fp, ",\n"); fprintf(summary->fp, " {\n"); - fprintf(summary->fp, " \"message\": \"%s\",\n", error->error_msg); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->message); if (error->description) fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); From 9ae1f435776a38695e663f94439f4067d2ed3886 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 17:42:52 +0000 Subject: [PATCH 08/10] in benchmark mode run iteratively In benchmark mode, when the number of runs was not explicitly specified in the test itself, run a reasonably number of iterations. We do this by measuring one run of the test, then using that data to determine how many iterations we should run to fit within 3 seconds. (With a minimum number of iterations to ensure that we get some data, and a maximum number to deal with poor precision for fast test runs.) The 3 second number, and 10 iteration minimum, were chosen by consulting the hyperfine defaults. --- clar.c | 39 ++++++++++++++++++++++++++------------- clar.h | 10 ++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/clar.c b/clar.c index cf63f2a..34ab39c 100644 --- a/clar.c +++ b/clar.c @@ -87,6 +87,7 @@ typedef struct stat STAT_T; #endif +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #include "clar.h" @@ -329,18 +330,10 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - int runs, i; - - runs = (test->runs > 0) ? test->runs : 1; - - _clar.last_report->times = (runs > 1) ? - calloc(runs, sizeof(double)) : - &_clar.last_report->time_mean; - - if (!_clar.last_report->times) - clar_abort("Failed to allocate report times.\n"); + int runs = test->runs, i = 0; _clar.last_report->start = time(NULL); + _clar.last_report->times = &_clar.last_report->time_mean; CL_TRACE(CL_TRACE__TEST__BEGIN); @@ -356,16 +349,36 @@ clar_run_test( CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - for (i = 0; i < runs; i++) { + do { struct clar_counter start, end; + double elapsed; clar_counter_now(&start); test->ptr(); clar_counter_now(&end); + elapsed = clar_counter_diff(&start, &end); + + /* + * unless the number of runs was explicitly given + * in benchmark mode, use the first run as a sample + * to determine how many runs we should attempt + */ + if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) { + runs = MAX(CLAR_BENCHMARK_RUN_MIN, (CLAR_BENCHMARK_RUN_TIME / elapsed)); + runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs); + } + + if (i == 0 && runs > 1) { + _clar.last_report->times = calloc(runs, sizeof(double)); + + if (_clar.last_report->times == NULL) + clar_abort("Failed to allocate report times.\n"); + } + _clar.last_report->runs++; - _clar.last_report->times[i] = clar_counter_diff(&start, &end); - } + _clar.last_report->times[i] = elapsed; + } while(++i < runs); CL_TRACE(CL_TRACE__TEST__RUN_END); } diff --git a/clar.h b/clar.h index 0c19755..b6e5069 100644 --- a/clar.h +++ b/clar.h @@ -18,6 +18,16 @@ # define CLAR_MAX_PATH PATH_MAX #endif +/* + * In benchmark mode, by default, clar will run the test repeatedly for + * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least + * `CLAR_BENCHMARK_RUN_MIN` iterations. + */ + +#define CLAR_BENCHMARK_RUN_TIME 3.0 +#define CLAR_BENCHMARK_RUN_MIN 10 +#define CLAR_BENCHMARK_RUN_MAX 30000000 + #ifndef CLAR_SELFTEST # define CLAR_CURRENT_FILE __FILE__ # define CLAR_CURRENT_LINE __LINE__ From 201f40ad9fd4313567dcae606cd1913a8b8bec73 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 19 Jan 2025 10:53:32 +0000 Subject: [PATCH 09/10] introduce a reset function For multi-run tests (benchmarks), we introduce a `reset` function. By default, between each run of a test, the initialization will be called at startup, and the cleanup will be called at finish. A benchmark may wish to set up multi-run state at the beginning of the invocation (in initialization), and keep a steady state through all test runs. Users can now add a `reset` function so that initialization occurs only at the beginning of all runs. --- clar.c | 15 ++++++++++++++- generate.py | 8 ++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/clar.c b/clar.c index 34ab39c..35166e4 100644 --- a/clar.c +++ b/clar.c @@ -202,6 +202,7 @@ struct clar_func { struct clar_suite { const char *name; struct clar_func initialize; + struct clar_func reset; struct clar_func cleanup; const struct clar_func *tests; size_t test_count; @@ -328,6 +329,7 @@ clar_run_test( const struct clar_suite *suite, const struct clar_func *test, const struct clar_func *initialize, + const struct clar_func *reset, const struct clar_func *cleanup) { int runs = test->runs, i = 0; @@ -353,6 +355,17 @@ clar_run_test( struct clar_counter start, end; double elapsed; + if (i > 0 && reset->ptr != NULL) { + reset->ptr(); + } else if (i > 0) { + if (_clar.local_cleanup != NULL) + _clar.local_cleanup(_clar.local_cleanup_payload); + if (cleanup->ptr != NULL) + cleanup->ptr(); + if (initialize->ptr != NULL) + initialize->ptr(); + } + clar_counter_now(&start); test->ptr(); clar_counter_now(&end); @@ -481,7 +494,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.last_report = report; - clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup); + clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) return; diff --git a/generate.py b/generate.py index 7775b07..7843beb 100755 --- a/generate.py +++ b/generate.py @@ -32,6 +32,9 @@ def render(self): for initializer in self.module.initializers: out += "extern %s;\n" % initializer['declaration'] + if self.module.reset: + out += "extern %s;\n" % self.module.reset['declaration'] + if self.module.cleanup: out += "extern %s;\n" % self.module.cleanup['declaration'] @@ -63,12 +66,14 @@ def render(self): { "${clean_name}", ${initialize}, + ${reset}, ${cleanup}, ${cb_ptr}, ${cb_count}, ${enabled} }""" ).substitute( clean_name = name, initialize = self._render_callback(initializer), + reset = self._render_callback(self.module.reset), cleanup = self._render_callback(self.module.cleanup), cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), @@ -109,6 +114,7 @@ def parse(self, contents): self.callbacks = [] self.initializers = [] + self.reset = None self.cleanup = None for (declaration, symbol, short_name, options) in regex.findall(contents): @@ -156,6 +162,8 @@ def parse(self, contents): if short_name.startswith('initialize'): self.initializers.append(data) + elif short_name == 'reset': + self.reset = data elif short_name == 'cleanup': self.cleanup = data else: From 8577d170602957c4e18540c45bd519e5aed6efee Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 19 Jan 2025 01:21:25 +0000 Subject: [PATCH 10/10] Update README with benchmark information --- README.md | 112 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 4159598..d2d681e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,13 @@ Come out and Clar ================= -In Catalan, "clar" means clear, easy to perceive. Using clar will make it -easy to test and make clear the quality of your code. +Clar is a minimal C unit testing and benchmarking framework. It +provides a simple mechanism for writing tests and asserting +postconditions, while providing support for TAP and JUnit style +outputs. -> _Historical note_ -> -> Originally the clar project was named "clay" because the word "test" has its -> roots in the latin word *"testum"*, meaning "earthen pot", and *"testa"*, -> meaning "piece of burned clay"? -> -> This is because historically, testing implied melting metal in a pot to -> check its quality. Clay is what tests are made of. +In Catalan, "clar" means clear, easy to perceive. Using Clar will make it +easy to test and make clear the quality of your code. ## Quick Usage Overview @@ -19,9 +15,7 @@ Clar is a minimal C unit testing framework. It's been written to replace the old framework in [libgit2][libgit2], but it's both very versatile and straightforward to use. -Can you count to funk? - -- **Zero: Initialize test directory** +1. **Initialize test directory** ~~~~ sh $ mkdir tests @@ -29,7 +23,7 @@ Can you count to funk? $ cp $CLAR_ROOT/example/*.c tests ~~~~ -- **One: Write some tests** +2. **Write some tests** File: tests/adding.c: @@ -59,7 +53,7 @@ Can you count to funk? } ~~~~~ -- **Two: Build the test executable** +3. **Build the test executable** ~~~~ sh $ cd tests @@ -68,7 +62,7 @@ Can you count to funk? $ gcc -I. clar.c main.c adding.c -o testit ~~~~ -- **Funk: Funk it.** +4. **Run the tests** ~~~~ sh $ ./testit @@ -319,6 +313,92 @@ void test_example__a_test_with_auxiliary_methods(void) } ~~~~ +## Benchmarks + +The clar mixer (`generate.py`) and runner can also be used to support +simple benchmark capabilities. When running in benchmark mode, Clar +will run each test multiple times in succession, using a high-resolution +platform timer to measure the elapsed time of each run. + +By default, Clar will run each test repeatedly for 3 seconds (with +a minimum of 10 runs), but you can define the explicit number of +runs for each test in the definition. + +By default, Clar will run the initialization and cleanup functions +before _each_ test run. This allows for consistent setup and teardown +behavior, and predictability with existing test setups. However, you +can avoid this additional overhead by defining a _reset_ function. +This will be called between test runs instead of the cleanup and +re-initialization; in this case, initialization will occur only +before all test runs, and cleanup will be performed only when all +test runs are complete. + +To configure a benchmark application instead of a test application: + +1. **Set clar into benchmark mode in your main function** + + ~~~~ c + int main(int argc, char *argv[]) + { + clar_test_set_mode(CL_TEST_BENCHMARK); + clar_test_init(argc, argv); + res = clar_test_run(); + clar_test_shutdown(); + return res; + } + ~~~~ + +2. **Optionally, set up your initialization, cleanup, and reset + functions** + + ~~~~ c + void test_foo__initialize(void) + { + global_data = malloc(1024 * 1024 * 1024); + memset(global_data, 0, 1024 * 1024 * 1024); + } + + void test_foo__reset(void) + { + memset(global_data, 0, 1024 * 1024 * 1024); + } + + void test_foo__cleanup(void) + { + global_data = malloc(1024 * 1024 * 1024); + } + ~~~~ + +3. **Optionally, configure tests with a specific run number** + + ~~~~ c + /* Run this test 500 times */ + void test_foo__bar(void) + /* [clar]:runs=500 */ + { + bar(); + } + ~~~~ + +3. **Run the benchmarks** + + When running in benchmark mode, you'll see timings output; if you + write a summary file, it will be a JSON file that contains the + time information. + + ~~~~ sh + $ ./benchmarks -r/path/to/results.json + Started benchmarks (mean time ± stddev / min time … max time): + + foo::bar: 24.75 ms ± 1.214 ms / range: 24.41 ms … 38.06 ms (500 runs) + foo::baz: 24.67 ms ± 248.2 μs / range: 24.41 ms … 25.41 ms (478 runs) + foo::qux: 25.98 ms ± 333.0 μs / range: 25.64 ms … 26.82 ms (112 runs) + ~~~~ + +Note: you can change the prefix of the test function names from `test_` +to something of your choice by using the `--prefix=...` option for +the `generate.py` mixer script. + About Clar ==========