add benchmark mode

An application can provide _benchmarks_ instead of _tests_. Benchmarks
can run multiple times, will calculate the times of each run, and some
simple additional data (mean, min, max, etc). This information will be
displayed and will optionally be emitted in the summary output.

Test hosts can indicate that they're benchmarks (not tests) by setting
the mode before parsing the arguments. This will switch the output and
summary format types.

Author: ethomson

clar.c

@@ -146,6 +146,7 @@ struct clar_summary {
 };
 
 static struct {
+	enum cl_test_mode test_mode;
 	enum cl_test_status test_status;
 
 	const char *active_suite;
@@ -159,6 +160,7 @@ static struct {
 	int suites_ran;
 
 	enum cl_output_format output_format;
+	enum cl_summary_format summary_format;
 
 	int report_errors_only;
 	int exit_on_error;
@@ -639,6 +641,14 @@ clar_test_init(int argc, char **argv)
 {
 	const char *summary_env;
 
+	if (_clar.test_mode == CL_TEST_BENCHMARK) {
+		_clar.output_format = CL_OUTPUT_TIMING;
+		_clar.summary_format = CL_SUMMARY_JSON;
+	} else {
+		_clar.output_format = CL_OUTPUT_CLAP;
+		_clar.summary_format = CL_SUMMARY_JUNIT;
+	}
+
 	if (argc > 1)
 		clar_parse_args(argc, argv);
 
@@ -665,6 +675,12 @@ clar_test_init(int argc, char **argv)
 	clar_tempdir_init();
 }
 
+void
+clar_test_set_mode(enum cl_test_mode mode)
+{
+	_clar.test_mode = mode;
+}
+
 int
 clar_test_run(void)
 {

clar.h

@@ -28,6 +28,11 @@
 # define CLAR_CURRENT_FUNC "func"
 #endif
 
+enum cl_test_mode {
+	CL_TEST_STANDARD,
+	CL_TEST_BENCHMARK,
+};
+
 enum cl_test_status {
 	CL_TEST_OK,
 	CL_TEST_FAILURE,
@@ -38,10 +43,17 @@ enum cl_test_status {
 enum cl_output_format {
 	CL_OUTPUT_CLAP,
 	CL_OUTPUT_TAP,
+	CL_OUTPUT_TIMING,
+};
+
+enum cl_summary_format {
+	CL_SUMMARY_JUNIT,
+	CL_SUMMARY_JSON,
 };
 
 /** Setup clar environment */
 void clar_test_init(int argc, char *argv[]);
+void clar_test_set_mode(enum cl_test_mode mode);
 int clar_test_run(void);
 void clar_test_shutdown(void);
 

clar/print.h

@@ -194,6 +194,115 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg)
 	fflush(stdout);
 }
 
+/* timings format: useful for benchmarks */
+
+static void clar_print_timing_init(int test_count, int suite_count, const char *suite_names)
+{
+	(void)test_count;
+	(void)suite_count;
+	(void)suite_names;
+
+	printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n");
+}
+
+static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count)
+{
+	(void)test_count;
+	(void)suite_count;
+	(void)error_count;
+}
+
+static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error)
+{
+	(void)num;
+	(void)report;
+	(void)error;
+}
+
+static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number)
+{
+	(void)test_number;
+
+	printf("%s::%s:  ", suite_name, test_name);
+	fflush(stdout);
+}
+
+static void clar_print_timing_time(double t)
+{
+	static const char *units[] = { "sec", "ms", "μs", "ns" };
+	static const int units_len = sizeof(units) / sizeof(units[0]);
+	int unit = 0, exponent = 0, digits;
+
+	while (t < 1.0 && unit < units_len - 1) {
+		t *= 1000.0;
+		unit++;
+	}
+
+	while (t > 0.0 && t < 1.0 && exponent < 10) {
+		t *= 10.0;
+		exponent++;
+	}
+
+	digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1);
+
+	printf("%.*f", digits, t);
+
+	if (exponent > 0)
+		printf("e-%d", exponent);
+
+	printf(" %s", units[unit]);
+}
+
+static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report)
+{
+	const struct clar_error *error = _clar.last_report->errors;
+
+	(void)suite_name;
+	(void)test_name;
+	(void)test_number;
+
+	switch(report->status) {
+	case CL_TEST_OK:
+		clar_print_timing_time(report->time_mean);
+
+		if (report->runs > 1) {
+			printf(" ± ");
+			clar_print_timing_time(report->time_stddev);
+
+			printf(" / range: ");
+			clar_print_timing_time(report->time_min);
+			printf(" … ");
+			clar_print_timing_time(report->time_max);
+			printf("  (%d runs)", report->runs);
+		}
+
+		printf("\n");
+		break;
+	case CL_TEST_FAILURE:
+		printf("failed: %s\n", error->error_msg);
+		break;
+	case CL_TEST_SKIP:
+	case CL_TEST_NOTRUN:
+		printf("skipped\n");
+		break;
+	}
+
+	fflush(stdout);
+}
+
+static void clar_print_timing_suite_start(const char *suite_name, int suite_index)
+{
+	if (_clar.verbosity == 1)
+		printf("\n%s", suite_name);
+
+	(void)suite_index;
+}
+
+static void clar_print_timing_onabort(const char *fmt, va_list arg)
+{
+	vfprintf(stderr, fmt, arg);
+}
+
 /* indirection between protocol output selection */
 
 #define PRINT(FN, ...) do { \
@@ -204,6 +313,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg)
 			case CL_OUTPUT_TAP: \
 				clar_print_tap_##FN (__VA_ARGS__); \
 				break; \
+			case CL_OUTPUT_TIMING: \
+				clar_print_timing_##FN (__VA_ARGS__); \
+				break; \
 			default: \
 				abort(); \
 		} \