test(runtime): [develop] measure all suites and cover protocol boundaries

.github/workflows/regressions.yml

@@ -32,7 +32,7 @@ jobs:
         run: |
           set -eu
           sudo apt-get update
-          sudo apt-get install -y autoconf automake libtool help2man libmariadb-dev mariadb-client libsnmp-dev snmpd libssl-dev libcap2-bin lcov
+          sudo apt-get install -y autoconf automake libtool help2man libmariadb-dev libcmocka-dev mariadb-client libsnmp-dev snmpd libssl-dev libcap2-bin lcov
       - name: Build and test production code
         run: |
           set -eu

Makefile.am

@@ -121,6 +121,25 @@ coverage-fault-source-list:
 # Production and linked tests share runtime objects. Report real object names.
 coverage-object-list:
 	@printf '%s\n' $(spine_OBJECTS) $(test_spine_regressions_OBJECTS) $(test_spine_faults_OBJECTS) | LC_ALL=C sort -u
+
+# The cmocka suites exercise shipped code, including socket/error paths via
+# syscall interception. Keep their measured producers separate from the
+# unmodified production-linked regression/fault producers above.
+.PHONY: coverage-unit-program-list coverage-unit-object-list
+coverage-unit-program-list:
+	@printf '%s\n' $(filter tests/unit/%,$(check_PROGRAMS))
+
+.PHONY: coverage-helper-program-list coverage-helper-object-list
+coverage-helper-program-list:
+	@printf '%s\n' tests/support/php_test_server$(EXEEXT)
+coverage-helper-object-list:
+	@printf '%s\n' $(tests_support_php_test_server_OBJECTS)
+
+# spine_runtime.c contains only test globals and generates no executable
+# coverage profile. All production sources remain in coverage-object-list.
+coverage-unit-object-list:
+	@printf '%s\n' $(filter-out tests/support/spine_runtime.o tests/support/%-spine_runtime.o, \
+	 $(foreach program,$(filter tests/unit/%,$(check_PROGRAMS)),$($(subst /,_,$(program))_OBJECTS))) | LC_ALL=C sort -u
 # Docker targets — require Dockerfile and docker/dev.Dockerfile (from PR #401)
 .PHONY: docker docker-dev verify cppcheck
 

Makefile.in

@@ -4664,6 +4664,25 @@ coverage-fault-source-list:
 # Production and linked tests share runtime objects. Report real object names.
 coverage-object-list:
 	@printf '%s\n' $(spine_OBJECTS) $(test_spine_regressions_OBJECTS) $(test_spine_faults_OBJECTS) | LC_ALL=C sort -u
+
+# The cmocka suites exercise shipped code, including socket/error paths via
+# syscall interception. Keep their measured producers separate from the
+# unmodified production-linked regression/fault producers above.
+.PHONY: coverage-unit-program-list coverage-unit-object-list
+coverage-unit-program-list:
+	@printf '%s\n' $(filter tests/unit/%,$(check_PROGRAMS))
+
+.PHONY: coverage-helper-program-list coverage-helper-object-list
+coverage-helper-program-list:
+	@printf '%s\n' tests/support/php_test_server$(EXEEXT)
+coverage-helper-object-list:
+	@printf '%s\n' $(tests_support_php_test_server_OBJECTS)
+
+# spine_runtime.c contains only test globals and generates no executable
+# coverage profile. All production sources remain in coverage-object-list.
+coverage-unit-object-list:
+	@printf '%s\n' $(filter-out tests/support/spine_runtime.o tests/support/%-spine_runtime.o, \
+	 $(foreach program,$(filter tests/unit/%,$(check_PROGRAMS)),$($(subst /,_,$(program))_OBJECTS))) | LC_ALL=C sort -u
 # Docker targets — require Dockerfile and docker/dev.Dockerfile (from PR #401)
 .PHONY: docker docker-dev verify cppcheck
 

src/ping/icmp_shared.c

@@ -352,7 +352,7 @@ static void icmp_shared_dispatch(int family, const unsigned char *reply, ssize_t
 		else {
 			const struct sockaddr_in6 *source = (const struct sockaddr_in6 *) from;
 
-			if (memcmp(&source->sin6_addr, &waiter->peer6, sizeof(waiter->peer6)) != 0 ||
+			if (memcmp(source->sin6_addr.s6_addr, waiter->peer6.s6_addr, sizeof(waiter->peer6.s6_addr)) != 0 ||
 				!icmp6_reply_matches(reply, len, waiter->id, waiter->seq, TRUE)) {
 				continue;
 			}

tests/contracts/test_spine_contracts.c

@@ -7,6 +7,7 @@
  */
 #include "internal/common.h"
 #include "app/spine.h"
+#include "app/startup_internal.h"
 #include <stdint.h>
 #include <sys/resource.h>
 
@@ -253,13 +254,62 @@ static void test_fatal_signal_contracts(void) {
 	assert(WIFEXITED(status) && WEXITSTATUS(status) == 0);
 }
 
+static void test_startup_output_contracts(void) {
+	config_t previous = set;
+	char output[DBL_BUFSIZE];
+
+	set.logging.log_destination = LOGDEST_STDOUT;
+	set.poller.threads = 3;
+	for (int debug = 0; debug <= 1; debug++) {
+		set.logging.log_level = debug ? POLLER_VERBOSITY_DEBUG : POLLER_VERBOSITY_LOW;
+		for (int hidden = 0; hidden <= 1; hidden++) {
+			set.console.stdout_notty = hidden;
+			for (int remote = 0; remote <= 1; remote++) {
+				for (int secondary = 0; secondary <= 1; secondary++) {
+					FILE *capture = tmpfile();
+					int saved_stdout;
+					size_t count;
+
+					assert(capture != NULL && fflush(stdout) == 0);
+					saved_stdout = dup(STDOUT_FILENO);
+					assert(saved_stdout >= 0 && dup2(fileno(capture), STDOUT_FILENO) == STDOUT_FILENO);
+					set.poller.poller_id = secondary ? 2 : 1;
+					set.hosts.has_device_0 = secondary;
+					report_startup(remote ? REMOTE : LOCAL);
+					report_poll_statistics(0.0, 7);
+					assert(fflush(stdout) == 0 && dup2(saved_stdout, STDOUT_FILENO) == STDOUT_FILENO);
+					assert(close(saved_stdout) == 0);
+					rewind(capture);
+					count = fread(output, 1, sizeof(output) - 1, capture);
+					assert(count < sizeof(output) - 1 && !ferror(capture));
+					output[count] = '\0';
+					assert(fclose(capture) == 0);
+					assert((strstr(output, "Version " VERSION " starting") != NULL) == (debug || !hidden));
+					assert((strstr(output, "Threads: 3, Devices: 7") != NULL) == (debug || !hidden));
+					assert((strstr(output, "Sending entries to ") != NULL) == (secondary && (debug || !hidden)));
+					if (secondary && (debug || !hidden)) {
+						assert(strstr(output, remote ? "remote database in 'online' mode" : "local database in 'offline', or 'recovery' mode") != NULL);
+					}
+					if (debug) {
+						assert(strstr(output, secondary ? "Device 0 Poller Items found." : "No Device 0 Poller Items found.") != NULL);
+						assert(strstr(output, mysql_thread_safe() ? "MySQL is Thread Safe!" : "MySQL is NOT Thread Safe!") != NULL);
+					}
+				}
+			}
+		}
+	}
+	set = previous;
+	puts("production startup and statistics output contracts passed");
+}
+
 void test_additional_contracts(void) {
 	test_keyword_roundtrips();
 	test_lock_contracts();
 	test_php_roundrobin();
 	test_legacy_ip_predicate();
 	test_multipart_boundaries();
 	test_fatal_signal_contracts();
+	test_startup_output_contracts();
 	puts("production additional contracts passed");
 }
 

tests/contracts/test_spine_settings_contracts.c

@@ -52,3 +52,81 @@ void test_settings_write_contracts(MYSQL *mysql) {
 	set = previous;
 	puts("production settings write outcome regressions passed");
 }
+
+/* Only the disposable fixture settings/poller tables are changed. Keep exact
+ * copies because read_config_options() opens its own connection and therefore
+ * cannot see a transaction on this test's connection. */
+void test_config_option_boundaries(MYSQL *mysql) {
+	config_t previous = set;
+	char query[LRG_BUFSIZE];
+
+	assert(db_insert(mysql, LOCAL, "CREATE TEMPORARY TABLE spine_saved_option_settings AS SELECT name,value FROM settings"));
+	assert(db_insert(mysql, LOCAL, "CREATE TEMPORARY TABLE spine_saved_option_poller AS SELECT id,threads FROM poller WHERE id=1"));
+	set.poller.poller_id = 1;
+	set.poller.threads_set = FALSE;
+	set.logging.log_destination = LOGDEST_STDOUT;
+	set.logging.log_level = POLLER_VERBOSITY_NONE;
+	set.hosts.host_id_list[0] = '\0';
+	for (int high = 0; high <= 1; high++) {
+		assert(db_insert(mysql, LOCAL, "DELETE FROM settings"));
+		spine_snprintf(query, sizeof(query),
+			"INSERT INTO settings(name,value) VALUES "
+			"('log_verbosity','%d'),('log_destination','STDOUT'),"
+			"('path_webroot','%s'),('path_cactilog','%s'),('path_php_binary','/fixture/php'),"
+			"('default_datechar','%d'),('default_date_format','%d'),"
+			"('availability_method','%d'),('ping_recovery_count','2'),('ping_failure_count','3'),"
+			"('ping_method','%d'),('ping_retries','4'),('snmp_retries','2'),"
+			"('poller_interval','%d'),('concurrent_processes','2'),('script_timeout','%d'),"
+			"('selective_device_debug','42'),('spine_log_level','1'),('php_servers','%d'),"
+			"('active_profiles','%d'),('total_snmp_ports','%d'),('max_get_size','%d'),"
+			"('log_perror','on'),('log_pwarn','on'),('log_pstats','on'),"
+			"('boost_redirect','on'),('boost_rrd_update_enable','on')",
+			POLLER_VERBOSITY_DEBUG, high ? "/fixture/web" : "", high ? "/fixture/log" : "",
+			high ? GDC_MAX + 1 : GDC_MIN - 1, high ? GD_MAX + 1 : GD_MIN - 1,
+			AVAIL_NONE, PING_TCP, high ? 60 : 0, high ? 30 : 2,
+			high ? MAX_PHP_SERVERS + 1 : 0, high ? 2 : -1, high ? 2 : -1,
+			high ? 999 : 16);
+		assert(db_insert(mysql, LOCAL, query));
+		spine_snprintf(query, sizeof(query), "UPDATE poller SET threads=%d WHERE id=1", high ? MAX_THREADS + 1 : 3);
+		assert(db_insert(mysql, LOCAL, query));
+		read_config_options();
+		assert(set.logging.log_level == POLLER_VERBOSITY_DEBUG);
+		assert(set.logging.log_destination == LOGDEST_STDOUT);
+		assert(strcmp(set.logging.path_logfile, high ? "/fixture/log" : "") == 0);
+		assert(set.logging.log_datetime_separator == GDC_DEFAULT && set.logging.log_datetime_format == GD_DEFAULT);
+		assert(strcmp(set.php.path_php, "/fixture/php") == 0);
+		assert(set.availability.availability_method == AVAIL_NONE && set.availability.ping_method == PING_TCP);
+		assert(set.availability.ping_recovery_count == 2 && set.availability.ping_failure_count == 3);
+		assert(set.availability.ping_retries == 4 && set.snmp.snmp_retries == 2);
+		assert(set.poller.threads == (high ? MAX_THREADS : 3));
+		assert(set.poller.poller_interval == (high ? 60 : 0) && set.poller.num_parent_processes == 2);
+		assert(set.php.script_timeout == (high ? 30 : 5));
+		assert(set.php.php_servers == (high ? MAX_PHP_SERVERS : 1));
+		assert(set.poller.active_profiles == (high ? 2 : 0) && set.snmp.total_snmp_ports == (high ? 2 : 0));
+		assert(set.snmp.snmp_max_get_size == (high ? 128 : 16));
+		assert(set.logging.log_perror && set.logging.log_pwarn && set.logging.log_pstats);
+		assert(set.boost.boost_enabled && set.boost.boost_redirect);
+		assert(strcmp(set.logging.selective_device_debug, "42") == 0 && set.logging.spine_log_level == 1);
+		assert(!set.php.php_required); /* The fixture has no script-server items. */
+		/* The preceding CLI override remains authoritative across reloads. */
+		assert(set.availability.ping_timeout == 777);
+	}
+	assert(db_insert(mysql, LOCAL, "DELETE FROM settings"));
+	assert(db_insert(mysql, LOCAL, "INSERT INTO settings(name,value) VALUES('log_destination','STDOUT')"));
+	read_config_options();
+	/* getsetting() represents absent rows as owned empty strings. Preserve
+	 * that contract: numeric reads become zero before their existing clamps. */
+	assert(set.php.script_timeout == 5 && set.php.php_servers == 1);
+	assert(set.snmp.snmp_max_get_size == 0 && set.snmp.snmp_retries == 0);
+	assert(set.poller.active_profiles == 0 && set.snmp.total_snmp_ports == 0);
+	assert(set.poller.poller_interval == 0 && set.poller.num_parent_processes == 0);
+	assert(!set.logging.log_perror && !set.logging.log_pwarn && !set.logging.log_pstats);
+	assert(!set.boost.boost_enabled && !set.boost.boost_redirect);
+	assert(set.availability.ping_timeout == 777);
+	assert(db_insert(mysql, LOCAL, "DELETE FROM settings"));
+	assert(db_insert(mysql, LOCAL, "INSERT INTO settings(name,value) SELECT name,value FROM spine_saved_option_settings"));
+	assert(db_insert(mysql, LOCAL, "UPDATE poller p JOIN spine_saved_option_poller s ON p.id=s.id SET p.threads=s.threads"));
+	assert(db_insert(mysql, LOCAL, "DROP TEMPORARY TABLE spine_saved_option_settings,spine_saved_option_poller"));
+	set = previous;
+	puts("production configuration option bounds and defaults passed");
+}

tests/regression/test_spine_faults.c

@@ -75,6 +75,14 @@ static const char *fake_fail_prefix;
 static int fake_item_rows;
 static char *fake_item_row[21];
 static int fake_queries;
+static int fake_schedule_writes;
+static int fake_item_selects;
+static bool fake_device_row;
+static bool fake_device_row_pending;
+static char *fake_host_row[37];
+static poller_thread_t *fake_completion_peer;
+static void *fake_poll_worker(void *argument);
+static void start_test_worker(pthread_t *worker, void *(*start)(void *), void *argument);
 static int fake_result_storage;
 #define FAKE_RESULT ((MYSQL_RES *) (void *) &fake_result_storage)
 static int interrupted_queries;
@@ -187,7 +195,22 @@ size_t __wrap_strftime(char *output, size_t capacity, const char *format, const
 int __wrap_mysql_query(MYSQL *mysql, const char *query) {
 	if (fake_database) {
 		fake_queries++;
+		if (strncmp(query, "UPDATE poller_item", strlen("UPDATE poller_item")) == 0) fake_schedule_writes++;
+		if (strncmp(query, "SELECT ", 7) == 0 && strstr(query, " FROM poller_item ") != NULL) fake_item_selects++;
+		/* Pause the first partition's error write while another partition
+		 * finishes. Joining keeps this scripted server's state serialized. */
+		if (fake_completion_peer != NULL && strncmp(query, "INSERT INTO host_errors", strlen("INSERT INTO host_errors")) == 0) {
+			poller_thread_t *peer = fake_completion_peer;
+			pthread_t worker;
+			fake_completion_peer = NULL;
+			fake_rows = 0;
+			start_test_worker(&worker, fake_poll_worker, peer);
+			assert(pthread_join(worker, NULL) == 0);
+			assert(fake_schedule_writes == 0 && !details[peer->device_counter]->complete);
+		}
+		fake_device_row_pending = fake_device_row && strncmp(query, "SELECT SQL_NO_CACHE id,", strlen("SELECT SQL_NO_CACHE id,")) == 0;
 		fake_rows_current = fake_rows_prefix != NULL && strncmp(query, fake_rows_prefix, strlen(fake_rows_prefix)) == 0 ? fake_rows_matched : fake_rows;
+		if (fake_device_row_pending) fake_rows_current = 1;
 		if (strncmp(query, "SET SESSION sql_mode", strlen("SET SESSION sql_mode")) == 0) {
 			fake_errno_value = fake_session_mode_fails ? 1146 : 0;
 		} else if (fake_fail_prefix != NULL && strncmp(query, fake_fail_prefix, strlen(fake_fail_prefix)) == 0) {
@@ -286,6 +309,10 @@ my_ulonglong __wrap_mysql_num_rows(MYSQL_RES *result) {
 
 MYSQL_ROW __wrap_mysql_fetch_row(MYSQL_RES *result) {
 	if (result == FAKE_RESULT) {
+		if (fake_device_row_pending) {
+			fake_device_row_pending = FALSE;
+			return fake_host_row;
+		}
 		if (fake_item_rows <= 0) return NULL;
 		fake_item_rows--;
 		return fake_item_row;
@@ -870,6 +897,11 @@ static void fake_script(const int *errors, int count) {
 	fake_fail_connect_number = 0;
 	fake_connects = 0;
 	fake_queries = 0;
+	fake_schedule_writes = 0;
+	fake_item_selects = 0;
+	fake_device_row = FALSE;
+	fake_device_row_pending = FALSE;
+	fake_completion_peer = NULL;
 	fake_session_mode_fails = FALSE;
 	fake_ping_reconnects = FALSE;
 	fake_rows = 0;
@@ -1072,15 +1104,18 @@ static void test_fake_database_poll_host(void) {
 	details = device_list;
 	fake_database = TRUE;
 	set.logging.log_destination = 0;
-	set.poller.threads = 1;
+	set.poller.threads = 2;
 	set.poller.poller_id = 1;
 	set.poller.active_profiles = 2;
 	set.poller.poller_interval = 5;
 	set.availability.ping_only = FALSE;
-	db_pool_local = calloc(1, sizeof(*db_pool_local));
+	db_pool_local = calloc(2, sizeof(*db_pool_local));
 	assert(db_pool_local != NULL);
 	fake_script(NULL, 0);
 	assert(db_connect(LOCAL, &db_pool_local[0].mysql));
+	assert(db_connect(LOCAL, &db_pool_local[1].mysql));
+	db_pool_local[1].id = 1;
+	db_pool_local[1].free = FALSE;
 	db_pool_local[0].free = FALSE;
 	run_fake_poll(&device, 0);
 	assert(device.poll_failed && !device.complete && set.exit.exit_code == EXIT_FAILURE);
@@ -1106,6 +1141,17 @@ static void test_fake_database_poll_host(void) {
 	assert_fake_poll_failed(&device);
 	assert(fake_queries == 1);
 
+	/* A failed reindex read stops collection and keeps the device due. */
+	fake_script(NULL, 0);
+	fake_device_row = TRUE;
+	fake_host_row[0] = "902";
+	fake_host_row[3] = "0";
+	fake_host_row[14] = "0"; /* AVAIL_NONE: no network access. */
+	fake_fail_prefix = "SELECT SQL_NO_CACHE data_query_id";
+	run_fake_poll(&device, 902);
+	assert_fake_poll_failed(&device);
+	assert(fake_item_selects == 0 && fake_schedule_writes == 0);
+
 	/* A device deleted since selection returns quietly, uncounted. */
 	fake_script(NULL, 0);
 	memset(&device, 0, sizeof(device));
@@ -1153,9 +1199,39 @@ static void test_fake_database_poll_host(void) {
 		fprintf(stderr, "completion write '%s' failing: complete=%d exit=%d queries=%d\n",
 			completion_writes[failing], device.complete, set.exit.exit_code, fake_queries);
 		assert(!device.complete && set.exit.exit_code == EXIT_FAILURE && db_pool_local[0].free);
+		assert(fake_schedule_writes == (failing == 0 ? 1 : 0));
 	}
+	db_pool_local[1].free = TRUE;
+	/* The final partition must wait for the first partition's pending
+	 * bookkeeping outcome before it can advance the shared schedule. */
+	for (int fail_write = 0; fail_write < 2; fail_write++) {
+		poller_thread_t first = {0};
+		poller_thread_t second = {0};
+		pthread_t worker;
+		fake_script(NULL, 0);
+		fake_rows = 1;
+		fake_item_rows = 1;
+		fake_fail_prefix = fail_write ? "INSERT INTO host_errors" : NULL;
+		memset(&device, 0, sizeof(device));
+		device.host_threads = 2;
+		first.host_thread = 1;
+		second.host_thread = 2;
+		STRNCOPY(first.host_time, "1791244800");
+		STRNCOPY(second.host_time, "1791244800");
+		set.exit.exit_code = EXIT_SUCCESS;
+		fake_completion_peer = &second;
+		start_test_worker(&worker, fake_poll_worker, &first);
+		assert(pthread_join(worker, NULL) == 0);
+		assert(fake_completion_peer == NULL && device.threads_complete == 2);
+		assert(device.complete == !fail_write && device.output_failed == fail_write);
+		assert(fake_schedule_writes == !fail_write);
+		assert(set.exit.exit_code == (fail_write ? EXIT_FAILURE : EXIT_SUCCESS));
+		assert(db_pool_local[0].free && db_pool_local[1].free);
+	}
+
 	assert(spine_permits_destroy(&available_scripts) == 0);
 
+	db_disconnect(&db_pool_local[1].mysql);
 	db_disconnect(&db_pool_local[0].mysql);
 	free(db_pool_local);
 	fake_database = FALSE;