perf: stream the slow query log during import to fix the 10GB memory footprint

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 --- 2.6 ---
 
+* perf: Stream the slow query log a line at a time during import (`fgets()`) instead of slurping the whole file into memory with `file()`, so importing a multi-gigabyte slow query log no longer costs many times its own size in PHP array overhead - the detail rows were already flushed to the database in `SLOWLOG_IMPORT_BATCH_SIZE` batches as parsing proceeds
+* bug: Distinguish a read error from end-of-file when streaming the import - `fgets()` returns `false` for both, so a mid-file I/O error was being treated as a clean EOF and the partial import was finalized (and post-processed) as successful. `import_logfile()` now checks `feof()` before closing the handle and fails the import (status 3, "Read Error - Import Aborted, Log May Be Truncated") instead of ingesting a truncated log
 * refactor: Move the slowlog_functions.php library file into includes/ and switch every file inclusion from include/include_once to require/require_once for fail-fast consistency (references updated across setup.php, slowlog.php, import_log.php, and the test suite)
 * refactor: Move all schema management (table definitions, create, upgrade, and drop helpers) into includes/database.php (the thold model) with the Cacti copyright header; setup.php's install/uninstall/upgrade paths now require that file and delegate to it, keeping the api_plugin_db_table_create() create path and the db_update_table() upgrade refresh unchanged
 * dev: Enforce patch coverage of changed lines in CI and remove the inert COMPOSER_ROOT_VERSION env from the Pest step

includes/slowlog_functions.php

@@ -910,8 +910,11 @@ function import_logfile(string $logfile, string $description = 'Imported using i
 	$logfile = trim($logfile);
 
 	if (file_exists($logfile)) {
-		// suck the log through a straw
-		$entries    = file($logfile) ?: [];
+		// Stream the log a line at a time rather than slurping the whole file into
+		// memory with file() - a multi-gigabyte slow query log would otherwise cost
+		// many times its own size in PHP array overhead (the detail rows are already
+		// flushed to the database in SLOWLOG_IMPORT_BATCH_SIZE batches as we go).
+		$fh = fopen($logfile, 'r');
 
 		// denotes that the log beginning has been found
 		$start      = false;
@@ -920,7 +923,7 @@ function import_logfile(string $logfile, string $description = 'Imported using i
 		$records    = [];
 		$sql_prefix = 'INSERT INTO plugin_slowlog_details (logid, date, user, host, ip_address, query_time, lock_time, thread_id, `schema`, qc_hit, rows_sent, rows_examined, rows_affected, bytes_sent, oquery, query) VALUES ';
 
-		if (cacti_sizeof($entries)) {
+		if ($fh !== false) {
 			// variables related to each slowlog entry
 			$date          = 0;
 			$user          = '';
@@ -942,7 +945,7 @@ function import_logfile(string $logfile, string $description = 'Imported using i
 			$lines         = 0;
 			$query_start   = false;
 
-			foreach ($entries as $l) {
+			while (($l = fgets($fh)) !== false) {
 				if ($start && substr($l, 0, 1) != '#') {
 					$query_start = true;
 				}
@@ -1114,6 +1117,29 @@ function import_logfile(string $logfile, string $description = 'Imported using i
 				}
 			}
 
+			// fgets() returns false on both EOF and a read error, so a mid-file I/O
+			// failure would otherwise look like a clean end-of-log and get finalized as a
+			// successful (but partial) import. Check feof() before closing: if the loop
+			// stopped on a read error, fail the import instead of ingesting and
+			// post-processing a truncated log.
+			$read_failed = !feof($fh);
+
+			fclose($fh);
+
+			if ($read_failed) {
+				if ($logid !== null) {
+					db_execute_prepared('UPDATE plugin_slowlog
+						SET import_status = 3,
+						import_text_status = ?
+						WHERE logid = ?',
+						[__('Read Error - Import Aborted, Log May Be Truncated', 'slowlog'), $logid]);
+				} else {
+					print "FATAL: Read error while importing '$logfile' - aborting before end of file\n";
+				}
+
+				return;
+			}
+
 			if ($query != '') {
 				if ($length != -1) {
 					$query = substr($query, 0, $length);

locales/po/cacti.pot

@@ -22,6 +22,10 @@ msgstr ""
 msgid "ERROR: Invalid --logid value '%s', must be a positive integer"
 msgstr ""
 
+#: includes/slowlog_functions.php
+msgid "Read Error - Import Aborted, Log May Be Truncated"
+msgstr ""
+
 #: includes/slowlog_functions.php
 msgid "Bad File Format - No Slow Query Log Entries Found"
 msgstr ""

tests/Integration/ImportLogfileReadErrorTest.php

@@ -0,0 +1,118 @@
+<?php
+/*
+ +-------------------------------------------------------------------------+
+ | Copyright (C) 2004-2026 The Cacti Group                                 |
+ |                                                                         |
+ | This program is free software; you can redistribute it and/or           |
+ | modify it under the terms of the GNU General Public License             |
+ | as published by the Free Software Foundation; either version 2          |
+ | of the License, or (at your option) any later version.                  |
+ |                                                                         |
+ | This program is distributed in the hope that it will be useful,         |
+ | but WITHOUT ANY WARRANTY; without even the implied warranty of          |
+ | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           |
+ | GNU General Public License for more details.                            |
+ +-------------------------------------------------------------------------+
+ | Cacti: The Complete RRDtool-based Graphing Solution                     |
+ +-------------------------------------------------------------------------+
+ | http://www.cacti.net/                                                   |
+ +-------------------------------------------------------------------------+
+*/
+
+/*
+ * Covers import_logfile()'s read-error handling. Streaming the log with
+ * fgets() can't tell EOF from an I/O error - both return false - so a
+ * mid-file read failure must not be mistaken for a clean end-of-log and
+ * finalized as a successful (but truncated) import.
+ *
+ * SlowlogReadErrorStream simulates exactly that: fopen() succeeds, the first
+ * fgets() returns false, and feof() stays false (as it would on a real read
+ * error), so import_logfile() must take its failure path.
+ */
+
+uses(TestCase::class);
+
+if (!class_exists('SlowlogReadErrorStream')) {
+	class SlowlogReadErrorStream {
+		/** @var resource|null */
+		public $context;
+
+		public function stream_open($path, $mode, $options, &$opened_path) {
+			return true;
+		}
+
+		public function stream_read($count) {
+			// A failed read, not EOF: false (not '') keeps feof() false.
+			return false;
+		}
+
+		public function stream_eof() {
+			return false;
+		}
+
+		public function stream_stat() {
+			return array('size' => 1);
+		}
+
+		public function url_stat($path, $flags) {
+			return array('size' => 1);
+		}
+
+		public function stream_close() {
+		}
+	}
+}
+
+beforeEach(function () {
+	TestCase::loadPluginSource('includes/slowlog_functions.php');
+
+	if (in_array('slowlogfail', stream_get_wrappers(), true)) {
+		stream_wrapper_unregister('slowlogfail');
+	}
+
+	stream_wrapper_register('slowlogfail', 'SlowlogReadErrorStream');
+});
+
+afterEach(function () {
+	if (in_array('slowlogfail', stream_get_wrappers(), true)) {
+		stream_wrapper_unregister('slowlogfail');
+	}
+});
+
+it('marks the parent record as failed (status 3) when the log read errors mid-stream', function () {
+	// 4242 is a pre-created parent logid, as the web upload handler supplies.
+	import_logfile('slowlogfail://log', 'test', -1, '', false, false, null, 4242);
+
+	$failed = null;
+
+	foreach ($GLOBALS['__test_db_calls'] as $call) {
+		if ($call['fn'] === 'db_execute_prepared' && strpos($call['sql'], 'import_status = 3') !== false) {
+			$failed = $call;
+		}
+	}
+
+	expect($failed)->not->toBeNull();
+	expect($failed['params'])->toContain(4242);
+});
+
+it('does not finalize the import as successful (status 1) when the log read errors mid-stream', function () {
+	import_logfile('slowlogfail://log', 'test', -1, '', false, false, null, 4242);
+
+	foreach ($GLOBALS['__test_db_calls'] as $call) {
+		expect($call['sql'])->not->toContain('import_status = 1');
+	}
+});
+
+it('prints a FATAL read error for a CLI import with no pre-created parent record', function () {
+	ob_start();
+	import_logfile('slowlogfail://log', 'test', -1, '', false, false, null, null);
+	$output = ob_get_clean();
+
+	expect($output)->toContain('FATAL: Read error');
+
+	// With no parent logid there is nothing to mark as failed - it must not
+	// have recorded a status-3 update either.
+	foreach ($GLOBALS['__test_db_calls'] as $call) {
+		expect($call['sql'])->not->toContain('import_status = 3');
+	}
+});