From 87ae07aa169ab7a1e4b9b610d0b5fc324d62977f Mon Sep 17 00:00:00 2001 From: Andi Wieser Date: Sat, 22 Aug 2026 01:30:09 +0200 Subject: [PATCH] fix: keep the mutant --filter under the kernel's per-argument limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutant's covering tests are emitted one regex fragment per test and joined into a SINGLE `--filter=` argv element. That element is unbounded, and on a class most of a suite reaches it runs past MAX_ARG_STRLEN — PAGE_SIZE * 32, 131072 bytes on x86_64, a kernel constant `ulimit` does not move. The child then dies with `posix_spawn() failed: Argument list too long` before running a single test, so every mutation in that class is lost. Measured at 172,779 bytes for one mutation in a class with 2,076 covering tests. macOS has no per-argument cap, so it reproduces only on Linux — in practice on CI, on the run whose score people publish. Support/FilterArgument applies two encodings in order: 1. Factor the class prefix: `Cls::(.*)a|Cls::(.*)b` becomes `Cls::(.*)(a|b)`. Lossless — both forms select exactly the same tests — and 34% shorter on a four-class set. The inner parentheses are load-bearing: without them the alternation binds to the whole pattern rather than the tail, and the filter selects tests it was never given. 2. If it still does not fit, collapse a class to its bare `Cls::` prefix, heaviest first, stopping the moment it fits. That is a strict superset of the original selection, so it can only run MORE tests and can never turn a killed mutant into a survivor. A collapse is never silent: `widened` names each class and how many extra tests it now selects, and MutationTest reports it once per process rather than once per mutation. A filter that quietly widened would make the score certify more than the run measured. If even a fully collapsed filter does not fit it throws, because dropping the filter would run the whole suite per mutant and read as a fast green. Closes #1771 (pestphp/pest). --- src/MutationTest.php | 27 +++- src/Support/FilterArgument.php | 184 ++++++++++++++++++++++ tests/Unit/Support/FilterArgumentTest.php | 170 ++++++++++++++++++++ 3 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 src/Support/FilterArgument.php create mode 100644 tests/Unit/Support/FilterArgumentTest.php diff --git a/src/MutationTest.php b/src/MutationTest.php index 7bf0918..57677fd 100644 --- a/src/MutationTest.php +++ b/src/MutationTest.php @@ -9,6 +9,7 @@ use Pest\Mutate\Plugins\Mutate; use Pest\Mutate\Repositories\TelemetryRepository; use Pest\Mutate\Support\Configuration\Configuration; +use Pest\Mutate\Support\FilterArgument; use Pest\Mutate\Support\MutationTestResult; use Pest\Support\Container; use Symfony\Component\Process\Exception\ProcessTimedOutException; @@ -18,6 +19,14 @@ class MutationTest { private MutationTestResult $result = MutationTestResult::None; + /** + * Widenings already reported, so a class that had to collapse says so once per + * process rather than once per mutation in it. + * + * @var array + */ + private static array $reportedWidenings = []; + private ?float $start = null; private ?float $finish = null; @@ -87,12 +96,28 @@ public function start(array $coveredLines, Configuration $configuration, array $ // remove coverage arguments from the original arguments $filteredArguments = array_filter($originalArguments, fn (string $argument): bool => ! str_starts_with($argument, '--coverage')); + // A fragment per covering test, joined into ONE argv element, runs past the + // kernel's per-element cap on a widely-covered class — see FilterArgument. + $filter = FilterArgument::for(array_values($filters)); + + $notice = $filter->notice(); + + if ($notice !== null) { + $key = implode(',', array_keys($filter->widened)); + + if (! isset(self::$reportedWidenings[$key])) { + self::$reportedWidenings[$key] = true; + + fwrite(STDERR, $notice.PHP_EOL); + } + } + // TODO: filter arguments to remove unnecessary stuff (Teamcity, Coverage, etc.) $process = new Process( command: [ ...$filteredArguments, '--bail', - '--filter="'.implode('|', $filters).'"', + $filter->argument, ], env: $envs, timeout: $this->calculateTimeout(), diff --git a/src/Support/FilterArgument.php b/src/Support/FilterArgument.php new file mode 100644 index 0000000..794d466 --- /dev/null +++ b/src/Support/FilterArgument.php @@ -0,0 +1,184 @@ + $widened class => how many extra tests it now selects + */ + private function __construct( + public string $argument, + public array $widened, + ) {} + + /** + * @param list $filters one regex fragment per covering test + * + * @throws RuntimeException when even a fully collapsed filter does not fit + */ + public static function for(array $filters, int $budget = self::BUDGET_BYTES): self + { + /** @var array> $byClass */ + $byClass = []; + + foreach ($filters as $filter) { + // A fragment whose shape is not recognised is kept verbatim under a reserved + // key. If the shape upstream builds ever changes, this degrades to today's + // behaviour rather than silently dropping a selection. + if (preg_match('/^([A-Za-z0-9_]+)::\(\.\*\)(.*)$/s', $filter, $matches) === 1) { + $byClass[$matches[1]][] = $matches[2]; + + continue; + } + + $byClass["\0raw"][] = $filter; + } + + /** @var array $collapsed */ + $collapsed = []; + + $render = static function () use (&$byClass, &$collapsed): string { + $parts = []; + + /** + * @var string $class + * @var list $tails + */ + foreach ($byClass as $class => $tails) { + if ($class === "\0raw") { + foreach ($tails as $tail) { + $parts[] = $tail; + } + + continue; + } + + if (isset($collapsed[$class])) { + $parts[] = $class.'::'; + + continue; + } + + $parts[] = $class.'::(.*)('.implode('|', array_unique($tails)).')'; + } + + return '--filter="'.implode('|', $parts).'"'; + }; + + $argument = $render(); + + if (strlen($argument) <= $budget) { + return new self($argument, []); + } + + // Heaviest class first, so the fewest collapses buy the most room. + /** @var array $weights */ + $weights = []; + + foreach ($byClass as $class => $tails) { + if ($class !== "\0raw") { + $weights[$class] = strlen(implode('|', array_unique($tails))); + } + } + + arsort($weights); + + /** @var array $widened */ + $widened = []; + + foreach (array_keys($weights) as $class) { + $collapsed[$class] = true; + $widened[$class] = count(array_unique($byClass[$class])); + $argument = $render(); + + if (strlen($argument) <= $budget) { + break; + } + } + + if (strlen($argument) > $budget) { + // Dropping the filter entirely would run the whole suite per mutant and read + // as a fast green, so this fails loudly instead. + throw new RuntimeException(sprintf( + 'The mutation filter is %d bytes with every class collapsed to its prefix, over the %d-byte budget.', + strlen($argument), + $budget, + )); + } + + return new self($argument, $widened); + } + + /** + * A human-readable note for a run whose filter had to widen, or null when none did. + */ + public function notice(): ?string + { + if ($this->widened === []) { + return null; + } + + return sprintf( + 'Mutation filter collapsed %d class(es) to a bare prefix to fit within %d bytes: %s. '. + 'This selects a superset of the covering tests, so it cannot fabricate a kill — '. + 'provided the ordinary suite is green before mutation starts.', + count($this->widened), + strlen($this->argument), + implode(', ', array_map( + static fn (string $class, int $count): string => $class.' (+'.$count.' tests)', + array_keys($this->widened), + array_values($this->widened), + )), + ); + } +} diff --git a/tests/Unit/Support/FilterArgumentTest.php b/tests/Unit/Support/FilterArgumentTest.php new file mode 100644 index 0000000..32507fd --- /dev/null +++ b/tests/Unit/Support/FilterArgumentTest.php @@ -0,0 +1,170 @@ +argument)->toBe('--filter="SizeHelperTest::(.*)(it_is_small|it_is_large)"') + ->and($filter->widened)->toBe([]); +}); + +it('factors losslessly — the factored form selects exactly what the naive form selected', function (): void { + $fragments = [ + 'SizeHelperTest::(.*)it_is_small', + 'SizeHelperTest::(.*)it_is_large', + 'AgeHelperTest::(.*)it_is_adult', + ]; + + $names = [ + 'Tests\Unit\SizeHelperTest::it_is_small', + 'Tests\Unit\SizeHelperTest::it_is_large', + 'Tests\Unit\AgeHelperTest::it_is_adult', + 'Tests\Unit\SizeHelperTest::it_is_medium', // never selected by either form + 'Tests\Unit\OtherTest::it_is_small', // wrong class, same tail + ]; + + $naive = '--filter="'.implode('|', $fragments).'"'; + $factored = FilterArgument::for($fragments)->argument; + + $before = array_values(array_filter($names, fn (string $n): bool => selects($naive, $n))); + $after = array_values(array_filter($names, fn (string $n): bool => selects($factored, $n))); + + expect($after)->toBe($before)->toHaveCount(3); +}); + +it('keeps the inner parentheses, without which the alternation would bind to the whole pattern', function (): void { + // ⚠️ THE ONE DETAIL THAT MAKES FACTORING SAFE. `Cls::(.*)a|b` is `(Cls::(.*)a)|(b)`, + // so a bare `b` anywhere matches and the filter runs tests it was never given. + $broken = '--filter="SizeHelperTest::(.*)it_is_small|it_is_large"'; + $correct = FilterArgument::for([ + 'SizeHelperTest::(.*)it_is_small', + 'SizeHelperTest::(.*)it_is_large', + ])->argument; + + expect(selects($broken, 'Tests\Unit\OtherTest::it_is_large'))->toBeTrue() + ->and(selects($correct, 'Tests\Unit\OtherTest::it_is_large'))->toBeFalse(); +}); + +it('leaves a fragment it does not recognise exactly as it found it', function (): void { + $filter = FilterArgument::for([ + 'SizeHelperTest::(.*)it_is_small', + 'something upstream started emitting', + ]); + + expect($filter->argument)->toContain('something upstream started emitting'); +}); + +it('de-duplicates repeated tails rather than repeating them', function (): void { + $filter = FilterArgument::for([ + 'SizeHelperTest::(.*)it_is_small', + 'SizeHelperTest::(.*)it_is_small', + ]); + + expect($filter->argument)->toBe('--filter="SizeHelperTest::(.*)(it_is_small)"'); +}); + +it('stays under the kernel cap on a filter that would otherwise exceed it', function (): void { + // The measured shape of the defect: one class reached by a very large number of tests. + // 2,076 covering tests produced a 172,779-byte argument against a 131,072-byte cap. + $fragments = []; + + for ($i = 0; $i < 3000; $i++) { + $fragments[] = 'ComponentRenderTest::(.*)it_renders_the_component_variant_number_'.$i; + } + + $naive = '--filter="'.implode('|', $fragments).'"'; + + expect(strlen($naive))->toBeGreaterThan(131072); + + $filter = FilterArgument::for($fragments); + + expect(strlen($filter->argument))->toBeLessThanOrEqual(FilterArgument::BUDGET_BYTES); +}); + +it('collapses the heaviest class first, so the fewest collapses buy the most room', function (): void { + $fragments = ['SmallTest::(.*)it_does_one_thing']; + + for ($i = 0; $i < 4000; $i++) { + $fragments[] = 'HugeTest::(.*)it_renders_a_very_long_test_name_number_'.$i; + } + + $filter = FilterArgument::for($fragments); + + expect(array_keys($filter->widened))->toBe(['HugeTest']) + ->and($filter->argument)->toContain('SmallTest::(.*)(it_does_one_thing)') + ->and($filter->argument)->toContain('HugeTest::'); +}); + +it('widens rather than narrows — every originally-selected test still matches', function (): void { + $fragments = []; + + for ($i = 0; $i < 4000; $i++) { + $fragments[] = 'HugeTest::(.*)it_renders_a_very_long_test_name_number_'.$i; + } + + $filter = FilterArgument::for($fragments); + + expect($filter->widened)->not->toBe([]); + + foreach ([0, 1999, 3999] as $i) { + expect(selects($filter->argument, 'Tests\Unit\HugeTest::it_renders_a_very_long_test_name_number_'.$i)) + ->toBeTrue(); + } +}); + +it('reports what it widened instead of widening silently', function (): void { + $fragments = []; + + for ($i = 0; $i < 4000; $i++) { + $fragments[] = 'HugeTest::(.*)it_renders_a_very_long_test_name_number_'.$i; + } + + $filter = FilterArgument::for($fragments); + + expect($filter->widened)->toBe(['HugeTest' => 4000]) + ->and($filter->notice())->toContain('HugeTest (+4000 tests)') + ->and($filter->notice())->toContain('superset'); +}); + +it('says nothing when nothing was widened', function (): void { + expect(FilterArgument::for(['SizeHelperTest::(.*)it_is_small'])->notice())->toBeNull(); +}); + +it('fails loudly rather than returning a filter that still does not fit', function (): void { + // Dropping the filter would run the whole suite per mutant and read as a fast green, + // so the one thing this must never do is return something over budget. + $fragments = []; + + for ($i = 0; $i < 200; $i++) { + $fragments[] = 'Test'.str_repeat('X', 400).$i.'::(.*)it_does_something'; + } + + expect(fn (): FilterArgument => FilterArgument::for($fragments, 1000)) + ->toThrow(RuntimeException::class, 'over the 1000-byte budget'); +}); + +it('handles an empty selection without inventing one', function (): void { + expect(FilterArgument::for([])->argument)->toBe('--filter=""'); +});