diff --git a/src/main/java/com/williamcallahan/javachat/cli/DocumentProcessor.java b/src/main/java/com/williamcallahan/javachat/cli/DocumentProcessor.java index f3cbd0d6..112389fd 100644 --- a/src/main/java/com/williamcallahan/javachat/cli/DocumentProcessor.java +++ b/src/main/java/com/williamcallahan/javachat/cli/DocumentProcessor.java @@ -59,6 +59,7 @@ public class DocumentProcessor { private static final String LOG_PROCESSING_SET = "Processing documentation set"; private static final String LOG_FILES_TO_PROCESS = "Files to process: {}"; private static final String LOG_DUPLICATES_SKIPPED = " Skipped {} duplicate files (already in Qdrant)"; + private static final String LOG_EXCLUDED_SKIPPED = " Excluded {} files (intentionally not indexed)"; private static final String LOG_SKIP_PATH_ESCAPE = "Skipping documentation set (path escaped base directory)"; private static final String LOG_SKIP_DIR_NOT_FOUND = "Skipping documentation set (directory not found)"; private static final String LOG_SKIP_NO_ELIGIBLE = "Skipping documentation set (no eligible files)"; @@ -68,6 +69,7 @@ public class DocumentProcessor { private static final String LOG_PROCESSED_STATS = "Processed {} files in {}s ({} files/sec) ({})"; private static final String LOG_TOTAL_PROCESSED = "Total new documents processed: {}"; private static final String LOG_TOTAL_DUPLICATES = "Total duplicates skipped: {}"; + private static final String LOG_TOTAL_EXCLUDED = "Total excluded files: {}"; private static final String LOG_TOTAL_FAILED = "Documentation sets FAILED: {}"; private static final String LOG_NEXT_STEP_QDRANT = "1. Verify in Qdrant Dashboard"; private static final String LOG_NEXT_STEP_RETRIEVAL = "2. Test retrieval"; @@ -194,8 +196,10 @@ void processDocumentationSets(final Path basePath, final List private IngestionTotals accumulateOutcome(final IngestionTotals totals, final ProcessingOutcome outcome) { return switch (outcome) { - case ProcessingOutcome.Success success -> totals.addSuccess(success.processed(), success.duplicates()); - case ProcessingOutcome.Failed failed -> totals.addFailed(failed.processed(), failed.duplicates()); + case ProcessingOutcome.Success success -> + totals.addSuccess(success.processed(), success.duplicates(), success.excluded()); + case ProcessingOutcome.Failed failed -> + totals.addFailed(failed.processed(), failed.duplicates(), failed.excluded()); }; } @@ -206,14 +210,14 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc if (LOGGER.isWarnEnabled()) { LOGGER.warn(LOG_SKIP_PATH_ESCAPE); } - return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0); + return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0); } if (!Files.exists(docsPath) || !Files.isDirectory(docsPath)) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(LOG_SKIP_DIR_NOT_FOUND); } - return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0); + return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0); } try { @@ -222,7 +226,7 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc if (LOGGER.isDebugEnabled()) { LOGGER.debug(LOG_SKIP_NO_ELIGIBLE); } - return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0); + return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0); } if (LOGGER.isInfoEnabled()) { @@ -240,11 +244,17 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc final int failureCount = outcome.backlog().failedFiles(); final long duplicates = outcome.backlog().skippedFiles(); + final long excluded = outcome.backlog().excludedFiles(); if (duplicates > 0) { if (LOGGER.isInfoEnabled()) { LOGGER.info(LOG_DUPLICATES_SKIPPED, duplicates); } } + if (excluded > 0) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(LOG_EXCLUDED_SKIPPED, excluded); + } + } if (outcome.backlog().pendingFiles() > 0 && LOGGER.isWarnEnabled()) { LOGGER.warn( "Ingestion stopped with {} pending file(s)", @@ -255,15 +265,15 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc if (LOGGER.isWarnEnabled()) { LOGGER.warn("Ingestion completed with {} file failures", failureCount); } - return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates); + return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates, excluded); } if (outcome.backlog().pendingFiles() > 0) { - return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates); + return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates, excluded); } String safeDocumentationSet = docSet.indexedDocSet().replace('\r', '?').replace('\n', '?'); LOGGER.info(LOG_DOCSET_POSTCONDITION, safeDocumentationSet); - return new ProcessingOutcome.Success(processed, duplicates); + return new ProcessingOutcome.Success(processed, duplicates, excluded); } catch (IOException | UncheckedIOException | IngestionAlreadyRunningException processingFailure) { if (LOGGER.isErrorEnabled()) { @@ -272,7 +282,7 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc if (LOGGER.isDebugEnabled()) { LOGGER.debug(LOG_STACK_TRACE, processingFailure); } - return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0); + return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0); } } @@ -454,6 +464,7 @@ private void logTotals(final String title, final IngestionTotals totals) { LOGGER.info(LOG_BANNER_LINE); LOGGER.info(LOG_TOTAL_PROCESSED, totals.processed()); LOGGER.info(LOG_TOTAL_DUPLICATES, totals.duplicates()); + LOGGER.info(LOG_TOTAL_EXCLUDED, totals.excluded()); } } @@ -503,15 +514,17 @@ private static boolean envBooleanOrDefault(final String key, final boolean fallb /** * Accumulated totals across all documentation sets, tracking successes and failures separately. */ - private record IngestionTotals(long processed, long duplicates, int failedSets) { - static final IngestionTotals ZERO = new IngestionTotals(0, 0, 0); + private record IngestionTotals(long processed, long duplicates, long excluded, int failedSets) { + static final IngestionTotals ZERO = new IngestionTotals(0, 0, 0, 0); - IngestionTotals addSuccess(final long newProcessed, final long newDuplicates) { - return new IngestionTotals(processed + newProcessed, duplicates + newDuplicates, failedSets); + IngestionTotals addSuccess(final long newProcessed, final long newDuplicates, final long newExcluded) { + return new IngestionTotals( + processed + newProcessed, duplicates + newDuplicates, excluded + newExcluded, failedSets); } - IngestionTotals addFailed(final long newProcessed, final long newDuplicates) { - return new IngestionTotals(processed + newProcessed, duplicates + newDuplicates, failedSets + 1); + IngestionTotals addFailed(final long newProcessed, final long newDuplicates, final long newExcluded) { + return new IngestionTotals( + processed + newProcessed, duplicates + newDuplicates, excluded + newExcluded, failedSets + 1); } } @@ -519,9 +532,9 @@ IngestionTotals addFailed(final long newProcessed, final long newDuplicates) { * Outcome of processing a single documentation set - distinguishes success, skip, and failure. */ private sealed interface ProcessingOutcome { - record Success(long processed, long duplicates) implements ProcessingOutcome {} + record Success(long processed, long duplicates, long excluded) implements ProcessingOutcome {} - record Failed(String setName, long processed, long duplicates) implements ProcessingOutcome {} + record Failed(String setName, long processed, long duplicates, long excluded) implements ProcessingOutcome {} } /** diff --git a/src/main/java/com/williamcallahan/javachat/cli/GitHubRepoProcessor.java b/src/main/java/com/williamcallahan/javachat/cli/GitHubRepoProcessor.java index 9f1e3835..2d04c39a 100644 --- a/src/main/java/com/williamcallahan/javachat/cli/GitHubRepoProcessor.java +++ b/src/main/java/com/williamcallahan/javachat/cli/GitHubRepoProcessor.java @@ -190,7 +190,7 @@ private IngestionWalkSummary walkAndProcess( activeFileUrls.add(fileProcessingOutcome.fileUrl()); switch (fileProcessingOutcome.outcome()) { case LocalDocsFileOutcome.Processed _ -> processedCount++; - case LocalDocsFileOutcome.Skipped _ -> skippedCount++; + case LocalDocsFileOutcome.Skipped _, LocalDocsFileOutcome.Excluded _ -> skippedCount++; case LocalDocsFileOutcome.Failed failed -> { failedCount++; failed.failure() diff --git a/src/main/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatus.java b/src/main/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatus.java index 4c4c4e77..ce8c2a8d 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatus.java +++ b/src/main/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatus.java @@ -9,7 +9,8 @@ * @param eligibleFiles ingestable files discovered beneath the selected directory * @param inspectedFiles files that reached a terminal per-file outcome * @param processedFiles files that produced newly indexed chunks - * @param skippedFiles files already indexed or intentionally excluded + * @param skippedFiles files already indexed (unchanged duplicates that retain their Qdrant points) + * @param excludedFiles files intentionally excluded from indexing (held zero Qdrant points) * @param failedFiles files that reached a typed failure * @param pendingFiles files still waiting to be inspected * @param inProgressFiles files currently owned by the active ingestion batch @@ -21,6 +22,7 @@ public record IngestionBacklogStatus( int inspectedFiles, int processedFiles, int skippedFiles, + int excludedFiles, int failedFiles, int pendingFiles, int inProgressFiles, @@ -47,11 +49,13 @@ public enum Lifecycle { requireNonNegative("inspectedFiles", inspectedFiles); requireNonNegative("processedFiles", processedFiles); requireNonNegative("skippedFiles", skippedFiles); + requireNonNegative("excludedFiles", excludedFiles); requireNonNegative("failedFiles", failedFiles); requireNonNegative("pendingFiles", pendingFiles); requireNonNegative("inProgressFiles", inProgressFiles); - if (inspectedFiles != processedFiles + skippedFiles + failedFiles) { - throw new IllegalArgumentException("Inspected files must equal processed, skipped, and failed files"); + if (inspectedFiles != processedFiles + skippedFiles + excludedFiles + failedFiles) { + throw new IllegalArgumentException( + "Inspected files must equal processed, skipped, excluded, and failed files"); } if (eligibleFiles != inspectedFiles + pendingFiles + inProgressFiles) { throw new IllegalArgumentException("Eligible files must equal inspected, pending, and in-progress files"); @@ -67,14 +71,14 @@ public enum Lifecycle { */ public static IngestionBacklogStatus notStarted(String directory, int eligibleFiles) { return new IngestionBacklogStatus( - Lifecycle.NOT_STARTED, eligibleFiles, 0, 0, 0, 0, eligibleFiles, 0, directory); + Lifecycle.NOT_STARTED, eligibleFiles, 0, 0, 0, 0, 0, eligibleFiles, 0, directory); } /** * Creates the initial durable state after a process claims the directory. */ public static IngestionBacklogStatus running(String directory, int eligibleFiles) { - return new IngestionBacklogStatus(Lifecycle.RUNNING, eligibleFiles, 0, 0, 0, 0, eligibleFiles, 0, directory); + return new IngestionBacklogStatus(Lifecycle.RUNNING, eligibleFiles, 0, 0, 0, 0, 0, eligibleFiles, 0, directory); } /** @@ -96,6 +100,7 @@ public IngestionBacklogStatus startBatch(int batchFileCount) { inspectedFiles, processedFiles, skippedFiles, + excludedFiles, failedFiles, pendingFiles - batchFileCount, batchFileCount, @@ -105,11 +110,13 @@ public IngestionBacklogStatus startBatch(int batchFileCount) { /** * Records all terminal outcomes returned by the active batch. */ - public IngestionBacklogStatus completeBatch(int processedCount, int skippedCount, int failedCount) { + public IngestionBacklogStatus completeBatch( + int processedCount, int skippedCount, int excludedCount, int failedCount) { requireNonNegative("processedCount", processedCount); requireNonNegative("skippedCount", skippedCount); + requireNonNegative("excludedCount", excludedCount); requireNonNegative("failedCount", failedCount); - int terminalOutcomeCount = processedCount + skippedCount + failedCount; + int terminalOutcomeCount = processedCount + skippedCount + excludedCount + failedCount; if (terminalOutcomeCount > inProgressFiles) { throw new IllegalArgumentException("Batch outcomes exceed the in-progress file count"); } @@ -120,6 +127,7 @@ public IngestionBacklogStatus completeBatch(int processedCount, int skippedCount inspectedFiles + terminalOutcomeCount, processedFiles + processedCount, skippedFiles + skippedCount, + excludedFiles + excludedCount, failedFiles + failedCount, pendingFiles + unattemptedCount, 0, @@ -140,6 +148,7 @@ public IngestionBacklogStatus finish() { inspectedFiles, processedFiles, skippedFiles, + excludedFiles, failedFiles, pendingFiles, 0, @@ -161,15 +170,16 @@ public IngestionBacklogStatus abandon() { } if (failedFiles > 0) { return new IngestionBacklogStatus( - Lifecycle.PARTIAL, eligibleFiles, 0, 0, 0, 0, eligibleFiles, 0, directory); + Lifecycle.PARTIAL, eligibleFiles, 0, 0, 0, 0, 0, eligibleFiles, 0, directory); } - int terminalSuccessCount = processedFiles + skippedFiles; + int terminalSuccessCount = processedFiles + skippedFiles + excludedFiles; return new IngestionBacklogStatus( Lifecycle.PARTIAL, eligibleFiles, terminalSuccessCount, processedFiles, skippedFiles, + excludedFiles, 0, eligibleFiles - terminalSuccessCount, 0, @@ -194,6 +204,7 @@ public IngestionBacklogStatus resume() { abandonedBacklog.inspectedFiles, abandonedBacklog.processedFiles, abandonedBacklog.skippedFiles, + abandonedBacklog.excludedFiles, 0, abandonedBacklog.pendingFiles, 0, diff --git a/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionService.java b/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionService.java index 3459aa1b..3aa5d9ca 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionService.java +++ b/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionService.java @@ -105,6 +105,7 @@ public IngestionLocalOutcome ingestLocalDirectory(String rootDirectory, FileLimi int batchProcessedCount = 0; int batchSkippedCount = 0; + int batchExcludedCount = 0; int batchFailedCount = 0; for (LocalDocsFileOutcome fileOutcome : fileProcessor.processBatch(realSelectedRoot, fileBatch, ingestionIdentities)) { @@ -112,12 +113,15 @@ public IngestionLocalOutcome ingestLocalDirectory(String rootDirectory, FileLimi batchProcessedCount++; } else if (fileOutcome.failure().isPresent()) { batchFailedCount++; + } else if (fileOutcome instanceof LocalDocsFileOutcome.Excluded) { + batchExcludedCount++; } else { batchSkippedCount++; } fileOutcome.failure().ifPresent(failures::add); } - backlogStatus = backlogStatus.completeBatch(batchProcessedCount, batchSkippedCount, batchFailedCount); + backlogStatus = backlogStatus.completeBatch( + batchProcessedCount, batchSkippedCount, batchExcludedCount, batchFailedCount); ingestionRunStore.write(realSelectedRoot, backlogStatus, eligibleFileInventory.inventoryFingerprint()); runStopped = batchFailedCount > 0; selectedFileIndex = batchEndIndex; diff --git a/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessor.java b/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessor.java index 7c28bbcc..353ca21e 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessor.java +++ b/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessor.java @@ -435,7 +435,7 @@ private ReindexDecision inspectExistingMarker(MarkerContext markerContext) { expectedPointUuids(expectedChunkHashes)); if (hasExactPointIds && expectedChunkHashes.isEmpty()) { INDEXING_LOG.debug("[INDEXING] Skipping unchanged excluded Java API page"); - return ReindexDecision.terminal(LocalDocsFileOutcome.skippedFile()); + return ReindexDecision.terminal(LocalDocsFileOutcome.excludedFile()); } if (hasExactPointIds) { INDEXING_LOG.debug("[INDEXING] Skipping unchanged file (already ingested)"); @@ -721,7 +721,7 @@ private LocalDocsFileOutcome processExcludedPage(MarkerContext markerContext, bo failureFactory.failure(markerContext.file(), "marker-transition", markerTransitionException)); } INDEXING_LOG.info("[INDEXING] Excluded documentation page from indexing"); - return LocalDocsFileOutcome.skippedFile(); + return LocalDocsFileOutcome.excludedFile(); } private static boolean isNavigationOnlyDocument(org.jsoup.nodes.Document parsedDocument) { diff --git a/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileOutcome.java b/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileOutcome.java index 13af22a7..be3e017f 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileOutcome.java +++ b/src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileOutcome.java @@ -8,7 +8,10 @@ * Outcome of processing a single local docs file. */ public sealed interface LocalDocsFileOutcome - permits LocalDocsFileOutcome.Processed, LocalDocsFileOutcome.Skipped, LocalDocsFileOutcome.Failed { + permits LocalDocsFileOutcome.Processed, + LocalDocsFileOutcome.Skipped, + LocalDocsFileOutcome.Excluded, + LocalDocsFileOutcome.Failed { /** * Returns true when the file contributed new chunks to the destination. @@ -29,11 +32,26 @@ static LocalDocsFileOutcome processedFile() { /** * Returns a skipped outcome for files that were unchanged or already indexed. + * + *

Skipped files retain their existing Qdrant points and are genuine duplicates of content + * already stored in the vector index. Intentionally excluded pages that hold zero Qdrant points + * use {@link #excludedFile()} instead.

*/ static LocalDocsFileOutcome skippedFile() { return Skipped.INSTANCE; } + /** + * Returns an excluded outcome for a file that was intentionally not indexed. + * + *

Excluded pages (for example Javadoc class-use index pages or frameset/navigation shells) are + * not upserted into Qdrant and may have their existing points deleted, so the URL ends up with + * zero Qdrant points — the opposite of an already-indexed duplicate.

+ */ + static LocalDocsFileOutcome excludedFile() { + return Excluded.INSTANCE; + } + /** * Returns a failed outcome carrying the typed local ingestion failure. */ @@ -70,6 +88,20 @@ public Optional failure() { } } + record Excluded() implements LocalDocsFileOutcome { + private static final Excluded INSTANCE = new Excluded(); + + @Override + public boolean processed() { + return false; + } + + @Override + public Optional failure() { + return Optional.empty(); + } + } + record Failed(IngestionLocalFailure detail) implements LocalDocsFileOutcome { public Failed { Objects.requireNonNull(detail, "detail"); diff --git a/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorExcludedCategoryLogTest.java b/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorExcludedCategoryLogTest.java new file mode 100644 index 00000000..eb703917 --- /dev/null +++ b/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorExcludedCategoryLogTest.java @@ -0,0 +1,133 @@ +package com.williamcallahan.javachat.cli; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import ch.qos.logback.classic.Logger; +import com.williamcallahan.javachat.application.ingestion.FileLimit; +import com.williamcallahan.javachat.application.ingestion.LocalDocumentationIngestionUseCase; +import com.williamcallahan.javachat.domain.ingestion.IngestionBacklogStatus; +import com.williamcallahan.javachat.domain.ingestion.IngestionLocalOutcome; +import com.williamcallahan.javachat.service.ProgressTracker; +import com.williamcallahan.javachat.support.logging.ExpectedLogEvents; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; + +/** + * Verifies the CLI reports intentionally-excluded files as a distinct category instead of + * conflating them with already-in-Qdrant duplicates. + * + *

Precondition: {@link IngestionBacklogStatus#skippedFiles()} must count only genuine + * already-indexed duplicates while {@link IngestionBacklogStatus#excludedFiles()} counts + * intentionally-excluded pages, so the operator-facing labels stay honest. + */ +class DocumentProcessorExcludedCategoryLogTest { + private static final FileLimit EXPECTED_CLI_FILE_LIMIT = new FileLimit(Integer.MAX_VALUE); + private static final String DOCUMENT_PROCESSING_COMPLETE = "DOCUMENT PROCESSING COMPLETE"; + private static final String TOTAL_PROCESSED_THREE_DOCUMENTS = "Total new documents processed: 3"; + private static final String TOTAL_DUPLICATES_THREE = "Total duplicates skipped: 3"; + private static final String TOTAL_EXCLUDED_THREE = "Total excluded files: 3"; + + private final Logger documentProcessorLogger = (Logger) LoggerFactory.getLogger(DocumentProcessor.class); + private ExpectedLogEvents documentProcessorLogEvents; + + @BeforeEach + void captureDocumentProcessorLogs() { + documentProcessorLogEvents = ExpectedLogEvents.capture(documentProcessorLogger); + } + + @AfterEach + void restoreDocumentProcessorLogs() { + documentProcessorLogEvents.close(); + } + + @Test + void reportsExcludedFilesSeparatelyFromAlreadyIndexedDuplicates(@TempDir Path temporaryDirectory) + throws IOException { + Path firstDocumentationDirectory = temporaryDirectory.resolve("java-api-with-class-use"); + Path secondDocumentationDirectory = temporaryDirectory.resolve("clean-documentation"); + createEligibleDocument(firstDocumentationDirectory, "page.html"); + createEligibleDocument(secondDocumentationDirectory, "page.html"); + + LocalDocumentationIngestionUseCase ingestionService = mock(LocalDocumentationIngestionUseCase.class); + ProgressTracker progressTracker = mock(ProgressTracker.class); + when(progressTracker.formatPercent()).thenReturn("0%"); + when(ingestionService.ingestLocalDirectory(firstDocumentationDirectory.toString(), EXPECTED_CLI_FILE_LIMIT)) + .thenReturn(IngestionLocalOutcome.fromBacklog( + new IngestionBacklogStatus( + IngestionBacklogStatus.Lifecycle.COMPLETE, + 6, + 6, + 1, + 2, + 3, + 0, + 0, + 0, + firstDocumentationDirectory.toString()), + firstDocumentationDirectory.toString(), + List.of())); + when(ingestionService.ingestLocalDirectory(secondDocumentationDirectory.toString(), EXPECTED_CLI_FILE_LIMIT)) + .thenReturn(IngestionLocalOutcome.fromBacklog( + new IngestionBacklogStatus( + IngestionBacklogStatus.Lifecycle.COMPLETE, + 3, + 3, + 2, + 1, + 0, + 0, + 0, + 0, + secondDocumentationDirectory.toString()), + secondDocumentationDirectory.toString(), + List.of())); + + DocumentProcessor documentProcessor = new DocumentProcessor(ingestionService, progressTracker); + String firstDirectoryName = Objects.requireNonNull( + firstDocumentationDirectory.getFileName(), "firstDocumentationDirectory file name") + .toString(); + String secondDirectoryName = Objects.requireNonNull( + secondDocumentationDirectory.getFileName(), "secondDocumentationDirectory file name") + .toString(); + List documentationSets = List.of( + new DocumentationSet("Java API with class-use", firstDirectoryName, firstDirectoryName), + new DocumentationSet("Clean documentation", secondDirectoryName, secondDirectoryName)); + + documentProcessor.processDocumentationSets(temporaryDirectory, documentationSets); + + assertTrue(containsLogMessage(DOCUMENT_PROCESSING_COMPLETE)); + assertTrue(containsLogMessage(" Skipped 2 duplicate files (already in Qdrant)")); + assertTrue(containsLogMessage(" Excluded 3 files (intentionally not indexed)")); + assertTrue(containsLogMessage(" Skipped 1 duplicate files (already in Qdrant)")); + assertTrue(containsLogMessage(TOTAL_PROCESSED_THREE_DOCUMENTS)); + assertTrue(containsLogMessage(TOTAL_DUPLICATES_THREE)); + assertTrue(containsLogMessage(TOTAL_EXCLUDED_THREE)); + assertFalse(containsLogMessage(" Skipped 5 duplicate files (already in Qdrant)")); + assertFalse(containsLogMessage("Total duplicates skipped: 5")); + assertFalse(containsLogMessage("Total duplicates skipped: 6")); + assertFalse(containsLogMessage(" Excluded 0 files (intentionally not indexed)")); + } + + private boolean containsLogMessage(final String expectedMessage) { + return documentProcessorLogEvents.events().stream() + .map(logEvent -> logEvent.getFormattedMessage()) + .anyMatch(expectedMessage::equals); + } + + private static void createEligibleDocument(final Path documentationDirectory, final String documentFileName) + throws IOException { + Files.createDirectories(documentationDirectory); + Files.writeString(documentationDirectory.resolve(documentFileName), "Test"); + } +} diff --git a/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorFailureContractTest.java b/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorFailureContractTest.java index cc9bfc5f..f5ef04e5 100644 --- a/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorFailureContractTest.java +++ b/src/test/java/com/williamcallahan/javachat/cli/DocumentProcessorFailureContractTest.java @@ -40,6 +40,7 @@ class DocumentProcessorFailureContractTest { private static final String DOCUMENT_PROCESSING_FAILED = "DOCUMENT PROCESSING FAILED"; private static final String TOTAL_PROCESSED_ONE_DOCUMENT = "Total new documents processed: 1"; private static final String TOTAL_DUPLICATES_ZERO_DOCUMENTS = "Total duplicates skipped: 0"; + private static final String TOTAL_EXCLUDED_ZERO_DOCUMENTS = "Total excluded files: 0"; private static final String TOTAL_FAILED_ONE_SET = "Documentation sets FAILED: 1"; private static final String FAILURE_PHASE = "synthetic-ingestion"; private static final String SENSITIVE_FAILURE_DETAILS = "api-key=synthetic-private-value"; @@ -81,6 +82,7 @@ void failsCliAndSuppressesSuccessMarkerWhenDocumentationSetReportsFileFailure(@T 2, 1, 0, + 0, 1, 1, 0, @@ -100,6 +102,7 @@ void failsCliAndSuppressesSuccessMarkerWhenDocumentationSetReportsFileFailure(@T 0, 0, 0, + 0, successfulDocumentationDirectory.toString()), successfulDocumentationDirectory.toString(), List.of())); @@ -130,6 +133,7 @@ void failsCliAndSuppressesSuccessMarkerWhenDocumentationSetReportsFileFailure(@T assertTrue(containsLogMessage(DOCUMENT_PROCESSING_FAILED)); assertTrue(containsLogMessage(TOTAL_PROCESSED_ONE_DOCUMENT)); assertTrue(containsLogMessage(TOTAL_DUPLICATES_ZERO_DOCUMENTS)); + assertTrue(containsLogMessage(TOTAL_EXCLUDED_ZERO_DOCUMENTS)); assertTrue(containsLogMessage(TOTAL_FAILED_ONE_SET)); assertTrue(containsLogMessage( "File failed (phase=" + FAILURE_PHASE + "): " + failedDocument + "??" + FORGED_LOG_LINE)); diff --git a/src/test/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatusTest.java b/src/test/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatusTest.java index 884f0c1d..d078fc18 100644 --- a/src/test/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatusTest.java +++ b/src/test/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatusTest.java @@ -11,7 +11,7 @@ class IngestionBacklogStatusTest { void resumeRestartsTheMarkerBackedInventoryAfterFailure() { IngestionBacklogStatus partialBacklog = IngestionBacklogStatus.running("java-21", 4) .startBatch(3) - .completeBatch(1, 1, 1) + .completeBatch(1, 1, 0, 1) .finish(); IngestionBacklogStatus resumedBacklog = partialBacklog.resume(); @@ -21,8 +21,41 @@ void resumeRestartsTheMarkerBackedInventoryAfterFailure() { assertEquals(0, resumedBacklog.inspectedFiles()); assertEquals(0, resumedBacklog.processedFiles()); assertEquals(0, resumedBacklog.skippedFiles()); + assertEquals(0, resumedBacklog.excludedFiles()); assertEquals(0, resumedBacklog.failedFiles()); assertEquals(4, resumedBacklog.pendingFiles()); assertEquals(0, resumedBacklog.inProgressFiles()); } + + @Test + void completeBatchAccumulatesExcludedFilesSeparatelyFromSkippedDuplicates() { + IngestionBacklogStatus completeBacklog = IngestionBacklogStatus.running("java-21", 3) + .startBatch(3) + .completeBatch(1, 1, 1, 0) + .finish(); + + assertEquals(IngestionBacklogStatus.Lifecycle.COMPLETE, completeBacklog.lifecycle()); + assertEquals(1, completeBacklog.processedFiles()); + assertEquals(1, completeBacklog.skippedFiles()); + assertEquals(1, completeBacklog.excludedFiles()); + assertEquals(0, completeBacklog.failedFiles()); + assertEquals(0, completeBacklog.pendingFiles()); + assertEquals(3, completeBacklog.inspectedFiles()); + } + + @Test + void abandonRetainsExcludedFilesInTerminalSuccessPrefixAfterInterruption() { + IngestionBacklogStatus interruptedBacklog = + IngestionBacklogStatus.running("java-21", 3).startBatch(2).completeBatch(1, 0, 1, 0); + + IngestionBacklogStatus abandonedBacklog = interruptedBacklog.abandon(); + + assertEquals(IngestionBacklogStatus.Lifecycle.PARTIAL, abandonedBacklog.lifecycle()); + assertEquals(1, abandonedBacklog.processedFiles()); + assertEquals(0, abandonedBacklog.skippedFiles()); + assertEquals(1, abandonedBacklog.excludedFiles()); + assertEquals(0, abandonedBacklog.failedFiles()); + assertEquals(2, abandonedBacklog.inspectedFiles()); + assertEquals(1, abandonedBacklog.pendingFiles()); + } } diff --git a/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionServiceTest.java b/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionServiceTest.java index 5e4733af..1700193e 100644 --- a/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionServiceTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionServiceTest.java @@ -257,7 +257,7 @@ void resumesAtNextPendingFileAcrossBoundedRuns(@TempDir Path temporaryDirectory) .thenReturn( Optional.empty(), Optional.of(new IngestionBacklogStatus( - IngestionBacklogStatus.Lifecycle.PARTIAL, 2, 1, 0, 1, 0, 1, 0, "java"))); + IngestionBacklogStatus.Lifecycle.PARTIAL, 2, 1, 0, 1, 0, 0, 1, 0, "java"))); LocalDocsDirectoryIngestionService directoryIngestionService = new LocalDocsDirectoryIngestionService( fileProcessor, ingestionRunStore, configuredDocumentationRoot.toString()); diff --git a/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessorTest.java b/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessorTest.java index fff31daf..7caef66a 100644 --- a/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessorTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessorTest.java @@ -945,6 +945,7 @@ void shouldPruneAndMarkClassUsePageAsExcludedWithoutRetryingOrQuarantining(@Temp assertFalse(firstOutcome.processed()); assertTrue(firstOutcome.failure().isEmpty()); + assertTrue(firstOutcome instanceof LocalDocsFileOutcome.Excluded); verify(ingestionFixture.hybridVectorService) .deleteByUrl(any(QdrantCollectionKind.class), eq(expectedClassUseUrl)); verify(ingestionFixture.ingestedFilePruneService) @@ -971,6 +972,7 @@ void shouldPruneAndMarkClassUsePageAsExcludedWithoutRetryingOrQuarantining(@Temp assertFalse(repeatedOutcome.processed()); assertTrue(repeatedOutcome.failure().isEmpty()); + assertTrue(repeatedOutcome instanceof LocalDocsFileOutcome.Excluded); verify(ingestionFixture.ingestedFilePruneService, times(1)) .pruneObsoleteLocalStateAfterReplacement(expectedClassUseUrl, staleIngestionRecord, List.of()); verify(ingestionFixture.fileIngestionMarkerStore, times(1)).markFileIngested(eq(expectedClassUseUrl), any()); @@ -1003,6 +1005,7 @@ void shouldMarkGenericFramesetNavigationPageAsExcluded(@TempDir Path temporaryDi assertFalse(outcome.processed()); assertTrue(outcome.failure().isEmpty()); + assertTrue(outcome instanceof LocalDocsFileOutcome.Excluded); ArgumentCaptor markerCaptor = ArgumentCaptor.forClass(FileIngestionRecord.class); verify(ingestionFixture.fileIngestionMarkerStore).markFileIngested(eq(expectedUrl), markerCaptor.capture()); assertTrue(markerCaptor.getValue().chunkHashes().isEmpty()); @@ -1055,6 +1058,7 @@ void shouldMarkInteractiveApiReferenceShellAsExcludedAndContinueBatch(@TempDir P assertEquals(2, outcomes.size()); assertFalse(outcomes.getFirst().processed()); assertTrue(outcomes.getFirst().failure().isEmpty()); + assertTrue(outcomes.getFirst() instanceof LocalDocsFileOutcome.Excluded); assertTrue(outcomes.getLast().processed()); assertTrue(outcomes.getLast().failure().isEmpty()); ArgumentCaptor markerCaptor = ArgumentCaptor.forClass(FileIngestionRecord.class); diff --git a/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalIngestionRunStoreTest.java b/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalIngestionRunStoreTest.java index d7c31acc..70f9676f 100644 --- a/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalIngestionRunStoreTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/ingestion/LocalIngestionRunStoreTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.williamcallahan.javachat.application.ingestion.IngestionAlreadyRunningException; import com.williamcallahan.javachat.domain.ingestion.IngestionBacklogStatus; import com.williamcallahan.javachat.service.LocalStoreService; @@ -11,7 +12,11 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -30,7 +35,7 @@ void persistsPartialBacklogAndPublishesCounts(@TempDir Path temporaryDirectory) Path documentationDirectory = temporaryDirectory.resolve("docs").toAbsolutePath(); IngestionBacklogStatus partialBacklog = IngestionBacklogStatus.running(documentationDirectory.toString(), 4) .startBatch(3) - .completeBatch(1, 1, 1) + .completeBatch(1, 1, 0, 1) .finish(); runStore.write(documentationDirectory, partialBacklog, INVENTORY_FINGERPRINT); @@ -53,7 +58,7 @@ void invalidatesCheckpointWhenInventoryIdentityChanges(@TempDir Path temporaryDi Path documentationDirectory = temporaryDirectory.resolve("docs").toAbsolutePath(); IngestionBacklogStatus partialBacklog = IngestionBacklogStatus.running(documentationDirectory.toString(), 2) .startBatch(1) - .completeBatch(1, 0, 0) + .completeBatch(1, 0, 0, 0) .finish(); runStore.write(documentationDirectory, partialBacklog, INVENTORY_FINGERPRINT); @@ -133,7 +138,7 @@ void doesNotOverwriteCheckpointCompletedBeforeAbandonedLockAcquisition(@TempDir IngestionBacklogStatus.running("docs", 3).startBatch(3); runStore.write(documentationDirectory, runningBacklog, INVENTORY_FINGERPRINT); IngestionBacklogStatus completedBacklog = - runningBacklog.completeBatch(2, 1, 0).finish(); + runningBacklog.completeBatch(2, 1, 0, 0).finish(); ownerCompletion.set(() -> { try { runStore.write(documentationDirectory, completedBacklog, INVENTORY_FINGERPRINT); @@ -173,6 +178,49 @@ void distinctDirectoriesWithPreviouslyCollidingSafeNamesHaveIndependentClaims(@T } } + @Test + void readsCheckpointWrittenBeforeExcludedFilesFieldWasIntroduced(@TempDir Path temporaryDirectory) + throws IOException { + SimpleMeterRegistry meterRegistry = new SimpleMeterRegistry(); + LocalIngestionRunStore runStore = runStore(temporaryDirectory, meterRegistry); + Path documentationDirectory = temporaryDirectory.resolve("docs").toAbsolutePath(); + IngestionBacklogStatus partialBacklog = IngestionBacklogStatus.running(documentationDirectory.toString(), 2) + .startBatch(2) + .completeBatch(0, 1, 0, 0) + .finish(); + runStore.write(documentationDirectory, partialBacklog, INVENTORY_FINGERPRINT); + + Path progressFile = soleProgressFile(temporaryDirectory); + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode checkpointNode = (ObjectNode) objectMapper.readTree(progressFile.toFile()); + ObjectNode backlogNode = (ObjectNode) checkpointNode.get("backlog"); + backlogNode.remove("excludedFiles"); + objectMapper.writeValue(progressFile.toFile(), checkpointNode); + + IngestionBacklogStatus readBacklog = + runStore.read(documentationDirectory, INVENTORY_FINGERPRINT).orElseThrow(); + + assertEquals(IngestionBacklogStatus.Lifecycle.PARTIAL, readBacklog.lifecycle()); + assertEquals(1, readBacklog.inspectedFiles()); + assertEquals(0, readBacklog.processedFiles()); + assertEquals(1, readBacklog.skippedFiles()); + assertEquals(0, readBacklog.excludedFiles()); + assertEquals(0, readBacklog.failedFiles()); + assertEquals(1, readBacklog.pendingFiles()); + } + + private static Path soleProgressFile(Path temporaryDirectory) throws IOException { + Path indexDirectory = temporaryDirectory.resolve("qwen3-embedding-4b-2560/local/index"); + List progressFiles = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(indexDirectory, "local-ingestion-*.json")) { + stream.forEach(progressFiles::add); + } + if (progressFiles.size() != 1) { + throw new IllegalStateException("Expected exactly one progress file, found " + progressFiles.size()); + } + return progressFiles.getFirst(); + } + private static LocalIngestionRunStore runStore(Path temporaryDirectory, SimpleMeterRegistry meterRegistry) { return runStore(temporaryDirectory, meterRegistry, new ObjectMapper()); }