fix(ingestion): report intentionally-excluded docs separately from Qdrant duplicates

src/main/java/com/williamcallahan/javachat/cli/DocumentProcessor.java

@@ -59,6 +59,7 @@ public class DocumentProcessor {
     private static final String LOG_PROCESSING_SET = "Processing documentation set";
     private static final String LOG_FILES_TO_PROCESS = "Files to process: {}";
     private static final String LOG_DUPLICATES_SKIPPED = "  Skipped {} duplicate files (already in Qdrant)";
+    private static final String LOG_EXCLUDED_SKIPPED = "  Excluded {} files (intentionally not indexed)";
     private static final String LOG_SKIP_PATH_ESCAPE = "Skipping documentation set (path escaped base directory)";
     private static final String LOG_SKIP_DIR_NOT_FOUND = "Skipping documentation set (directory not found)";
     private static final String LOG_SKIP_NO_ELIGIBLE = "Skipping documentation set (no eligible files)";
@@ -68,6 +69,7 @@ public class DocumentProcessor {
     private static final String LOG_PROCESSED_STATS = "Processed {} files in {}s ({} files/sec) ({})";
     private static final String LOG_TOTAL_PROCESSED = "Total new documents processed: {}";
     private static final String LOG_TOTAL_DUPLICATES = "Total duplicates skipped: {}";
+    private static final String LOG_TOTAL_EXCLUDED = "Total excluded files: {}";
     private static final String LOG_TOTAL_FAILED = "Documentation sets FAILED: {}";
     private static final String LOG_NEXT_STEP_QDRANT = "1. Verify in Qdrant Dashboard";
     private static final String LOG_NEXT_STEP_RETRIEVAL = "2. Test retrieval";
@@ -194,8 +196,10 @@ void processDocumentationSets(final Path basePath, final List<DocumentationSet>
 
     private IngestionTotals accumulateOutcome(final IngestionTotals totals, final ProcessingOutcome outcome) {
         return switch (outcome) {
-            case ProcessingOutcome.Success success -> totals.addSuccess(success.processed(), success.duplicates());
-            case ProcessingOutcome.Failed failed -> totals.addFailed(failed.processed(), failed.duplicates());
+            case ProcessingOutcome.Success success ->
+                totals.addSuccess(success.processed(), success.duplicates(), success.excluded());
+            case ProcessingOutcome.Failed failed ->
+                totals.addFailed(failed.processed(), failed.duplicates(), failed.excluded());
         };
     }
 
@@ -206,14 +210,14 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc
             if (LOGGER.isWarnEnabled()) {
                 LOGGER.warn(LOG_SKIP_PATH_ESCAPE);
             }
-            return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0);
+            return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0);
         }
 
         if (!Files.exists(docsPath) || !Files.isDirectory(docsPath)) {
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.debug(LOG_SKIP_DIR_NOT_FOUND);
             }
-            return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0);
+            return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0);
         }
 
         try {
@@ -222,7 +226,7 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc
                 if (LOGGER.isDebugEnabled()) {
                     LOGGER.debug(LOG_SKIP_NO_ELIGIBLE);
                 }
-                return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0);
+                return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0);
             }
 
             if (LOGGER.isInfoEnabled()) {
@@ -240,11 +244,17 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc
 
             final int failureCount = outcome.backlog().failedFiles();
             final long duplicates = outcome.backlog().skippedFiles();
+            final long excluded = outcome.backlog().excludedFiles();
             if (duplicates > 0) {
                 if (LOGGER.isInfoEnabled()) {
                     LOGGER.info(LOG_DUPLICATES_SKIPPED, duplicates);
                 }
             }
+            if (excluded > 0) {
+                if (LOGGER.isInfoEnabled()) {
+                    LOGGER.info(LOG_EXCLUDED_SKIPPED, excluded);
+                }
+            }
             if (outcome.backlog().pendingFiles() > 0 && LOGGER.isWarnEnabled()) {
                 LOGGER.warn(
                         "Ingestion stopped with {} pending file(s)",
@@ -255,15 +265,15 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc
                 if (LOGGER.isWarnEnabled()) {
                     LOGGER.warn("Ingestion completed with {} file failures", failureCount);
                 }
-                return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates);
+                return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates, excluded);
             }
             if (outcome.backlog().pendingFiles() > 0) {
-                return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates);
+                return new ProcessingOutcome.Failed(docSet.displayName(), processed, duplicates, excluded);
             }
             String safeDocumentationSet =
                     docSet.indexedDocSet().replace('\r', '?').replace('\n', '?');
             LOGGER.info(LOG_DOCSET_POSTCONDITION, safeDocumentationSet);
-            return new ProcessingOutcome.Success(processed, duplicates);
+            return new ProcessingOutcome.Success(processed, duplicates, excluded);
 
         } catch (IOException | UncheckedIOException | IngestionAlreadyRunningException processingFailure) {
             if (LOGGER.isErrorEnabled()) {
@@ -272,7 +282,7 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.debug(LOG_STACK_TRACE, processingFailure);
             }
-            return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0);
+            return new ProcessingOutcome.Failed(docSet.displayName(), 0, 0, 0);
         }
     }
 
@@ -454,6 +464,7 @@ private void logTotals(final String title, final IngestionTotals totals) {
             LOGGER.info(LOG_BANNER_LINE);
             LOGGER.info(LOG_TOTAL_PROCESSED, totals.processed());
             LOGGER.info(LOG_TOTAL_DUPLICATES, totals.duplicates());
+            LOGGER.info(LOG_TOTAL_EXCLUDED, totals.excluded());
         }
     }
 
@@ -503,25 +514,27 @@ private static boolean envBooleanOrDefault(final String key, final boolean fallb
     /**
      * Accumulated totals across all documentation sets, tracking successes and failures separately.
      */
-    private record IngestionTotals(long processed, long duplicates, int failedSets) {
-        static final IngestionTotals ZERO = new IngestionTotals(0, 0, 0);
+    private record IngestionTotals(long processed, long duplicates, long excluded, int failedSets) {
+        static final IngestionTotals ZERO = new IngestionTotals(0, 0, 0, 0);
 
-        IngestionTotals addSuccess(final long newProcessed, final long newDuplicates) {
-            return new IngestionTotals(processed + newProcessed, duplicates + newDuplicates, failedSets);
+        IngestionTotals addSuccess(final long newProcessed, final long newDuplicates, final long newExcluded) {
+            return new IngestionTotals(
+                    processed + newProcessed, duplicates + newDuplicates, excluded + newExcluded, failedSets);
         }
 
-        IngestionTotals addFailed(final long newProcessed, final long newDuplicates) {
-            return new IngestionTotals(processed + newProcessed, duplicates + newDuplicates, failedSets + 1);
+        IngestionTotals addFailed(final long newProcessed, final long newDuplicates, final long newExcluded) {
+            return new IngestionTotals(
+                    processed + newProcessed, duplicates + newDuplicates, excluded + newExcluded, failedSets + 1);
         }
     }
 
     /**
      * Outcome of processing a single documentation set - distinguishes success, skip, and failure.
      */
     private sealed interface ProcessingOutcome {
-        record Success(long processed, long duplicates) implements ProcessingOutcome {}
+        record Success(long processed, long duplicates, long excluded) implements ProcessingOutcome {}
 
-        record Failed(String setName, long processed, long duplicates) implements ProcessingOutcome {}
+        record Failed(String setName, long processed, long duplicates, long excluded) implements ProcessingOutcome {}
     }
 
     /**

src/main/java/com/williamcallahan/javachat/cli/GitHubRepoProcessor.java

@@ -190,7 +190,7 @@ private IngestionWalkSummary walkAndProcess(
                 activeFileUrls.add(fileProcessingOutcome.fileUrl());
                 switch (fileProcessingOutcome.outcome()) {
                     case LocalDocsFileOutcome.Processed _ -> processedCount++;
-                    case LocalDocsFileOutcome.Skipped _ -> skippedCount++;
+                    case LocalDocsFileOutcome.Skipped _, LocalDocsFileOutcome.Excluded _ -> skippedCount++;
                     case LocalDocsFileOutcome.Failed failed -> {
                         failedCount++;
                         failed.failure()

src/main/java/com/williamcallahan/javachat/domain/ingestion/IngestionBacklogStatus.java

@@ -9,7 +9,8 @@
  * @param eligibleFiles ingestable files discovered beneath the selected directory
  * @param inspectedFiles files that reached a terminal per-file outcome
  * @param processedFiles files that produced newly indexed chunks
- * @param skippedFiles files already indexed or intentionally excluded
+ * @param skippedFiles files already indexed (unchanged duplicates that retain their Qdrant points)
+ * @param excludedFiles files intentionally excluded from indexing (held zero Qdrant points)
  * @param failedFiles files that reached a typed failure
  * @param pendingFiles files still waiting to be inspected
  * @param inProgressFiles files currently owned by the active ingestion batch
@@ -21,6 +22,7 @@ public record IngestionBacklogStatus(
         int inspectedFiles,
         int processedFiles,
         int skippedFiles,
+        int excludedFiles,
         int failedFiles,
         int pendingFiles,
         int inProgressFiles,
@@ -47,11 +49,13 @@ public enum Lifecycle {
         requireNonNegative("inspectedFiles", inspectedFiles);
         requireNonNegative("processedFiles", processedFiles);
         requireNonNegative("skippedFiles", skippedFiles);
+        requireNonNegative("excludedFiles", excludedFiles);
         requireNonNegative("failedFiles", failedFiles);
         requireNonNegative("pendingFiles", pendingFiles);
         requireNonNegative("inProgressFiles", inProgressFiles);
-        if (inspectedFiles != processedFiles + skippedFiles + failedFiles) {
-            throw new IllegalArgumentException("Inspected files must equal processed, skipped, and failed files");
+        if (inspectedFiles != processedFiles + skippedFiles + excludedFiles + failedFiles) {
+            throw new IllegalArgumentException(
+                    "Inspected files must equal processed, skipped, excluded, and failed files");
         }
         if (eligibleFiles != inspectedFiles + pendingFiles + inProgressFiles) {
             throw new IllegalArgumentException("Eligible files must equal inspected, pending, and in-progress files");
@@ -67,14 +71,14 @@ public enum Lifecycle {
      */
     public static IngestionBacklogStatus notStarted(String directory, int eligibleFiles) {
         return new IngestionBacklogStatus(
-                Lifecycle.NOT_STARTED, eligibleFiles, 0, 0, 0, 0, eligibleFiles, 0, directory);
+                Lifecycle.NOT_STARTED, eligibleFiles, 0, 0, 0, 0, 0, eligibleFiles, 0, directory);
     }
 
     /**
      * Creates the initial durable state after a process claims the directory.
      */
     public static IngestionBacklogStatus running(String directory, int eligibleFiles) {
-        return new IngestionBacklogStatus(Lifecycle.RUNNING, eligibleFiles, 0, 0, 0, 0, eligibleFiles, 0, directory);
+        return new IngestionBacklogStatus(Lifecycle.RUNNING, eligibleFiles, 0, 0, 0, 0, 0, eligibleFiles, 0, directory);
     }
 
     /**
@@ -96,6 +100,7 @@ public IngestionBacklogStatus startBatch(int batchFileCount) {
                 inspectedFiles,
                 processedFiles,
                 skippedFiles,
+                excludedFiles,
                 failedFiles,
                 pendingFiles - batchFileCount,
                 batchFileCount,
@@ -105,11 +110,13 @@ public IngestionBacklogStatus startBatch(int batchFileCount) {
     /**
      * Records all terminal outcomes returned by the active batch.
      */
-    public IngestionBacklogStatus completeBatch(int processedCount, int skippedCount, int failedCount) {
+    public IngestionBacklogStatus completeBatch(
+            int processedCount, int skippedCount, int excludedCount, int failedCount) {
         requireNonNegative("processedCount", processedCount);
         requireNonNegative("skippedCount", skippedCount);
+        requireNonNegative("excludedCount", excludedCount);
         requireNonNegative("failedCount", failedCount);
-        int terminalOutcomeCount = processedCount + skippedCount + failedCount;
+        int terminalOutcomeCount = processedCount + skippedCount + excludedCount + failedCount;
         if (terminalOutcomeCount > inProgressFiles) {
             throw new IllegalArgumentException("Batch outcomes exceed the in-progress file count");
         }
@@ -120,6 +127,7 @@ public IngestionBacklogStatus completeBatch(int processedCount, int skippedCount
                 inspectedFiles + terminalOutcomeCount,
                 processedFiles + processedCount,
                 skippedFiles + skippedCount,
+                excludedFiles + excludedCount,
                 failedFiles + failedCount,
                 pendingFiles + unattemptedCount,
                 0,
@@ -140,6 +148,7 @@ public IngestionBacklogStatus finish() {
                 inspectedFiles,
                 processedFiles,
                 skippedFiles,
+                excludedFiles,
                 failedFiles,
                 pendingFiles,
                 0,
@@ -161,15 +170,16 @@ public IngestionBacklogStatus abandon() {
         }
         if (failedFiles > 0) {
             return new IngestionBacklogStatus(
-                    Lifecycle.PARTIAL, eligibleFiles, 0, 0, 0, 0, eligibleFiles, 0, directory);
+                    Lifecycle.PARTIAL, eligibleFiles, 0, 0, 0, 0, 0, eligibleFiles, 0, directory);
         }
-        int terminalSuccessCount = processedFiles + skippedFiles;
+        int terminalSuccessCount = processedFiles + skippedFiles + excludedFiles;
         return new IngestionBacklogStatus(
                 Lifecycle.PARTIAL,
                 eligibleFiles,
                 terminalSuccessCount,
                 processedFiles,
                 skippedFiles,
+                excludedFiles,
                 0,
                 eligibleFiles - terminalSuccessCount,
                 0,
@@ -194,6 +204,7 @@ public IngestionBacklogStatus resume() {
                 abandonedBacklog.inspectedFiles,
                 abandonedBacklog.processedFiles,
                 abandonedBacklog.skippedFiles,
+                abandonedBacklog.excludedFiles,
                 0,
                 abandonedBacklog.pendingFiles,
                 0,

src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsDirectoryIngestionService.java

@@ -105,19 +105,23 @@ public IngestionLocalOutcome ingestLocalDirectory(String rootDirectory, FileLimi
 
                 int batchProcessedCount = 0;
                 int batchSkippedCount = 0;
+                int batchExcludedCount = 0;
                 int batchFailedCount = 0;
                 for (LocalDocsFileOutcome fileOutcome :
                         fileProcessor.processBatch(realSelectedRoot, fileBatch, ingestionIdentities)) {
                     if (fileOutcome.processed()) {
                         batchProcessedCount++;
                     } else if (fileOutcome.failure().isPresent()) {
                         batchFailedCount++;
+                    } else if (fileOutcome instanceof LocalDocsFileOutcome.Excluded) {
+                        batchExcludedCount++;
                     } else {
                         batchSkippedCount++;
                     }
                     fileOutcome.failure().ifPresent(failures::add);
                 }
-                backlogStatus = backlogStatus.completeBatch(batchProcessedCount, batchSkippedCount, batchFailedCount);
+                backlogStatus = backlogStatus.completeBatch(
+                        batchProcessedCount, batchSkippedCount, batchExcludedCount, batchFailedCount);
                 ingestionRunStore.write(realSelectedRoot, backlogStatus, eligibleFileInventory.inventoryFingerprint());
                 runStopped = batchFailedCount > 0;
                 selectedFileIndex = batchEndIndex;

src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileIngestionProcessor.java

@@ -435,7 +435,7 @@ private ReindexDecision inspectExistingMarker(MarkerContext markerContext) {
                             expectedPointUuids(expectedChunkHashes));
             if (hasExactPointIds && expectedChunkHashes.isEmpty()) {
                 INDEXING_LOG.debug("[INDEXING] Skipping unchanged excluded Java API page");
-                return ReindexDecision.terminal(LocalDocsFileOutcome.skippedFile());
+                return ReindexDecision.terminal(LocalDocsFileOutcome.excludedFile());
             }
             if (hasExactPointIds) {
                 INDEXING_LOG.debug("[INDEXING] Skipping unchanged file (already ingested)");
@@ -721,7 +721,7 @@ private LocalDocsFileOutcome processExcludedPage(MarkerContext markerContext, bo
                     failureFactory.failure(markerContext.file(), "marker-transition", markerTransitionException));
         }
         INDEXING_LOG.info("[INDEXING] Excluded documentation page from indexing");
-        return LocalDocsFileOutcome.skippedFile();
+        return LocalDocsFileOutcome.excludedFile();
     }
 
     private static boolean isNavigationOnlyDocument(org.jsoup.nodes.Document parsedDocument) {

src/main/java/com/williamcallahan/javachat/service/ingestion/LocalDocsFileOutcome.java

@@ -8,7 +8,10 @@
  * Outcome of processing a single local docs file.
  */
 public sealed interface LocalDocsFileOutcome
-        permits LocalDocsFileOutcome.Processed, LocalDocsFileOutcome.Skipped, LocalDocsFileOutcome.Failed {
+        permits LocalDocsFileOutcome.Processed,
+                LocalDocsFileOutcome.Skipped,
+                LocalDocsFileOutcome.Excluded,
+                LocalDocsFileOutcome.Failed {
 
     /**
      * Returns true when the file contributed new chunks to the destination.
@@ -29,11 +32,26 @@ static LocalDocsFileOutcome processedFile() {
 
     /**
      * Returns a skipped outcome for files that were unchanged or already indexed.
+     *
+     * <p>Skipped files retain their existing Qdrant points and are genuine duplicates of content
+     * already stored in the vector index. Intentionally excluded pages that hold zero Qdrant points
+     * use {@link #excludedFile()} instead.</p>
      */
     static LocalDocsFileOutcome skippedFile() {
         return Skipped.INSTANCE;
     }
 
+    /**
+     * Returns an excluded outcome for a file that was intentionally not indexed.
+     *
+     * <p>Excluded pages (for example Javadoc class-use index pages or frameset/navigation shells) are
+     * not upserted into Qdrant and may have their existing points deleted, so the URL ends up with
+     * zero Qdrant points — the opposite of an already-indexed duplicate.</p>
+     */
+    static LocalDocsFileOutcome excludedFile() {
+        return Excluded.INSTANCE;
+    }
+
     /**
      * Returns a failed outcome carrying the typed local ingestion failure.
      */
@@ -70,6 +88,20 @@ public Optional<IngestionLocalFailure> failure() {
         }
     }
 
+    record Excluded() implements LocalDocsFileOutcome {
+        private static final Excluded INSTANCE = new Excluded();
+
+        @Override
+        public boolean processed() {
+            return false;
+        }
+
+        @Override
+        public Optional<IngestionLocalFailure> failure() {
+            return Optional.empty();
+        }
+    }
+
     record Failed(IngestionLocalFailure detail) implements LocalDocsFileOutcome {
         public Failed {
             Objects.requireNonNull(detail, "detail");