fix: route debug output at test run level

wenytang-ms · wenytang-ms · commit 474ef23f1c91 · 2026-07-15T15:55:59.000+08:00
Correlate debug sessions with a unique launch marker, mirror non-telemetry DAP output at run level, and preserve structured TestNG runner errors while suppressing control protocol noise.

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;

Copilot-Session: 8db89d6b-bcb8-42d2-9f14-78aab176fcd0
diff --git a/src/runners/baseRunner/BaseRunner.ts b/src/runners/baseRunner/BaseRunner.ts
@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 import * as iconv from 'iconv-lite';
+import { randomUUID } from 'crypto';
 import { AddressInfo, createServer, Server, Socket } from 'net';
 import * as os from 'os';
 import { CancellationToken, debug, DebugAdapterTracker, DebugConfiguration, DebugSession, Disposable, ProviderResult } from 'vscode';
@@ -12,6 +13,8 @@ import { ITestRunnerInternal } from '../ITestRunner';
 import { RunnerResultAnalyzer } from './RunnerResultAnalyzer';
 import { IExecutionConfig, IRunTestContext } from '../../java-test-runner.api';
 
+const JAVA_TEST_RUN_ID: string = '__javaTestRunId';
+
 export abstract class BaseRunner implements ITestRunnerInternal {
     protected server: Server;
     protected socket: Socket;
@@ -56,37 +59,26 @@ export abstract class BaseRunner implements ITestRunnerInternal {
         // So we force to use internal console here to make sure the session is still under debugger's control.
         launchConfiguration.console = 'internalConsole';
 
-        // The debuggee's stdout/stderr are delivered by java-debug as standard DAP `output` events,
-        // which by default only surface in the Debug Console. Attach a tracker to the test's own debug
-        // session and forward those output events into the Test Results view, so the program output shows
-        // up next to the test results instead of being split across two separate surfaces.
+        const testRunId: string = randomUUID();
+        launchConfiguration[JAVA_TEST_RUN_ID] = testRunId;
+        const isTestSession: (session: DebugSession) => boolean = (session: DebugSession): boolean =>
+            session.configuration[JAVA_TEST_RUN_ID] === testRunId;
+
+        // Mirror the test session's user-visible Debug Console output in Test Results.
         this.disposables.push(debug.registerDebugAdapterTrackerFactory('java', {
             createDebugAdapterTracker: (session: DebugSession): ProviderResult<DebugAdapterTracker> => {
-                if (session.name !== launchConfiguration.name) {
+                if (!isTestSession(session)) {
                     return undefined;
                 }
                 return {
-                    onDidSendMessage: (message: any): void => {
-                        if (message?.type === 'event' && message.event === 'output') {
-                            const category: string | undefined = message.body?.category;
-                            // `telemetry` output events are not meant for the user.
-                            if (category === 'telemetry') {
-                                return;
-                            }
-                            const output: string | undefined = message.body?.output;
-                            if (output) {
-                                // Let the analyzer attribute the output to the running test when possible.
-                                this.runnerResultAnalyzer.appendProgramOutput(output);
-                            }
-                        }
-                    },
+                    onDidSendMessage: (message: any): void => this.handleDebugAdapterMessage(message),
                 };
             },
         }));
 
         let debugSession: DebugSession | undefined;
         this.disposables.push(debug.onDidStartDebugSession((session: DebugSession) => {
-            if (session.name === launchConfiguration.name) {
+            if (!debugSession && isTestSession(session)) {
                 debugSession = session;
             }
         }));
@@ -108,7 +100,7 @@ export abstract class BaseRunner implements ITestRunnerInternal {
             return await new Promise<void>((resolve: () => void): void => {
                 this.disposables.push(
                     debug.onDidTerminateDebugSession((session: DebugSession): void => {
-                        if (launchConfiguration.name === session.name) {
+                        if (session.id === debugSession?.id) {
                             debugSession = undefined;
                             this.tearDown();
                             if (data.length > 0) {
@@ -125,6 +117,19 @@ export abstract class BaseRunner implements ITestRunnerInternal {
         }));
     }
 
+    protected handleDebugAdapterMessage(message: any): void {
+        if (message?.type !== 'event' || message.event !== 'output' || message.body?.category === 'telemetry') {
+            return;
+        }
+
+        const output: unknown = message.body?.output;
+        if (typeof output !== 'string' || output.length === 0) {
+            return;
+        }
+
+        this.testContext.testRun.appendOutput(output.replace(/\r?\n/g, '\r\n'));
+    }
+
     public async tearDown(): Promise<void> {
         try {
             if (this.socket) {
diff --git a/src/runners/baseRunner/RunnerResultAnalyzer.ts b/src/runners/baseRunner/RunnerResultAnalyzer.ts
@@ -10,45 +10,12 @@ export abstract class RunnerResultAnalyzer {
     // Track parent test item states to update them when all children complete
     protected parentStates: Map<TestItem, ParentItemState> = new Map();
 
-    // Test items that are currently executing. Used to attribute program output
-    // (captured from the debug session) to the running test when it is unambiguous.
-    private runningItems: Set<TestItem> = new Set();
-
     constructor(protected testContext: IRunTestContext) { }
 
     public abstract analyzeData(data: string): void;
     public abstract processData(data: string): void;
     protected testMessageLocation: Location | undefined;
 
-    /**
-     * Record that a test item has started executing.
-     */
-    protected markItemStarted(item: TestItem): void {
-        this.runningItems.add(item);
-    }
-
-    /**
-     * Record that a test item has finished executing.
-     */
-    protected markItemFinished(item: TestItem): void {
-        this.runningItems.delete(item);
-    }
-
-    /**
-     * Forward program output (captured from the test's debug session as DAP `output`
-     * events) into the Test Results view. When exactly one test is currently running,
-     * the output is attributed to that test item so it shows up under the test in the
-     * explorer; otherwise (idle, or several tests running in parallel where attribution
-     * would only be a guess) it is appended to the run as a whole.
-     */
-    public appendProgramOutput(output: string): void {
-        const normalized: string = output.replace(/\r?\n/g, '\r\n');
-        const item: TestItem | undefined = this.runningItems.size === 1
-            ? this.runningItems.values().next().value
-            : undefined;
-        this.testContext.testRun.appendOutput(normalized, undefined, item);
-    }
-
     /**
      * Return a string array which contains the stacktraces that need to be filtered.
      * All the stacktraces which include the element in the return array will be removed.
diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts
@@ -76,7 +76,6 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer {
             this.setDurationAtStart(this.getCurrentState(item));
             setTestState(this.testContext.testRun, item, this.getCurrentState(item).resultState);
             this.updateParentOnChildStart(item);
-            this.markItemStarted(item);
         } else if (data.startsWith(MessageId.TestEnd)) {
             const item: TestItem | undefined = this.getTestItem(data.substr(MessageId.TestEnd.length));
             if (!item) {
@@ -86,7 +85,6 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer {
             this.calcDurationAtEnd(currentState);
             this.determineResultStateAtEnd(data, currentState);
             setTestState(this.testContext.testRun, item, currentState.resultState, undefined, currentState.duration);
-            this.markItemFinished(item);
             const itemData: ITestItemData | undefined = dataCache.get(item);
             if (itemData?.testLevel === TestLevel.Method) {
                 this.updateParentOnChildComplete(item, currentState.resultState);
diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts
@@ -10,6 +10,7 @@ import { IRunTestContext, TestLevel, TestResultState } from '../../java-test-run
 const TEST_START: string = 'testStarted';
 const TEST_FAIL: string = 'testFailed';
 const TEST_FINISH: string = 'testFinished';
+const TEST_ERROR: string = 'error';
 
 export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
 
@@ -47,15 +48,25 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
             try {
                 this.processData(match[1]);
             } catch (error) {
-                this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${match[1]}\n`);
+                this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${match[1]}\r\n`);
             }
         }
     }
 
     public processData(data: string): void {
         const outputData: ITestNGOutputData = JSON.parse(data) as ITestNGOutputData;
 
-        const id: string = `${this.projectName}@${outputData.attributes.name}`;
+        if (outputData.name === TEST_ERROR) {
+            this.processRunnerError(outputData.attributes);
+            return;
+        }
+
+        const attributes: ITestNGAttributes | undefined = outputData.attributes;
+        if (!attributes?.name) {
+            return;
+        }
+
+        const id: string = `${this.projectName}@${attributes.name}`;
         if (outputData.name === TEST_START) {
             this.initializeCache();
             const item: TestItem | undefined = this.getTestItem(id);
@@ -66,7 +77,6 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
             this.currentTestState = TestResultState.Running;
             this.testContext.testRun.started(item);
             this.updateParentOnChildStart(item);
-            this.markItemStarted(item);
         } else if (outputData.name === TEST_FAIL) {
             const item: TestItem | undefined = this.getTestItem(id);
             if (!item) {
@@ -75,11 +85,11 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
             this.currentTestState = TestResultState.Failed;
             const testMessages: TestMessage[] = [];
 
-            if (outputData.attributes.trace) {
+            if (attributes.trace) {
                 const markdownTrace: MarkdownString = new MarkdownString();
                 markdownTrace.isTrusted = true;
                 markdownTrace.supportHtml = true;
-                for (const line of outputData.attributes.trace.split(/\r?\n/)) {
+                for (const line of attributes.trace.split(/\r?\n/)) {
                     this.processStackTrace(line, markdownTrace, this.currentItem, this.projectName);
                 }
 
@@ -92,19 +102,18 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
                 }
                 testMessages.push(testMessage);
             }
-            const duration: number = Number.parseInt(outputData.attributes.duration, 10);
+            const duration: number | undefined = this.parseDuration(attributes.duration);
             setTestState(this.testContext.testRun, item, this.currentTestState, testMessages, duration);
         } else if (outputData.name === TEST_FINISH) {
-            const item: TestItem | undefined = this.getTestItem(data);
+            const item: TestItem | undefined = this.getTestItem(id);
             if (!item) {
                 return;
             }
             if (this.currentTestState === TestResultState.Running) {
                 this.currentTestState = TestResultState.Passed;
             }
-            const duration: number = Number.parseInt(outputData.attributes.duration, 10);
+            const duration: number | undefined = this.parseDuration(attributes.duration);
             setTestState(this.testContext.testRun, item, this.currentTestState, undefined, duration);
-            this.markItemFinished(item);
             const itemData: ITestItemData | undefined = dataCache.get(item);
             if (itemData?.testLevel === TestLevel.Method) {
                 this.updateParentOnChildComplete(item, this.currentTestState);
@@ -126,6 +135,26 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
         this.currentItem = undefined;
     }
 
+    private processRunnerError(attributes: ITestNGAttributes | undefined): void {
+        let message: string = attributes?.message || 'Failed to run TestNG tests.';
+        if (attributes?.trace) {
+            message += `\n${attributes.trace}`;
+        }
+        const testMessage: TestMessage = new TestMessage(message);
+        for (const item of this.testContext.testItems) {
+            this.testContext.testRun.errored(item, testMessage);
+        }
+    }
+
+    private parseDuration(duration: string | undefined): number | undefined {
+        if (!duration) {
+            return undefined;
+        }
+
+        const parsed: number = Number.parseInt(duration, 10);
+        return Number.isNaN(parsed) ? undefined : parsed;
+    }
+
     protected getStacktraceFilter(): string[] {
         return [
             'com.microsoft.java.test.runner.',
@@ -142,20 +171,14 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer {
 }
 
 interface ITestNGOutputData {
-    attributes: ITestNGAttributes;
-    type: TestOutputType;
+    attributes?: ITestNGAttributes;
     name: string;
 }
 
-enum TestOutputType {
-    Info,
-    Error,
-}
-
 interface ITestNGAttributes {
-    name: string;
-    duration: string;
-    location: string;
-    message: string;
-    trace: string;
+    name?: string;
+    duration?: string;
+    location?: string;
+    message?: string;
+    trace?: string;
 }
diff --git a/test/suite/TestNGAnalyzer.test.ts b/test/suite/TestNGAnalyzer.test.ts
@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+'use strict';
+
+import * as assert from 'assert';
+import * as sinon from 'sinon';
+import { TestController, TestMessage, TestRunRequest, tests, workspace } from 'vscode';
+import { TestNGRunnerResultAnalyzer } from '../../src/runners/testngRunner/TestNGRunnerResultAnalyzer';
+import { IRunTestContext, TestKind } from '../../src/java-test-runner.api';
+import { generateTestItem } from './utils';
+
+// tslint:disable: only-arrow-functions
+suite('TestNG Runner Analyzer Tests', () => {
+
+    let testController: TestController;
+
+    setup(() => {
+        testController = tests.createTestController('testngTestController', 'Mock TestNG');
+    });
+
+    teardown(() => {
+        testController.dispose();
+    });
+
+    test('surfaces runner errors as structured test errors', () => {
+        const testItem = generateTestItem(testController, 'testng@example.SampleTest#test', TestKind.TestNG);
+        const testRun = testController.createTestRun(new TestRunRequest([testItem], []));
+        const erroredSpy = sinon.spy(testRun, 'errored');
+        const runnerContext: IRunTestContext = {
+            isDebug: false,
+            kind: TestKind.TestNG,
+            projectName: 'testng',
+            testItems: [testItem],
+            testRun,
+            workspaceFolder: workspace.workspaceFolders?.[0]!,
+        };
+        const analyzer = new TestNGRunnerResultAnalyzer(runnerContext);
+        const trace = 'java.lang.ClassNotFoundException: example.SampleTest';
+
+        analyzer.processData(JSON.stringify({
+            name: 'error',
+            attributes: {
+                message: 'Failed to run TestNG tests',
+                trace,
+            },
+        }));
+
+        sinon.assert.calledOnce(erroredSpy);
+        sinon.assert.calledWith(erroredSpy, testItem, sinon.match.instanceOf(TestMessage));
+        const testMessage = erroredSpy.firstCall.args[1] as TestMessage;
+        assert.strictEqual(testMessage.message, `Failed to run TestNG tests\n${trace}`);
+    });
+
+    test('ignores control messages without test attributes', () => {
+        const testItem = generateTestItem(testController, 'testng@example.SampleTest#test', TestKind.TestNG);
+        const testRun = testController.createTestRun(new TestRunRequest([testItem], []));
+        const appendOutputSpy = sinon.spy(testRun, 'appendOutput');
+        const runnerContext: IRunTestContext = {
+            isDebug: false,
+            kind: TestKind.TestNG,
+            projectName: 'testng',
+            testItems: [testItem],
+            testRun,
+            workspaceFolder: workspace.workspaceFolders?.[0]!,
+        };
+        const analyzer = new TestNGRunnerResultAnalyzer(runnerContext);
+
+        analyzer.analyzeData('@@<TestRunner-{"name":"reporterAttached"}-TestRunner>');
+
+        sinon.assert.notCalled(appendOutputSpy);
+    });
+});
diff --git a/test/suite/baseRunner.test.ts b/test/suite/baseRunner.test.ts
