Skip to content

Commit c5ce275

Browse files
authored
e2e-test: browser in parallel & enable Currents (#5442)
### Summary This PR enables the Currents dashboard integration to evaluate the tool’s usefulness. Additionally, the browser is now configured to run on multiple ports, allowing browser tests to execute in parallel. I also consolidated matrix runs into a single machine and enabled multiple threads/workers. This adjustment seems to align better with the current setup and is likely more cost-efficient. ### QA Notes I performed numerous test runs to monitor the potential for increased flakiness with parallel workers. The results are promising overall. While I observed some minor flakiness, these issues appear manageable and can be addressed with further adjustments.
1 parent 7a118ad commit c5ce275

File tree

14 files changed

+775
-74
lines changed

14 files changed

+775
-74
lines changed

.github/workflows/e2e-test-release-run-ubuntu.yml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,15 @@ jobs:
8888
env:
8989
POSITRON_PY_VER_SEL: 3.10.12
9090
POSITRON_R_VER_SEL: 4.4.0
91-
id: electron-smoke-tests
91+
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
92+
CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
93+
COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
94+
PWTEST_BLOB_DO_NOT_REMOVE: 1
95+
CURRENTS_TAG: "electron,release,${{ inputs.e2e_grep }}"
96+
id: electron-e2e-tests
9297
run: |
9398
export DISPLAY=:10
94-
BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
99+
BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 3 --grep=${{ env.E2E_GREP }}
95100
96101
- name: Upload Playwright Report to S3
97102
if: ${{ !cancelled() }}

.github/workflows/positron-full-test.yml

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,6 @@ jobs:
9696
e2e-electron-tests:
9797
runs-on: ubuntu-latest-8x
9898
timeout-minutes: 60
99-
strategy:
100-
fail-fast: false
101-
matrix:
102-
shardIndex: [1, 2]
103-
shardTotal: [2]
10499
env:
105100
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
106101
POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -131,26 +126,32 @@ jobs:
131126
env:
132127
POSITRON_PY_VER_SEL: 3.10.12
133128
POSITRON_R_VER_SEL: 4.4.0
129+
CURRENTS_PROJECT_ID: ZOs5z2
130+
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
131+
CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
132+
COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
133+
PWTEST_BLOB_DO_NOT_REMOVE: 1
134+
CURRENTS_TAG: "electron"
134135
id: electron-tests
135-
run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 1 --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
136+
run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 3
136137

137138
- name: Upload blob report
138139
if: ${{ !cancelled() }}
139140
uses: actions/upload-artifact@v4
140141
with:
141-
name: blob-report-electron-${{ matrix.shardIndex }}
142+
name: blob-report-electron
142143
path: blob-report
143144
retention-days: 14
144145

145146
- name: Upload junit report
146147
if: ${{ !cancelled() }}
147148
uses: actions/upload-artifact@v4
148149
with:
149-
name: junit-report-electron-${{ matrix.shardIndex }}
150+
name: junit-report-electron
150151
path: test-results/junit.xml
151152

152153
e2e-browser-tests:
153-
runs-on: ubuntu-latest-4x
154+
runs-on: ubuntu-latest-8x
154155
timeout-minutes: 50
155156
env:
156157
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -188,8 +189,14 @@ jobs:
188189
env:
189190
POSITRON_PY_VER_SEL: 3.10.12
190191
POSITRON_R_VER_SEL: 4.4.0
192+
CURRENTS_PROJECT_ID: ZOs5z2
193+
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
194+
CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
195+
COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
196+
PWTEST_BLOB_DO_NOT_REMOVE: 1
197+
CURRENTS_TAG: "chromium"
191198
id: browser-tests
192-
run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 1
199+
run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 2
193200

194201
- name: Upload blob report
195202
if: ${{ !cancelled() }}

.github/workflows/positron-merge-to-branch.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,13 @@ jobs:
7070
env:
7171
POSITRON_PY_VER_SEL: 3.10.12
7272
POSITRON_R_VER_SEL: 4.4.0
73+
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
74+
CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
75+
COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
76+
PWTEST_BLOB_DO_NOT_REMOVE: 1
77+
CURRENTS_TAG: "electron,${{ inputs.e2e_grep }}"
7378
id: e2e-playwright-tests
74-
run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
79+
run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep=${{ env.E2E_GREP }}
7580

7681
- name: Upload Playwright Report to S3
7782
if: ${{ success() || failure() }}

.github/workflows/positron-windows-nightly.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,23 +102,29 @@ jobs:
102102
env:
103103
POSITRON_PY_VER_SEL: 3.10.10
104104
POSITRON_R_VER_SEL: 4.4.0
105+
CURRENTS_PROJECT_ID: ZOs5z2
106+
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
107+
CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
108+
COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
109+
PWTEST_BLOB_DO_NOT_REMOVE: 1
110+
CURRENTS_TAG: "electron,@win"
105111
if: ${{ !cancelled() }}
106112
id: e2e-win-electron-tests
107-
run: npx playwright test --project e2e-electron --grep "@win" --workers 1
113+
run: npx playwright test --project e2e-electron --grep "@win" --workers 2
108114

109115
- name: Upload blob report
110116
if: ${{ !cancelled() }}
111117
uses: actions/upload-artifact@v4
112118
with:
113-
name: blob-report-electron-${{ matrix.shardIndex }}
119+
name: blob-report-electron
114120
path: blob-report
115121
retention-days: 14
116122

117123
- name: Upload junit report
118124
if: ${{ !cancelled() }}
119125
uses: actions/upload-artifact@v4
120126
with:
121-
name: junit-report-electron-${{ matrix.shardIndex }}
127+
name: junit-report-electron
122128
path: test-results/junit.xml
123129

124130
e2e-report:

package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"update-localization-extension": "node build/npm/update-localization-extension.js",
4747
"e2e": "yarn e2e-electron",
4848
"e2e-electron": "npx playwright test --project e2e-electron",
49-
"e2e-browser": "npx playwright test --project e2e-browser --workers 1",
49+
"e2e-browser": "npx playwright test --project e2e-browser",
5050
"e2e-pr": "npx playwright test --project e2e-electron --grep @pr",
5151
"e2e-win": "npx playwright test --project e2e-electron --grep @win",
5252
"e2e-failed": "npx playwright test --last-failed",
@@ -129,6 +129,7 @@
129129
"yazl": "^2.4.3"
130130
},
131131
"devDependencies": {
132+
"@currents/playwright": "^1.8.0",
132133
"@midleman/github-actions-reporter": "^1.9.5",
133134
"@playwright/test": "^1.49.0",
134135
"@swc/core": "1.3.62",

playwright.config.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import { defineConfig } from '@playwright/test';
77
import { CustomTestOptions } from './test/smoke/src/areas/positron/_test.setup';
88
import type { GitHubActionOptions } from '@midleman/github-actions-reporter';
9+
import { currentsReporter } from '@currents/playwright';
910

1011
/**
1112
* See https://playwright.dev/docs/test-configuration.
@@ -37,7 +38,13 @@ export default defineConfig<CustomTestOptions>({
3738
includeResults: ['fail', 'flaky']
3839
}],
3940
['junit', { outputFile: 'test-results/junit.xml' }],
40-
['list'], ['html'], ['blob']
41+
['list'], ['html'], ['blob'],
42+
currentsReporter({
43+
ciBuildId: process.env.CURRENTS_CI_BUILD_ID || Date.now().toString(),
44+
recordKey: process.env.CURRENTS_RECORD_KEY || '',
45+
projectId: 'ZOs5z2',
46+
disableTitleTags: true,
47+
}),
4148
]
4249
: [
4350
['list'],

test/automation/src/playwrightBrowser.ts

Lines changed: 80 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -30,66 +30,115 @@ export async function launch(options: LaunchOptions): Promise<{ serverProcess: C
3030
};
3131
}
3232

33+
// --- Start Positron ---
34+
// Modified `launchServer` function to add support for multiple ports to enable parallel test
35+
// execution of browser tests. Also added helper functions: `getServerArgs`, `resolveServerLocation`,
36+
// and `startServer` to make this code easier to read.
3337
async function launchServer(options: LaunchOptions) {
3438
const { userDataDir, codePath, extensionsPath, logger, logsPath } = options;
3539
const serverLogsPath = join(logsPath, 'server');
3640
const codeServerPath = codePath ?? process.env.VSCODE_REMOTE_SERVER_PATH;
3741
const agentFolder = userDataDir;
42+
3843
await measureAndLog(() => mkdirp(agentFolder), `mkdirp(${agentFolder})`, logger);
3944

4045
const env = {
4146
VSCODE_REMOTE_SERVER_PATH: codeServerPath,
42-
...process.env
47+
...process.env,
4348
};
4449

50+
const maxRetries = 10;
51+
let serverProcess: ChildProcess | null = null;
52+
let endpoint: string | undefined;
53+
54+
for (let attempts = 0; attempts < maxRetries; attempts++) {
55+
const currentPort = port++;
56+
const args = getServerArgs(currentPort, extensionsPath, agentFolder, serverLogsPath, options.verbose);
57+
const serverLocation = resolveServerLocation(codeServerPath, logger);
58+
59+
logger.log(`Attempting to start server on port ${currentPort}`);
60+
logger.log(`Command: '${serverLocation}' ${args.join(' ')}`);
61+
62+
try {
63+
serverProcess = await startServer(serverLocation, args, env, logger);
64+
endpoint = await measureAndLog(
65+
() => waitForEndpoint(serverProcess!, logger),
66+
'waitForEndpoint(serverProcess)',
67+
logger
68+
);
69+
70+
logger.log(`Server started successfully on port ${currentPort}`);
71+
break; // Exit loop on success
72+
} catch (error) {
73+
if ((error as Error).message.includes('EADDRINUSE')) {
74+
logger.log(`Port ${currentPort} is already in use. Retrying...`);
75+
serverProcess?.kill();
76+
} else {
77+
throw error; // Rethrow non-port-related errors
78+
}
79+
}
80+
}
81+
82+
if (!serverProcess || !endpoint) {
83+
throw new Error('Failed to launch the server after multiple attempts.');
84+
}
85+
86+
return { serverProcess, endpoint };
87+
}
88+
89+
function getServerArgs(
90+
port: number,
91+
extensionsPath: string,
92+
agentFolder: string,
93+
logsPath: string,
94+
verbose?: boolean
95+
): string[] {
4596
const args = [
4697
'--disable-telemetry',
4798
'--disable-workspace-trust',
48-
`--port=${port++}`,
99+
`--port=${port}`,
49100
'--enable-smoke-test-driver',
50101
`--extensions-dir=${extensionsPath}`,
51102
`--server-data-dir=${agentFolder}`,
52103
'--accept-server-license-terms',
53-
`--logsPath=${serverLogsPath}`,
54-
// --- Start Positron ---
55-
`--connection-token`,
56-
`dev-token`
57-
// --- End Positron ---
104+
`--logsPath=${logsPath}`,
105+
'--connection-token',
106+
'dev-token',
58107
];
59108

60-
if (options.verbose) {
109+
if (verbose) {
61110
args.push('--log=trace');
62111
}
63112

64-
let serverLocation: string | undefined;
113+
return args;
114+
}
115+
116+
function resolveServerLocation(codeServerPath: string | undefined, logger: Logger): string {
65117
if (codeServerPath) {
66118
const { serverApplicationName } = require(join(codeServerPath, 'product.json'));
67-
serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
68-
69-
logger.log(`Starting built server from '${serverLocation}'`);
70-
} else {
71-
serverLocation = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
72-
73-
logger.log(`Starting server out of sources from '${serverLocation}'`);
119+
const serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
120+
logger.log(`Using built server from '${serverLocation}'`);
121+
return serverLocation;
74122
}
75123

76-
logger.log(`Storing log files into '${serverLogsPath}'`);
77-
78-
logger.log(`Command line: '${serverLocation}' ${args.join(' ')}`);
79-
const shell: boolean = (process.platform === 'win32');
80-
const serverProcess = spawn(
81-
serverLocation,
82-
args,
83-
{ env, shell }
84-
);
85-
86-
logger.log(`Started server for browser smoke tests (pid: ${serverProcess.pid})`);
124+
const scriptPath = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
125+
logger.log(`Using source server from '${scriptPath}'`);
126+
return scriptPath;
127+
}
87128

88-
return {
89-
serverProcess,
90-
endpoint: await measureAndLog(() => waitForEndpoint(serverProcess, logger), 'waitForEndpoint(serverProcess)', logger)
91-
};
129+
async function startServer(
130+
serverLocation: string,
131+
args: string[],
132+
env: NodeJS.ProcessEnv,
133+
logger: Logger
134+
): Promise<ChildProcess> {
135+
logger.log(`Starting server: ${serverLocation}`);
136+
const serverProcess = spawn(serverLocation, args, { env, shell: process.platform === 'win32' });
137+
logger.log(`Server started (pid: ${serverProcess.pid})`);
138+
return serverProcess;
92139
}
140+
// --- End Positron ---
141+
93142

94143
async function launchBrowser(options: LaunchOptions, endpoint: string) {
95144
const { logger, workspacePath, tracing, headless } = options;

test/automation/src/positron/positronNotebooks.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ export class PositronNotebooks {
8282
}
8383

8484
async assertCellOutput(text: string): Promise<void> {
85-
await expect(this.frameLocator.getByText(text)).toBeVisible();
85+
await expect(this.frameLocator.getByText(text)).toBeVisible({ timeout: 15000 });
8686
}
8787

8888
async closeNotebookWithoutSaving() {

test/automation/src/positron/positronVariables.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ export class PositronVariables {
5757
}
5858

5959
async doubleClickVariableRow(variableName: string) {
60-
6160
const desiredRow = await this.waitForVariableRow(variableName);
6261
await desiredRow.dblclick();
6362
}

0 commit comments

Comments
 (0)