test(wtr): improve reliability (#5581)

* test(wtr): add coverage reports

* chore(ci): always upload coverage

* test(wtr): improve concurrency

hope it's not too flaky

* chore(wtr): lower coverage thresholds

* chore(wtr): lower hydration coverage thresholds

* test(wtr): fix concurrency

* chore(ci): cache playwright deps

* chore(ci): maybe fix version check

* chore: debug

* chore: debug

* chore: debug

* chore: fix playwright version detection

* chore(wtr): change default dir to monorepo root

* chore(ci): clean up coverage

* test(wtr): revert concurrency

not that reliable

* chore(ci): always do playwright install

* chore(ci): actually, who cares about coverage reports

* chore(ci): time out playwright install

* chore(ci): refine timeout

* chore: okay 3 minutes

* chore(ci): try again on flaky steps

* chore(ci): try to make conditions work

* chore(ci): don't fail CI if flaky steps fail first time

* test(wtr): enable all tests

* chore(ci): give playwright retry an extra minute

* chore(ci): restore coverage reports

Author: wjhsf

.github/workflows/web-test-runner.yml

@@ -14,14 +14,8 @@ env:
     COVERAGE: '1'
     NODE_VERSION: '20.19.4'
 
-defaults:
-    run:
-        working-directory: packages/@lwc/integration-not-karma
-
 jobs:
     # We run tests in parallel to speed up CI, with an arbitrary goal of ~20 minutes per job.
-    # TODO: upload result artifacts
-
     integration-tests:
         name: Integration tests (${{ matrix.shadow_mode }} shadow; ${{ matrix.env || 'default env' }})
         runs-on: ubuntu-22.04
@@ -71,12 +65,33 @@ jobs:
                   node-version: ${{ env.NODE_VERSION }}
                   cache: 'yarn'
 
+            - name: Get installed playwright version
+              id: playwright-version
+              run: |
+                  version="$(yarn --silent --json list --depth 0 playwright 2>/dev/null | jq -r '.data.trees[0].name|split("@")[1]')"
+                  echo "version=$version" >> "$GITHUB_OUTPUT"
+
+            - name: Restore playwright cache
+              uses: actions/cache@v4
+              with:
+                  path: |
+                      ~/.cache/ms-playwright
+                  key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}
+
             - name: Install dependencies
               run: yarn install --frozen-lockfile
-              working-directory: ./
 
             - name: Install playwright dependencies
+              id: install-playwright
+              continue-on-error: true
               run: yarn playwright install chrome firefox webkit --with-deps
+              timeout-minutes: 3 # Sometimes seems to just hang forever? :|
+
+            # If installing playwright times out, wait 1 minute then try again
+            - name: Install playwright dependencies (fallback)
+              if: ${{ steps.install-playwright.outcome != 'success' }}
+              run: sleep 60 && yarn playwright install chrome firefox webkit --with-deps
+              timeout-minutes: 4
 
             - name: Set env vars
               run: echo '${{ matrix.env }}' | tr ' ' '\n' >> "$GITHUB_ENV"
@@ -93,13 +108,24 @@ jobs:
                   echo "NODE_ENV_FOR_TEST=$NODE_ENV_FOR_TEST"
 
             - name: Run integration tests
+              id: test-integration
+              continue-on-error: true
               run: yarn test
+              working-directory: packages/@lwc/integration-not-karma
+
+            # Because sometimes things are flaky
+            - name: Run integration tests (fallback)
+              if: ${{ steps.test-integration.outcome == 'failure' }}
+              run: yarn run test
+              working-directory: packages/@lwc/integration-not-karma
 
             - name: Upload coverage report
               # Upload coverage even if the test step failed
               if: always()
               run: |
-                  [ -f coverage/index.html ] && awk '/<table/,/<\/table>/' coverage/index.html >> "$GITHUB_STEP_SUMMARY"
+                  if [ -f coverage/index.html ]; then
+                    awk '/<table/,/<\/table>/' coverage/index.html >> "$GITHUB_STEP_SUMMARY"
+                  fi
 
     hydration-tests:
         name: Hydration tests (${{ matrix.engine_server && 'engine-server' || 'SSR v2' }}; ${{ matrix.env || 'default env' }})
@@ -134,12 +160,33 @@ jobs:
                   node-version: ${{ env.NODE_VERSION }}
                   cache: 'yarn'
 
+            - name: Get installed playwright version
+              id: playwright-version
+              run: |
+                  version="$(yarn --silent --json list --depth 0 playwright 2>/dev/null | jq -r '.data.trees[0].name|split("@")[1]')"
+                  echo "version=$version" >> "$GITHUB_OUTPUT"
+
+            - name: Restore playwright cache
+              uses: actions/cache@v4
+              with:
+                  path: |
+                      ~/.cache/ms-playwright
+                  key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}
+
             - name: Install dependencies
               run: yarn install --frozen-lockfile
-              working-directory: ./
 
             - name: Install playwright dependencies
+              id: install-playwright
+              continue-on-error: true
               run: yarn playwright install chrome firefox webkit --with-deps
+              timeout-minutes: 3 # Sometimes seems to just hang forever? :|
+
+            # If installing playwright times out, wait 1 minute then try again
+            - name: Install playwright dependencies (fallback)
+              if: ${{ steps.install-playwright.outcome != 'success' }}
+              run: sleep 60 && yarn playwright install chrome firefox webkit --with-deps
+              timeout-minutes: 4
 
             - name: Set env vars
               run: echo '${{ matrix.env }}' | tr ' ' '\n' >> "$GITHUB_ENV"
@@ -154,10 +201,21 @@ jobs:
                   echo "ENGINE_SERVER=$ENGINE_SERVER"
 
             - name: Run hydration tests
-              run: yarn test:hydration
+              id: test-hydration
+              continue-on-error: true
+              run: yarn run test:hydration
+              working-directory: packages/@lwc/integration-not-karma
+
+            # Because sometimes things are flaky
+            - name: Run hydration tests (fallback)
+              if: ${{ steps.test-hydration.outcome == 'failure' }}
+              run: yarn run test:hydration
+              working-directory: packages/@lwc/integration-not-karma
 
             - name: Upload coverage report
               # Upload coverage even if the test step failed
               if: always()
               run: |
-                  [ -f coverage/index.html ] && awk '/<table/,/<\/table>/' coverage/index.html >> "$GITHUB_STEP_SUMMARY"
+                  if [ -f coverage/index.html ]; then
+                    awk '/<table/,/<\/table>/' coverage/index.html >> "$GITHUB_STEP_SUMMARY"
+                  fi

packages/@lwc/integration-not-karma/configs/hydration.js

@@ -12,7 +12,7 @@ const baseConfig = createConfig({
 /** @type {import("@web/test-runner").TestRunnerConfig} */
 export default {
     ...baseConfig,
-    files: ['test-hydration/**/*.spec.js', '!test-hydration/synthetic-shadow/index.spec.js'],
+    files: ['test-hydration/**/*.spec.js'],
     plugins: [...baseConfig.plugins, hydrationTestPlugin],
     coverageConfig: {
         ...baseConfig.coverageConfig,

packages/@lwc/integration-not-karma/configs/integration.js

@@ -13,7 +13,7 @@ const baseConfig = createConfig({
 /** @type {import("@web/test-runner").TestRunnerConfig} */
 export default {
     ...baseConfig,
-    files: ['test/**/*.spec.js', '!test/custom-elements/index.spec.js'],
+    files: ['test/**/*.spec.js'],
     plugins: [
         ...baseConfig.plugins,
         // Only used for the `sanitizeAttribute` test