Improve macOS integration test performance (#727)

marcins · web-flow · commit 2f29297bd550 · 2025-07-31T13:57:00.000+10:00
diff --git a/packages/core/integration-tests/test/cache.ts b/packages/core/integration-tests/test/cache.ts
@@ -40,6 +40,7 @@ import {deserialize} from '@atlaspack/build-cache';
 import {hashString} from '@atlaspack/rust';
 import {getAllEnvironments} from '@atlaspack/rust';
 import type {FeatureFlags} from '@atlaspack/feature-flags';
+import os from 'os';
 
 let inputDir: string;
 let packageManager = new NodePackageManager(inputFS, '/');
@@ -56,6 +57,7 @@ let packageManager = new NodePackageManager(inputFS, '/');
       {
         inputFS: overlayFS,
         shouldDisableCache: false,
+        watchBackend: os.platform() === 'darwin' ? 'fs-events' : undefined,
         featureFlags: {
           ...featureFlags,
           cachePerformanceImprovements,
diff --git a/packages/core/test-utils/package.json b/packages/core/test-utils/package.json
@@ -24,14 +24,17 @@
     "@atlaspack/fs": "2.15.16",
     "@atlaspack/package-manager": "2.14.21",
     "@atlaspack/utils": "2.17.3",
+    "async": "^3.2.6",
     "chalk": "^4.1.0",
     "expect": "^29.7.0",
     "ncp": "^2.0.0",
     "nullthrows": "^1.1.1",
     "posthtml": "^0.16.5",
     "posthtml-parser": "^0.10.1",
     "resolve": "^1.12.0",
+    "rimraf": "^5.0.5",
     "ws": "^7.0.0",
+    "@types/async": "^3.2.6",
     "tempy": "^1.0.0",
     "why-is-node-running": "^3.2.2"
   }
diff --git a/packages/core/test-utils/src/utils.ts b/packages/core/test-utils/src/utils.ts
@@ -40,6 +40,8 @@ import {LMDBLiteCache} from '@atlaspack/cache';
 import tempy from 'tempy';
 import {FILE_CONFIG_NO_REPORTERS} from './paths';
 
+import {queue} from 'async';
+
 export let cacheDir: string = tempy.directory();
 export let cache: LMDBLiteCache = new LMDBLiteCache(cacheDir);
 cache.ensure();
@@ -1328,6 +1330,52 @@ export async function assertNoFilePathInCache(
   fs: FileSystem,
   dir: string,
   projectRoot: string,
+) {
+  // For debugging purposes, log all instances of the projectRoot in the cache.
+  // Otherwise, fail the test if one is found.
+  if (process.env.ATLASPACK_DEBUG_CACHE_FILEPATH != null)
+    return assertNoFilePathInCacheDebug(fs, dir, projectRoot);
+
+  const processFile = async (filePath: string) => {
+    let contents = await fs.readFile(filePath);
+    if (contents.includes(projectRoot)) {
+      throw new Error(
+        `Found projectRoot ${projectRoot} in cache file ${filePath}`,
+      );
+    }
+  };
+  // This "optimal" concurrency value was determined by using hyperfine locally on a subset of tests
+  // that use the cache.
+  const q = queue(processFile, 50);
+
+  const queueDir = async (dir: string) => {
+    const files = await fs.readdir(dir);
+    for (const file of files) {
+      const fullPath = path.join(dir, file);
+      const stat = await fs.stat(fullPath);
+      if (stat.isFile() && !file.endsWith('.txt')) {
+        q.push(fullPath);
+      } else if (stat.isDirectory()) {
+        await queueDir(fullPath);
+      }
+    }
+  };
+  await queueDir(dir);
+
+  q.error((err: Error) => {
+    assert.fail(err);
+  });
+  if (q.length() > 0) {
+    await q.drain();
+  }
+}
+
+// This function is called from the assertNoFilePathInCache function when the
+// ATLASPACK_DEBUG_CACHE_FILEPATH environment variable is set.
+async function assertNoFilePathInCacheDebug(
+  fs: FileSystem,
+  dir: string,
+  projectRoot: string,
 ) {
   let entries = await fs.readdir(dir);
   for (let entry of entries) {
@@ -1343,38 +1391,29 @@ export async function assertNoFilePathInCache(
     } else if (stat.isFile()) {
       let contents = await fs.readFile(fullPath);
 
-      // For debugging purposes, log all instances of the projectRoot in the cache.
-      // Otherwise, fail the test if one is found.
-      if (process.env.ATLASPACK_DEBUG_CACHE_FILEPATH != null) {
-        if (contents.includes(projectRoot)) {
-          let deserialized;
-          try {
-            deserialized = v8.deserialize(contents);
-          } catch (err: any) {
-            // rudimentary detection of binary files
-            if (!contents.includes(0)) {
-              deserialized = contents.toString();
-            } else {
-              deserialized = contents;
-            }
+      if (contents.includes(projectRoot)) {
+        let deserialized;
+        try {
+          deserialized = v8.deserialize(contents);
+        } catch (err: any) {
+          // rudimentary detection of binary files
+          if (!contents.includes(0)) {
+            deserialized = contents.toString();
+          } else {
+            deserialized = contents;
           }
+        }
 
-          if (deserialized != null) {
-            // eslint-disable-next-line no-console
-            console.log(
-              `Found projectRoot ${projectRoot} in cache file ${fullPath}`,
-            );
-            // eslint-disable-next-line no-console
-            console.log(
-              require('util').inspect(deserialized, {depth: 50, colors: true}),
-            );
-          }
+        if (deserialized != null) {
+          // eslint-disable-next-line no-console
+          console.log(
+            `Found projectRoot ${projectRoot} in cache file ${fullPath}`,
+          );
+          // eslint-disable-next-line no-console
+          console.log(
+            require('util').inspect(deserialized, {depth: 50, colors: true}),
+          );
         }
-      } else {
-        assert(
-          !contents.includes(projectRoot),
-          `Found projectRoot ${projectRoot} in cache file ${fullPath}`,
-        );
       }
     }
   }
diff --git a/packages/dev/mocha-profiler/package.json b/packages/dev/mocha-profiler/package.json
@@ -4,7 +4,8 @@
   "description": "Profile mocha tests",
   "license": "(MIT OR Apache-2.0)",
   "private": true,
-  "main": "src/index.js",
+  "main": "./lib/index.js",
+  "source": "./src/index.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/atlassian-labs/atlaspack.git"
diff --git a/yarn.lock b/yarn.lock
@@ -4047,6 +4047,11 @@
   dependencies:
     tslib "^2.4.0"
 
+"@types/async@^3.2.6":
+  version "3.2.24"
+  resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.24.tgz#3a96351047575bbcf2340541b2d955a35339608f"
+  integrity sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==
+
 "@types/babel__core@*", "@types/babel__core@^7.12.2":
   version "7.20.5"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.5.tgz#3df15f27ba85319caa07ba08d0721889bb39c017"
@@ -5364,6 +5369,11 @@ async@^3.2.3:
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66"
   integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==
 
+async@^3.2.6:
+  version "3.2.6"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.6.tgz#1b0728e14929d51b85b449b7f06e27c1145e38ce"
+  integrity sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -16465,7 +16475,7 @@ string-natural-compare@^3.0.1:
   resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4"
   integrity sha512-n3sPwynL1nwKi3WJ6AIsClwBMa0zTi54fn2oLU6ndfTSIO05xaznjSf15PcBZU6FNWbmN5Q6cxT4V5hGvB4taw==
 
-"string-width-cjs@npm:string-width@^4.2.0":
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -16491,15 +16501,6 @@ string-width@^1.0.1, string-width@^1.0.2:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^4.0.0"
 
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
 string-width@^5.0.1, string-width@^5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794"
@@ -16587,7 +16588,7 @@ stringify-object@^3.3.0:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -16608,13 +16609,6 @@ strip-ansi@^4.0.0:
   dependencies:
     ansi-regex "^3.0.0"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1, strip-ansi@^7.1.0:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45"
@@ -18246,7 +18240,7 @@ workerpool@6.1.0:
   resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.0.tgz#a8e038b4c94569596852de7a8ea4228eefdeb37b"
   integrity sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -18272,15 +18266,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
