From 896f76c89d9e7c1215701dea0303ddede7fbc969 Mon Sep 17 00:00:00 2001 From: Panda Wu Date: Sat, 3 Oct 2026 21:08:31 +0800 Subject: [PATCH] Derive test counts from judge fixtures Repeated browser totals and literal report comparisons drift when judge cases change. Resolve flow judges before browser startup and reuse their counts. Validate live report totals against current fixtures while comparing passing counts with the saved capture. Keep its total historical so case additions do not require updates. --- scripts/browser-check.cjs | 277 ++++++++++-------- scripts/judge-case-counts.cjs | 61 ++++ scripts/report-parity-check.sh | 16 +- tests/browser/judge-case-counts.test.js | 355 ++++++++++++++++++++++++ 4 files changed, 592 insertions(+), 117 deletions(-) create mode 100644 scripts/judge-case-counts.cjs create mode 100644 tests/browser/judge-case-counts.test.js diff --git a/scripts/browser-check.cjs b/scripts/browser-check.cjs index a29c8ac3..38d64b5a 100644 --- a/scripts/browser-check.cjs +++ b/scripts/browser-check.cjs @@ -17,17 +17,50 @@ const { listRoomParticipants, removeParticipant, } = require("./livekit-room-service.cjs"); +const { problemPages, judgeCaseCount } = require("./judge-case-counts.cjs"); + +const PROBLEM_IDS = { + twoSum: "two-sum", + validParentheses: "valid-parentheses", + minStack: "min-stack", + treeIterator: "binary-search-tree-iterator", + mergeSortedArray: "merge-sorted-array", + limitDuplicates: "remove-duplicates-from-sorted-array-ii", + mergeLists: "merge-two-sorted-lists", + copyRandomList: "copy-list-with-random-pointer", + rotateList: "rotate-list", + invertTree: "invert-binary-tree", + buildTree: "construct-binary-tree-from-preorder-and-inorder-traversal", + connectTreeLevels: "populating-next-right-pointers-in-each-node-ii", + lowestCommonAncestor: "lowest-common-ancestor-of-a-binary-tree", + zigzagTraversal: "binary-tree-zigzag-level-order-traversal", + validateSearchTree: "validate-binary-search-tree", + cloneGraph: "clone-graph", + courseOrder: "course-schedule-ii", + trie: "implement-trie-prefix-tree", + wordDictionary: "design-add-and-search-words-data-structure", + combinationSum: "combination-sum", + permutations: "permutations", + generateParentheses: "generate-parentheses", + nQueens: "n-queens-ii", + wordSearch: "word-search", + sortedArrayToTree: "convert-sorted-array-to-binary-search-tree", +}; + +// Resolve every flow's judge before launching the browser or starting an interview. +const scenarios = new Map( + Object.values(PROBLEM_IDS).map((problemId) => [ + problemId, + { ...problemPages[problemId], caseCount: judgeCaseCount(problemId) }, + ]), +); // The browser loads scenario names, while the checks stay keyed by the stable // bank ids that identify their judges and candidate programs. Reading the // generated map makes that boundary explicit instead of letting a removed // problem file decide which scenario each flow happens to exercise. -const problemPages = JSON.parse( - fs.readFileSync(`${__dirname}/../web/problem-pages.json`, "utf8"), -); - function scenario(problemId) { - const entry = problemPages[problemId]; + const entry = scenarios.get(problemId); if (!entry) throw new Error(`no scenario page for ${problemId}`); return entry; } @@ -415,7 +448,7 @@ function stopProcessGroup(child) { /// because the source was right and the order it ran in was not. async function checkWhiteboardInterview(page, pageErrors) { const before = pageErrors.length; - await page.goto(interviewUrl("two-sum", "whiteboard"), { + await page.goto(interviewUrl(PROBLEM_IDS.twoSum, "whiteboard"), { waitUntil: "domcontentloaded", }); @@ -922,7 +955,8 @@ async function isolateRustAgent( // instead of the agent output above. agentFailure.catch(() => {}); } - async function runAndExpectPassing(expected, timeout = 30000) { + async function runAndExpectPassing(problemId, timeout = 30000) { + const expected = scenario(problemId).caseCount; await page.getByRole("button", { name: /Run tests/ }).click(); await page .getByRole("button", { name: "Run tests" }) @@ -955,7 +989,7 @@ async function isolateRustAgent( // The browser launches with --use-fake-ui-for-media-stream, which is why // no other flow in this file grants permissions. await page.setViewportSize({ width: 1440, height: 900 }); - await page.goto(interviewUrl("two-sum"), { + await page.goto(interviewUrl(PROBLEM_IDS.twoSum), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); @@ -1097,19 +1131,22 @@ async function isolateRustAgent( await page.getByText("Choose a specific problem instead").click(); await page.getByRole("checkbox", { name: "Easy" }).check(); await page - .getByText(scenarioTitle("valid-parentheses"), { exact: true }) + .getByText(scenarioTitle(PROBLEM_IDS.validParentheses), { exact: true }) .waitFor(); return; } if (mode === "offline") { await checkWhiteboardInterview(page, pageErrors); - await page.goto(interviewUrl("two-sum"), { + await page.goto(interviewUrl(PROBLEM_IDS.twoSum), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("two-sum"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.twoSum), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await checkEditorNewlines(page); @@ -1143,13 +1180,16 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return out; } `); - await runAndExpectPassing(5, 120000); - await page.goto(interviewUrl("min-stack"), { + await runAndExpectPassing(PROBLEM_IDS.twoSum, 120000); + await page.goto(interviewUrl(PROBLEM_IDS.minStack), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("min-stack"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.minStack), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await page.getByRole("button", { name: "C++" }).click(); @@ -1172,14 +1212,14 @@ public: int getMin() { return minimums.back(); } }; `); - await runAndExpectPassing(5, 120000); - await page.goto(interviewUrl("binary-search-tree-iterator"), { + await runAndExpectPassing(PROBLEM_IDS.minStack, 120000); + await page.goto(interviewUrl(PROBLEM_IDS.treeIterator), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("binary-search-tree-iterator"), + name: scenarioTitle(PROBLEM_IDS.treeIterator), level: 1, }) .waitFor(); @@ -1202,7 +1242,7 @@ public: public boolean hasNext() { return !stack.isEmpty(); } } `); - await runAndExpectPassing(5, 120000); + await runAndExpectPassing(PROBLEM_IDS.treeIterator, 120000); return; } if (compilerExplorerBaseUrl !== "__default__") { @@ -1234,7 +1274,9 @@ function matchDisputedCharge() { `); await page.getByRole("button", { name: /Run tests/ }).click(); await page.getByRole("button", { name: "Run tests" }).waitFor(); - await page.getByText("Test results · 0/5").waitFor(); + await page + .getByText(`Test results · 0/${scenario(PROBLEM_IDS.twoSum).caseCount}`) + .waitFor(); await page.getByLabel("Code editor") .fill(`function matchDisputedCharge(nums, target) { const seen = new Map(); @@ -1246,7 +1288,7 @@ function matchDisputedCharge() { return []; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.twoSum); await page.getByRole("button", { name: "C++" }).click(); await page.getByLabel("Code editor").fill(`class Solution { public: @@ -1261,7 +1303,7 @@ public: } }; `); - await runAndExpectPassing(5, 120000); + await runAndExpectPassing(PROBLEM_IDS.twoSum, 120000); await page.getByLabel("Code editor").fill(`class Solution { public: vector matchDisputedCharge(vector& nums, int target) { @@ -1298,7 +1340,7 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return out; } `); - await runAndExpectPassing(5, 120000); + await runAndExpectPassing(PROBLEM_IDS.twoSum, 120000); await page.getByRole("button", { name: "Python" }).click(); await page.getByLabel("Code editor").fill(`class Solution: def matchDisputedCharge(self, nums, target): @@ -1310,18 +1352,18 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { seen[value] = i return [] `); - await runAndExpectPassing(5, 120000); + await runAndExpectPassing(PROBLEM_IDS.twoSum, 120000); await page.getByRole("button", { name: "Transcript" }).click(); await page.locator("p").filter({ hasText: /^Jim$/ }).first().waitFor(); await page.locator("p").filter({ hasText: /^You$/ }).first().waitFor(); - await page.goto(interviewUrl("merge-sorted-array"), { + await page.goto(interviewUrl(PROBLEM_IDS.mergeSortedArray), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("merge-sorted-array"), + name: scenarioTitle(PROBLEM_IDS.mergeSortedArray), level: 1, }) .waitFor(); @@ -1341,15 +1383,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { } } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.mergeSortedArray); - await page.goto(interviewUrl("remove-duplicates-from-sorted-array-ii"), { + await page.goto(interviewUrl(PROBLEM_IDS.limitDuplicates), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("remove-duplicates-from-sorted-array-ii"), + name: scenarioTitle(PROBLEM_IDS.limitDuplicates), level: 1, }) .waitFor(); @@ -1368,7 +1410,11 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { `); await page.getByRole("button", { name: /Run tests/ }).click(); await page.getByRole("button", { name: "Run tests" }).waitFor(); - await page.getByText("Test results · 0/5").waitFor(); + await page + .getByText( + `Test results · 0/${scenario(PROBLEM_IDS.limitDuplicates).caseCount}`, + ) + .waitFor(); await page.getByLabel("Code editor") .fill(`function capRepeatsAtTwo(nums) { let write = 0; @@ -1380,15 +1426,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return write; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.limitDuplicates); - await page.goto(interviewUrl("merge-two-sorted-lists"), { + await page.goto(interviewUrl(PROBLEM_IDS.mergeLists), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("merge-two-sorted-lists"), + name: scenarioTitle(PROBLEM_IDS.mergeLists), level: 1, }) .waitFor(); @@ -1412,15 +1458,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return dummy.next; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.mergeLists); - await page.goto(interviewUrl("copy-list-with-random-pointer"), { + await page.goto(interviewUrl(PROBLEM_IDS.copyRandomList), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("copy-list-with-random-pointer"), + name: scenarioTitle(PROBLEM_IDS.copyRandomList), level: 1, }) .waitFor(); @@ -1438,14 +1484,17 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return copies.get(head); } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.copyRandomList); - await page.goto(interviewUrl("rotate-list"), { + await page.goto(interviewUrl(PROBLEM_IDS.rotateList), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("rotate-list"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.rotateList), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await page.getByRole("button", { name: "JavaScript" }).click(); @@ -1468,15 +1517,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return next; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.rotateList); - await page.goto(interviewUrl("invert-binary-tree"), { + await page.goto(interviewUrl(PROBLEM_IDS.invertTree), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("invert-binary-tree"), + name: scenarioTitle(PROBLEM_IDS.invertTree), level: 1, }) .waitFor(); @@ -1490,20 +1539,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return root; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.invertTree); - await page.goto( - interviewUrl( - "construct-binary-tree-from-preorder-and-inorder-traversal", - ), - { waitUntil: "domcontentloaded" }, - ); + await page.goto(interviewUrl(PROBLEM_IDS.buildTree), { + waitUntil: "domcontentloaded", + }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle( - "construct-binary-tree-from-preorder-and-inorder-traversal", - ), + name: scenarioTitle(PROBLEM_IDS.buildTree), level: 1, }) .waitFor(); @@ -1525,16 +1569,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return build(0, inorder.length - 1); } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.buildTree); - await page.goto( - interviewUrl("populating-next-right-pointers-in-each-node-ii"), - { waitUntil: "domcontentloaded" }, - ); + await page.goto(interviewUrl(PROBLEM_IDS.connectTreeLevels), { + waitUntil: "domcontentloaded", + }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("populating-next-right-pointers-in-each-node-ii"), + name: scenarioTitle(PROBLEM_IDS.connectTreeLevels), level: 1, }) .waitFor(); @@ -1555,15 +1598,15 @@ int* matchDisputedCharge(int* nums, int numsSize, int target, int* returnSize) { return root; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.connectTreeLevels); - await page.goto(interviewUrl("binary-search-tree-iterator"), { + await page.goto(interviewUrl(PROBLEM_IDS.treeIterator), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("binary-search-tree-iterator"), + name: scenarioTitle(PROBLEM_IDS.treeIterator), level: 1, }) .waitFor(); @@ -1592,15 +1635,15 @@ OrderedCursor.prototype.hasNext = function() { return this.stack.length > 0; }; `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.treeIterator); - await page.goto(interviewUrl("lowest-common-ancestor-of-a-binary-tree"), { + await page.goto(interviewUrl(PROBLEM_IDS.lowestCommonAncestor), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("lowest-common-ancestor-of-a-binary-tree"), + name: scenarioTitle(PROBLEM_IDS.lowestCommonAncestor), level: 1, }) .waitFor(); @@ -1615,16 +1658,15 @@ OrderedCursor.prototype.hasNext = function() { return left || right; } `); - await runAndExpectPassing(6); + await runAndExpectPassing(PROBLEM_IDS.lowestCommonAncestor); - await page.goto( - interviewUrl("binary-tree-zigzag-level-order-traversal"), - { waitUntil: "domcontentloaded" }, - ); + await page.goto(interviewUrl(PROBLEM_IDS.zigzagTraversal), { + waitUntil: "domcontentloaded", + }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("binary-tree-zigzag-level-order-traversal"), + name: scenarioTitle(PROBLEM_IDS.zigzagTraversal), level: 1, }) .waitFor(); @@ -1652,15 +1694,15 @@ OrderedCursor.prototype.hasNext = function() { return rows; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.zigzagTraversal); - await page.goto(interviewUrl("validate-binary-search-tree"), { + await page.goto(interviewUrl(PROBLEM_IDS.validateSearchTree), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("validate-binary-search-tree"), + name: scenarioTitle(PROBLEM_IDS.validateSearchTree), level: 1, }) .waitFor(); @@ -1676,14 +1718,17 @@ OrderedCursor.prototype.hasNext = function() { return valid(root, -Infinity, Infinity); } `); - await runAndExpectPassing(7); + await runAndExpectPassing(PROBLEM_IDS.validateSearchTree); - await page.goto(interviewUrl("clone-graph"), { + await page.goto(interviewUrl(PROBLEM_IDS.cloneGraph), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("clone-graph"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.cloneGraph), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await page.getByRole("button", { name: "JavaScript" }).click(); @@ -1701,15 +1746,15 @@ OrderedCursor.prototype.hasNext = function() { return clone(node); } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.cloneGraph); - await page.goto(interviewUrl("course-schedule-ii"), { + await page.goto(interviewUrl(PROBLEM_IDS.courseOrder), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("course-schedule-ii"), + name: scenarioTitle(PROBLEM_IDS.courseOrder), level: 1, }) .waitFor(); @@ -1738,15 +1783,15 @@ OrderedCursor.prototype.hasNext = function() { return order.length === migrationCount ? order : []; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.courseOrder); - await page.goto(interviewUrl("implement-trie-prefix-tree"), { + await page.goto(interviewUrl(PROBLEM_IDS.trie), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("implement-trie-prefix-tree"), + name: scenarioTitle(PROBLEM_IDS.trie), level: 1, }) .waitFor(); @@ -1784,16 +1829,15 @@ CommandIndex.prototype.find = function(text) { return node; }; `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.trie); - await page.goto( - interviewUrl("design-add-and-search-words-data-structure"), - { waitUntil: "domcontentloaded" }, - ); + await page.goto(interviewUrl(PROBLEM_IDS.wordDictionary), { + waitUntil: "domcontentloaded", + }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("design-add-and-search-words-data-structure"), + name: scenarioTitle(PROBLEM_IDS.wordDictionary), level: 1, }) .waitFor(); @@ -1830,15 +1874,15 @@ PatternLexicon.prototype.search = function(word) { return dfs(this, 0); }; `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.wordDictionary); - await page.goto(interviewUrl("combination-sum"), { + await page.goto(interviewUrl(PROBLEM_IDS.combinationSum), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("combination-sum"), + name: scenarioTitle(PROBLEM_IDS.combinationSum), level: 1, }) .waitFor(); @@ -1863,14 +1907,17 @@ PatternLexicon.prototype.search = function(word) { return results; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.combinationSum); - await page.goto(interviewUrl("permutations"), { + await page.goto(interviewUrl(PROBLEM_IDS.permutations), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("permutations"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.permutations), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await page.getByRole("button", { name: "JavaScript" }).click(); @@ -1895,15 +1942,15 @@ PatternLexicon.prototype.search = function(word) { return results; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.permutations); - await page.goto(interviewUrl("generate-parentheses"), { + await page.goto(interviewUrl(PROBLEM_IDS.generateParentheses), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("generate-parentheses"), + name: scenarioTitle(PROBLEM_IDS.generateParentheses), level: 1, }) .waitFor(); @@ -1924,14 +1971,17 @@ PatternLexicon.prototype.search = function(word) { return results; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.generateParentheses); - await page.goto(interviewUrl("n-queens-ii"), { + await page.goto(interviewUrl(PROBLEM_IDS.nQueens), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("n-queens-ii"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.nQueens), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await page.getByRole("button", { name: "JavaScript" }).click(); @@ -1960,14 +2010,17 @@ PatternLexicon.prototype.search = function(word) { return count; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.nQueens); - await page.goto(interviewUrl("word-search"), { + await page.goto(interviewUrl(PROBLEM_IDS.wordSearch), { waitUntil: "domcontentloaded", }); await clearMediaGate(page); await page - .getByRole("heading", { name: scenarioTitle("word-search"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.wordSearch), + level: 1, + }) .waitFor(); await page.getByText("Offline", { exact: true }).waitFor(); await page.getByRole("button", { name: "JavaScript" }).click(); @@ -1994,16 +2047,15 @@ PatternLexicon.prototype.search = function(word) { return false; } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.wordSearch); - await page.goto( - interviewUrl("convert-sorted-array-to-binary-search-tree"), - { waitUntil: "domcontentloaded" }, - ); + await page.goto(interviewUrl(PROBLEM_IDS.sortedArrayToTree), { + waitUntil: "domcontentloaded", + }); await clearMediaGate(page); await page .getByRole("heading", { - name: scenarioTitle("convert-sorted-array-to-binary-search-tree"), + name: scenarioTitle(PROBLEM_IDS.sortedArrayToTree), level: 1, }) .waitFor(); @@ -2019,7 +2071,7 @@ PatternLexicon.prototype.search = function(word) { return build(0, nums.length - 1); } `); - await runAndExpectPassing(5); + await runAndExpectPassing(PROBLEM_IDS.sortedArrayToTree); await page.getByRole("button", { name: "End interview" }).click(); await page @@ -2072,7 +2124,7 @@ PatternLexicon.prototype.search = function(word) { } } - await page.goto(interviewUrl("two-sum")); + await page.goto(interviewUrl(PROBLEM_IDS.twoSum)); await clearMediaGate(page); if (credentialed && !rustAgentIdentity) { roomName = await page @@ -2133,7 +2185,10 @@ PatternLexicon.prototype.search = function(word) { ); } const problemTitle = await page - .getByRole("heading", { name: scenarioTitle("two-sum"), level: 1 }) + .getByRole("heading", { + name: scenarioTitle(PROBLEM_IDS.twoSum), + level: 1, + }) .innerText(); // Scoped to the pill. The captions element also renders the literal // word "Listening" as its placeholder, so an unscoped text match hits diff --git a/scripts/judge-case-counts.cjs b/scripts/judge-case-counts.cjs new file mode 100644 index 00000000..4f3863b4 --- /dev/null +++ b/scripts/judge-case-counts.cjs @@ -0,0 +1,61 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +const webRoot = path.join(__dirname, "..", "web"); +const problemPages = JSON.parse( + fs.readFileSync(path.join(webRoot, "problem-pages.json"), "utf8"), +); + +function judgeCaseCount(problemId) { + const page = problemPages[problemId]?.page; + if (!page) throw new Error(`no scenario page for ${problemId}`); + const fixture = JSON.parse( + fs.readFileSync(path.join(webRoot, "judges", `${page}.json`), "utf8"), + ); + if (!Array.isArray(fixture.cases) || !fixture.cases.length) + throw new Error(`${problemId}: judge has no cases`); + return fixture.cases.length; +} + +function validateJudgeCaseCounts() { + const problemIds = Object.keys(problemPages); + if (!problemIds.length) throw new Error("no judge fixtures to check"); + for (const problemId of problemIds) judgeCaseCount(problemId); + return problemIds.length; +} + +function reportCaseCounts(resultText) { + const result = + typeof resultText === "string" + ? /^Test results · (\d+)\/(\d+)$/.exec(resultText) + : null; + const passed = Number(result?.[1]); + const total = Number(result?.[2]); + if ( + !result || + !Number.isSafeInteger(passed) || + !Number.isSafeInteger(total) || + total <= 0 || + passed > total + ) + throw new Error(`invalid report result ${JSON.stringify(resultText)}`); + return { passed, total }; +} + +function validateReportCaseCount(resultText, problemId) { + const result = reportCaseCounts(resultText); + const actual = judgeCaseCount(problemId); + if (result.total !== actual) + throw new Error( + `${problemId}: report result ${JSON.stringify(resultText)} does not match ${actual} judge cases`, + ); + return result; +} + +module.exports = { + problemPages, + judgeCaseCount, + validateJudgeCaseCounts, + reportCaseCounts, + validateReportCaseCount, +}; diff --git a/scripts/report-parity-check.sh b/scripts/report-parity-check.sh index df35e031..36467269 100755 --- a/scripts/report-parity-check.sh +++ b/scripts/report-parity-check.sh @@ -10,8 +10,9 @@ cleanup() trap cleanup EXIT INT TERM if [ "${REPORT_PARITY_CHECK_VALIDATE_FIXTURES_ONLY:-}" ]; then - PY_CAPTURE="$ROOT/tests/golden/report-python.json" node << 'NODE' + PY_CAPTURE="$ROOT/tests/golden/report-python.json" CASE_COUNTS_MODULE="$ROOT/scripts/judge-case-counts.cjs" node << 'NODE' const fs = require("fs"); +const { reportCaseCounts } = require(process.env.CASE_COUNTS_MODULE); const capture = JSON.parse(fs.readFileSync(process.env.PY_CAPTURE, "utf8")); function assert(condition, message) { if (!condition) { @@ -20,7 +21,7 @@ function assert(condition, message) { } } assert(capture.problemTitle === "Two Sum", "report reference problem mismatch"); -assert(capture.testResultText && capture.testResultText !== "Couldn't run your code", "report reference missing test result"); +assert(reportCaseCounts(capture.testResultText).passed === 0, "report reference should have zero passing cases"); assert(Array.isArray(capture.reportKeys), "report reference missing keys"); for (const key of ["codingScore", "communicationScore", "decision", "summary", "codingFeedback", "communicationFeedback", "hintsUsed"]) { assert(capture.reportKeys.includes(key), `report reference missing ${key}`); @@ -49,8 +50,9 @@ run_capture() run_capture "$TMP/rust.json" -PY_CAPTURE="$ROOT/tests/golden/report-python.json" RUST_CAPTURE="$TMP/rust.json" WEB_ROOT="$ROOT/web" node << 'NODE' +PY_CAPTURE="$ROOT/tests/golden/report-python.json" RUST_CAPTURE="$TMP/rust.json" WEB_ROOT="$ROOT/web" CASE_COUNTS_MODULE="$ROOT/scripts/judge-case-counts.cjs" node << 'NODE' const fs = require("fs"); +const { reportCaseCounts, validateReportCaseCount } = require(process.env.CASE_COUNTS_MODULE); const python = JSON.parse(fs.readFileSync(process.env.PY_CAPTURE, "utf8")); const rust = JSON.parse(fs.readFileSync(process.env.RUST_CAPTURE, "utf8")); // The Python reference was captured when the heading was the published title. @@ -85,9 +87,11 @@ assert(rust.problemTitle === scenario, "rust problem mismatch"); // The title above and the heading it matched are read from the same map, so // they agree even when the map has the published title in it. assert(rust.problemTitle !== python.problemTitle, "rust interview shows the published title"); -assert(python.testResultText === rust.testResultText, "test result mismatch"); -assert(python.testResultText !== "Couldn't run your code", "test run setup failed"); +// The saved total is historical; only the live total follows the current judge. +const referenceResults = reportCaseCounts(python.testResultText); +const liveResults = validateReportCaseCount(rust.testResultText, "two-sum"); +assert(referenceResults.passed === liveResults.passed, "passing test count mismatch"); assert(Array.isArray(python.reportKeys), "python reference missing report keys"); assert(python.reportKeys.join(",") === shape(rust.report, "rust"), "report key mismatch"); -console.log(`report parity comparison passed: problem=${python.problemTitle} tests=${python.testResultText} rustDecision=${rust.report.decision} rustScores=${rust.report.codingScore}/${rust.report.communicationScore}`); +console.log(`report parity comparison passed: problem=${python.problemTitle} tests=${rust.testResultText} rustDecision=${rust.report.decision} rustScores=${rust.report.codingScore}/${rust.report.communicationScore}`); NODE diff --git a/tests/browser/judge-case-counts.test.js b/tests/browser/judge-case-counts.test.js new file mode 100644 index 00000000..353ab0c4 --- /dev/null +++ b/tests/browser/judge-case-counts.test.js @@ -0,0 +1,355 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import vm from "node:vm"; + +import counts from "../../scripts/judge-case-counts.cjs"; + +test("every scenario resolves to a judge with cases", () => { + const checked = counts.validateJudgeCaseCounts(); + assert.equal(checked, Object.keys(counts.problemPages).length); + assert.ok(checked > 0); +}); + +function replaceJudge(t, fixture) { + const file = fileURLToPath( + new URL( + `../../web/judges/${counts.problemPages["two-sum"].page}.json`, + import.meta.url, + ), + ); + const readFile = fs.readFileSync; + t.mock.method(fs, "readFileSync", (path, ...args) => + path === file ? JSON.stringify(fixture()) : readFile(path, ...args), + ); +} + +function browserCheck() { + const file = new URL("../../scripts/browser-check.cjs", import.meta.url); + const require = createRequire(file); + let launches = 0; + const context = vm.createContext({ + require: (name) => + name === "test-playwright" + ? { + chromium: { + launch: () => { + launches += 1; + // Stop at the browser boundary without starting a real interview. + return new Promise(() => {}); + }, + }, + } + : require(name), + process: { + env: { ...process.env, PLAYWRIGHT_PATH: "test-playwright" }, + }, + console, + }); + const script = new vm.Script(fs.readFileSync(file, "utf8"), { + filename: fileURLToPath(file), + }); + return { + run: () => script.runInContext(context), + evaluate: (code) => vm.runInContext(code, context), + launches: () => launches, + }; +} + +test("browser flows read their judges once before launch and reuse the counts", (t) => { + const cases = [ + { input: [], expected: [] }, + { input: [], expected: [] }, + ]; + const readFile = fs.readFileSync; + const judgeRoot = fileURLToPath( + new URL("../../web/judges/", import.meta.url), + ); + const judge = fileURLToPath( + new URL( + `../../web/judges/${counts.problemPages["two-sum"].page}.json`, + import.meta.url, + ), + ); + const reads = new Map(); + t.mock.method(fs, "readFileSync", (file, ...args) => { + if (typeof file === "string" && file.startsWith(judgeRoot)) + reads.set(file, (reads.get(file) ?? 0) + 1); + return file === judge ? JSON.stringify({ cases }) : readFile(file, ...args); + }); + const check = browserCheck(); + check.run(); + assert.equal(check.launches(), 1); + assert.equal(reads.size, check.evaluate("Object.values(PROBLEM_IDS).length")); + assert.ok([...reads.values()].every((count) => count === 1)); + cases.push({ input: [], expected: [] }); + for (let i = 0; i < 2; i++) + assert.equal(check.evaluate("scenario(PROBLEM_IDS.twoSum).caseCount"), 2); + assert.ok([...reads.values()].every((count) => count === 1)); + assert.throws( + () => check.evaluate("scenario(undefined)"), + /no scenario page for undefined/, + ); +}); + +test("a missing flow problem fails before the browser launches", (t) => { + const original = counts.problemPages["two-sum"]; + delete counts.problemPages["two-sum"]; + t.after(() => { + counts.problemPages["two-sum"] = original; + }); + const check = browserCheck(); + assert.throws(() => check.run(), /no scenario page for two-sum/); + assert.equal(check.launches(), 0); +}); + +test("a missing flow judge fails before the browser launches", (t) => { + const judge = fileURLToPath( + new URL( + `../../web/judges/${counts.problemPages["two-sum"].page}.json`, + import.meta.url, + ), + ); + const readFile = fs.readFileSync; + t.mock.method(fs, "readFileSync", (file, ...args) => + readFile(file === judge ? file + ".missing-test-fixture" : file, ...args), + ); + const check = browserCheck(); + assert.throws(() => check.run(), { code: "ENOENT" }); + assert.equal(check.launches(), 0); +}); + +test("an empty flow judge fails before the browser launches", (t) => { + replaceJudge(t, () => ({ cases: [] })); + const check = browserCheck(); + assert.throws(() => check.run(), /two-sum: judge has no cases/); + assert.equal(check.launches(), 0); +}); + +test("browser case counts follow changes to their judge fixture", (t) => { + const cases = [ + { input: [], expected: [] }, + { input: [], expected: [] }, + ]; + replaceJudge(t, () => ({ cases })); + assert.equal(counts.judgeCaseCount("two-sum"), 2); + cases.push({ input: [], expected: [] }); + assert.equal(counts.judgeCaseCount("two-sum"), 3); + assert.doesNotThrow(() => + counts.validateReportCaseCount("Test results · 0/3", "two-sum"), + ); + assert.throws( + () => counts.validateReportCaseCount("Test results · 0/2", "two-sum"), + /does not match 3 judge cases/, + ); +}); + +test("an unknown or omitted problem has no default case count", () => { + for (const problemId of ["unknown", undefined]) + assert.throws( + () => counts.judgeCaseCount(problemId), + /no scenario page for/, + ); +}); + +test("empty or malformed judge cases fail validation", (t) => { + let fixture; + replaceJudge(t, () => fixture); + for (fixture of [{}, { cases: [] }, { cases: {} }]) { + assert.throws( + () => counts.judgeCaseCount("two-sum"), + /two-sum: judge has no cases/, + ); + assert.throws( + () => counts.validateJudgeCaseCounts(), + /two-sum: judge has no cases/, + ); + } +}); + +const reportReference = JSON.parse( + fs.readFileSync( + new URL("../golden/report-python.json", import.meta.url), + "utf8", + ), +); + +function reportCheck(reference, live) { + const file = new URL("../../scripts/report-parity-check.sh", import.meta.url); + const blocks = [ + ...fs + .readFileSync(file, "utf8") + .matchAll(/node << 'NODE'\n([\s\S]*?)\nNODE/g), + ]; + assert.equal( + blocks.length, + 2, + "both report validation paths must be exercised", + ); + const require = createRequire(import.meta.url); + const errors = []; + const output = []; + const captureFiles = { + "saved-capture": reference, + "live-capture": live, + }; + vm.runInNewContext(blocks[live ? 1 : 0][1], { + require: (name) => + name === "fs" + ? { + readFileSync: (path, ...args) => + Object.hasOwn(captureFiles, path) + ? JSON.stringify(captureFiles[path]) + : fs.readFileSync(path, ...args), + } + : require(name), + process: { + env: { + PY_CAPTURE: "saved-capture", + RUST_CAPTURE: "live-capture", + WEB_ROOT: fileURLToPath(new URL("../../web", import.meta.url)), + CASE_COUNTS_MODULE: fileURLToPath( + new URL("../../scripts/judge-case-counts.cjs", import.meta.url), + ), + }, + exit: (code) => { + throw new Error(`report check exited ${code}: ${errors.join("\n")}`); + }, + }, + console: { + error: (message) => errors.push(message), + log: (message) => output.push(message), + }, + }); + return output; +} + +function liveReport(testResultText) { + return { + problemTitle: counts.problemPages["two-sum"].title, + testResultText, + report: { + codingScore: 0, + communicationScore: 50, + decision: "NO_HIRE", + summary: "Incomplete solution", + codingFeedback: { strengths: [], improvements: [] }, + communicationFeedback: { strengths: [], improvements: [] }, + hintsUsed: 0, + }, + }; +} + +test("report results separate passing counts from historical totals", () => { + assert.deepEqual(counts.reportCaseCounts("Test results · 0/4"), { + passed: 0, + total: 4, + }); + assert.deepEqual(counts.reportCaseCounts("Test results · 2/5"), { + passed: 2, + total: 5, + }); + assert.equal( + counts.reportCaseCounts(reportReference.testResultText).passed, + 0, + ); +}); + +test("malformed or impossible report counts fail validation", () => { + for (const result of [ + "Couldn't run your code", + "", + undefined, + null, + ["Test results · 0/5"], + "Test results · 6/5", + "Test results · 0/0", + "Test results · -1/5", + "Test results · 0/9007199254740992", + ]) { + assert.throws( + () => counts.reportCaseCounts(result), + /invalid report result/, + ); + assert.throws( + () => counts.validateReportCaseCount(result, "two-sum"), + /invalid report result/, + ); + } +}); + +test("a stale live report total fails validation", () => { + const total = counts.judgeCaseCount("two-sum"); + assert.deepEqual( + counts.validateReportCaseCount(`Test results · 0/${total}`, "two-sum"), + { + passed: 0, + total, + }, + ); + assert.throws( + () => + counts.validateReportCaseCount( + `Test results · 0/${total + 1}`, + "two-sum", + ), + /two-sum: report result.*does not match/, + ); +}); + +test("fixture-only report validation accepts historical totals after judge cases change", (t) => { + replaceJudge(t, () => ({ cases: Array.from({ length: 6 }, () => ({})) })); + assert.doesNotThrow(() => reportCheck(reportReference)); +}); + +test("fixture-only report validation rejects invalid results and nonzero passing counts", () => { + for (const testResultText of [ + undefined, + "Couldn't run your code", + "Test results · 5/4", + ]) + assert.throws( + () => reportCheck({ ...reportReference, testResultText }), + /invalid report result/, + ); + assert.throws( + () => + reportCheck({ ...reportReference, testResultText: "Test results · 1/4" }), + /zero passing cases/, + ); +}); + +test("live report validation follows the current judge instead of the saved total", (t) => { + replaceJudge(t, () => ({ cases: Array.from({ length: 6 }, () => ({})) })); + const output = reportCheck(reportReference, liveReport("Test results · 0/6")); + assert.equal(output.length, 1); + assert.match(output[0], /tests=Test results · 0\/6/); + assert.throws( + () => reportCheck(reportReference, liveReport("Test results · 0/5")), + /does not match 6 judge cases/, + ); + for (const result of ["Test results · 1/6", "Test results · 6/6"]) + assert.throws( + () => reportCheck(reportReference, liveReport(result)), + /passing test count mismatch/, + ); + for (const result of [undefined, "Couldn't run your code"]) + assert.throws( + () => reportCheck(reportReference, liveReport(result)), + /invalid report result/, + ); +}); + +test("report result changes retain the report shape and problem checks", () => { + const result = `Test results · 0/${counts.judgeCaseCount("two-sum")}`; + const live = liveReport(result); + assert.throws( + () => + reportCheck(reportReference, { ...live, problemTitle: "Other problem" }), + /rust problem mismatch/, + ); + live.report.error = true; + assert.throws(() => reportCheck(reportReference, live), /fallback report/); +});