From b62938c9f4769c9ad0c9e421780513020bfd0215 Mon Sep 17 00:00:00 2001 From: helen229 Date: Fri, 17 Jul 2026 15:09:26 -0700 Subject: [PATCH 1/2] Rename azsdk-evals directory to evals --- eng/common/pipelines/live-eval.yml | 2 +- eng/common/pipelines/workflow-eval.yml | 8 ++++---- eng/common/scripts/allow-relative-links.txt | 4 ++-- .../scripts/eval/test/init-eval-git-fixtures.test.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/common/pipelines/live-eval.yml b/eng/common/pipelines/live-eval.yml index adcb3d560e..4b4edd6bfa 100644 --- a/eng/common/pipelines/live-eval.yml +++ b/eng/common/pipelines/live-eval.yml @@ -25,7 +25,7 @@ extends: # Shared mock/live builder; select the live tier (real Cli MCP). mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml TestType: live - vallyRoot: azsdk-evals + vallyRoot: evals evalGlobs: - 'evals/workflow-scenarios/live/*.eval.yaml' # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated. diff --git a/eng/common/pipelines/workflow-eval.yml b/eng/common/pipelines/workflow-eval.yml index faa42cfdee..6ab82162fb 100644 --- a/eng/common/pipelines/workflow-eval.yml +++ b/eng/common/pipelines/workflow-eval.yml @@ -1,5 +1,5 @@ # Hermetic workflow-scenario eval CI: runs the unit-tool + mock workflow-scenario evals in -# azsdk-evals against the mock MCP (live tier runs in live-eval.yml). +# evals against the mock MCP (live tier runs in live-eval.yml). trigger: branches: @@ -7,7 +7,7 @@ trigger: - main paths: include: - - azsdk-evals/** + - evals/** - tools/azsdk-cli/Azure.Sdk.Tools.Mock/** # Cli tool catalog + skills feed the eval results, so changes here must retrigger. - tools/azsdk-cli/Azure.Sdk.Tools.Cli/** @@ -25,7 +25,7 @@ pr: - main paths: include: - - azsdk-evals/** + - evals/** - tools/azsdk-cli/Azure.Sdk.Tools.Mock/** - tools/azsdk-cli/Azure.Sdk.Tools.Cli/** - .github/skills/** @@ -48,7 +48,7 @@ extends: # Shared mock/live builder; select the mock tier. mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml TestType: mock - vallyRoot: azsdk-evals + vallyRoot: evals # Per-shard job timeout (report-only tier). shardTimeoutInMinutes: 20 # Pass-rate gate for `vally eval`. This tier is report-only (failOnFailedTests defaults false), diff --git a/eng/common/scripts/allow-relative-links.txt b/eng/common/scripts/allow-relative-links.txt index 0813f14360..084448ef1e 100644 --- a/eng/common/scripts/allow-relative-links.txt +++ b/eng/common/scripts/allow-relative-links.txt @@ -9,5 +9,5 @@ AGENTS.md # Allow relative links for all files under the eng folder (e.g. engineering system scripts and templates). eng/** -# Allow relative links for the top-level eval suite (azsdk-evals is a cross-cutting asset at repo root). -azsdk-evals/** +# Allow relative links for the top-level eval suite (evals is a cross-cutting asset at repo root). +evals/** diff --git a/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts b/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts index 76ce954301..28e3ad626a 100644 --- a/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts +++ b/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts @@ -111,7 +111,7 @@ describe("getEvalGitFixtures discovery", () => { // contain the Vally suite they skip. describe("Folder-level invariant for real git fixtures", () => { const repoRoot = path.resolve(here, "../../../../.."); - const vallyRoot = path.join(repoRoot, "azsdk-evals"); + const vallyRoot = path.join(repoRoot, "evals"); const vallyEvals = path.join(vallyRoot, "evals"); const present = fs.existsSync(vallyEvals); From 4bbbd2d1ad4874136f25a4901e6ff78904115c5b Mon Sep 17 00:00:00 2001 From: helen229 Date: Mon, 20 Jul 2026 09:42:27 -0700 Subject: [PATCH 2/2] Flatten eval suite directory layout --- eng/common/pipelines/live-eval.yml | 2 +- .../templates/stages/archetype-eval.yml | 4 +- eng/common/scripts/eval/collect-stimuli.ts | 4 +- .../scripts/eval/init-eval-git-fixtures.ts | 4 +- eng/common/scripts/eval/lib/glob.ts | 2 +- .../scripts/eval/test/collect-stimuli.test.ts | 60 +++++++++---------- .../eval/test/init-eval-git-fixtures.test.ts | 18 +++--- 7 files changed, 47 insertions(+), 47 deletions(-) diff --git a/eng/common/pipelines/live-eval.yml b/eng/common/pipelines/live-eval.yml index 4b4edd6bfa..e69e480055 100644 --- a/eng/common/pipelines/live-eval.yml +++ b/eng/common/pipelines/live-eval.yml @@ -27,7 +27,7 @@ extends: TestType: live vallyRoot: evals evalGlobs: - - 'evals/workflow-scenarios/live/*.eval.yaml' + - 'workflows/live/*.eval.yaml' # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated. UseAzSdkAuthentication: true failOnFailedTests: true diff --git a/eng/common/pipelines/templates/stages/archetype-eval.yml b/eng/common/pipelines/templates/stages/archetype-eval.yml index 68d40e9589..4e29b4c314 100644 --- a/eng/common/pipelines/templates/stages/archetype-eval.yml +++ b/eng/common/pipelines/templates/stages/archetype-eval.yml @@ -15,8 +15,8 @@ parameters: # collect-stimuli.js mirrors this list as its no-args fallback — keep them in sync. type: object default: - - 'evals/tools/*.eval.yaml' - - 'evals/workflow-scenarios/mock/*.eval.yaml' + - 'tools/*.eval.yaml' + - 'workflows/mock/*.eval.yaml' - name: mcpSetupTemplate # SWAP POINT: builds/installs the MCP server(s) under artifacts/mcp/. Required (no default): # each repo passes its own build template. This repo uses the shared mock/live builder diff --git a/eng/common/scripts/eval/collect-stimuli.ts b/eng/common/scripts/eval/collect-stimuli.ts index 260e4006fd..d13d028c90 100644 --- a/eng/common/scripts/eval/collect-stimuli.ts +++ b/eng/common/scripts/eval/collect-stimuli.ts @@ -10,8 +10,8 @@ import { globFiles } from "./lib/glob.ts"; // Fallback patterns when run with no --pattern. Mirrors archetype-eval.yml's evalGlobs default. const DEFAULT_PATTERNS = [ - "evals/tools/*.eval.yaml", - "evals/workflow-scenarios/mock/*.eval.yaml", + "tools/*.eval.yaml", + "workflows/mock/*.eval.yaml", ]; const SANITIZE = /[^A-Za-z0-9]/g; diff --git a/eng/common/scripts/eval/init-eval-git-fixtures.ts b/eng/common/scripts/eval/init-eval-git-fixtures.ts index 8921b1004c..1a30e1f365 100644 --- a/eng/common/scripts/eval/init-eval-git-fixtures.ts +++ b/eng/common/scripts/eval/init-eval-git-fixtures.ts @@ -9,8 +9,8 @@ import { globFiles } from "./lib/glob.ts"; import { syncRepo } from "./sync-eval-git-repo.ts"; export const DEFAULT_PATTERNS = [ - "evals/tools/*.eval.yaml", - "evals/workflow-scenarios/mock/*.eval.yaml", + "tools/*.eval.yaml", + "workflows/mock/*.eval.yaml", ]; // Repos we know how to clone efficiently (cone-sparse to the spec folders the fixtures touch). diff --git a/eng/common/scripts/eval/lib/glob.ts b/eng/common/scripts/eval/lib/glob.ts index 84dd96a1ca..f600182736 100644 --- a/eng/common/scripts/eval/lib/glob.ts +++ b/eng/common/scripts/eval/lib/glob.ts @@ -62,7 +62,7 @@ function walk(dir, segments, out) { * glob `pattern`. Results are sorted for deterministic ordering. * * @param {string} root Absolute or relative base directory to glob from. - * @param {string} pattern Forward-slashed glob relative to `root` (e.g. "evals/tools/*.eval.yaml"). + * @param {string} pattern Forward-slashed glob relative to `root` (e.g. "tools/*.eval.yaml"). * @returns {string[]} Sorted absolute file paths. */ export function globFiles(root, pattern) { diff --git a/eng/common/scripts/eval/test/collect-stimuli.test.ts b/eng/common/scripts/eval/test/collect-stimuli.test.ts index ca0e6ead50..17deca9633 100644 --- a/eng/common/scripts/eval/test/collect-stimuli.test.ts +++ b/eng/common/scripts/eval/test/collect-stimuli.test.ts @@ -31,19 +31,19 @@ describe("collect-stimuli (default discovery)", () => { before(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-")); writeFile( - path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"), + path.join(root, "tools/prompt-to-tool-github.eval.yaml"), "tags:\n area: github" ); writeFile( - path.join(root, "evals/tools/add-arm-resource.eval.yaml"), + path.join(root, "tools/add-arm-resource.eval.yaml"), "tags:\n area: typespec" ); writeFile( - path.join(root, "evals/workflow-scenarios/mock/rename-client-property.eval.yaml"), + path.join(root, "workflows/mock/rename-client-property.eval.yaml"), "tags:\n area: typespec" ); writeFile( - path.join(root, "evals/workflow-scenarios/live/release-planner.eval.yaml"), + path.join(root, "workflows/live/release-planner.eval.yaml"), "tags:\n area: release-plan" ); }); @@ -70,7 +70,7 @@ describe("collect-stimuli (default discovery)", () => { const matrix = buildMatrix({ roots: [root] }); for (const entry of Object.values(matrix)) { assert.doesNotMatch(entry.evalArgs, /\\/); - assert.match(entry.evalArgs, /^-e evals\//); + assert.match(entry.evalArgs, /^-e (tools|workflows)\//); } }); @@ -94,15 +94,15 @@ describe("collect-stimuli (area grouping)", () => { before(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-area-")); writeFile( - path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"), + path.join(root, "tools/prompt-to-tool-github.eval.yaml"), "tags:\n area: github" ); writeFile( - path.join(root, "evals/tools/add-arm-resource.eval.yaml"), + path.join(root, "tools/add-arm-resource.eval.yaml"), "tags:\n area: typespec" ); writeFile( - path.join(root, "evals/workflow-scenarios/mock/rename-client-property.eval.yaml"), + path.join(root, "workflows/mock/rename-client-property.eval.yaml"), "tags:\n area: typespec" ); }); @@ -127,11 +127,11 @@ describe("collect-stimuli (area grouping)", () => { const collideRoot = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-area-collide-")); try { writeFile( - path.join(collideRoot, "evals/tools/a.eval.yaml"), + path.join(collideRoot, "tools/a.eval.yaml"), "tags:\n area: release-plan" ); writeFile( - path.join(collideRoot, "evals/tools/b.eval.yaml"), + path.join(collideRoot, "tools/b.eval.yaml"), "tags:\n area: release_plan" ); assert.throws( @@ -149,8 +149,8 @@ describe("collect-stimuli (area with an untagged eval)", () => { before(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-ut-")); - writeFile(path.join(root, "evals/tools/tagged.eval.yaml"), "tags:\n area: github"); - writeFile(path.join(root, "evals/tools/untagged.eval.yaml"), "no tags here"); + writeFile(path.join(root, "tools/tagged.eval.yaml"), "tags:\n area: github"); + writeFile(path.join(root, "tools/untagged.eval.yaml"), "no tags here"); }); after(() => fs.rmSync(root, { recursive: true, force: true })); @@ -159,7 +159,7 @@ describe("collect-stimuli (area with an untagged eval)", () => { const { result: matrix } = withWarnings((warn) => buildMatrix({ roots: [root], - patterns: ["evals/tools/*.eval.yaml"], + patterns: ["tools/*.eval.yaml"], warn, }) ); @@ -172,7 +172,7 @@ describe("collect-stimuli (area with an untagged eval)", () => { const { result: matrix } = withWarnings((warn) => buildMatrix({ roots: [root], - patterns: ["evals/tools/*.eval.yaml"], + patterns: ["tools/*.eval.yaml"], warn, }) ); @@ -183,7 +183,7 @@ describe("collect-stimuli (area with an untagged eval)", () => { const { warnings } = withWarnings((warn) => buildMatrix({ roots: [root], - patterns: ["evals/tools/*.eval.yaml"], + patterns: ["tools/*.eval.yaml"], warn, }) ); @@ -197,11 +197,11 @@ describe("collect-stimuli (overlapping patterns)", () => { before(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-overlap-")); writeFile( - path.join(root, "evals/tools/add-arm-resource.eval.yaml"), + path.join(root, "tools/add-arm-resource.eval.yaml"), "tags:\n area: typespec" ); writeFile( - path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"), + path.join(root, "tools/prompt-to-tool-github.eval.yaml"), "tags:\n area: github" ); }); @@ -211,7 +211,7 @@ describe("collect-stimuli (overlapping patterns)", () => { it("does not emit a duplicate -e flag for a file matched by multiple patterns", () => { const matrix = buildMatrix({ roots: [root], - patterns: ["evals/tools/*.eval.yaml", "evals/tools/add-arm-resource.eval.yaml"], + patterns: ["tools/*.eval.yaml", "tools/add-arm-resource.eval.yaml"], }); const count = (matrix.area_typespec.evalArgs.match(/add-arm-resource/g) || []).length; assert.equal(count, 1); @@ -226,11 +226,11 @@ describe("collect-stimuli (multiple eval roots: repo + common)", () => { repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-repo-")); commonRoot = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-common-")); writeFile( - path.join(repoRoot, "evals/tools/repo-specific.eval.yaml"), + path.join(repoRoot, "tools/repo-specific.eval.yaml"), "tags:\n area: repo" ); writeFile( - path.join(commonRoot, "evals/tools/shared-scenario.eval.yaml"), + path.join(commonRoot, "tools/shared-scenario.eval.yaml"), "tags:\n area: shared" ); }); @@ -243,7 +243,7 @@ describe("collect-stimuli (multiple eval roots: repo + common)", () => { it("collects evals from both roots into one matrix", () => { const matrix = buildMatrix({ roots: [repoRoot, commonRoot], - patterns: ["evals/tools/*.eval.yaml"], + patterns: ["tools/*.eval.yaml"], }); const keys = Object.keys(matrix); assert.equal(keys.length, 2); @@ -254,11 +254,11 @@ describe("collect-stimuli (multiple eval roots: repo + common)", () => { it("computes each file's relative path against its own root", () => { const matrix = buildMatrix({ roots: [repoRoot, commonRoot], - patterns: ["evals/tools/*.eval.yaml"], + patterns: ["tools/*.eval.yaml"], }); assert.equal( matrix.area_shared.evalArgs, - "-e evals/tools/shared-scenario.eval.yaml" + "-e tools/shared-scenario.eval.yaml" ); }); }); @@ -275,11 +275,11 @@ describe("collect-stimuli (pathBase anchors scattered roots to one run root)", ( runRoot = path.join(parent, "project"); scatteredRoot = path.join(parent, "extra"); writeFile( - path.join(runRoot, "evals/tools/in-project.eval.yaml"), + path.join(runRoot, "tools/in-project.eval.yaml"), "tags:\n area: inproject" ); writeFile( - path.join(scatteredRoot, "evals/out-of-tree.eval.yaml"), + path.join(scatteredRoot, "workflows/out-of-tree.eval.yaml"), "tags:\n area: scattered" ); }); @@ -290,28 +290,28 @@ describe("collect-stimuli (pathBase anchors scattered roots to one run root)", ( const matrix = buildMatrix({ roots: [runRoot, scatteredRoot], pathBase: runRoot, - patterns: ["evals/**/*.eval.yaml", "evals/*.eval.yaml"], + patterns: ["tools/**/*.eval.yaml", "workflows/*.eval.yaml"], }); // The in-project file stays a simple relative path; the scattered one walks up. assert.equal( matrix.area_inproject.evalArgs, - "-e evals/tools/in-project.eval.yaml" + "-e tools/in-project.eval.yaml" ); assert.equal( matrix.area_scattered.evalArgs, - "-e ../extra/evals/out-of-tree.eval.yaml" + "-e ../extra/workflows/out-of-tree.eval.yaml" ); }); it("falls back to per-root relative paths when no pathBase is given", () => { const matrix = buildMatrix({ roots: [scatteredRoot], - patterns: ["evals/*.eval.yaml"], + patterns: ["workflows/*.eval.yaml"], }); // Without a base, the path is relative to the root it was found under (no `../`). assert.equal( matrix.area_scattered.evalArgs, - "-e evals/out-of-tree.eval.yaml" + "-e workflows/out-of-tree.eval.yaml" ); }); }); diff --git a/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts b/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts index 28e3ad626a..a1804a4f73 100644 --- a/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts +++ b/eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts @@ -17,12 +17,12 @@ describe("getEvalGitFixtures discovery", () => { before(() => { // Throwaway eval tree so the tests do not depend on real eval content. root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-fixtures-test-")); - fs.mkdirSync(path.join(root, "evals/tools"), { recursive: true }); - fs.mkdirSync(path.join(root, "evals/workflow-scenarios/mock"), { recursive: true }); + fs.mkdirSync(path.join(root, "tools"), { recursive: true }); + fs.mkdirSync(path.join(root, "workflows/mock"), { recursive: true }); // A unit eval with NO git fixture. fs.writeFileSync( - path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"), + path.join(root, "tools/prompt-to-tool-github.eval.yaml"), "tags:\n area: github\nstimuli:\n - name: x\n" ); @@ -45,7 +45,7 @@ describe("getEvalGitFixtures discovery", () => { "", ].join("\n"); fs.writeFileSync( - path.join(root, "evals/workflow-scenarios/mock/release-planner-workflows.eval.yaml"), + path.join(root, "workflows/mock/release-planner-workflows.eval.yaml"), mock ); }); @@ -76,7 +76,7 @@ describe("getEvalGitFixtures discovery", () => { }); it("is a no-op when the scanned suite declares no git fixtures", () => { - const fixtures = getEvalGitFixtures({ root, patterns: ["evals/tools/*.eval.yaml"] }); + const fixtures = getEvalGitFixtures({ root, patterns: ["tools/*.eval.yaml"] }); assert.equal(fixtures.length, 0); }); @@ -90,7 +90,7 @@ describe("getEvalGitFixtures discovery", () => { " source: ../../../../../../artifacts/specs-cache/some-other-repo", "", ].join("\n"); - const file = path.join(root, "evals/workflow-scenarios/mock/no-ref.eval.yaml"); + const file = path.join(root, "workflows/mock/no-ref.eval.yaml"); fs.writeFileSync(file, noRef); try { const fixtures = dedupeFixtures(getEvalGitFixtures({ root })); @@ -112,8 +112,8 @@ describe("getEvalGitFixtures discovery", () => { describe("Folder-level invariant for real git fixtures", () => { const repoRoot = path.resolve(here, "../../../../.."); const vallyRoot = path.join(repoRoot, "evals"); - const vallyEvals = path.join(vallyRoot, "evals"); - const present = fs.existsSync(vallyEvals); + const evalRoots = [path.join(vallyRoot, "tools"), path.join(vallyRoot, "workflows")]; + const present = evalRoots.some((root) => fs.existsSync(root)); const expectedCacheRoot = path .join(repoRoot, "artifacts", "specs-cache") @@ -122,7 +122,7 @@ describe("Folder-level invariant for real git fixtures", () => { function collectRealFixtures() { const srcRegex = /^\s*source:\s*(\.\.\S+)/gm; const results = []; - const stack = [vallyEvals]; + const stack = evalRoots.filter((root) => fs.existsSync(root)); while (stack.length > 0) { const dir = stack.pop(); for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {