From b91d6c81062b513c0eba3fe0d87a62f115440cc0 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 30 May 2026 11:46:56 -0400 Subject: [PATCH 1/3] fix(storyboard): add pattern validation checks --- scripts/conformance-replay.ts | 31 +++ src/lib/testing/storyboard/types.ts | 21 ++ src/lib/testing/storyboard/validations.ts | 104 ++++++++- test/lib/storyboard-drift.test.js | 29 +-- test/lib/storyboard-validations.test.js | 209 ++++++++++++++++++ .../storyboard-version-negotiation.test.js | 72 ++++++ 6 files changed, 449 insertions(+), 17 deletions(-) diff --git a/scripts/conformance-replay.ts b/scripts/conformance-replay.ts index e0487c91f..2fe351606 100644 --- a/scripts/conformance-replay.ts +++ b/scripts/conformance-replay.ts @@ -218,12 +218,14 @@ const IMPLEMENTED_CHECKS = new Set([ 'response_schema', 'field_present', 'field_value', + 'field_pattern', // Envelope-scoped variants (adcp#3429). Runtime semantics are identical to // the un-prefixed checks because the SDK's response unwrappers expose // envelope fields at the structuredContent surface (`status`, `task_id`, // `replayed`, etc.); the distinction is for static drift detection. 'envelope_field_present', 'envelope_field_value', + 'envelope_field_pattern', // Cardinality assertions (adcp#4685 / adcp-client#1830). 'array_length', ]); @@ -331,6 +333,35 @@ async function runStep( allPassed = false; result.failures.push(`${check} ${v.path}: got ${JSON.stringify(actual)}, want ${JSON.stringify(v.value)}`); } + } else if (check === 'field_pattern' || check === 'envelope_field_pattern') { + const pattern = (v as { pattern?: unknown }).pattern; + if (typeof pattern !== 'string' || pattern.length === 0) { + allPassed = false; + result.failures.push(`${check} ${v.path}: misconfigured (pattern must be a non-empty string)`); + } else { + let re: RegExp | undefined; + try { + re = new RegExp(pattern); + } catch (err) { + allPassed = false; + result.failures.push( + `${check} ${v.path}: invalid pattern ${JSON.stringify(pattern)} (${ + err instanceof Error ? err.message : String(err) + })` + ); + } + if (re) { + const actual = getByPath(structured, v.path as string); + if (typeof actual !== 'string') { + allPassed = false; + const ty = actual === undefined ? 'undefined' : actual === null ? 'null' : typeof actual; + result.failures.push(`${check} ${v.path}: got ${ty}, want string matching ${JSON.stringify(pattern)}`); + } else if (!re.test(actual)) { + allPassed = false; + result.failures.push(`${check} ${v.path}: got ${JSON.stringify(actual)}, want /${pattern}/`); + } + } + } } else if (check === 'array_length') { const hasExact = typeof v.value === 'number'; const hasMin = typeof (v as { min?: unknown }).min === 'number'; diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index 96d9bb806..b56c4a315 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -736,6 +736,25 @@ export type StoryboardValidationCheck = | 'envelope_field_value_or_absent' | 'field_value' | 'field_value_or_absent' + /** + * Assert that a string field in the task payload matches a JavaScript + * regular expression source declared in `pattern`. Fails when the path is + * missing, the value is not a string, the pattern is invalid, or the regex + * does not match. Failure output carries `expected: { pattern }` and + * `actual` as the observed value (or null when missing). Added for + * runner-output-contract v2.5.0. + */ + | 'field_pattern' + /** + * Envelope-scoped `field_pattern`. The path resolves against the union of + * protocol-envelope fields (`status`, `task_id`, etc.) and string-valued + * version-envelope fields such as `adcp_version`. Runtime matching is the + * same as `field_pattern`; the distinct kind lets static linting validate + * against the envelope schemas instead of the task payload schema. Numeric + * envelope fields such as `adcp_major_version` should use + * `envelope_field_value`. + */ + | 'envelope_field_pattern' // Wildcard-aware membership check. `path` may include `[*]` segments that // expand to every array element via `resolvePathAll`. Passes when ANY // resolved value matches `value` (or any of `allowed_values`). Lets @@ -1062,6 +1081,8 @@ export interface StoryboardValidation { value?: unknown; /** Accepted values for list-match checks (passes if actual matches any). */ allowed_values?: unknown[]; + /** JavaScript regular expression source for field_pattern checks. */ + pattern?: string; description: string; // ─── refs_resolve fields ─────────────────────────────────── /** Source refs (the refs being checked). */ diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index d020cc058..c66e52a5c 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -253,19 +253,24 @@ function runValidation(validation: StoryboardValidation, ctx: ValidationContext) case 'envelope_field_absent': case 'envelope_field_value': case 'envelope_field_value_or_absent': + case 'envelope_field_pattern': // Envelope-scoped variants — runtime semantics identical to the // un-prefixed checks (TaskResult exposes envelope fields like - // `status`, `task_id` at the surface level). The distinct check types - // exist primarily so static drift detection can walk the envelope - // schema instead of the per-tool response. See adcp#3429. + // `status`, `task_id` and version-envelope fields like `adcp_version` + // at the surface level). The distinct check types exist primarily so + // static drift detection can walk the envelope schemas instead of the + // per-tool response. See adcp#3429 and adcp#5195. if (validation.check === 'envelope_field_present') return validateFieldPresent(validation, resolveTarget(ctx)); if (validation.check === 'envelope_field_absent') return validateFieldAbsent(validation, resolveTarget(ctx)); if (validation.check === 'envelope_field_value') return validateFieldValue(validation, resolveTarget(ctx)); + if (validation.check === 'envelope_field_pattern') return validateFieldPattern(validation, resolveTarget(ctx)); return validateFieldValueOrAbsent(validation, resolveTarget(ctx)); case 'field_value': return validateFieldValue(validation, resolveTarget(ctx)); case 'field_value_or_absent': return validateFieldValueOrAbsent(validation, resolveTarget(ctx)); + case 'field_pattern': + return validateFieldPattern(validation, resolveTarget(ctx)); case 'field_contains': return validateFieldContains(validation, resolveTarget(ctx)); case 'status_code': @@ -1007,6 +1012,99 @@ function isTaskEnvelopeStatus(value: unknown): boolean { return typeof value === 'string' && TASK_ENVELOPE_STATUSES.has(value); } +// ──────────────────────────────────────────────────────────── +// field_pattern / envelope_field_pattern: check a string matches +// a storyboard-authored JavaScript regular expression source. +// ──────────────────────────────────────────────────────────── + +function validateFieldPattern(validation: StoryboardValidation, taskResult: TaskResult): ValidationResult { + const checkName = validation.check; + if (!validation.path) { + return { + check: checkName, + passed: false, + description: validation.description, + path: validation.path, + error: `No path specified for ${checkName} validation`, + json_pointer: null, + expected: 'path must be set in storyboard validation entry', + actual: null, + }; + } + + const pointer = toJsonPointer(validation.path); + const expected = { pattern: validation.pattern }; + + if (typeof validation.pattern !== 'string' || validation.pattern.length === 0) { + return { + check: checkName, + passed: false, + description: validation.description, + path: validation.path, + error: `${checkName} requires a non-empty \`pattern\` string`, + json_pointer: pointer, + expected: { pattern: 'non-empty JavaScript regular expression source' }, + actual: validation.pattern ?? null, + }; + } + + let re: RegExp; + try { + re = new RegExp(validation.pattern); + } catch (err) { + return { + check: checkName, + passed: false, + description: validation.description, + path: validation.path, + error: `Invalid ${checkName} pattern ${JSON.stringify(validation.pattern)}: ${ + err instanceof Error ? err.message : String(err) + }`, + json_pointer: pointer, + expected, + actual: validation.pattern, + }; + } + + const actual = resolvePath(taskResult.data, validation.path); + if (typeof actual !== 'string') { + return { + check: checkName, + passed: false, + description: validation.description, + path: validation.path, + error: + actual === undefined || actual === null + ? `Field not found at path: ${validation.path}` + : `Expected string at path: ${validation.path}, got ${Array.isArray(actual) ? 'array' : typeof actual}`, + json_pointer: pointer, + expected, + actual: actual ?? null, + }; + } + + if (re.test(actual)) { + return { + check: checkName, + passed: true, + description: validation.description, + path: validation.path, + json_pointer: pointer, + }; + } + + return { + check: checkName, + passed: false, + description: validation.description, + path: validation.path, + error: `Expected string at path ${validation.path} to match pattern ${JSON.stringify(validation.pattern)}, got ${JSON.stringify(actual)}`, + json_pointer: pointer, + expected, + actual, + }; +} + // ──────────────────────────────────────────────────────────── // field_contains: wildcard-aware membership check // diff --git a/test/lib/storyboard-drift.test.js b/test/lib/storyboard-drift.test.js index 6428700c1..bf3a406de 100644 --- a/test/lib/storyboard-drift.test.js +++ b/test/lib/storyboard-drift.test.js @@ -1,8 +1,8 @@ /** * Schema drift detection for storyboard YAML validations. * - * Catches when field_present / field_value / field_value_or_absent paths in - * storyboard YAML reference fields that don't exist in the corresponding + * Catches when field_present / field_value / field_value_or_absent / + * field_pattern paths in storyboard YAML reference fields that don't exist in the corresponding * Zod response schemas, when context extractors reference tasks without * schemas, and when `field_value_or_absent` is asserted on a path the * response schema already marks required (the tolerance is meaningless — @@ -17,14 +17,12 @@ const { listAllComplianceStoryboards } = require('../../dist/lib/testing/storybo const { parsePath } = require('../../dist/lib/testing/storyboard/path.js'); const { TOOL_RESPONSE_SCHEMAS } = require('../../dist/lib/utils/response-schemas.js'); const { CONTEXT_EXTRACTORS } = require('../../dist/lib/testing/storyboard/context.js'); -// `envelope_field_present` (and `envelope_field_value{,_or_absent}`) -// validations walk the v3 protocol envelope — `status`, `task_id`, -// `message`, `replayed`, `governance_context`, `timestamp`, `context_id`, -// `push_notification_config` — instead of per-tool response schemas. -// `errors` and `adcp_version` are NOT envelope fields (errors lives inside -// `payload` per the per-tool response schema; adcp_major_version is on the -// request, not on either response surface). Added per adcp#3429. -const { ProtocolEnvelopeSchema } = require('../../dist/lib/types/schemas.generated.js'); +// `envelope_field_*` validations walk the v3 protocol + version envelopes +// (`status`, `task_id`, `message`, `replayed`, `adcp_version`, etc.) instead +// of per-tool response schemas. `errors` lives inside `payload` per the +// per-tool response schema. Added per adcp#3429 and adcp#5195. +const { AdCPVersionEnvelopeSchema, ProtocolEnvelopeSchema } = require('../../dist/lib/types/schemas.generated.js'); +const EnvelopeSchema = ProtocolEnvelopeSchema.merge(AdCPVersionEnvelopeSchema); // Runner-internal tasks with no agent-facing schema. const HARNESS_TASKS = new Set([ @@ -233,7 +231,9 @@ function collectFieldValidations(storyboards) { v.check === 'field_value' || v.check === 'envelope_field_value' || v.check === 'field_value_or_absent' || - v.check === 'envelope_field_value_or_absent') && + v.check === 'envelope_field_value_or_absent' || + v.check === 'field_pattern' || + v.check === 'envelope_field_pattern') && v.path ) { if (ENVELOPE_PATHS.has(v.path)) continue; // protocol-level, not per-schema @@ -372,7 +372,8 @@ describe('storyboard schema drift', () => { v => v.check === 'envelope_field_present' || v.check === 'envelope_field_value' || - v.check === 'envelope_field_value_or_absent' + v.check === 'envelope_field_value_or_absent' || + v.check === 'envelope_field_pattern' ); for (const entry of envelopeValidations) { @@ -383,10 +384,10 @@ describe('storyboard schema drift', () => { { skip }, () => { const segments = parsePath(entry.path); - const reachable = isPathReachable(ProtocolEnvelopeSchema, segments); + const reachable = isPathReachable(EnvelopeSchema, segments); assert.ok( reachable, - `Path "${entry.path}" is not reachable in protocol-envelope.json. ` + + `Path "${entry.path}" is not reachable in protocol-envelope.json or version-envelope.json. ` + `Segments: ${JSON.stringify(segments)}` ); } diff --git a/test/lib/storyboard-validations.test.js b/test/lib/storyboard-validations.test.js index 1166167b4..85151f9f2 100644 --- a/test/lib/storyboard-validations.test.js +++ b/test/lib/storyboard-validations.test.js @@ -191,6 +191,215 @@ describe('envelope_field_value_or_absent (adcp#3429)', () => { }); }); +describe('field_pattern / envelope_field_pattern (adcp-client#2121)', () => { + it('field_pattern passes when the payload field matches the pattern', () => { + const taskResult = { success: true, data: { creative: { asset_url: 'https://cdn.example/ad-123.png' } } }; + const [result] = runOne( + [ + { + check: 'field_pattern', + path: 'creative.asset_url', + pattern: '^https://cdn\\.example/.+\\.png$', + description: 'asset URL has expected host and extension', + }, + ], + 'build_creative', + taskResult + ); + assert.strictEqual(result.passed, true, result.error); + assert.strictEqual(result.check, 'field_pattern'); + }); + + it('field_pattern fails with expected pattern and actual value on mismatch', () => { + const taskResult = { success: true, data: { creative: { asset_url: 'http://cdn.example/ad-123.gif' } } }; + const [result] = runOne( + [ + { + check: 'field_pattern', + path: 'creative.asset_url', + pattern: '^https://cdn\\.example/.+\\.png$', + description: 'asset URL has expected host and extension', + }, + ], + 'build_creative', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '^https://cdn\\.example/.+\\.png$' }); + assert.strictEqual(result.actual, 'http://cdn.example/ad-123.gif'); + }); + + it('field_pattern reports null actual when the payload field is missing', () => { + const taskResult = { success: true, data: { creative: {} } }; + const [result] = runOne( + [ + { + check: 'field_pattern', + path: 'creative.asset_url', + pattern: '^https://', + description: 'asset URL is present', + }, + ], + 'build_creative', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '^https://' }); + assert.strictEqual(result.actual, null); + assert.match(result.error, /Field not found at path: creative\.asset_url/); + }); + + it('field_pattern fails when the payload field is not a string', () => { + const taskResult = { success: true, data: { creative: { asset_url: 123 } } }; + const [result] = runOne( + [ + { + check: 'field_pattern', + path: 'creative.asset_url', + pattern: '^https://', + description: 'asset URL is string', + }, + ], + 'build_creative', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '^https://' }); + assert.strictEqual(result.actual, 123); + assert.match(result.error, /Expected string at path: creative\.asset_url, got number/); + }); + + it('field_pattern rejects missing pattern configuration', () => { + const taskResult = { success: true, data: { creative: { asset_url: 'https://cdn.example/ad-123.png' } } }; + const [result] = runOne( + [ + { + check: 'field_pattern', + path: 'creative.asset_url', + description: 'asset URL is string', + }, + ], + 'build_creative', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: 'non-empty JavaScript regular expression source' }); + assert.strictEqual(result.actual, null); + assert.match(result.error, /field_pattern requires a non-empty `pattern` string/); + }); + + it('field_pattern rejects invalid regex sources', () => { + const taskResult = { success: true, data: { creative: { asset_url: 'https://cdn.example/ad-123.png' } } }; + const [result] = runOne( + [ + { + check: 'field_pattern', + path: 'creative.asset_url', + pattern: '(', + description: 'asset URL is string', + }, + ], + 'build_creative', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '(' }); + assert.strictEqual(result.actual, '('); + assert.match(result.error, /Invalid field_pattern pattern/); + }); + + it('envelope_field_pattern passes for adcp_version from the version envelope', () => { + const taskResult = { + success: true, + data: { + status: 'completed', + adcp_version: '3.1-rc.3', + adcp: { major_versions: [3], supported_versions: ['3.1-rc.3'] }, + }, + }; + const [result] = runOne( + [ + { + check: 'envelope_field_pattern', + path: 'adcp_version', + pattern: '^\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$', + description: 'adcp_version has release precision', + }, + ], + 'get_adcp_capabilities', + taskResult + ); + assert.strictEqual(result.passed, true, result.error); + assert.strictEqual(result.check, 'envelope_field_pattern'); + }); + + it('envelope_field_pattern fails with expected pattern and actual value on version-envelope mismatch', () => { + const taskResult = { + success: true, + data: { + status: 'completed', + adcp_version: '3.1.0-rc.3', + adcp: { major_versions: [3], supported_versions: ['3.1-rc.3'] }, + }, + }; + const [result] = runOne( + [ + { + check: 'envelope_field_pattern', + path: 'adcp_version', + pattern: '^\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$', + description: 'adcp_version has release precision', + }, + ], + 'get_adcp_capabilities', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '^\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$' }); + assert.strictEqual(result.actual, '3.1.0-rc.3'); + }); + + it('envelope_field_pattern reports null actual when the envelope field is missing', () => { + const taskResult = { success: true, data: { status: 'completed', adcp: { major_versions: [3] } } }; + const [result] = runOne( + [ + { + check: 'envelope_field_pattern', + path: 'adcp_version', + pattern: '^\\d+\\.\\d+', + description: 'adcp_version is present', + }, + ], + 'get_adcp_capabilities', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '^\\d+\\.\\d+' }); + assert.strictEqual(result.actual, null); + assert.match(result.error, /Field not found at path: adcp_version/); + }); + + it('envelope_field_pattern fails when the version-envelope field is not a string', () => { + const taskResult = { success: true, data: { status: 'completed', adcp_major_version: 3 } }; + const [result] = runOne( + [ + { + check: 'envelope_field_pattern', + path: 'adcp_major_version', + pattern: '^3$', + description: 'adcp_major_version is string-shaped', + }, + ], + 'get_adcp_capabilities', + taskResult + ); + assert.strictEqual(result.passed, false); + assert.deepStrictEqual(result.expected, { pattern: '^3$' }); + assert.strictEqual(result.actual, 3); + assert.match(result.error, /Expected string at path: adcp_major_version, got number/); + }); +}); + describe('field_value_or_absent media-buy status collision (adcp-client#1961)', () => { it('treats flat MCP envelope status completed as absent for the deprecated media-buy status field', () => { const taskResult = { diff --git a/test/lib/storyboard-version-negotiation.test.js b/test/lib/storyboard-version-negotiation.test.js index 4339775a1..2c51cf54f 100644 --- a/test/lib/storyboard-version-negotiation.test.js +++ b/test/lib/storyboard-version-negotiation.test.js @@ -104,6 +104,78 @@ describe('storyboard runner AdCP version negotiation', () => { assert.strictEqual(options.versionEnvelope, 'auto'); }); + test('version_negotiation evaluates envelope_field_pattern instead of forward-compatible not_applicable', async () => { + const { runStoryboard } = require('../../dist/lib/testing/storyboard/runner.js'); + const { createTestClient } = require('../../dist/lib/testing/client.js'); + + const storyboard = { + id: 'version_negotiation', + version: '1.0.0', + adcp_version: CURRENT_PRERELEASE_VERSION, + title: 'Version negotiation', + category: 'universal', + summary: '', + narrative: '', + agent: { interaction_model: '*', capabilities: [] }, + caller: { role: 'buyer_agent' }, + phases: [ + { + id: 'capabilities', + title: 'Capabilities', + steps: [ + { + id: 'get_capabilities_with_version', + title: 'Get capabilities with version envelope', + task: 'get_adcp_capabilities', + sample_request: { + context: { + correlation_id: 'version_negotiation--get_capabilities_with_version', + }, + }, + validations: [ + { + check: 'envelope_field_pattern', + path: 'adcp_version', + pattern: '^\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$', + description: 'Seller echoes release-precision adcp_version on the response envelope', + }, + ], + }, + ], + }, + ], + }; + + const client = createTestClient('https://example.invalid/mcp', 'mcp', { + adcpVersion: CURRENT_PRERELEASE_VERSION, + versionEnvelope: 'auto', + }); + client.getAgentInfo = async () => ({ name: 'Test', tools: [{ name: 'get_adcp_capabilities' }] }); + client.getAdcpCapabilities = async () => ({ + success: true, + data: { + status: 'completed', + adcp_version: CURRENT_PRERELEASE_RELEASE_PRECISION, + adcp: { major_versions: [3], supported_versions: [CURRENT_PRERELEASE_RELEASE_PRECISION] }, + supported_protocols: ['media_buy'], + context: { correlation_id: 'version_negotiation--get_capabilities_with_version' }, + }, + }); + + const result = await runStoryboard('https://example.invalid/mcp', storyboard, { + protocol: 'mcp', + agentTools: ['get_adcp_capabilities'], + _profile: { name: 'Test', tools: ['get_adcp_capabilities'] }, + _client: client, + }); + + const validation = result.phases[0].steps[0].validations.find(v => v.check === 'envelope_field_pattern'); + assert.ok(validation, 'envelope_field_pattern validation should be present'); + assert.strictEqual(validation.passed, true, validation.error); + assert.strictEqual(validation.not_applicable, undefined); + assert.strictEqual(result.validations_not_applicable, undefined); + }); + test('caller-supplied version envelope mode wins', () => { const { applyStoryboardVersionOptions } = require('../../dist/lib/testing/storyboard/index.js'); From 0269716c391094dfa792a4815a2d6c094c77237e Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 30 May 2026 11:55:30 -0400 Subject: [PATCH 2/3] chore: add changeset for storyboard pattern checks --- .changeset/storyboard-pattern-validations.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/storyboard-pattern-validations.md diff --git a/.changeset/storyboard-pattern-validations.md b/.changeset/storyboard-pattern-validations.md new file mode 100644 index 000000000..dcc85d608 --- /dev/null +++ b/.changeset/storyboard-pattern-validations.md @@ -0,0 +1,7 @@ +--- +"@adcp/sdk": patch +--- + +fix(storyboard): add regex-backed field pattern validations + +Storyboard validations now support `field_pattern` and `envelope_field_pattern` checks for string fields, with consistent handling for missing fields, non-string values, invalid regex sources, conformance replay, and schema drift detection. From 831c2b34bd06182f4d9ea2903960ee53f2ad05d4 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 30 May 2026 12:13:24 -0400 Subject: [PATCH 3/3] fix(storyboard): address pattern validation review --- .changeset/storyboard-pattern-validations.md | 2 +- src/lib/testing/storyboard/validations.ts | 18 ++++++------- test/lib/storyboard-drift.test.js | 27 +++++++++++++++++--- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.changeset/storyboard-pattern-validations.md b/.changeset/storyboard-pattern-validations.md index dcc85d608..638e2e0f9 100644 --- a/.changeset/storyboard-pattern-validations.md +++ b/.changeset/storyboard-pattern-validations.md @@ -1,5 +1,5 @@ --- -"@adcp/sdk": patch +'@adcp/sdk': minor --- fix(storyboard): add regex-backed field pattern validations diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index c66e52a5c..e44a8e855 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -250,21 +250,21 @@ function runValidation(validation: StoryboardValidation, ctx: ValidationContext) case 'field_absent': return validateFieldAbsent(validation, resolveTarget(ctx)); case 'envelope_field_present': - case 'envelope_field_absent': - case 'envelope_field_value': - case 'envelope_field_value_or_absent': - case 'envelope_field_pattern': // Envelope-scoped variants — runtime semantics identical to the // un-prefixed checks (TaskResult exposes envelope fields like // `status`, `task_id` and version-envelope fields like `adcp_version` // at the surface level). The distinct check types exist primarily so // static drift detection can walk the envelope schemas instead of the // per-tool response. See adcp#3429 and adcp#5195. - if (validation.check === 'envelope_field_present') return validateFieldPresent(validation, resolveTarget(ctx)); - if (validation.check === 'envelope_field_absent') return validateFieldAbsent(validation, resolveTarget(ctx)); - if (validation.check === 'envelope_field_value') return validateFieldValue(validation, resolveTarget(ctx)); - if (validation.check === 'envelope_field_pattern') return validateFieldPattern(validation, resolveTarget(ctx)); + return validateFieldPresent(validation, resolveTarget(ctx)); + case 'envelope_field_absent': + return validateFieldAbsent(validation, resolveTarget(ctx)); + case 'envelope_field_value': + return validateFieldValue(validation, resolveTarget(ctx)); + case 'envelope_field_value_or_absent': return validateFieldValueOrAbsent(validation, resolveTarget(ctx)); + case 'envelope_field_pattern': + return validateFieldPattern(validation, resolveTarget(ctx)); case 'field_value': return validateFieldValue(validation, resolveTarget(ctx)); case 'field_value_or_absent': @@ -1062,7 +1062,7 @@ function validateFieldPattern(validation: StoryboardValidation, taskResult: Task }`, json_pointer: pointer, expected, - actual: validation.pattern, + actual: validation.pattern ?? null, }; } diff --git a/test/lib/storyboard-drift.test.js b/test/lib/storyboard-drift.test.js index bf3a406de..c635de8fb 100644 --- a/test/lib/storyboard-drift.test.js +++ b/test/lib/storyboard-drift.test.js @@ -267,7 +267,7 @@ describe('storyboard schema drift', () => { it('found field validations to check', () => { assert.ok( fieldValidations.length > 0, - 'Expected at least one field_present, field_value, or field_value_or_absent validation' + 'Expected at least one field_present, field_value, field_value_or_absent, or field_pattern validation' ); }); @@ -364,8 +364,8 @@ describe('storyboard schema drift', () => { // `task_id`, `message`, `replayed`, `governance_context`, `timestamp`, // `context_id`, `push_notification_config`) using the envelope-scoped // checks so the drift detector knows to walk `protocol-envelope.json` - // rather than the per-tool response schema. `errors` and `adcp_version` - // are NOT envelope fields — keep them on the un-prefixed checks. + // and `version-envelope.json` rather than the per-tool response schema. + // `errors` lives inside `payload`, not on either envelope. // `envelope_field_absent` is excluded here — absence checks have no schema // target (see the `field_absent / envelope_field_absent` block above). const envelopeValidations = fieldValidations.filter( @@ -416,6 +416,27 @@ describe('storyboard schema drift', () => { } }); + describe('field_pattern paths are reachable in response schemas', () => { + const patternValidations = fieldValidations.filter(v => v.check === 'field_pattern'); + + for (const entry of patternValidations) { + const schema = TOOL_RESPONSE_SCHEMAS[entry.task]; + if (!schema) continue; + + const key = `${entry.storyboard}/${entry.step}:${entry.path}`; + const skip = skipReason(key); + it(`${entry.storyboard}/${entry.step}: ${entry.path} exists in ${entry.task} schema`, { skip }, () => { + const segments = parsePath(entry.path); + const reachable = isPathReachable(schema, segments); + assert.ok( + reachable, + `Path "${entry.path}" is not reachable in ${entry.task} response schema. ` + + `Segments: ${JSON.stringify(segments)}` + ); + }); + } + }); + describe('field_value_or_absent paths are reachable in response schemas', () => { const tolerantValidations = fieldValidations.filter(v => v.check === 'field_value_or_absent');