fix(storyboard): add pattern validation checks

.changeset/storyboard-pattern-validations.md

@@ -0,0 +1,7 @@
+---
+'@adcp/sdk': minor
+---
+
+fix(storyboard): add regex-backed field pattern validations
+
+Storyboard validations now support `field_pattern` and `envelope_field_pattern` checks for string fields, with consistent handling for missing fields, non-string values, invalid regex sources, conformance replay, and schema drift detection.

scripts/conformance-replay.ts

@@ -218,12 +218,14 @@ const IMPLEMENTED_CHECKS = new Set([
   'response_schema',
   'field_present',
   'field_value',
+  'field_pattern',
   // Envelope-scoped variants (adcp#3429). Runtime semantics are identical to
   // the un-prefixed checks because the SDK's response unwrappers expose
   // envelope fields at the structuredContent surface (`status`, `task_id`,
   // `replayed`, etc.); the distinction is for static drift detection.
   'envelope_field_present',
   'envelope_field_value',
+  'envelope_field_pattern',
   // Cardinality assertions (adcp#4685 / adcp-client#1830).
   'array_length',
 ]);
@@ -331,6 +333,35 @@ async function runStep(
         allPassed = false;
         result.failures.push(`${check} ${v.path}: got ${JSON.stringify(actual)}, want ${JSON.stringify(v.value)}`);
       }
+    } else if (check === 'field_pattern' || check === 'envelope_field_pattern') {
+      const pattern = (v as { pattern?: unknown }).pattern;
+      if (typeof pattern !== 'string' || pattern.length === 0) {
+        allPassed = false;
+        result.failures.push(`${check} ${v.path}: misconfigured (pattern must be a non-empty string)`);
+      } else {
+        let re: RegExp | undefined;
+        try {
+          re = new RegExp(pattern);
+        } catch (err) {
+          allPassed = false;
+          result.failures.push(
+            `${check} ${v.path}: invalid pattern ${JSON.stringify(pattern)} (${
+              err instanceof Error ? err.message : String(err)
+            })`
+          );
+        }
+        if (re) {
+          const actual = getByPath(structured, v.path as string);
+          if (typeof actual !== 'string') {
+            allPassed = false;
+            const ty = actual === undefined ? 'undefined' : actual === null ? 'null' : typeof actual;
+            result.failures.push(`${check} ${v.path}: got ${ty}, want string matching ${JSON.stringify(pattern)}`);
+          } else if (!re.test(actual)) {
+            allPassed = false;
+            result.failures.push(`${check} ${v.path}: got ${JSON.stringify(actual)}, want /${pattern}/`);
+          }
+        }
+      }
     } else if (check === 'array_length') {
       const hasExact = typeof v.value === 'number';
       const hasMin = typeof (v as { min?: unknown }).min === 'number';

src/lib/testing/storyboard/types.ts

@@ -736,6 +736,25 @@ export type StoryboardValidationCheck =
   | 'envelope_field_value_or_absent'
   | 'field_value'
   | 'field_value_or_absent'
+  /**
+   * Assert that a string field in the task payload matches a JavaScript
+   * regular expression source declared in `pattern`. Fails when the path is
+   * missing, the value is not a string, the pattern is invalid, or the regex
+   * does not match. Failure output carries `expected: { pattern }` and
+   * `actual` as the observed value (or null when missing). Added for
+   * runner-output-contract v2.5.0.
+   */
+  | 'field_pattern'
+  /**
+   * Envelope-scoped `field_pattern`. The path resolves against the union of
+   * protocol-envelope fields (`status`, `task_id`, etc.) and string-valued
+   * version-envelope fields such as `adcp_version`. Runtime matching is the
+   * same as `field_pattern`; the distinct kind lets static linting validate
+   * against the envelope schemas instead of the task payload schema. Numeric
+   * envelope fields such as `adcp_major_version` should use
+   * `envelope_field_value`.
+   */
+  | 'envelope_field_pattern'
   // Wildcard-aware membership check. `path` may include `[*]` segments that
   // expand to every array element via `resolvePathAll`. Passes when ANY
   // resolved value matches `value` (or any of `allowed_values`). Lets
@@ -1062,6 +1081,8 @@ export interface StoryboardValidation {
   value?: unknown;
   /** Accepted values for list-match checks (passes if actual matches any). */
   allowed_values?: unknown[];
+  /** JavaScript regular expression source for field_pattern checks. */
+  pattern?: string;
   description: string;
   // ─── refs_resolve fields ───────────────────────────────────
   /** Source refs (the refs being checked). */

src/lib/testing/storyboard/validations.ts

@@ -250,22 +250,27 @@ function runValidation(validation: StoryboardValidation, ctx: ValidationContext)
     case 'field_absent':
       return validateFieldAbsent(validation, resolveTarget(ctx));
     case 'envelope_field_present':
+      // Envelope-scoped variants — runtime semantics identical to the
+      // un-prefixed checks (TaskResult exposes envelope fields like
+      // `status`, `task_id` and version-envelope fields like `adcp_version`
+      // at the surface level). The distinct check types exist primarily so
+      // static drift detection can walk the envelope schemas instead of the
+      // per-tool response. See adcp#3429 and adcp#5195.
+      return validateFieldPresent(validation, resolveTarget(ctx));
     case 'envelope_field_absent':
+      return validateFieldAbsent(validation, resolveTarget(ctx));
     case 'envelope_field_value':
+      return validateFieldValue(validation, resolveTarget(ctx));
     case 'envelope_field_value_or_absent':
-      // Envelope-scoped variants — runtime semantics identical to the
-      // un-prefixed checks (TaskResult exposes envelope fields like
-      // `status`, `task_id` at the surface level). The distinct check types
-      // exist primarily so static drift detection can walk the envelope
-      // schema instead of the per-tool response. See adcp#3429.
-      if (validation.check === 'envelope_field_present') return validateFieldPresent(validation, resolveTarget(ctx));
-      if (validation.check === 'envelope_field_absent') return validateFieldAbsent(validation, resolveTarget(ctx));
-      if (validation.check === 'envelope_field_value') return validateFieldValue(validation, resolveTarget(ctx));
       return validateFieldValueOrAbsent(validation, resolveTarget(ctx));
+    case 'envelope_field_pattern':
+      return validateFieldPattern(validation, resolveTarget(ctx));
     case 'field_value':
       return validateFieldValue(validation, resolveTarget(ctx));
     case 'field_value_or_absent':
       return validateFieldValueOrAbsent(validation, resolveTarget(ctx));
+    case 'field_pattern':
+      return validateFieldPattern(validation, resolveTarget(ctx));
     case 'field_contains':
       return validateFieldContains(validation, resolveTarget(ctx));
     case 'status_code':
@@ -1007,6 +1012,99 @@ function isTaskEnvelopeStatus(value: unknown): boolean {
   return typeof value === 'string' && TASK_ENVELOPE_STATUSES.has(value);
 }
 
+// ────────────────────────────────────────────────────────────
+// field_pattern / envelope_field_pattern: check a string matches
+// a storyboard-authored JavaScript regular expression source.
+// ────────────────────────────────────────────────────────────
+
+function validateFieldPattern(validation: StoryboardValidation, taskResult: TaskResult): ValidationResult {
+  const checkName = validation.check;
+  if (!validation.path) {
+    return {
+      check: checkName,
+      passed: false,
+      description: validation.description,
+      path: validation.path,
+      error: `No path specified for ${checkName} validation`,
+      json_pointer: null,
+      expected: 'path must be set in storyboard validation entry',
+      actual: null,
+    };
+  }
+
+  const pointer = toJsonPointer(validation.path);
+  const expected = { pattern: validation.pattern };
+
+  if (typeof validation.pattern !== 'string' || validation.pattern.length === 0) {
+    return {
+      check: checkName,
+      passed: false,
+      description: validation.description,
+      path: validation.path,
+      error: `${checkName} requires a non-empty \`pattern\` string`,
+      json_pointer: pointer,
+      expected: { pattern: 'non-empty JavaScript regular expression source' },
+      actual: validation.pattern ?? null,
+    };
+  }
+
+  let re: RegExp;
+  try {
+    re = new RegExp(validation.pattern);
+  } catch (err) {
+    return {
+      check: checkName,
+      passed: false,
+      description: validation.description,
+      path: validation.path,
+      error: `Invalid ${checkName} pattern ${JSON.stringify(validation.pattern)}: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+      json_pointer: pointer,
+      expected,
+      actual: validation.pattern ?? null,
+    };
+  }
+
+  const actual = resolvePath(taskResult.data, validation.path);
+  if (typeof actual !== 'string') {
+    return {
+      check: checkName,
+      passed: false,
+      description: validation.description,
+      path: validation.path,
+      error:
+        actual === undefined || actual === null
+          ? `Field not found at path: ${validation.path}`
+          : `Expected string at path: ${validation.path}, got ${Array.isArray(actual) ? 'array' : typeof actual}`,
+      json_pointer: pointer,
+      expected,
+      actual: actual ?? null,
+    };
+  }
+
+  if (re.test(actual)) {
+    return {
+      check: checkName,
+      passed: true,
+      description: validation.description,
+      path: validation.path,
+      json_pointer: pointer,
+    };
+  }
+
+  return {
+    check: checkName,
+    passed: false,
+    description: validation.description,
+    path: validation.path,
+    error: `Expected string at path ${validation.path} to match pattern ${JSON.stringify(validation.pattern)}, got ${JSON.stringify(actual)}`,
+    json_pointer: pointer,
+    expected,
+    actual,
+  };
+}
+
 // ────────────────────────────────────────────────────────────
 // field_contains: wildcard-aware membership check
 //

test/lib/storyboard-drift.test.js

@@ -1,8 +1,8 @@
 /**
  * Schema drift detection for storyboard YAML validations.
  *
- * Catches when field_present / field_value / field_value_or_absent paths in
- * storyboard YAML reference fields that don't exist in the corresponding
+ * Catches when field_present / field_value / field_value_or_absent /
+ * field_pattern paths in storyboard YAML reference fields that don't exist in the corresponding
  * Zod response schemas, when context extractors reference tasks without
  * schemas, and when `field_value_or_absent` is asserted on a path the
  * response schema already marks required (the tolerance is meaningless —
@@ -17,14 +17,12 @@ const { listAllComplianceStoryboards } = require('../../dist/lib/testing/storybo
 const { parsePath } = require('../../dist/lib/testing/storyboard/path.js');
 const { TOOL_RESPONSE_SCHEMAS } = require('../../dist/lib/utils/response-schemas.js');
 const { CONTEXT_EXTRACTORS } = require('../../dist/lib/testing/storyboard/context.js');
-// `envelope_field_present` (and `envelope_field_value{,_or_absent}`)
-// validations walk the v3 protocol envelope — `status`, `task_id`,
-// `message`, `replayed`, `governance_context`, `timestamp`, `context_id`,
-// `push_notification_config` — instead of per-tool response schemas.
-// `errors` and `adcp_version` are NOT envelope fields (errors lives inside
-// `payload` per the per-tool response schema; adcp_major_version is on the
-// request, not on either response surface). Added per adcp#3429.
-const { ProtocolEnvelopeSchema } = require('../../dist/lib/types/schemas.generated.js');
+// `envelope_field_*` validations walk the v3 protocol + version envelopes
+// (`status`, `task_id`, `message`, `replayed`, `adcp_version`, etc.) instead
+// of per-tool response schemas. `errors` lives inside `payload` per the
+// per-tool response schema. Added per adcp#3429 and adcp#5195.
+const { AdCPVersionEnvelopeSchema, ProtocolEnvelopeSchema } = require('../../dist/lib/types/schemas.generated.js');
+const EnvelopeSchema = ProtocolEnvelopeSchema.merge(AdCPVersionEnvelopeSchema);
 
 // Runner-internal tasks with no agent-facing schema.
 const HARNESS_TASKS = new Set([
@@ -233,7 +231,9 @@ function collectFieldValidations(storyboards) {
               v.check === 'field_value' ||
               v.check === 'envelope_field_value' ||
               v.check === 'field_value_or_absent' ||
-              v.check === 'envelope_field_value_or_absent') &&
+              v.check === 'envelope_field_value_or_absent' ||
+              v.check === 'field_pattern' ||
+              v.check === 'envelope_field_pattern') &&
             v.path
           ) {
             if (ENVELOPE_PATHS.has(v.path)) continue; // protocol-level, not per-schema
@@ -267,7 +267,7 @@ describe('storyboard schema drift', () => {
   it('found field validations to check', () => {
     assert.ok(
       fieldValidations.length > 0,
-      'Expected at least one field_present, field_value, or field_value_or_absent validation'
+      'Expected at least one field_present, field_value, field_value_or_absent, or field_pattern validation'
     );
   });
 
@@ -364,15 +364,16 @@ describe('storyboard schema drift', () => {
     // `task_id`, `message`, `replayed`, `governance_context`, `timestamp`,
     // `context_id`, `push_notification_config`) using the envelope-scoped
     // checks so the drift detector knows to walk `protocol-envelope.json`
-    // rather than the per-tool response schema. `errors` and `adcp_version`
-    // are NOT envelope fields — keep them on the un-prefixed checks.
+    // and `version-envelope.json` rather than the per-tool response schema.
+    // `errors` lives inside `payload`, not on either envelope.
     // `envelope_field_absent` is excluded here — absence checks have no schema
     // target (see the `field_absent / envelope_field_absent` block above).
     const envelopeValidations = fieldValidations.filter(
       v =>
         v.check === 'envelope_field_present' ||
         v.check === 'envelope_field_value' ||
-        v.check === 'envelope_field_value_or_absent'
+        v.check === 'envelope_field_value_or_absent' ||
+        v.check === 'envelope_field_pattern'
     );
 
     for (const entry of envelopeValidations) {
@@ -383,10 +384,10 @@ describe('storyboard schema drift', () => {
         { skip },
         () => {
           const segments = parsePath(entry.path);
-          const reachable = isPathReachable(ProtocolEnvelopeSchema, segments);
+          const reachable = isPathReachable(EnvelopeSchema, segments);
           assert.ok(
             reachable,
-            `Path "${entry.path}" is not reachable in protocol-envelope.json. ` +
+            `Path "${entry.path}" is not reachable in protocol-envelope.json or version-envelope.json. ` +
               `Segments: ${JSON.stringify(segments)}`
           );
         }
@@ -415,6 +416,27 @@ describe('storyboard schema drift', () => {
     }
   });
 
+  describe('field_pattern paths are reachable in response schemas', () => {
+    const patternValidations = fieldValidations.filter(v => v.check === 'field_pattern');
+
+    for (const entry of patternValidations) {
+      const schema = TOOL_RESPONSE_SCHEMAS[entry.task];
+      if (!schema) continue;
+
+      const key = `${entry.storyboard}/${entry.step}:${entry.path}`;
+      const skip = skipReason(key);
+      it(`${entry.storyboard}/${entry.step}: ${entry.path} exists in ${entry.task} schema`, { skip }, () => {
+        const segments = parsePath(entry.path);
+        const reachable = isPathReachable(schema, segments);
+        assert.ok(
+          reachable,
+          `Path "${entry.path}" is not reachable in ${entry.task} response schema. ` +
+            `Segments: ${JSON.stringify(segments)}`
+        );
+      });
+    }
+  });
+
   describe('field_value_or_absent paths are reachable in response schemas', () => {
     const tolerantValidations = fieldValidations.filter(v => v.check === 'field_value_or_absent');