From bf1e8a894d81e9931faedf678c424233fe0dc8f9 Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Thu, 7 May 2026 15:36:41 -0400 Subject: [PATCH 01/10] Issue #799 Transform PoC --- .../common_grants_sdk/extensions/README.md | 152 +++++++++++- .../common_grants_sdk/extensions/__init__.py | 31 +++ .../common_grants_sdk/extensions/plugin.py | 69 +++++- .../extensions/transforms.py | 121 ++++++++++ .../common_grants_sdk/extensions/types.py | 160 ++++++++++++ .../common_grants_sdk/utils/transformation.py | 67 ++++- lib/python-sdk/examples/README.md | 52 ++++ .../examples/plugins/grants_gov/__init__.py | 14 ++ .../examples/plugins/grants_gov/cg_config.py | 141 +++++++++++ lib/python-sdk/examples/transforms.py | 136 +++++++++++ lib/python-sdk/tests/extensions/__init__.py | 0 .../tests/extensions/test_plugin.py | 116 +++++++++ .../tests/extensions/test_transforms.py | 228 ++++++++++++++++++ lib/python-sdk/tests/extensions/test_types.py | 228 ++++++++++++++++++ .../tests/utils/test_transformation.py | 81 +++++++ 15 files changed, 1581 insertions(+), 15 deletions(-) create mode 100644 lib/python-sdk/common_grants_sdk/extensions/transforms.py create mode 100644 lib/python-sdk/common_grants_sdk/extensions/types.py create mode 100644 lib/python-sdk/examples/plugins/grants_gov/__init__.py create mode 100644 lib/python-sdk/examples/plugins/grants_gov/cg_config.py create mode 100644 lib/python-sdk/examples/transforms.py create mode 100644 lib/python-sdk/tests/extensions/__init__.py create mode 100644 lib/python-sdk/tests/extensions/test_plugin.py create mode 100644 lib/python-sdk/tests/extensions/test_transforms.py create mode 100644 lib/python-sdk/tests/extensions/test_types.py diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 9083276ce..df0f0355b 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -22,6 +22,10 @@ The `common-grants/sdk/extensions` module contains the utilities for working wit - [Defining a plugin](#defining-a-plugin) - [Publishing a plugin](#publishing-a-plugin) - [Combining Plugins](#combining-plugins) +- [Bidirectional Transforms](#bidirectional-transforms) + - [Defining transforms](#defining-transforms) + - [Mapping format](#mapping-format) + - [Using transforms](#using-transforms) - [Using plugins with the API client](#using-plugins-with-the-api-client) - [Best practices](#best-practices) - [Field naming](#field-naming) @@ -37,11 +41,15 @@ The `common-grants/sdk/extensions` module contains the utilities for working wit Here are some key concepts that are used to define custom fields and plugins that extend base schemas from the CommonGrants protocol. | Concept | Description | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | -| **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | -| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | -| **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | +| **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | +| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | +| **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | +| **`PluginMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | +| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict and returns a `TransformResult`. | +| **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | +| **`ObjectSchemasInput`**| Bundles a `to_common` and `from_common` callable for a single object type. Passed to `define_plugin()` via the `transform_schemas` parameter. | @@ -433,6 +441,140 @@ Prefer unique, namespaced field names so `"error"` is never triggered. After building your package, import the plugin in a test file and confirm that `.extensions` keys and `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. +## Bidirectional Transforms + +Plugins can define bidirectional mappings between a source system's native data format and the CommonGrants format. These transforms are authored as plain Python dicts and compiled into callable functions by `build_transforms()`. + +### Defining transforms + +Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `transform_schemas`: + +```python +from common_grants_sdk.extensions import ( + CustomFieldSpec, + ObjectSchemasInput, + PluginMeta, + build_transforms, + define_plugin, +) +from common_grants_sdk.schemas.pydantic.fields import CustomFieldType + +to_common, from_common = build_transforms( + to_common_mapping={ + "title": {"field": "data.opportunity_title"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": {"posted": "open", "archived": "closed", "forecasted": "forecasted"}, + "default": "custom", + } + }, + "description": {"const": "The opportunity is currently accepting applications"}, + }, + "funding": { + "minAwardAmount": { + "amount": {"field": "data.summary.award_floor"}, + "currency": {"const": "USD"}, + }, + }, + }, + from_common_mapping={ + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "match": { + "field": "status.value", + "case": {"open": "posted", "closed": "archived", "forecasted": "forecasted"}, + "default": "custom", + } + }, + } + }, +) + +plugin = define_plugin( + extensions={ + "Opportunity": { + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Unique identifier in legacy database", + ), + } + }, + meta=PluginMeta( + name="my-system", + version="0.1.0", + source_system="my-system.example.gov", + capabilities=["customFields", "transforms"], + ), + transform_schemas={ + "Opportunity": ObjectSchemasInput( + to_common=to_common, + from_common=from_common, + ) + }, +) +``` + +Both directions must be provided explicitly. `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. + +### Mapping format + +A mapping dict describes how to build an output object from a source dict. Each leaf node is either a literal value or a single-key dict that invokes a named handler. + +| Handler | Syntax | Description | +|---|---|---| +| `const` | `{"const": "USD"}` | Returns a fixed literal value, ignoring source data | +| `field` | `{"field": "data.summary.award_floor"}` | Extracts a value using a dot-notation path | +| `match` | `{"match": {"field": "...", "case": {...}, "default": "..."}}` | Case-based lookup on a field value (canonical ADR name) | +| `switch` | `{"switch": {...}}` | Alias for `match`, kept for backward compatibility | +| `numberToString` | `{"numberToString": "data.summary.award_floor"}` | Extracts a numeric value and coerces it to a string | +| `stringToNumber` | `{"stringToNumber": "some.string.field"}` | Extracts a string and coerces it to `int` or `float` | + +Bare non-dict values (strings, numbers, booleans) in a mapping are treated as literals and passed through unchanged. Use `{"const": ...}` when you want a literal value inside a dict node that might otherwise be mistaken for a field name. + +You can also register custom handlers by passing a `handlers` dict to `build_transforms()`: + +```python +def handle_upper(data, field_path): + val = get_from_path(data, field_path) + return val.upper() if isinstance(val, str) else val + +to_common, from_common = build_transforms( + to_common_mapping={"title": {"upper": "data.opportunity_title"}}, + from_common_mapping={...}, + handlers={"upper": handle_upper}, +) +``` + +Custom handlers are merged with the defaults; they cannot override built-in handler names. + +### Using transforms + +The compiled callables are stored on the plugin's `transform_schemas` dict, keyed by object name. Each callable takes a data dict and returns a `TransformResult`: + +```python +opp_transforms = plugin.transform_schemas["Opportunity"] + +# Source system → CommonGrants +result = opp_transforms.to_common(native_data) +if result.errors: + for err in result.errors: + print(f"[{err.path}] {err}") +else: + cg_data = result.result + +# CommonGrants → source system +result = opp_transforms.from_common(cg_data) +native_data = result.result +``` + +`TransformResult.errors` is always a list (empty on success). A non-empty errors list means the transform encountered a problem but still returned a partial result in `result`. + +See `examples/transforms.py` for a complete working example with roundtrip verification. + + ## Using plugins with the API client Pass a plugin's extended schema to the API client via the `schema` parameter. The client uses it to hydrate API responses into fully typed models. The `schema` parameter accepts any `Type[OpportunityBase]` subclass. diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 87a089683..d60446878 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -2,8 +2,24 @@ from .plugin import Plugin, PluginConfig, define_plugin from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions, merge_extensions +from .transforms import build_transforms +from .types import ( + ClientConfig, + Handler, + ObjectMappings, + ObjectSchemas, + ObjectSchemasInput, + PluginCapability, + PluginError, + PluginExtensions, + PluginExtensionsMeta, + PluginExtensionsSchema, + PluginMeta, + TransformResult, +) __all__ = [ + # Existing exports (unchanged) "ConflictStrategy", "CustomFieldSpec", "Plugin", @@ -11,4 +27,19 @@ "SchemaExtensions", "define_plugin", "merge_extensions", + # New: build_transforms + "build_transforms", + # New: ADR-0022 types + "ClientConfig", + "Handler", + "ObjectMappings", + "ObjectSchemas", + "ObjectSchemasInput", + "PluginCapability", + "PluginError", + "PluginExtensions", + "PluginExtensionsMeta", + "PluginExtensionsSchema", + "PluginMeta", + "TransformResult", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 7b4670f34..c7dbaefac 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -1,28 +1,81 @@ """Plugin configuration and composition APIs.""" +from __future__ import annotations + from dataclasses import dataclass -from typing import Generic, TypeVar +from typing import Any, Callable, Generic, TypeVar from .specs import SchemaExtensions +from .types import ClientConfig, ObjectSchemas, ObjectSchemasInput, PluginMeta T = TypeVar("T") @dataclass(frozen=True) class PluginConfig: - """Build-time plugin config discoverable by the generator.""" + """Build-time plugin config discoverable by the code generator. + + extensions: custom field declarations (read by generate.py — do not rename). + meta: optional plugin identity and capability declaration. + transform_schemas: optional bidirectional transform callables per object. + Stored as ObjectSchemasInput (not compiled to ObjectSchemas) in the PoC. + Full compilation with model_validate wrapping is a TODO for the real SDK. + + TODO (full SDK): add get_client, filters. + """ extensions: SchemaExtensions + meta: PluginMeta | None = None + transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None -@dataclass(frozen=True) +@dataclass class Plugin(Generic[T]): - """Runtime plugin container with both extension specs and generated schemas.""" + """Runtime plugin container with extension specs and generated schemas. + + extensions: SchemaExtensions used by generate.py (do not rename or reorder — + the generated __init__.py constructs Plugin(extensions=..., schemas=...)). + schemas: generated _Schemas object (typed Pydantic model classes from generate.py). + NOTE: there is a naming collision: ADR-0022 also calls its runtime transform + dict "schemas". These are different concepts sharing the same name — a design + question to resolve in the full SDK (see Design Finding #1 in the spec). + transform_schemas: ADR-0022 runtime transform dict; named distinctly from + `schemas` to avoid collision with the generated schemas field in the PoC. + + TODO (full SDK): memoize get_client. + """ extensions: SchemaExtensions - schemas: T + schemas: T # generated _Schemas object — keep as positional for generate.py compat + meta: PluginMeta | None = None + get_client: Callable[[ClientConfig], Any] | None = None # TODO: memoize + # PoC stores ObjectSchemasInput here (no compilation yet); full SDK will store + # ObjectSchemas after model_validate wrapping. Annotated as the union so both + # the current PoC usage and the future compiled form are type-safe. + transform_schemas: ( + dict[str, ObjectSchemasInput[Any, Any] | ObjectSchemas[Any, Any]] | None + ) = None + filters: dict[str, dict[str, Any]] | None = None + + +def define_plugin( + extensions: SchemaExtensions, + meta: PluginMeta | None = None, + transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, + # TODO (full SDK): get_client, filters +) -> PluginConfig: + """Create a PluginConfig object consumed by the code generator. + Backward-compatible: existing callers passing only `extensions` are unaffected. + New params are stored as-is — no compilation occurs in the PoC. -def define_plugin(extensions: SchemaExtensions) -> PluginConfig: - """Create a plugin config object consumed by the code generator.""" - return PluginConfig(extensions=extensions) + TODO (full SDK): + - Auto-generate transforms from extensions.schemas[obj].mappings when no + explicit to_common/from_common is supplied. + - Wrap transform output with model_validate. + """ + return PluginConfig( + extensions=extensions, + meta=meta, + transform_schemas=transform_schemas, + ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py new file mode 100644 index 000000000..dacb22aab --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -0,0 +1,121 @@ +"""build_transforms() — generates to_common/from_common callables from mapping dicts. + +Using this utility is optional — plugin authors may provide plain hand-written +callables instead. + +Mappings are validated at call time. Custom handler names are +registered per call only; name collisions with defaults raise at call time +rather than silently shadowing them. +""" + +from __future__ import annotations + +from typing import Any, Callable + +from common_grants_sdk.utils.transformation import ( + DEFAULT_HANDLERS, + transform_from_mapping, +) + +from .types import Handler, PluginError, TransformResult + + +def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> None: + """Walk the mapping tree and raise ValueError on structural malformation. + + For each dict node: + - If a key is a known handler, the corresponding value is a runtime-only + handler argument and is NOT recursed into. + - All other keys are output field names (always valid); their values are + recursed into. + + Raises ValueError if any node is not a dict, string, number, boolean, or None + (e.g. a list where a scalar or dict is expected). + + Note: this function cannot detect intended-but-unknown handler invocations + because unknown keys are indistinguishable from output field names at static + analysis time. That detection is deferred to the full SDK. + """ + if mapping is None or isinstance(mapping, (str, int, float, bool)): + return # primitives and None are valid literals + + if not isinstance(mapping, dict): + raise ValueError( + f"Invalid mapping node at '{path}': expected dict, str, number, or bool, " + f"got {type(mapping).__name__}" + ) + + for key, value in mapping.items(): + current_path = f"{path}.{key}" if path else key + if key in known_handlers: + # Handler invocation — argument is runtime-only, do not recurse + continue + _validate_mapping(value, known_handlers, current_path) + + +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = None, +) -> tuple[ + Callable[[Any], TransformResult[Any]], + Callable[[Any], TransformResult[Any]], +]: + """Generate to_common and from_common callables from mapping dicts. + + Args: + to_common_mapping: mapping from native source → CommonGrants. + from_common_mapping: mapping from CommonGrants → native source. + handlers: Optional additional handlers registered for this call only. + Keys must not collide with DEFAULT_HANDLERS (raises ValueError if they do). + + Returns: + A (to_common, from_common) tuple. Each callable accepts a dict and returns + TransformResult[Any]. Failures surface as PluginError entries in + TransformResult.errors rather than being raised. + + Raises: + ValueError: At call time if handler names collide with defaults, + or if either mapping has structural malformation. + + TODO (full SDK): + - Validate field-path resolvability at call time (requires sample data or + schema introspection). + """ + # Custom handler names must not shadow defaults + if handlers: + collisions = set(handlers) & set(DEFAULT_HANDLERS) + if collisions: + raise ValueError( + f"build_transforms: handler names collide with defaults: {sorted(collisions)}" + ) + + merged = {**DEFAULT_HANDLERS, **(handlers or {})} + known = set(merged) + + # Validate mapping structure at call time + _validate_mapping(to_common_mapping, known) + _validate_mapping(from_common_mapping, known) + + def to_common(native: Any) -> TransformResult[Any]: + try: + result = transform_from_mapping(native, to_common_mapping, handlers=merged) + # TODO (full SDK): run model_validate on result and append validation + # failures to errors. Belongs in define_plugin() wrapper which knows + # the target Pydantic model, not here. + return TransformResult(result=result, errors=[]) + except Exception as exc: + error = PluginError(str(exc), path=None, source_value=native, cause=exc) + return TransformResult(result={}, errors=[error]) + + def from_common(common: Any) -> TransformResult[Any]: + try: + result = transform_from_mapping( + common, from_common_mapping, handlers=merged + ) + return TransformResult(result=result, errors=[]) + except Exception as exc: + error = PluginError(str(exc), path=None, source_value=common, cause=exc) + return TransformResult(result={}, errors=[error]) + + return to_common, from_common diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py new file mode 100644 index 000000000..e574bae11 --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -0,0 +1,160 @@ +"""Plugin framework types for the CommonGrants Python SDK.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Generic, Literal, TypeVar + +from pydantic import BaseModel, ConfigDict, Field + +TNative = TypeVar("TNative") +TCommon = TypeVar("TCommon") +T = TypeVar("T") + +# Capability enum — Literal rather than StrEnum to stay JSON-safe. +PluginCapability = Literal["customFields", "customFilters", "transforms", "client"] + +# Type aliases +Handler = Callable[[Any, Any], Any] +ClientConfig = dict[str, Any] + + +class PluginError(Exception): + """Structured transformation error per ADR-0022 Decision #9. + + Carries field path, handler name, source value, and underlying cause so + consumers can reason about failures programmatically without parsing error text. + + Note: source_value may contain PII when transforming applicant data. + Adopters are responsible for redacting it before logging or re-raising. + The SDK does not redact by default. + """ + + def __init__( + self, + message: str, + *, + path: str | None = None, + handler: str | None = None, + source_value: Any = None, + cause: BaseException | None = None, + ) -> None: + super().__init__(message) + self.path = path + self.handler = handler + self.source_value = source_value + self.cause = cause + + +@dataclass +class TransformResult(Generic[T]): + """Unconditional return shape for to_common / from_common (ADR-0022 Decision #7). + + result: the transformed value (may be partial on error). + errors: aggregated PluginErrors; empty on full success. + + Consumers apply their own strict-vs-lenient rule for what counts as success: + - Strict: treat any non-empty errors as failure. + - Lenient: use result despite warnings; inspect errors for context. + """ + + result: T + errors: list[PluginError] + + +class PluginMeta(BaseModel): + """Plugin identity and capability declaration.""" + + model_config = ConfigDict(populate_by_name=True) + + name: str + version: str | None = None + # Serialises as "sourceSystem" in JSON (camelCase per ADR-0022 language-agnostic config). + source_system: str = Field(alias="sourceSystem") + capabilities: list[PluginCapability] | None = None + + +class ObjectMappings(BaseModel): + """ADR-0017 mapping dicts for a single object, stored in the serializable extensions config. + + Each direction is author-provided — build_transforms() does not invert one into + the other because many-to-one handlers like switch are not reversible (Decision #6). + """ + + model_config = ConfigDict(populate_by_name=True) + + to_common: dict[str, Any] | None = Field(default=None, alias="toCommon") + from_common: dict[str, Any] | None = Field(default=None, alias="fromCommon") + + +class PluginExtensionsMeta(BaseModel): + """All-optional mirror of PluginMeta for use inside the serializable PluginExtensions. + + All fields are Optional so partial metadata can be declared without requiring + a full PluginMeta. If PluginMeta gains new required fields, update this class + manually — drift can be caught with: + assert PluginMeta.model_fields.keys() == PluginExtensionsMeta.model_fields.keys() + """ + + model_config = ConfigDict(populate_by_name=True) + + name: str | None = None + version: str | None = None + source_system: str | None = Field(default=None, alias="sourceSystem") + capabilities: list[PluginCapability] | None = None + + +class PluginExtensionsSchema(BaseModel): + """Per-object config inside extensions.schemas. + + custom_fields: custom field declarations (merged by merge_extensions). + mappings: optional ADR-0017 declarative mappings. When present and no explicit + to_common / from_common is supplied in schemas[obj], define_plugin() will + auto-invoke build_transforms() on these (TODO — ADR-0022 Decision #6). + """ + + model_config = ConfigDict(populate_by_name=True) + + custom_fields: dict[str, Any] | None = Field(default=None, alias="customFields") + mappings: ObjectMappings | None = None + + +class PluginExtensions(BaseModel): + """Serializable portion of plugin config — safe to store as JSON. + + Used by merge_extensions() to combine declarations from multiple plugin packages. + """ + + model_config = ConfigDict(populate_by_name=True) + + meta: PluginExtensionsMeta | None = None + schemas: dict[str, PluginExtensionsSchema] | None = None + + +@dataclass +class ObjectSchemasInput(Generic[TNative, TCommon]): + """Input type provided by plugin authors inside define_plugin(schemas=...). + + Plugin authors supply to_common and from_common as plain callables — either + hand-written or generated via build_transforms(). native defaults to + dict[str, Any] if omitted. + """ + + native: type[TNative] | None = None + to_common: Callable[[TNative], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TNative]] | None = None + + +@dataclass +class ObjectSchemas(Generic[TNative, TCommon]): + """Runtime compiled type produced by define_plugin() — not provided directly by authors. + + In the PoC, define_plugin() stores ObjectSchemasInput as-is; full compilation + (adding common from the base CG model, wrapping with model_validate) is a TODO + for the real SDK (ADR-0022 Decision #7). + """ + + native: type[TNative] + common: type[TCommon] + to_common: Callable[[TNative], TransformResult[TCommon]] + from_common: Callable[[TCommon], TransformResult[TNative]] diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index 10153a71e..dea7395f6 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -64,10 +64,69 @@ def switch_on_value(data: dict, switch_spec: dict) -> Any: return lookup.get(val, switch_spec.get("default")) +def const_value(_data: dict, value: Any) -> Any: + """ + Handles a const transformation by returning a fixed literal value. + + Args: + _data: The source data dictionary (unused) + value: The constant value to return + + Returns: + The constant value exactly as specified + """ + return value + + +def number_to_string(data: dict, field_path: str) -> str | None: + """ + Handles a numberToString transformation by extracting a numeric value and coercing it to a string. + + Args: + data: The source data dictionary + field_path: A dot-separated string representing the path to the numeric value + + Returns: + The value at the specified path converted to a string, or None if the path doesn't exist + """ + val = get_from_path(data, field_path) + return str(val) if val is not None else None + + +def string_to_number(data: dict, field_path: str) -> int | float | None: + """ + Handles a stringToNumber transformation by extracting a string value and coercing it to a number. + + Attempts integer conversion first; falls back to float for decimal strings. + + Args: + data: The source data dictionary + field_path: A dot-separated string representing the path to the string value + + Returns: + The value at the specified path converted to int or float, or None if the path doesn't exist + + Raises: + ValueError: If the extracted value cannot be converted to a number + """ + val = get_from_path(data, field_path) + if val is None: + return None + s = str(val) + try: + return int(s) + except ValueError: + return float(s) + + # Registry for handlers DEFAULT_HANDLERS: dict[str, handle_func] = { + "const": const_value, "field": pluck_field_value, - "switch": switch_on_value, + "match": switch_on_value, # ADR-0017 canonical name + "numberToString": number_to_string, + "stringToNumber": string_to_number, + "switch": switch_on_value, # alias kept for backward compatibility } @@ -83,8 +142,12 @@ def transform_from_mapping( The mapping supports both literal values and transformations keyed by the following reserved words: + - `const`: Returns a fixed literal value regardless of input data - `field`: Extracts a value from the data using a dot-notation path - - `switch`: Performs a case-based lookup based on a field value + - `match`: Performs a case-based lookup based on a field value (canonical) + - `numberToString`: Extracts a numeric value and coerces it to a string + - `stringToNumber`: Extracts a string value and coerces it to int or float + - `switch`: Alias for `match` (kept for backward compatibility) Args: data: The source data dictionary to transform diff --git a/lib/python-sdk/examples/README.md b/lib/python-sdk/examples/README.md index d624cffa5..7e57099c7 100644 --- a/lib/python-sdk/examples/README.md +++ b/lib/python-sdk/examples/README.md @@ -119,6 +119,58 @@ None ``` +# Bidirectional transforms example + +This example demonstrates the plugin transform framework: mapping source system data (grants.gov format) to the CommonGrants format and back again, with a roundtrip consistency check. No API server is required — the script runs entirely offline using sample data defined in the file itself. + +```bash +poetry run python examples/transforms.py +``` + +**Output Example:** +``` +============================================================ +SOURCE DATA (grants.gov format) +============================================================ +{ ... } + +============================================================ +to_common: grants.gov → CommonGrants +============================================================ +Errors: none + +Result: +{ + "title": "Research into conservation techniques", + "status": { "value": "open", "description": "The opportunity is currently accepting applications" }, + "funding": { + "minAwardAmount": { "amount": 10000, "currency": "USD" }, + "maxAwardAmount": { "amount": 100000, "currency": "USD" } + }, + ... +} + +============================================================ +from_common: CommonGrants → grants.gov +============================================================ +Errors: none + +Result: { ... } + +============================================================ +ROUNDTRIP CHECK +============================================================ + [PASS] title: 'Research into conservation techniques' -> 'Research into conservation techniques' + [PASS] status: 'posted' -> 'posted' + [PASS] award_floor: 10000 -> 10000 + [PASS] award_ceiling: 100000 -> 100000 + +Roundtrip result: ALL PASS +``` + +The transform mappings live in `examples/plugins/grants_gov/cg_config.py`. See the [extensions README](../common_grants_sdk/extensions/README.md#bidirectional-transforms) for a full explanation of the mapping format. + + # Plugin framework example This example uses the plugin framework to define four typed custom fields, generate static Pydantic models, and validate an API payload. diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py new file mode 100644 index 000000000..c8e987530 --- /dev/null +++ b/lib/python-sdk/examples/plugins/grants_gov/__init__.py @@ -0,0 +1,14 @@ +# This file is auto-generated. Do not edit it manually — it will be overwritten +# the next time `python -m common_grants_sdk.extensions.generate` is run. +from __future__ import annotations + +from common_grants_sdk.extensions import Plugin +from .cg_config import plugin +from .generated import schemas + +grants_gov = Plugin( + extensions=plugin.extensions, + schemas=schemas, +) + +__all__ = ["grants_gov", "schemas"] diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py new file mode 100644 index 000000000..1ab3bdf10 --- /dev/null +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -0,0 +1,141 @@ +"""Grants.gov sample plugin — bidirectional transform PoC. + +Demonstrates the plugin framework shape using the grants.gov scenario. + +Usage (from lib/python-sdk/): + poetry run python examples/transforms.py + +Code generation (generates typed custom-field schemas): + poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov +""" + +from common_grants_sdk.extensions import ( + CustomFieldSpec, + ObjectSchemasInput, + PluginMeta, + build_transforms, + define_plugin, +) +from common_grants_sdk.schemas.pydantic.fields import CustomFieldType + +# --------------------------------------------------------------------------- +# Bidirectional transforms +# +# Both directions are author-provided — build_transforms() does not invert +# one into the other because many-to-one handlers like switch are not +# reversible. +# +# Convention: field extraction uses {"field": "dot.notation.path"} — bare +# string values are treated as literals by transform_from_mapping(), not +# as field paths. See Design Finding #2 in the spec for the open question +# about which convention is canonical. +# --------------------------------------------------------------------------- + +to_common, from_common = build_transforms( + # to_common: grants.gov native → CommonGrants Opportunity + to_common_mapping={ + "title": {"field": "data.opportunity_title"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": { + "forecasted": "forecasted", + "posted": "open", + "archived": "closed", + }, + "default": "custom", + } + }, + "description": { + "const": "The opportunity is currently accepting applications" + }, + }, + "funding": { + "minAwardAmount": { + "amount": {"field": "data.summary.award_floor"}, + "currency": {"const": "USD"}, + }, + "maxAwardAmount": { + "amount": {"field": "data.summary.award_ceiling"}, + "currency": {"const": "USD"}, + }, + }, + "keyDates": { + "appOpens": { + "name": {"const": "Open Date"}, + "date": {"field": "data.summary.forecasted_post_date"}, + "description": {"const": "Applications begin being accepted"}, + }, + "appDeadline": { + "name": {"const": "Application Deadline"}, + "date": {"field": "data.summary.forecasted_close_date"}, + "description": { + "const": "Final submission deadline for all grant applications" + }, + }, + }, + }, + # from_common: CommonGrants Opportunity → grants.gov native + from_common_mapping={ + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "match": { + "field": "status.value", + "case": { + "open": "posted", + "closed": "archived", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "summary": { + "award_floor": {"field": "funding.minAwardAmount.amount"}, + "award_ceiling": {"field": "funding.maxAwardAmount.amount"}, + "forecasted_post_date": {"field": "keyDates.appOpens.date"}, + "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, + }, + } + }, +) + +# --------------------------------------------------------------------------- +# Plugin config +# --------------------------------------------------------------------------- + +plugin = define_plugin( + # extensions: SchemaExtensions — dict[str, dict[str, CustomFieldSpec]] + extensions={ + "Opportunity": { + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + name="Legacy ID", + description="Unique identifier in legacy database", + ), + "agencyName": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Agency", + description="Agency hosting the opportunity", + ), + "applicantTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + name="Applicant types", + description="Types of applicants eligible to apply", + ), + } + }, + meta=PluginMeta( + name="grants-gov", + version="0.1.0", + sourceSystem="grants.gov", + capabilities=["customFields", "transforms"], + ), + transform_schemas={ + "Opportunity": ObjectSchemasInput( + to_common=to_common, + from_common=from_common, + ) + }, +) diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py new file mode 100644 index 000000000..caa302dfd --- /dev/null +++ b/lib/python-sdk/examples/transforms.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Bidirectional transform PoC — plugin transformation interface. + +Demonstrates source (grants.gov) → CommonGrants and CommonGrants → source +bidirectional transformations using the grants.gov sample plugin. + +Run with (from lib/python-sdk/): + poetry run python examples/transforms.py +""" + +from __future__ import annotations + +import json +from typing import Any + +# When run as `poetry run python examples/transforms.py`, Python automatically +# adds the script's directory (examples/) to sys.path. Import from there using +# the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only +# works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. +from plugins.grants_gov.cg_config import plugin + +# --------------------------------------------------------------------------- +# Sample grants.gov source data +# --------------------------------------------------------------------------- + +SOURCE_DATA: dict[str, Any] = { + "data": { + "agency_name": "Department of Examples", + "opportunity_id": 12345, + "opportunity_number": "ABC-123-XYZ-001", + "opportunity_status": "posted", + "opportunity_title": "Research into conservation techniques", + "summary": { + "applicant_types": ["state_governments"], + "archive_date": "2025-05-01", + "award_ceiling": 100000, + "award_floor": 10000, + "forecasted_award_date": "2025-09-01", + "forecasted_close_date": "2025-07-15", + "forecasted_post_date": "2025-05-01", + }, + } +} + + +def _section(title: str) -> None: + print(f"\n{'=' * 60}") + print(title) + print("=" * 60) + + +def main() -> None: + assert plugin.transform_schemas is not None + opp = plugin.transform_schemas["Opportunity"] + + _section("SOURCE DATA (grants.gov format)") + print(json.dumps(SOURCE_DATA, indent=2)) + + # --- to_common: grants.gov → CommonGrants --- + assert opp.to_common is not None + cg_result = opp.to_common(SOURCE_DATA) + + _section("to_common: grants.gov → CommonGrants") + if cg_result.errors: + print(f"ERRORS ({len(cg_result.errors)}):") + for err in cg_result.errors: + print(f" [path={err.path}] {err}") + else: + print("Errors: none") + print("\nResult:") + print(json.dumps(cg_result.result, indent=2)) + + # --- from_common: CommonGrants → grants.gov --- + assert opp.from_common is not None + native_result = opp.from_common(cg_result.result) + + _section("from_common: CommonGrants → grants.gov") + if native_result.errors: + print(f"ERRORS ({len(native_result.errors)}):") + for err in native_result.errors: + print(f" [path={err.path}] {err}") + else: + print("Errors: none") + print("\nResult:") + print(json.dumps(native_result.result, indent=2)) + + # --- Roundtrip comparison --- + # Note: SOURCE_DATA contains fields not covered by the mappings (agency_name, + # opportunity_id, etc.). Those fields are intentionally absent from the roundtrip + # output — the mapping layer is selective by design. + _section("ROUNDTRIP CHECK") + checks = [ + ( + "title", + SOURCE_DATA["data"]["opportunity_title"], + native_result.result.get("data", {}).get("opportunity_title"), + ), + ( + "status", + SOURCE_DATA["data"]["opportunity_status"], + native_result.result.get("data", {}).get("opportunity_status"), + ), + ( + "award_floor", + SOURCE_DATA["data"]["summary"]["award_floor"], + native_result.result.get("data", {}).get("summary", {}).get("award_floor"), + ), + ( + "award_ceiling", + SOURCE_DATA["data"]["summary"]["award_ceiling"], + native_result.result.get("data", {}) + .get("summary", {}) + .get("award_ceiling"), + ), + ] + all_pass = True + for field, original, roundtripped in checks: + ok = original == roundtripped + if not ok: + all_pass = False + status = "PASS" if ok else "FAIL" + print(f" [{status}] {field}: {original!r} -> {roundtripped!r}") + + print(f"\nRoundtrip result: {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}") + + # --- Plugin metadata --- + _section("PLUGIN METADATA") + assert plugin.meta is not None + print(f"name: {plugin.meta.name}") + print(f"version: {plugin.meta.version}") + print(f"sourceSystem: {plugin.meta.source_system}") + print(f"capabilities: {plugin.meta.capabilities}") + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/tests/extensions/__init__.py b/lib/python-sdk/tests/extensions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py new file mode 100644 index 000000000..568c1cb5a --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -0,0 +1,116 @@ +"""Tests for expanded plugin.py — backward compat + new optional fields.""" + +from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin +from common_grants_sdk.extensions.specs import SchemaExtensions +from common_grants_sdk.extensions.types import ( + ObjectSchemasInput, + PluginMeta, + TransformResult, +) + +EXTENSIONS: SchemaExtensions = {} # minimal valid extensions + + +# --- PluginConfig backward compatibility --- + + +def test_define_plugin_existing_signature(): + """define_plugin(extensions=...) still returns PluginConfig — backward compat.""" + config = define_plugin(extensions=EXTENSIONS) + assert isinstance(config, PluginConfig) + assert config.extensions is EXTENSIONS + + +def test_plugin_config_extensions_only(): + """PluginConfig with only extensions (old callers) still works.""" + config = PluginConfig(extensions=EXTENSIONS) + assert config.meta is None + assert config.transform_schemas is None + + +# --- PluginConfig new fields --- + + +def test_define_plugin_with_meta(): + meta = PluginMeta(name="test", source_system="test-system") + config = define_plugin(extensions=EXTENSIONS, meta=meta) + assert config.meta is meta + assert config.meta.name == "test" + + +def test_define_plugin_with_transform_schemas(): + def passthrough(x): + return TransformResult(result=x, errors=[]) + + schemas = { + "Opportunity": ObjectSchemasInput( + to_common=passthrough, from_common=passthrough + ) + } + config = define_plugin(extensions=EXTENSIONS, transform_schemas=schemas) + assert config.transform_schemas is schemas + assert "Opportunity" in config.transform_schemas + + +def test_define_plugin_stores_input_as_is(): + """define_plugin stores ObjectSchemasInput as-is (no compilation in PoC).""" + + def passthrough(x): + return TransformResult(result=x, errors=[]) + + inp = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) + config = define_plugin( + extensions=EXTENSIONS, transform_schemas={"Opportunity": inp} + ) + stored = config.transform_schemas["Opportunity"] + assert stored is inp + assert stored.to_common is passthrough + + +# --- Plugin backward compatibility --- + + +def test_plugin_existing_fields(): + """Plugin(extensions=..., schemas=...) still works — backward compat.""" + plugin = Plugin(extensions=EXTENSIONS, schemas=object()) + assert plugin.extensions is EXTENSIONS + assert plugin.meta is None + assert plugin.get_client is None + assert plugin.transform_schemas is None + assert plugin.filters is None + + +def test_plugin_new_optional_fields_accept_values(): + meta = PluginMeta(name="p", source_system="s") + schemas = {"Opportunity": object()} + plugin = Plugin( + extensions=EXTENSIONS, + schemas=object(), + meta=meta, + transform_schemas=schemas, + ) + assert plugin.meta is meta + assert plugin.transform_schemas is schemas + + +# --- transform_schemas access pattern used by demo --- + + +def test_transform_schemas_callable_roundtrip(): + """The demo calls config.transform_schemas["Opportunity"].to_common(data).""" + + def always_transformed(_x): + return TransformResult(result={"transformed": True}, errors=[]) + + config = define_plugin( + extensions=EXTENSIONS, + transform_schemas={ + "Opportunity": ObjectSchemasInput( + to_common=always_transformed, from_common=always_transformed + ) + }, + ) + opp = config.transform_schemas["Opportunity"] + result = opp.to_common({"raw": "data"}) + assert result.result == {"transformed": True} + assert result.errors == [] diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py new file mode 100644 index 000000000..cf22f7519 --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -0,0 +1,228 @@ +"""Tests for build_transforms() in common_grants_sdk.extensions.transforms.""" + +import pytest +from common_grants_sdk.extensions.transforms import build_transforms +from common_grants_sdk.extensions.types import PluginError, TransformResult + +# Shared source data matching the ADR-0017 grants.gov example +SOURCE_DATA = { + "data": { + "opportunity_title": "Research into conservation techniques", + "opportunity_status": "posted", + "summary": { + "award_floor": 10000, + "award_ceiling": 100000, + "forecasted_post_date": "2025-05-01", + "forecasted_close_date": "2025-07-15", + }, + } +} + +TO_COMMON_MAPPING = { + "title": {"field": "data.opportunity_title"}, + "status": { + "value": { + "switch": { + "field": "data.opportunity_status", + "case": { + "posted": "open", + "archived": "closed", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "description": "The opportunity is currently accepting applications", + }, + "funding": { + "minAwardAmount": { + "amount": {"field": "data.summary.award_floor"}, + "currency": "USD", + }, + }, +} + +FROM_COMMON_MAPPING = { + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "switch": { + "field": "status.value", + "case": { + "open": "posted", + "closed": "archived", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "summary": { + "award_floor": {"field": "funding.minAwardAmount.amount"}, + }, + } +} + + +# --- Call-time validation --- + + +def test_handler_collision_raises(): + """build_transforms raises if custom handler shadows a default handler name.""" + with pytest.raises(ValueError, match="collide with defaults"): + build_transforms( + TO_COMMON_MAPPING, + FROM_COMMON_MAPPING, + handlers={"field": lambda d, v: v}, # "field" is a default handler + ) + + +def test_handler_collision_raises_for_switch(): + with pytest.raises(ValueError, match="collide with defaults"): + build_transforms( + TO_COMMON_MAPPING, + FROM_COMMON_MAPPING, + handlers={"switch": lambda d, v: v}, + ) + + +def test_structural_error_list_node_raises(): + """build_transforms raises if a mapping node is a list (structural malformation).""" + bad_mapping = {"title": ["should", "not", "be", "a", "list"]} + with pytest.raises(ValueError, match="Invalid mapping node"): + build_transforms(bad_mapping, FROM_COMMON_MAPPING) + + +def test_valid_mapping_does_not_raise(): + """build_transforms does not raise on a well-formed mapping.""" + to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + assert callable(to_common) + assert callable(from_common) + + +# --- to_common transform --- + + +def test_to_common_returns_transform_result(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert isinstance(result, TransformResult) + + +def test_to_common_no_errors_on_valid_data(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert result.errors == [] + + +def test_to_common_maps_title(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert result.result["title"] == "Research into conservation techniques" + + +def test_to_common_maps_status_via_switch(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert result.result["status"]["value"] == "open" + + +def test_to_common_preserves_literal_constant(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert ( + result.result["status"]["description"] + == "The opportunity is currently accepting applications" + ) + + +def test_to_common_maps_nested_funding(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert result.result["funding"]["minAwardAmount"]["amount"] == 10000 + assert result.result["funding"]["minAwardAmount"]["currency"] == "USD" + + +# --- from_common transform --- + + +def test_from_common_returns_transform_result(): + to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + cg = to_common(SOURCE_DATA) + result = from_common(cg.result) + assert isinstance(result, TransformResult) + + +def test_from_common_no_errors_on_valid_data(): + to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + cg = to_common(SOURCE_DATA) + result = from_common(cg.result) + assert result.errors == [] + + +def test_from_common_roundtrip_title(): + to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + cg = to_common(SOURCE_DATA) + native = from_common(cg.result) + assert ( + native.result["data"]["opportunity_title"] + == "Research into conservation techniques" + ) + + +def test_from_common_roundtrip_status(): + """Status roundtrip: posted → open → posted.""" + to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + cg = to_common(SOURCE_DATA) + native = from_common(cg.result) + assert native.result["data"]["opportunity_status"] == "posted" + + +# --- Error surfacing --- + + +def test_exception_surfaces_as_plugin_error_not_raised(): + """Exceptions inside handlers surface as PluginError, not raised.""" + + def boom(data, _arg): + raise RuntimeError("handler exploded") + + to_common, _ = build_transforms( + {"title": {"boom": "anything"}}, + {}, + handlers={"boom": boom}, + ) + result = to_common(SOURCE_DATA) + assert isinstance(result, TransformResult) + assert len(result.errors) == 1 + assert isinstance(result.errors[0], PluginError) + assert "handler exploded" in str(result.errors[0]) + + +def test_structural_error_nested_list_path_reported(): + """_validate_mapping includes the field path in the error message for nested lists.""" + bad_mapping = {"funding": {"amount": [1, 2]}} + with pytest.raises(ValueError, match="funding.amount"): + build_transforms(bad_mapping, {}) + + +def test_custom_handler_registered_per_call(): + """Custom handlers apply only to the call they are registered on.""" + + def handle_upper(data, path): + parts = path.split(".") + val = data + for part in parts: + if isinstance(val, dict): + val = val.get(part) + else: + val = None + break + return str(val).upper() if val is not None else None + + to_common, _ = build_transforms( + {"title": {"upper": "data.opportunity_title"}}, + {}, + handlers={"upper": handle_upper}, + ) + result = to_common(SOURCE_DATA) + assert result.result["title"] == "RESEARCH INTO CONSERVATION TECHNIQUES" diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py new file mode 100644 index 000000000..7d2aa778d --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -0,0 +1,228 @@ +"""Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" + +import pytest +from common_grants_sdk.extensions.types import ( + ClientConfig, + Handler, + ObjectMappings, + ObjectSchemas, + ObjectSchemasInput, + PluginError, + PluginExtensions, + PluginExtensionsMeta, + PluginExtensionsSchema, + PluginMeta, + TransformResult, +) + + +def test_plugin_extensions_meta_mirrors_plugin_meta(): + """PluginExtensionsMeta must stay in sync with PluginMeta field names.""" + assert set(PluginMeta.model_fields.keys()) == set( + PluginExtensionsMeta.model_fields.keys() + ) + + +# --- PluginError --- + + +def test_plugin_error_is_exception(): + err = PluginError("something went wrong") + assert isinstance(err, Exception) + assert str(err) == "something went wrong" + + +def test_plugin_error_structured_fields(): + cause = ValueError("root cause") + err = PluginError( + "msg", + path="status.value", + handler="switch", + source_value={"x": 1}, + cause=cause, + ) + assert err.path == "status.value" + assert err.handler == "switch" + assert err.source_value == {"x": 1} + assert err.cause is cause + + +def test_plugin_error_defaults_to_none(): + err = PluginError("bare") + assert err.path is None + assert err.handler is None + assert err.source_value is None + assert err.cause is None + + +# --- TransformResult --- + + +def test_transform_result_success(): + result = TransformResult(result={"title": "hello"}, errors=[]) + assert result.result == {"title": "hello"} + assert result.errors == [] + + +def test_transform_result_with_errors(): + err = PluginError("bad") + result = TransformResult(result={}, errors=[err]) + assert len(result.errors) == 1 + assert result.errors[0] is err + + +# --- PluginMeta --- + + +def test_plugin_meta_required_fields(): + meta = PluginMeta(name="my-plugin", source_system="grants.gov") + assert meta.name == "my-plugin" + assert meta.source_system == "grants.gov" + assert meta.version is None + assert meta.capabilities is None + + +def test_plugin_meta_source_system_required(): + import pydantic + + with pytest.raises(pydantic.ValidationError): + PluginMeta(name="p") # missing source_system + + +def test_plugin_meta_camel_case_alias(): + """source_system serialises as sourceSystem in JSON.""" + meta = PluginMeta(name="p", sourceSystem="grants.gov") + assert meta.source_system == "grants.gov" + + +def test_plugin_meta_capabilities(): + meta = PluginMeta( + name="p", + source_system="grants.gov", + capabilities=["customFields", "transforms"], + ) + assert "customFields" in meta.capabilities + assert "transforms" in meta.capabilities + + +# --- ObjectMappings --- + + +def test_object_mappings_defaults_none(): + m = ObjectMappings() + assert m.to_common is None + assert m.from_common is None + + +def test_object_mappings_camel_aliases(): + m = ObjectMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) + assert m.to_common == {"title": "x"} + assert m.from_common == {"x": "title"} + + +# --- PluginExtensionsMeta --- + + +def test_plugin_extensions_meta_all_optional(): + m = PluginExtensionsMeta() + assert m.name is None + assert m.version is None + assert m.source_system is None + assert m.capabilities is None + + +def test_plugin_extensions_meta_camel_alias(): + m = PluginExtensionsMeta(sourceSystem="grants.gov") + assert m.source_system == "grants.gov" + + +# --- PluginExtensionsSchema --- + + +def test_plugin_extensions_schema_all_optional(): + s = PluginExtensionsSchema() + assert s.custom_fields is None + assert s.mappings is None + + +def test_plugin_extensions_schema_with_mappings(): + mappings = ObjectMappings(toCommon={"a": "b"}) + s = PluginExtensionsSchema(mappings=mappings) + assert s.mappings.to_common == {"a": "b"} + + +# --- PluginExtensions --- + + +def test_plugin_extensions_all_optional(): + ext = PluginExtensions() + assert ext.meta is None + assert ext.schemas is None + + +def test_plugin_extensions_with_schema(): + schema = PluginExtensionsSchema(customFields={"legacyId": {}}) + ext = PluginExtensions(schemas={"Opportunity": schema}) + assert ext.schemas["Opportunity"].custom_fields == {"legacyId": {}} + + +# --- ObjectSchemasInput --- + + +def test_object_schemas_input_all_optional(): + inp = ObjectSchemasInput() + assert inp.native is None + assert inp.to_common is None + assert inp.from_common is None + + +def test_object_schemas_input_with_callables(): + def passthrough(x): + return TransformResult(result=x, errors=[]) + + inp = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) + assert inp.to_common is passthrough + + +# --- ObjectSchemas --- + + +def test_object_schemas_required_fields(): + def passthrough(x): + return TransformResult(result=x, errors=[]) + + schemas = ObjectSchemas( + native=dict, + common=dict, + to_common=passthrough, + from_common=passthrough, + ) + assert schemas.native is dict + assert schemas.common is dict + assert schemas.to_common is passthrough + + +def test_object_schemas_all_fields_required(): + def passthrough(x): + return TransformResult(result=x, errors=[]) + + with pytest.raises(TypeError): + ObjectSchemas( + native=dict, common=dict, to_common=passthrough + ) # missing from_common + + +# --- Handler and ClientConfig type aliases --- + + +def test_handler_is_callable(): + def identity(data, spec): + return spec + + h: Handler = identity + assert h({}, "val") == "val" + + +def test_client_config_is_dict(): + cfg: ClientConfig = {"api_key": "abc", "timeout": 10} + assert cfg["api_key"] == "abc" diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 5365e665f..451d6f56a 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -233,6 +233,87 @@ def handle_type(data, type_spec): assert result == {"id_str": "12345"} +def test_const_string(input_data): + """Test const handler returns a fixed string value.""" + mapping = {"currency": {"const": "USD"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"currency": "USD"} + + +def test_const_number(input_data): + """Test const handler returns a fixed numeric value.""" + mapping = {"version": {"const": 1}} + result = transform_from_mapping(input_data, mapping) + assert result == {"version": 1} + + +def test_const_ignores_source_data(input_data): + """Test const handler is independent of source data.""" + mapping = {"x": {"const": "fixed"}} + result = transform_from_mapping({}, mapping) + assert result == {"x": "fixed"} + + +def test_match_key_alias(input_data): + """Test match key (ADR-0017 canonical name) works identically to switch.""" + mapping = { + "status": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } + } + } + result = transform_from_mapping(input_data, mapping) + assert result == {"status": "open"} + + +def test_number_to_string(input_data): + """Test numberToString handler coerces a numeric field to a string.""" + mapping = {"floor_str": {"numberToString": "summary.award_floor"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"floor_str": "10000"} + + +def test_number_to_string_missing_field(input_data): + """Test numberToString returns None when the field path does not exist.""" + mapping = {"x": {"numberToString": "nonexistent.path"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"x": None} + + +def test_string_to_number_integer(input_data): + """Test stringToNumber handler coerces a string integer field to int.""" + data = {**input_data, "amount_str": "50000"} + result = transform_from_mapping(data, {"amount": {"stringToNumber": "amount_str"}}) + assert result == {"amount": 50000} + assert isinstance(result["amount"], int) + + +def test_string_to_number_float(input_data): + """Test stringToNumber handler coerces a decimal string to float.""" + data = {**input_data, "rate_str": "3.14"} + result = transform_from_mapping(data, {"rate": {"stringToNumber": "rate_str"}}) + assert result == {"rate": 3.14} + assert isinstance(result["rate"], float) + + +def test_string_to_number_missing_field(input_data): + """Test stringToNumber returns None when the field path does not exist.""" + mapping = {"x": {"stringToNumber": "nonexistent.path"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"x": None} + + +def test_string_to_number_invalid_raises(input_data): + """Test stringToNumber raises ValueError for non-numeric strings.""" + + data = {**input_data, "bad": "not-a-number"} + with pytest.raises(ValueError): + transform_from_mapping(data, {"x": {"stringToNumber": "bad"}}) + + def test_deeply_nested(input_data): """ Test transformation with deeply nested structures. From 3e8d260b41d65c965d6ec3b79731616fea5b8388 Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Thu, 7 May 2026 16:22:48 -0400 Subject: [PATCH 02/10] Fixing plugin in makefile --- lib/python-sdk/Makefile | 1 + lib/python-sdk/examples/plugins/grants_gov/__init__.py | 4 ++-- lib/python-sdk/examples/plugins/grants_gov/cg_config.py | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/python-sdk/Makefile b/lib/python-sdk/Makefile index ba7ff350d..a5048497c 100644 --- a/lib/python-sdk/Makefile +++ b/lib/python-sdk/Makefile @@ -29,3 +29,4 @@ checks: check-format check-lint check-types plugins: $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions + $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py index c8e987530..5ae38feac 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/__init__.py +++ b/lib/python-sdk/examples/plugins/grants_gov/__init__.py @@ -3,11 +3,11 @@ from __future__ import annotations from common_grants_sdk.extensions import Plugin -from .cg_config import plugin +from .cg_config import config from .generated import schemas grants_gov = Plugin( - extensions=plugin.extensions, + extensions=config.extensions, schemas=schemas, ) diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py index 1ab3bdf10..3c036ce70 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -139,3 +139,5 @@ ) }, ) + +config = plugin From 8850fb140f225f48496b47b1e8b0123b21650edf Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Fri, 8 May 2026 12:22:13 -0400 Subject: [PATCH 03/10] Adding custom handler example --- lib/python-sdk/examples/transforms.py | 82 +++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index caa302dfd..fe6e79fc5 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -19,6 +19,9 @@ # works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. from plugins.grants_gov.cg_config import plugin +from common_grants_sdk.extensions import build_transforms +from common_grants_sdk.utils.transformation import get_from_path + # --------------------------------------------------------------------------- # Sample grants.gov source data # --------------------------------------------------------------------------- @@ -43,6 +46,62 @@ } +# --------------------------------------------------------------------------- +# Custom handlers: join_fields and split_field +# +# join_fields concatenates multiple source field values with a configurable +# separator. Mapping spec: {"join": {"fields": ["a.b", "c.d"], "sep": " — "}} +# +# split_field is the inverse: it splits a single field on a separator and +# returns the element at the given index. +# Mapping spec: {"split": {"field": "label", "sep": " — ", "index": 0}} +# --------------------------------------------------------------------------- + + +def join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + """Custom handler that joins multiple field values with a separator.""" + sep = spec.get("sep", " ") + parts = [get_from_path(data, path) for path in spec.get("fields", [])] + values = [str(p) for p in parts if p is not None] + return sep.join(values) if values else None + + +def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + """Custom handler that splits a field value and returns the element at index.""" + value = get_from_path(data, spec.get("field", "")) + if value is None: + return None + parts = str(value).split(spec.get("sep", " ")) + index = spec.get("index", 0) + return parts[index] if index < len(parts) else None + + +# Transform that uses the custom handlers — built independently of the plugin +# above to show that build_transforms() accepts arbitrary handler dicts. +to_common_with_custom, from_common_with_custom = build_transforms( + to_common_mapping={ + "title": {"field": "data.opportunity_title"}, + "label": { + "join": { + "fields": ["data.opportunity_number", "data.opportunity_title"], + "sep": " — ", + } + }, + }, + from_common_mapping={ + "data": { + "opportunity_number": { + "split": {"field": "label", "sep": " — ", "index": 0} + }, + "opportunity_title": { + "split": {"field": "label", "sep": " — ", "index": 1} + }, + } + }, + handlers={"join": join_fields, "split": split_field}, +) + + def _section(title: str) -> None: print(f"\n{'=' * 60}") print(title) @@ -123,6 +182,29 @@ def main() -> None: print(f"\nRoundtrip result: {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}") + # --- Custom handler demo --- + _section("CUSTOM HANDLER DEMO (join / split)") + print("Custom handlers passed to build_transforms(): join, split\n") + + custom_cg = to_common_with_custom(SOURCE_DATA) + print("to_common (with join handler):") + print(json.dumps(custom_cg.result, indent=2)) + + custom_native = from_common_with_custom(custom_cg.result) + print("\nfrom_common (with split handler):") + print(json.dumps(custom_native.result, indent=2)) + + orig_num = SOURCE_DATA["data"]["opportunity_number"] + orig_title = SOURCE_DATA["data"]["opportunity_title"] + rt_num = custom_native.result.get("data", {}).get("opportunity_number") + rt_title = custom_native.result.get("data", {}).get("opportunity_title") + print( + f"\n [{'PASS' if orig_num == rt_num else 'FAIL'}] opportunity_number: {orig_num!r} -> {rt_num!r}" + ) + print( + f" [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}" + ) + # --- Plugin metadata --- _section("PLUGIN METADATA") assert plugin.meta is not None From c6af48ca1d1d0211a7cd0e90460abc356077f8cf Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Fri, 8 May 2026 15:16:48 -0400 Subject: [PATCH 04/10] Updating transform contract --- .../extensions/transforms.py | 32 ++++++- .../common_grants_sdk/extensions/types.py | 5 + .../common_grants_sdk/utils/transformation.py | 7 ++ lib/python-sdk/examples/transforms.py | 93 ++++++++++++++----- .../tests/extensions/test_transforms.py | 61 ++++++++++++ .../tests/utils/test_transformation.py | 19 ++++ .../governance/adr/0022-plugin-framework.mdx | 26 +++++- 7 files changed, 213 insertions(+), 30 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index dacb22aab..f69a486dd 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -12,6 +12,8 @@ from typing import Any, Callable +from pydantic import ValidationError + from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, transform_from_mapping, @@ -57,6 +59,7 @@ def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], handlers: dict[str, Handler] | None = None, + common_model: type | None = None, ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], @@ -68,6 +71,15 @@ def build_transforms( from_common_mapping: mapping from CommonGrants → native source. handlers: Optional additional handlers registered for this call only. Keys must not collide with DEFAULT_HANDLERS (raises ValueError if they do). + common_model: Optional Pydantic model class to validate the to_common output + against. Must be the fully extended generated model class (e.g. the + generated Opportunity from generated/schemas.py), NOT the base class + (e.g. OpportunityBase). Passing a base class will silently weaken + validation — custom_fields will only be checked against + dict[str, CustomField] rather than the typed container produced by the + plugin's custom field declarations. When provided, model_validate is + called on the transform result and any ValidationErrors are appended to + TransformResult.errors rather than raised. Returns: A (to_common, from_common) tuple. Each callable accepts a dict and returns @@ -100,14 +112,26 @@ def build_transforms( def to_common(native: Any) -> TransformResult[Any]: try: result = transform_from_mapping(native, to_common_mapping, handlers=merged) - # TODO (full SDK): run model_validate on result and append validation - # failures to errors. Belongs in define_plugin() wrapper which knows - # the target Pydantic model, not here. - return TransformResult(result=result, errors=[]) except Exception as exc: error = PluginError(str(exc), path=None, source_value=native, cause=exc) return TransformResult(result={}, errors=[error]) + if common_model is None: + return TransformResult(result=result, errors=[]) + + try: + validated = common_model.model_validate(result) + return TransformResult(result=validated, errors=[]) + except ValidationError as exc: + errors = [ + PluginError( + e["msg"], + path=".".join(str(loc) for loc in e["loc"]), + ) + for e in exc.errors() + ] + return TransformResult(result=result, errors=errors) + def from_common(common: Any) -> TransformResult[Any]: try: result = transform_from_mapping( diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index e574bae11..2ea19e38e 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -138,6 +138,11 @@ class ObjectSchemasInput(Generic[TNative, TCommon]): Plugin authors supply to_common and from_common as plain callables — either hand-written or generated via build_transforms(). native defaults to dict[str, Any] if omitted. + + common is intentionally absent here. It is injected by define_plugin() during + compilation from ObjectSchemasInput → ObjectSchemas, resolved from the generated + model classes produced by the code generator. Plugin authors never set it directly — + cg_config.py cannot import from generated/ (it is the input to generation). """ native: type[TNative] | None = None diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index dea7395f6..8da2bbe54 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -186,6 +186,13 @@ def transform_from_mapping( } ``` """ + # Normalize Pydantic model instances to plain dicts so that field path + # extraction works regardless of whether the caller passes a raw dict or a + # validated model (e.g. the output of to_common with common_model set). + # mode="json" matches the convention used by CommonGrantsBaseModel.dump_with_mapping. + if hasattr(data, "model_dump"): + data = data.model_dump(mode="json") + # Check for maximum depth # This is a sanity check to prevent stack overflow from deeply nested mappings # which may be a concern when running this function on third-party mappings diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index fe6e79fc5..bfa484616 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -18,6 +18,7 @@ # the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only # works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. from plugins.grants_gov.cg_config import plugin +from plugins.grants_gov.generated.schemas import Opportunity from common_grants_sdk.extensions import build_transforms from common_grants_sdk.utils.transformation import get_from_path @@ -29,10 +30,14 @@ SOURCE_DATA: dict[str, Any] = { "data": { "agency_name": "Department of Examples", + "created_at": "2025-01-15T09:00:00Z", + "last_modified_at": "2025-04-01T12:30:00Z", + "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", "opportunity_id": 12345, "opportunity_number": "ABC-123-XYZ-001", "opportunity_status": "posted", "opportunity_title": "Research into conservation techniques", + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "summary": { "applicant_types": ["state_governments"], "archive_date": "2025-05-01", @@ -76,29 +81,58 @@ def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: return parts[index] if index < len(parts) else None -# Transform that uses the custom handlers — built independently of the plugin -# above to show that build_transforms() accepts arbitrary handler dicts. +# Transform that uses the custom handlers and validates output against the generated +# Opportunity model. common_model=Opportunity (from generated/schemas.py) ensures +# model_validate runs against the extended class with typed custom fields +# (legacyId, agencyName, applicantTypes), not just the base OpportunityBase. to_common_with_custom, from_common_with_custom = build_transforms( to_common_mapping={ + "id": {"field": "data.opportunity_uuid"}, "title": {"field": "data.opportunity_title"}, + "description": {"field": "data.opportunity_description"}, + "createdAt": {"field": "data.created_at"}, + "lastModifiedAt": {"field": "data.last_modified_at"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": { + "posted": "open", + "archived": "closed", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + }, "label": { "join": { "fields": ["data.opportunity_number", "data.opportunity_title"], "sep": " — ", } }, + "customFields": { + "legacyId": { + "value": {"field": "data.opportunity_id"}, + }, + "agencyName": { + "value": {"field": "data.agency_name"}, + }, + "applicantTypes": { + "value": {"field": "data.summary.applicant_types"}, + }, + }, }, from_common_mapping={ "data": { - "opportunity_number": { - "split": {"field": "label", "sep": " — ", "index": 0} - }, - "opportunity_title": { - "split": {"field": "label", "sep": " — ", "index": 1} - }, + # label is produced by the join handler above but gets dropped by + # model_validate (it is not a CG field), so from_common maps directly + # from the standard CG title field instead. + "opportunity_title": {"field": "title"}, } }, handlers={"join": join_fields, "split": split_field}, + common_model=Opportunity, ) @@ -182,28 +216,37 @@ def main() -> None: print(f"\nRoundtrip result: {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}") - # --- Custom handler demo --- - _section("CUSTOM HANDLER DEMO (join / split)") - print("Custom handlers passed to build_transforms(): join, split\n") + # --- Custom handler + model_validate demo --- + _section("CUSTOM HANDLER + MODEL VALIDATE DEMO (join / split / extended Opportunity)") + print("Custom handlers: join, split") + print("common_model: generated Opportunity (with typed customFields)\n") custom_cg = to_common_with_custom(SOURCE_DATA) - print("to_common (with join handler):") - print(json.dumps(custom_cg.result, indent=2)) - - custom_native = from_common_with_custom(custom_cg.result) - print("\nfrom_common (with split handler):") - print(json.dumps(custom_native.result, indent=2)) - orig_num = SOURCE_DATA["data"]["opportunity_number"] + if custom_cg.errors: + print(f"ERRORS ({len(custom_cg.errors)}):") + for err in custom_cg.errors: + print(f" [path={err.path}] {err}") + else: + print("Validation: PASS — result is a typed Opportunity instance") + opp_instance = custom_cg.result + print(f"\n title: {opp_instance.title}") + print(f" id: {opp_instance.id}") + print(f" status: {opp_instance.status.value}") + if opp_instance.custom_fields: + cf = opp_instance.custom_fields + print(f"\n customFields (typed):") + if cf.legacy_id: + print(f" legacyId.value: {cf.legacy_id.value!r} ({type(cf.legacy_id.value).__name__})") + if cf.agency_name: + print(f" agencyName.value: {cf.agency_name.value!r} ({type(cf.agency_name.value).__name__})") + if cf.applicant_types: + print(f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})") + + custom_native = from_common_with_custom(custom_cg.result if not custom_cg.errors else {}) orig_title = SOURCE_DATA["data"]["opportunity_title"] - rt_num = custom_native.result.get("data", {}).get("opportunity_number") rt_title = custom_native.result.get("data", {}).get("opportunity_title") - print( - f"\n [{'PASS' if orig_num == rt_num else 'FAIL'}] opportunity_number: {orig_num!r} -> {rt_num!r}" - ) - print( - f" [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}" - ) + print(f"\n [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}") # --- Plugin metadata --- _section("PLUGIN METADATA") diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py index cf22f7519..23d419b80 100644 --- a/lib/python-sdk/tests/extensions/test_transforms.py +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -1,6 +1,7 @@ """Tests for build_transforms() in common_grants_sdk.extensions.transforms.""" import pytest +from pydantic import BaseModel from common_grants_sdk.extensions.transforms import build_transforms from common_grants_sdk.extensions.types import PluginError, TransformResult @@ -205,6 +206,66 @@ def test_structural_error_nested_list_path_reported(): build_transforms(bad_mapping, {}) +# --- model_validate via common_model --- + + +class _TitleModel(BaseModel): + title: str + + +class _StrictModel(BaseModel): + title: str + required_field: str # always missing from SOURCE_DATA transform output + + +def test_common_model_validates_result(): + """When common_model is provided, result is a model instance on success.""" + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_TitleModel, + ) + result = to_common(SOURCE_DATA) + assert result.errors == [] + assert isinstance(result.result, _TitleModel) + assert result.result.title == "Research into conservation techniques" + + +def test_common_model_validation_failure_surfaces_as_plugin_errors(): + """ValidationError from model_validate surfaces as PluginError entries, not raised.""" + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_StrictModel, + ) + result = to_common(SOURCE_DATA) + assert len(result.errors) >= 1 + assert all(isinstance(e, PluginError) for e in result.errors) + assert any("required_field" in (e.path or "") for e in result.errors) + + +def test_common_model_validation_failure_preserves_partial_result(): + """On validation failure, the raw transform result dict is still returned.""" + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_StrictModel, + ) + result = to_common(SOURCE_DATA) + assert result.result["title"] == "Research into conservation techniques" + + +def test_no_common_model_returns_dict(): + """Without common_model, result is a plain dict as before.""" + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + ) + result = to_common(SOURCE_DATA) + assert result.errors == [] + assert isinstance(result.result, dict) + + def test_custom_handler_registered_per_call(): """Custom handlers apply only to the call they are registered on.""" diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 451d6f56a..87f8bb446 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -1,4 +1,5 @@ import pytest +from pydantic import BaseModel from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, @@ -314,6 +315,24 @@ def test_string_to_number_invalid_raises(input_data): transform_from_mapping(data, {"x": {"stringToNumber": "bad"}}) +def test_pydantic_model_instance_is_normalized(): + """transform_from_mapping accepts a Pydantic model instance and extracts fields correctly.""" + + class Inner(BaseModel): + value: str + + class Source(BaseModel): + title: str + nested: Inner + + model = Source(title="hello", nested=Inner(value="world")) + result = transform_from_mapping(model, { + "out_title": {"field": "title"}, + "out_value": {"field": "nested.value"}, + }) + assert result == {"out_title": "hello", "out_value": "world"} + + def test_deeply_nested(input_data): """ Test transformation with deeply nested structures. diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index a6bcbee4d..9f75b94a1 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -102,6 +102,9 @@ interface ObjectSchemas { } // Input type — provided by plugin authors inside DefinePluginOptions.schemas +// common is intentionally absent: the plugin config file cannot import from generated/ +// since it is the input to generation. definePlugin() injects common during compilation +// from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. interface ObjectSchemasInput { native?: ZodType; // defaults to Record if omitted toCommon?: (native: TNative) => TransformResult; @@ -199,15 +202,24 @@ function mergeExtensions( // Handler signature matches ADR-0017 runtime conventions. type Handler = (value: unknown, context: unknown) => unknown; -// Utility: generates toCommon and fromCommon functions from separate declarative +/// Utility: generates toCommon and fromCommon functions from separate declarative // mapping objects (ADR-0017 format). Using this utility is optional — plugin authors // may provide plain hand-written functions instead. Mappings are validated at call // time (see Decision #7); the optional `handlers` argument registers custom handler // names for this call only (see Decision #8). +// When commonModel is provided, toCommon calls commonModel.parse (Zod) on its output +// and appends any validation errors to TransformResult.errors rather than throwing. +// commonModel must be the fully extended generated schema (e.g. the generated +// Opportunity with typed customFields), not the base schema — passing a base schema +// silently weakens validation of typed custom fields. +// The underlying mapping runtime normalizes model/schema instances to plain objects +// at the entry point, so fromCommon can receive the validated output of toCommon +// and field paths still resolve correctly. function buildTransforms( toCommonMapping: Record, // ADR-0017 mapping from native → CommonGrants fromCommonMapping: Record, // ADR-0017 mapping from CommonGrants → native handlers?: Record, + commonModel?: ZodType, // must be the generated extended schema, not the base ): { toCommon: (native: TNative) => TransformResult; fromCommon: (common: TCommon) => TransformResult; @@ -262,6 +274,9 @@ class ObjectSchemas(Generic[TNative, TCommon]): from_common: Callable[[TCommon], TransformResult[TNative]] # Input type — provided by plugin authors inside define_plugin(schemas=...) +# common is intentionally absent: cg_config.py cannot import from generated/ since +# it is the input to generation. define_plugin() injects common during compilation +# from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. @dataclass class ObjectSchemasInput(Generic[TNative, TCommon]): native: type[TNative] | None = None # defaults to dict[str, Any] if omitted @@ -362,10 +377,19 @@ Handler = Callable[[Any, Any], Any] # may provide plain hand-written callables instead. Mappings are validated at call # time (see Decision #7); the optional `handlers` argument registers custom handler # names for this call only (see Decision #8). +# When common_model is provided, to_common calls model_validate on its output and +# appends any ValidationErrors to TransformResult.errors rather than raising. +# common_model must be the fully extended generated model class (e.g. +# generated/schemas.py's Opportunity), not the base class — passing a base class +# silently weakens validation of typed custom fields. +# transform_from_mapping normalizes Pydantic model instances to plain dicts via +# model_dump(mode="json") at the entry point, so from_common can receive the +# validated model output of to_common and field paths still resolve correctly. def build_transforms( to_common_mapping: dict[str, Any], # ADR-0017 mapping from native → CommonGrants from_common_mapping: dict[str, Any], # ADR-0017 mapping from CommonGrants → native handlers: dict[str, Handler] | None = None, + common_model: type | None = None, # must be the generated extended model, not the base ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], From 0139cdeecbee3e4fd35d181ad37b38e2fa7b9827 Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Fri, 8 May 2026 15:22:01 -0400 Subject: [PATCH 05/10] Fixing make:checks errors --- .../extensions/transforms.py | 4 +-- lib/python-sdk/examples/transforms.py | 26 ++++++++++++++----- .../tests/utils/test_transformation.py | 11 +++++--- .../governance/adr/0022-plugin-framework.mdx | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index f69a486dd..6917338ad 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -12,7 +12,7 @@ from typing import Any, Callable -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, @@ -59,7 +59,7 @@ def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], handlers: dict[str, Handler] | None = None, - common_model: type | None = None, + common_model: type[BaseModel] | None = None, ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index bfa484616..9ee443648 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -217,7 +217,9 @@ def main() -> None: print(f"\nRoundtrip result: {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}") # --- Custom handler + model_validate demo --- - _section("CUSTOM HANDLER + MODEL VALIDATE DEMO (join / split / extended Opportunity)") + _section( + "CUSTOM HANDLER + MODEL VALIDATE DEMO (join / split / extended Opportunity)" + ) print("Custom handlers: join, split") print("common_model: generated Opportunity (with typed customFields)\n") @@ -235,18 +237,28 @@ def main() -> None: print(f" status: {opp_instance.status.value}") if opp_instance.custom_fields: cf = opp_instance.custom_fields - print(f"\n customFields (typed):") + print("\n customFields (typed):") if cf.legacy_id: - print(f" legacyId.value: {cf.legacy_id.value!r} ({type(cf.legacy_id.value).__name__})") + print( + f" legacyId.value: {cf.legacy_id.value!r} ({type(cf.legacy_id.value).__name__})" + ) if cf.agency_name: - print(f" agencyName.value: {cf.agency_name.value!r} ({type(cf.agency_name.value).__name__})") + print( + f" agencyName.value: {cf.agency_name.value!r} ({type(cf.agency_name.value).__name__})" + ) if cf.applicant_types: - print(f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})") + print( + f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})" + ) - custom_native = from_common_with_custom(custom_cg.result if not custom_cg.errors else {}) + custom_native = from_common_with_custom( + custom_cg.result if not custom_cg.errors else {} + ) orig_title = SOURCE_DATA["data"]["opportunity_title"] rt_title = custom_native.result.get("data", {}).get("opportunity_title") - print(f"\n [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}") + print( + f"\n [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}" + ) # --- Plugin metadata --- _section("PLUGIN METADATA") diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 87f8bb446..cbecdb52c 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -326,10 +326,13 @@ class Source(BaseModel): nested: Inner model = Source(title="hello", nested=Inner(value="world")) - result = transform_from_mapping(model, { - "out_title": {"field": "title"}, - "out_value": {"field": "nested.value"}, - }) + result = transform_from_mapping( + model, + { + "out_title": {"field": "title"}, + "out_value": {"field": "nested.value"}, + }, + ) assert result == {"out_title": "hello", "out_value": "world"} diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 9f75b94a1..414cc8509 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -389,7 +389,7 @@ def build_transforms( to_common_mapping: dict[str, Any], # ADR-0017 mapping from native → CommonGrants from_common_mapping: dict[str, Any], # ADR-0017 mapping from CommonGrants → native handlers: dict[str, Handler] | None = None, - common_model: type | None = None, # must be the generated extended model, not the base + common_model: type[BaseModel] | None = None, # must be the generated extended model, not the base ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], From bb1c7663a53dd9c5db8440e2800188e973681a04 Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Mon, 11 May 2026 13:48:41 -0400 Subject: [PATCH 06/10] Reducing duplicate tests and updating makefile --- lib/python-sdk/Makefile | 3 +- .../common_grants_sdk/extensions/__init__.py | 2 - .../common_grants_sdk/extensions/generate.py | 10 +- .../common_grants_sdk/extensions/plugin.py | 8 +- .../common_grants_sdk/extensions/types.py | 20 +-- lib/python-sdk/examples/README.md | 37 +++- .../examples/plugins/grants_gov/cg_config.py | 4 +- .../tests/extensions/test_plugin.py | 85 +++------- .../tests/extensions/test_transforms.py | 127 +++----------- lib/python-sdk/tests/extensions/test_types.py | 159 ++++-------------- 10 files changed, 119 insertions(+), 336 deletions(-) diff --git a/lib/python-sdk/Makefile b/lib/python-sdk/Makefile index a5048497c..bd9d4614e 100644 --- a/lib/python-sdk/Makefile +++ b/lib/python-sdk/Makefile @@ -28,5 +28,4 @@ check-types: plugins checks: check-format check-lint check-types plugins: - $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions - $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov + $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions examples/plugins/grants_gov diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index d60446878..7a9ef1185 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -14,7 +14,6 @@ PluginExtensions, PluginExtensionsMeta, PluginExtensionsSchema, - PluginMeta, TransformResult, ) @@ -40,6 +39,5 @@ "PluginExtensions", "PluginExtensionsMeta", "PluginExtensionsSchema", - "PluginMeta", "TransformResult", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py index 40eba082b..9657a84a1 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ b/lib/python-sdk/common_grants_sdk/extensions/generate.py @@ -487,13 +487,15 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "--plugin", - default=".", - help="Path to plugin directory containing cg_config.py (default: current directory)", + nargs="+", + default=["."], + help="One or more plugin directories containing cg_config.py (default: current directory)", ) args = parser.parse_args(argv) - generated_dir = generate_plugin(Path(args.plugin)) - print(f"Generated plugin schemas at {generated_dir}") + for plugin_path in args.plugin: + generated_dir = generate_plugin(Path(plugin_path)) + print(f"Generated plugin schemas at {generated_dir}") return 0 diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index c7dbaefac..eab8cb8f4 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -6,7 +6,7 @@ from typing import Any, Callable, Generic, TypeVar from .specs import SchemaExtensions -from .types import ClientConfig, ObjectSchemas, ObjectSchemasInput, PluginMeta +from .types import ClientConfig, ObjectSchemas, ObjectSchemasInput, PluginExtensionsMeta T = TypeVar("T") @@ -25,7 +25,7 @@ class PluginConfig: """ extensions: SchemaExtensions - meta: PluginMeta | None = None + meta: PluginExtensionsMeta | None = None transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None @@ -47,7 +47,7 @@ class Plugin(Generic[T]): extensions: SchemaExtensions schemas: T # generated _Schemas object — keep as positional for generate.py compat - meta: PluginMeta | None = None + meta: PluginExtensionsMeta | None = None get_client: Callable[[ClientConfig], Any] | None = None # TODO: memoize # PoC stores ObjectSchemasInput here (no compilation yet); full SDK will store # ObjectSchemas after model_validate wrapping. Annotated as the union so both @@ -60,7 +60,7 @@ class Plugin(Generic[T]): def define_plugin( extensions: SchemaExtensions, - meta: PluginMeta | None = None, + meta: PluginExtensionsMeta | None = None, transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, # TODO (full SDK): get_client, filters ) -> PluginConfig: diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 2ea19e38e..10d17494e 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -62,18 +62,6 @@ class TransformResult(Generic[T]): errors: list[PluginError] -class PluginMeta(BaseModel): - """Plugin identity and capability declaration.""" - - model_config = ConfigDict(populate_by_name=True) - - name: str - version: str | None = None - # Serialises as "sourceSystem" in JSON (camelCase per ADR-0022 language-agnostic config). - source_system: str = Field(alias="sourceSystem") - capabilities: list[PluginCapability] | None = None - - class ObjectMappings(BaseModel): """ADR-0017 mapping dicts for a single object, stored in the serializable extensions config. @@ -88,13 +76,7 @@ class ObjectMappings(BaseModel): class PluginExtensionsMeta(BaseModel): - """All-optional mirror of PluginMeta for use inside the serializable PluginExtensions. - - All fields are Optional so partial metadata can be declared without requiring - a full PluginMeta. If PluginMeta gains new required fields, update this class - manually — drift can be caught with: - assert PluginMeta.model_fields.keys() == PluginExtensionsMeta.model_fields.keys() - """ + """Plugin identity and capability declaration. All fields are optional.""" model_config = ConfigDict(populate_by_name=True) diff --git a/lib/python-sdk/examples/README.md b/lib/python-sdk/examples/README.md index 7e57099c7..4df942a2e 100644 --- a/lib/python-sdk/examples/README.md +++ b/lib/python-sdk/examples/README.md @@ -132,7 +132,28 @@ poetry run python examples/transforms.py ============================================================ SOURCE DATA (grants.gov format) ============================================================ -{ ... } +{ + "data": { + "agency_name": "Department of Examples", + "created_at": "2025-01-15T09:00:00Z", + "last_modified_at": "2025-04-01T12:30:00Z", + "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", + "opportunity_id": 12345, + "opportunity_number": "ABC-123-XYZ-001", + "opportunity_status": "posted", + "opportunity_title": "Research into conservation techniques", + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "summary": { + "applicant_types": ["state_governments"], + "archive_date": "2025-05-01", + "award_ceiling": 100000, + "award_floor": 10000, + "forecasted_award_date": "2025-09-01", + "forecasted_close_date": "2025-07-15", + "forecasted_post_date": "2025-05-01" + } + } +} ============================================================ to_common: grants.gov → CommonGrants @@ -155,7 +176,19 @@ from_common: CommonGrants → grants.gov ============================================================ Errors: none -Result: { ... } +Result: +{ + "data": { + "opportunity_title": "Research into conservation techniques", + "opportunity_status": "posted", + "summary": { + "award_floor": 10000, + "award_ceiling": 100000, + "forecasted_post_date": "2025-05-01", + "forecasted_close_date": "2025-07-15" + } + } +} ============================================================ ROUNDTRIP CHECK diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py index 3c036ce70..b3f28b5e3 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -12,7 +12,7 @@ from common_grants_sdk.extensions import ( CustomFieldSpec, ObjectSchemasInput, - PluginMeta, + PluginExtensionsMeta, build_transforms, define_plugin, ) @@ -126,7 +126,7 @@ ), } }, - meta=PluginMeta( + meta=PluginExtensionsMeta( name="grants-gov", version="0.1.0", sourceSystem="grants.gov", diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index 568c1cb5a..8fcadc799 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -4,41 +4,25 @@ from common_grants_sdk.extensions.specs import SchemaExtensions from common_grants_sdk.extensions.types import ( ObjectSchemasInput, - PluginMeta, + PluginExtensionsMeta, TransformResult, ) EXTENSIONS: SchemaExtensions = {} # minimal valid extensions -# --- PluginConfig backward compatibility --- - - -def test_define_plugin_existing_signature(): - """define_plugin(extensions=...) still returns PluginConfig — backward compat.""" +def test_define_plugin_backward_compat(): + """define_plugin(extensions=...) still returns PluginConfig with all optional fields None.""" config = define_plugin(extensions=EXTENSIONS) assert isinstance(config, PluginConfig) assert config.extensions is EXTENSIONS - - -def test_plugin_config_extensions_only(): - """PluginConfig with only extensions (old callers) still works.""" - config = PluginConfig(extensions=EXTENSIONS) assert config.meta is None assert config.transform_schemas is None -# --- PluginConfig new fields --- +def test_define_plugin_with_meta_and_schemas(): + meta = PluginExtensionsMeta(name="test", source_system="test-system") - -def test_define_plugin_with_meta(): - meta = PluginMeta(name="test", source_system="test-system") - config = define_plugin(extensions=EXTENSIONS, meta=meta) - assert config.meta is meta - assert config.meta.name == "test" - - -def test_define_plugin_with_transform_schemas(): def passthrough(x): return TransformResult(result=x, errors=[]) @@ -47,53 +31,27 @@ def passthrough(x): to_common=passthrough, from_common=passthrough ) } - config = define_plugin(extensions=EXTENSIONS, transform_schemas=schemas) + config = define_plugin(extensions=EXTENSIONS, meta=meta, transform_schemas=schemas) + assert config.meta is meta + assert config.meta.name == "test" assert config.transform_schemas is schemas - assert "Opportunity" in config.transform_schemas -def test_define_plugin_stores_input_as_is(): - """define_plugin stores ObjectSchemasInput as-is (no compilation in PoC).""" +def test_plugin_fields(): + """Plugin accepts all fields; optional ones default to None.""" + base = Plugin(extensions=EXTENSIONS, schemas=object()) + assert base.meta is None + assert base.get_client is None + assert base.transform_schemas is None + assert base.filters is None - def passthrough(x): - return TransformResult(result=x, errors=[]) - - inp = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) - config = define_plugin( - extensions=EXTENSIONS, transform_schemas={"Opportunity": inp} - ) - stored = config.transform_schemas["Opportunity"] - assert stored is inp - assert stored.to_common is passthrough - - -# --- Plugin backward compatibility --- - - -def test_plugin_existing_fields(): - """Plugin(extensions=..., schemas=...) still works — backward compat.""" - plugin = Plugin(extensions=EXTENSIONS, schemas=object()) - assert plugin.extensions is EXTENSIONS - assert plugin.meta is None - assert plugin.get_client is None - assert plugin.transform_schemas is None - assert plugin.filters is None - - -def test_plugin_new_optional_fields_accept_values(): - meta = PluginMeta(name="p", source_system="s") + meta = PluginExtensionsMeta(name="p", source_system="s") schemas = {"Opportunity": object()} - plugin = Plugin( - extensions=EXTENSIONS, - schemas=object(), - meta=meta, - transform_schemas=schemas, + full = Plugin( + extensions=EXTENSIONS, schemas=object(), meta=meta, transform_schemas=schemas ) - assert plugin.meta is meta - assert plugin.transform_schemas is schemas - - -# --- transform_schemas access pattern used by demo --- + assert full.meta is meta + assert full.transform_schemas is schemas def test_transform_schemas_callable_roundtrip(): @@ -110,7 +68,6 @@ def always_transformed(_x): ) }, ) - opp = config.transform_schemas["Opportunity"] - result = opp.to_common({"raw": "data"}) + result = config.transform_schemas["Opportunity"].to_common({"raw": "data"}) assert result.result == {"transformed": True} assert result.errors == [] diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py index 23d419b80..a8213dac4 100644 --- a/lib/python-sdk/tests/extensions/test_transforms.py +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -67,114 +67,58 @@ # --- Call-time validation --- -def test_handler_collision_raises(): +@pytest.mark.parametrize("name", ["field", "switch"]) +def test_handler_collision_raises(name): """build_transforms raises if custom handler shadows a default handler name.""" with pytest.raises(ValueError, match="collide with defaults"): build_transforms( TO_COMMON_MAPPING, FROM_COMMON_MAPPING, - handlers={"field": lambda d, v: v}, # "field" is a default handler + handlers={name: lambda d, v: v}, ) -def test_handler_collision_raises_for_switch(): - with pytest.raises(ValueError, match="collide with defaults"): +def test_structural_error_raises_with_path(): + """build_transforms raises on list nodes and includes the field path.""" + with pytest.raises(ValueError, match="Invalid mapping node"): build_transforms( - TO_COMMON_MAPPING, - FROM_COMMON_MAPPING, - handlers={"switch": lambda d, v: v}, + {"title": ["should", "not", "be", "a", "list"]}, FROM_COMMON_MAPPING ) - - -def test_structural_error_list_node_raises(): - """build_transforms raises if a mapping node is a list (structural malformation).""" - bad_mapping = {"title": ["should", "not", "be", "a", "list"]} - with pytest.raises(ValueError, match="Invalid mapping node"): - build_transforms(bad_mapping, FROM_COMMON_MAPPING) - - -def test_valid_mapping_does_not_raise(): - """build_transforms does not raise on a well-formed mapping.""" - to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - assert callable(to_common) - assert callable(from_common) + with pytest.raises(ValueError, match="funding.amount"): + build_transforms({"funding": {"amount": [1, 2]}}, {}) # --- to_common transform --- -def test_to_common_returns_transform_result(): +def test_to_common(): to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) result = to_common(SOURCE_DATA) assert isinstance(result, TransformResult) - - -def test_to_common_no_errors_on_valid_data(): - to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - result = to_common(SOURCE_DATA) assert result.errors == [] - - -def test_to_common_maps_title(): - to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - result = to_common(SOURCE_DATA) assert result.result["title"] == "Research into conservation techniques" - - -def test_to_common_maps_status_via_switch(): - to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - result = to_common(SOURCE_DATA) assert result.result["status"]["value"] == "open" - - -def test_to_common_preserves_literal_constant(): - to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - result = to_common(SOURCE_DATA) assert ( result.result["status"]["description"] == "The opportunity is currently accepting applications" ) - - -def test_to_common_maps_nested_funding(): - to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - result = to_common(SOURCE_DATA) assert result.result["funding"]["minAwardAmount"]["amount"] == 10000 assert result.result["funding"]["minAwardAmount"]["currency"] == "USD" -# --- from_common transform --- - +# --- from_common roundtrip --- -def test_from_common_returns_transform_result(): - to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - cg = to_common(SOURCE_DATA) - result = from_common(cg.result) - assert isinstance(result, TransformResult) - - -def test_from_common_no_errors_on_valid_data(): - to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - cg = to_common(SOURCE_DATA) - result = from_common(cg.result) - assert result.errors == [] - -def test_from_common_roundtrip_title(): +def test_from_common_roundtrip(): + """Status roundtrip: posted → open → posted.""" to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - cg = to_common(SOURCE_DATA) - native = from_common(cg.result) + native = from_common(to_common(SOURCE_DATA).result) + assert isinstance(native, TransformResult) + assert native.errors == [] assert ( native.result["data"]["opportunity_title"] == "Research into conservation techniques" ) - - -def test_from_common_roundtrip_status(): - """Status roundtrip: posted → open → posted.""" - to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) - cg = to_common(SOURCE_DATA) - native = from_common(cg.result) assert native.result["data"]["opportunity_status"] == "posted" @@ -193,19 +137,11 @@ def boom(data, _arg): handlers={"boom": boom}, ) result = to_common(SOURCE_DATA) - assert isinstance(result, TransformResult) assert len(result.errors) == 1 assert isinstance(result.errors[0], PluginError) assert "handler exploded" in str(result.errors[0]) -def test_structural_error_nested_list_path_reported(): - """_validate_mapping includes the field path in the error message for nested lists.""" - bad_mapping = {"funding": {"amount": [1, 2]}} - with pytest.raises(ValueError, match="funding.amount"): - build_transforms(bad_mapping, {}) - - # --- model_validate via common_model --- @@ -231,8 +167,8 @@ def test_common_model_validates_result(): assert result.result.title == "Research into conservation techniques" -def test_common_model_validation_failure_surfaces_as_plugin_errors(): - """ValidationError from model_validate surfaces as PluginError entries, not raised.""" +def test_common_model_validation_failure(): + """ValidationError surfaces as PluginError entries; raw dict is still returned.""" to_common, _ = build_transforms( {"title": {"field": "data.opportunity_title"}}, {}, @@ -242,30 +178,9 @@ def test_common_model_validation_failure_surfaces_as_plugin_errors(): assert len(result.errors) >= 1 assert all(isinstance(e, PluginError) for e in result.errors) assert any("required_field" in (e.path or "") for e in result.errors) - - -def test_common_model_validation_failure_preserves_partial_result(): - """On validation failure, the raw transform result dict is still returned.""" - to_common, _ = build_transforms( - {"title": {"field": "data.opportunity_title"}}, - {}, - common_model=_StrictModel, - ) - result = to_common(SOURCE_DATA) assert result.result["title"] == "Research into conservation techniques" -def test_no_common_model_returns_dict(): - """Without common_model, result is a plain dict as before.""" - to_common, _ = build_transforms( - {"title": {"field": "data.opportunity_title"}}, - {}, - ) - result = to_common(SOURCE_DATA) - assert result.errors == [] - assert isinstance(result.result, dict) - - def test_custom_handler_registered_per_call(): """Custom handlers apply only to the call they are registered on.""" @@ -273,11 +188,7 @@ def handle_upper(data, path): parts = path.split(".") val = data for part in parts: - if isinstance(val, dict): - val = val.get(part) - else: - val = None - break + val = val.get(part) if isinstance(val, dict) else None return str(val).upper() if val is not None else None to_common, _ = build_transforms( diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index 7d2aa778d..044606ea2 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -2,8 +2,6 @@ import pytest from common_grants_sdk.extensions.types import ( - ClientConfig, - Handler, ObjectMappings, ObjectSchemas, ObjectSchemasInput, @@ -11,25 +9,20 @@ PluginExtensions, PluginExtensionsMeta, PluginExtensionsSchema, - PluginMeta, TransformResult, ) - -def test_plugin_extensions_meta_mirrors_plugin_meta(): - """PluginExtensionsMeta must stay in sync with PluginMeta field names.""" - assert set(PluginMeta.model_fields.keys()) == set( - PluginExtensionsMeta.model_fields.keys() - ) - - # --- PluginError --- -def test_plugin_error_is_exception(): +def test_plugin_error_is_exception_with_defaults(): err = PluginError("something went wrong") assert isinstance(err, Exception) assert str(err) == "something went wrong" + assert err.path is None + assert err.handler is None + assert err.source_value is None + assert err.cause is None def test_plugin_error_structured_fields(): @@ -47,74 +40,26 @@ def test_plugin_error_structured_fields(): assert err.cause is cause -def test_plugin_error_defaults_to_none(): - err = PluginError("bare") - assert err.path is None - assert err.handler is None - assert err.source_value is None - assert err.cause is None - - # --- TransformResult --- -def test_transform_result_success(): - result = TransformResult(result={"title": "hello"}, errors=[]) - assert result.result == {"title": "hello"} - assert result.errors == [] - +def test_transform_result(): + ok = TransformResult(result={"title": "hello"}, errors=[]) + assert ok.result == {"title": "hello"} + assert ok.errors == [] -def test_transform_result_with_errors(): err = PluginError("bad") - result = TransformResult(result={}, errors=[err]) - assert len(result.errors) == 1 - assert result.errors[0] is err - - -# --- PluginMeta --- - - -def test_plugin_meta_required_fields(): - meta = PluginMeta(name="my-plugin", source_system="grants.gov") - assert meta.name == "my-plugin" - assert meta.source_system == "grants.gov" - assert meta.version is None - assert meta.capabilities is None - - -def test_plugin_meta_source_system_required(): - import pydantic - - with pytest.raises(pydantic.ValidationError): - PluginMeta(name="p") # missing source_system - - -def test_plugin_meta_camel_case_alias(): - """source_system serialises as sourceSystem in JSON.""" - meta = PluginMeta(name="p", sourceSystem="grants.gov") - assert meta.source_system == "grants.gov" - - -def test_plugin_meta_capabilities(): - meta = PluginMeta( - name="p", - source_system="grants.gov", - capabilities=["customFields", "transforms"], - ) - assert "customFields" in meta.capabilities - assert "transforms" in meta.capabilities + partial = TransformResult(result={}, errors=[err]) + assert len(partial.errors) == 1 + assert partial.errors[0] is err # --- ObjectMappings --- -def test_object_mappings_defaults_none(): - m = ObjectMappings() - assert m.to_common is None - assert m.from_common is None - - -def test_object_mappings_camel_aliases(): +def test_object_mappings(): + assert ObjectMappings().to_common is None + assert ObjectMappings().from_common is None m = ObjectMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) assert m.to_common == {"title": "x"} assert m.from_common == {"x": "title"} @@ -123,15 +68,9 @@ def test_object_mappings_camel_aliases(): # --- PluginExtensionsMeta --- -def test_plugin_extensions_meta_all_optional(): - m = PluginExtensionsMeta() - assert m.name is None - assert m.version is None - assert m.source_system is None - assert m.capabilities is None - - -def test_plugin_extensions_meta_camel_alias(): +def test_plugin_extensions_meta(): + assert PluginExtensionsMeta().name is None + assert PluginExtensionsMeta().source_system is None m = PluginExtensionsMeta(sourceSystem="grants.gov") assert m.source_system == "grants.gov" @@ -139,28 +78,19 @@ def test_plugin_extensions_meta_camel_alias(): # --- PluginExtensionsSchema --- -def test_plugin_extensions_schema_all_optional(): - s = PluginExtensionsSchema() - assert s.custom_fields is None - assert s.mappings is None - - -def test_plugin_extensions_schema_with_mappings(): - mappings = ObjectMappings(toCommon={"a": "b"}) - s = PluginExtensionsSchema(mappings=mappings) +def test_plugin_extensions_schema(): + assert PluginExtensionsSchema().custom_fields is None + assert PluginExtensionsSchema().mappings is None + s = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) assert s.mappings.to_common == {"a": "b"} # --- PluginExtensions --- -def test_plugin_extensions_all_optional(): - ext = PluginExtensions() - assert ext.meta is None - assert ext.schemas is None - - -def test_plugin_extensions_with_schema(): +def test_plugin_extensions(): + assert PluginExtensions().meta is None + assert PluginExtensions().schemas is None schema = PluginExtensionsSchema(customFields={"legacyId": {}}) ext = PluginExtensions(schemas={"Opportunity": schema}) assert ext.schemas["Opportunity"].custom_fields == {"legacyId": {}} @@ -169,14 +99,10 @@ def test_plugin_extensions_with_schema(): # --- ObjectSchemasInput --- -def test_object_schemas_input_all_optional(): - inp = ObjectSchemasInput() - assert inp.native is None - assert inp.to_common is None - assert inp.from_common is None - +def test_object_schemas_input(): + assert ObjectSchemasInput().native is None + assert ObjectSchemasInput().to_common is None -def test_object_schemas_input_with_callables(): def passthrough(x): return TransformResult(result=x, errors=[]) @@ -187,42 +113,17 @@ def passthrough(x): # --- ObjectSchemas --- -def test_object_schemas_required_fields(): +def test_object_schemas(): def passthrough(x): return TransformResult(result=x, errors=[]) schemas = ObjectSchemas( - native=dict, - common=dict, - to_common=passthrough, - from_common=passthrough, + native=dict, common=dict, to_common=passthrough, from_common=passthrough ) assert schemas.native is dict assert schemas.common is dict - assert schemas.to_common is passthrough - - -def test_object_schemas_all_fields_required(): - def passthrough(x): - return TransformResult(result=x, errors=[]) with pytest.raises(TypeError): ObjectSchemas( native=dict, common=dict, to_common=passthrough ) # missing from_common - - -# --- Handler and ClientConfig type aliases --- - - -def test_handler_is_callable(): - def identity(data, spec): - return spec - - h: Handler = identity - assert h({}, "val") == "val" - - -def test_client_config_is_dict(): - cfg: ClientConfig = {"api_key": "abc", "timeout": 10} - assert cfg["api_key"] == "abc" From 84576f6da1f7fad0e1d3f5c3a2450329c18b8a52 Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Mon, 11 May 2026 15:46:01 -0400 Subject: [PATCH 07/10] Fixing pr comments. --- .../common_grants_sdk/extensions/README.md | 6 ++--- .../extensions/transforms.py | 25 +++++++++++++++++++ .../common_grants_sdk/extensions/types.py | 6 ++++- .../common_grants_sdk/utils/transformation.py | 14 ++++++++++- lib/python-sdk/examples/README.md | 15 +++++++++++ lib/python-sdk/examples/transforms.py | 12 +++++++-- lib/python-sdk/tests/extensions/test_types.py | 7 ++++-- .../tests/utils/test_transformation.py | 15 +++++++---- 8 files changed, 86 insertions(+), 14 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index df0f0355b..0a3786f8f 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -46,7 +46,7 @@ Here are some key concepts that are used to define custom fields and plugins tha | **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | | **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | | **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | -| **`PluginMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | +| **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | | **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict and returns a `TransformResult`. | | **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | | **`ObjectSchemasInput`**| Bundles a `to_common` and `from_common` callable for a single object type. Passed to `define_plugin()` via the `transform_schemas` parameter. | @@ -453,7 +453,7 @@ Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, fr from common_grants_sdk.extensions import ( CustomFieldSpec, ObjectSchemasInput, - PluginMeta, + PluginExtensionsMeta, build_transforms, define_plugin, ) @@ -502,7 +502,7 @@ plugin = define_plugin( ), } }, - meta=PluginMeta( + meta=PluginExtensionsMeta( name="my-system", version="0.1.0", source_system="my-system.example.gov", diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index 6917338ad..ea2d3ba7b 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -16,6 +16,7 @@ from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, + HandlerError, transform_from_mapping, ) @@ -81,6 +82,12 @@ def build_transforms( called on the transform result and any ValidationErrors are appended to TransformResult.errors rather than raised. + Note on result shape: when common_model is set, TransformResult.result + holds the validated Pydantic instance on success, or the raw transformed + dict on ValidationError (so callers can inspect the malformed data + alongside the errors). This is intentional — check TransformResult.errors + before consuming TransformResult.result. + Returns: A (to_common, from_common) tuple. Each callable accepts a dict and returns TransformResult[Any]. Failures surface as PluginError entries in @@ -112,6 +119,15 @@ def build_transforms( def to_common(native: Any) -> TransformResult[Any]: try: result = transform_from_mapping(native, to_common_mapping, handlers=merged) + except HandlerError as exc: + error = PluginError( + str(exc.cause), + path=None, + handler=exc.handler, + source_value=native, + cause=exc.cause, + ) + return TransformResult(result={}, errors=[error]) except Exception as exc: error = PluginError(str(exc), path=None, source_value=native, cause=exc) return TransformResult(result={}, errors=[error]) @@ -138,6 +154,15 @@ def from_common(common: Any) -> TransformResult[Any]: common, from_common_mapping, handlers=merged ) return TransformResult(result=result, errors=[]) + except HandlerError as exc: + error = PluginError( + str(exc.cause), + path=None, + handler=exc.handler, + source_value=common, + cause=exc.cause, + ) + return TransformResult(result={}, errors=[error]) except Exception as exc: error = PluginError(str(exc), path=None, source_value=common, cause=exc) return TransformResult(result={}, errors=[error]) diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 10d17494e..8be07358a 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -7,6 +7,8 @@ from pydantic import BaseModel, ConfigDict, Field +from .specs import CustomFieldSpec + TNative = TypeVar("TNative") TCommon = TypeVar("TCommon") T = TypeVar("T") @@ -97,7 +99,9 @@ class PluginExtensionsSchema(BaseModel): model_config = ConfigDict(populate_by_name=True) - custom_fields: dict[str, Any] | None = Field(default=None, alias="customFields") + custom_fields: dict[str, CustomFieldSpec] | None = Field( + default=None, alias="customFields" + ) mappings: ObjectMappings | None = None diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index 8da2bbe54..3c74c42cf 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -119,6 +119,15 @@ def string_to_number(data: dict, field_path: str) -> int | float | None: return float(s) +class HandlerError(Exception): + """Raised when a handler function raises, carrying the handler name for attribution.""" + + def __init__(self, handler: str, cause: Exception) -> None: + super().__init__(str(cause)) + self.handler = handler + self.cause = cause + + # Registry for handlers DEFAULT_HANDLERS: dict[str, handle_func] = { "const": const_value, @@ -220,7 +229,10 @@ def transform_node(node: Any, depth: int) -> Any: # Returns: `extract_field_value(data, "opportunity_status")` if k in handlers: handler_func = handlers[k] - return handler_func(data, v) + try: + return handler_func(data, v) + except Exception as exc: + raise HandlerError(k, exc) from exc # Otherwise, preserve the dictionary structure and # recursively apply the transformation to each value. diff --git a/lib/python-sdk/examples/README.md b/lib/python-sdk/examples/README.md index 4df942a2e..96b7bbc67 100644 --- a/lib/python-sdk/examples/README.md +++ b/lib/python-sdk/examples/README.md @@ -123,6 +123,21 @@ None This example demonstrates the plugin transform framework: mapping source system data (grants.gov format) to the CommonGrants format and back again, with a roundtrip consistency check. No API server is required — the script runs entirely offline using sample data defined in the file itself. +**Step 1:** Generate the typed models for the grants.gov plugin (only needed once, or after changing `cg_config.py`): + +```bash +cd lib/python-sdk +poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov +``` + +Or generate all example plugins at once with: + +```bash +make plugins +``` + +**Step 2:** Run the example: + ```bash poetry run python examples/transforms.py ``` diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index 9ee443648..030521058 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -4,7 +4,13 @@ Demonstrates source (grants.gov) → CommonGrants and CommonGrants → source bidirectional transformations using the grants.gov sample plugin. -Run with (from lib/python-sdk/): +Requires generated schemas (examples/plugins/grants_gov/generated/). +Generate them first (from lib/python-sdk/): + poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov +Or run all plugins at once: + make plugins + +Then run (from lib/python-sdk/): poetry run python examples/transforms.py """ @@ -214,7 +220,9 @@ def main() -> None: status = "PASS" if ok else "FAIL" print(f" [{status}] {field}: {original!r} -> {roundtripped!r}") - print(f"\nRoundtrip result: {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}") + print( + f"\nRoundtrip result ({len(checks)} mapped fields checked; unmapped fields dropped by design): {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}" + ) # --- Custom handler + model_validate demo --- _section( diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index 044606ea2..f44a3a86b 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -1,6 +1,7 @@ """Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" import pytest +from common_grants_sdk.extensions.specs import CustomFieldSpec from common_grants_sdk.extensions.types import ( ObjectMappings, ObjectSchemas, @@ -11,6 +12,7 @@ PluginExtensionsSchema, TransformResult, ) +from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType # --- PluginError --- @@ -91,9 +93,10 @@ def test_plugin_extensions_schema(): def test_plugin_extensions(): assert PluginExtensions().meta is None assert PluginExtensions().schemas is None - schema = PluginExtensionsSchema(customFields={"legacyId": {}}) + spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) + schema = PluginExtensionsSchema(customFields={"legacyId": spec}) ext = PluginExtensions(schemas={"Opportunity": schema}) - assert ext.schemas["Opportunity"].custom_fields == {"legacyId": {}} + assert ext.schemas["Opportunity"].custom_fields == {"legacyId": spec} # --- ObjectSchemasInput --- diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index cbecdb52c..57cad3070 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -3,6 +3,7 @@ from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, + HandlerError, transform_from_mapping, ) @@ -180,13 +181,15 @@ def test_extend_with_concat(input_data): - The handler works with both field values and constants """ - # Patch in a concat handler for this test def handle_concat(data, concat_spec): return "".join( str(transform_from_mapping(data, part)) for part in concat_spec["parts"] ) - DEFAULT_HANDLERS["concat"] = handle_concat + handlers = { + **DEFAULT_HANDLERS, + "concat": handle_concat, + } mapping = { "opportunity_code": { @@ -199,7 +202,7 @@ def handle_concat(data, concat_spec): } } } - result = transform_from_mapping(input_data, mapping) + result = transform_from_mapping(input_data, mapping, handlers=handlers) assert result == {"opportunity_code": "ABC-123-XYZ-001-12345"} @@ -308,11 +311,13 @@ def test_string_to_number_missing_field(input_data): def test_string_to_number_invalid_raises(input_data): - """Test stringToNumber raises ValueError for non-numeric strings.""" + """Test stringToNumber raises HandlerError (wrapping ValueError) for non-numeric strings.""" data = {**input_data, "bad": "not-a-number"} - with pytest.raises(ValueError): + with pytest.raises(HandlerError) as exc_info: transform_from_mapping(data, {"x": {"stringToNumber": "bad"}}) + assert exc_info.value.handler == "stringToNumber" + assert isinstance(exc_info.value.cause, ValueError) def test_pydantic_model_instance_is_normalized(): From ffd40518d85634f7e6da499132b3ee43f84920db Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Mon, 11 May 2026 16:15:09 -0400 Subject: [PATCH 08/10] Adding ADR fixes --- .../src/content/docs/governance/adr/0022-plugin-framework.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 414cc8509..607396b38 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -42,7 +42,7 @@ We decided to: 6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.`, `definePlugin()` invokes `buildTransforms()` automatically. Mappings for each direction are author-provided — `buildTransforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. -7. **`toCommon` / `fromCommon` return a `TransformResult` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. `definePlugin()` wraps the underlying transform output with runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`); validation failures surface as entries in `errors` rather than thrown exceptions. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. +7. **`toCommon` / `fromCommon` return a `TransformResult` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. In the current PoC, this validation is opt-in at the `buildTransforms()` call site via the `commonModel` / `common_model` parameter — when supplied, validation runs inside `toCommon` against the fully extended generated schema. In the full SDK, `definePlugin()` will additionally inject validation when auto-generating transforms from `extensions.schemas..mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. 8. **Custom handlers are registered per utility call, not globally.** `buildTransforms()` accepts an optional `handlers` argument for registering additional handler names. Per-call scoping keeps behavior explicit and testable; name collisions with the default set raise at `buildTransforms()` call time rather than silently shadowing them. Handler-name lookup must not resolve inherited attributes, because mapping JSON can be reconstituted from untrusted sources via `mergeExtensions()`. From 915c6c57443c05b1d90a65de49c0d31ffb0152c9 Mon Sep 17 00:00:00 2001 From: jcrichlake Date: Tue, 12 May 2026 10:33:03 -0400 Subject: [PATCH 09/10] PR comments rd 2 --- lib/python-sdk/.coverage | Bin 0 -> 53248 bytes .../extensions/transforms.py | 18 +++++++++++++++--- .../common_grants_sdk/utils/transformation.py | 10 ++++++++-- .../tests/extensions/test_transforms.py | 17 +++++++++++++++-- .../tests/utils/test_transformation.py | 11 +++++++++++ 5 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 lib/python-sdk/.coverage diff --git a/lib/python-sdk/.coverage b/lib/python-sdk/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..9ac35a7ec410f45428a8dc61ec83f94552ae0181 GIT binary patch literal 53248 zcmeI)%WoS+90%}SJGE=a)&^CT6@`+yKpHiUYgOWTfB-pER4Ro+4@kJ<^*G*kAMWlt zO%4d$B2_}_v7Au<9}bn^!Vz()=!qK=QhVU{+u8M_jjLYbsPJ9c&U$yApLzL}{ru7e zCsv{xgr1DWIb+E%P2&S048thVy-N3NE7HzF_JsbL6Z^AvOUCBif2!7(Mx}7qu)eNd zwCa_gs&6d*TJcIhEZ#55l1(SDK>z{}fWZG-V0fugtSv8_x4wyG+f{KWZ58Ur>PMe# zZ(ZINm$yE=ur2g4ak?yMtJlSr2!orVuR_svTqPX8MeTvz)WNq{uBSnb97041WF9iKyp(`oc|5vr~Vm2ay^ zm*TY3Sud~O{)MWnZ{$4rt49eW#i; zBj*;?u?D_#W1vQjzahpkHUP!D*0u~2*GglSN(RD014mtJqRiO?ba+PB;q$Zm4UdxPPQs(ebw$ukh789R zx^bvCPmY;Jh9N^9Xd*}hQb~`rw{c;q45IA@?diUj^q5G|0HL`c3)UU@HV>7E>tF}F zDwI2FGd-)Pg`@6B)ODJrLT!6>R@WJIfmU<0TOKYKinWy$^HwS8MrmuMe~MFGC~L_B zrqS$_r(`q;vx($+lVlcSJWFO{Bt1jbsTT{it(95TNyD|8>CW+CaiLf{dD0x}5yA5k z-;YgIf{%=cz*7ssix_`eM+rc1C`Y%6Cx+kwayOxwGTU>aWzf6NJ8(*~JdD8Es>lH7_CVvPo6L__qcG_yq0 zvpi#E>2LPQgj4loM-O*yl!+8?_S7@U>DB0C8LQawl;9vqVUmt@Q%1rL6`iEDbbT%V zyl%j))oSu1ze_&OFXe`-X0dkWj5!>ngHg{jt%w$NPiDnVpXR1I$x|oC{pBJ}xb$jg zvY)Ws1P4FWIq>ABTd=JavQX)3o$@-r!8fhP2Hmhh00Izz00bZa0SG_<0uX=z1R(I= z6Udu6v&irNbJib*^%p&0g8&2|009U<00Izz00bZa0SG|gr4*>-@~5oq7anJq%>2r7 z`Wt|^Hk;=*&z7lHIqQ*OJ+l6ODJzIVLjVF0fB*y_009U<00Izz00bZ~5~$=)nc2qx zxpID`lzs}JfB%2au#EUFyN$^4`mPG=J9KrRII8! zEDW!d=JNg@*Z-Sq!y+>XKmY;|fB*y_009U<00Izzz`OkNhA2 z0SG_<0uX=z1Rwwb2tWV=5SWKRK3^^A@Bbef*28(^h&&(w0SG_<0uX=z1Rwwb2tWV= z5I8h}Wpkladh^zAKUc)>e;Qq{@Zh`q=7V2eeY^OY{{H{5VLd*y+>tN@AOHafKmY;| zfB*y_009U<00KuRus|OQB;WrVOGj4-jNtEr7fs009U<00Izz00bZa0SG_< I0*5c~FLLaI$^ZZW literal 0 HcmV?d00001 diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index ea2d3ba7b..c2aaacdd4 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -27,13 +27,16 @@ def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> """Walk the mapping tree and raise ValueError on structural malformation. For each dict node: - - If a key is a known handler, the corresponding value is a runtime-only - handler argument and is NOT recursed into. + - If a key is a known handler, the node must contain ONLY that handler key. + The corresponding value is a runtime-only handler argument and is NOT + recursed into. - All other keys are output field names (always valid); their values are recursed into. Raises ValueError if any node is not a dict, string, number, boolean, or None - (e.g. a list where a scalar or dict is expected). + (e.g. a list where a scalar or dict is expected), or if a handler key appears + alongside sibling keys in the same dict (ambiguous — handler invocations must + be the sole key in their dict). Note: this function cannot detect intended-but-unknown handler invocations because unknown keys are indistinguishable from output field names at static @@ -48,6 +51,15 @@ def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> f"got {type(mapping).__name__}" ) + handler_keys = [k for k in mapping if k in known_handlers] + if handler_keys and len(mapping) > 1: + label = f" at '{path}'" if path else "" + raise ValueError( + f"Invalid mapping node{label}: handler key {handler_keys[0]!r} " + f"cannot have sibling keys {sorted(k for k in mapping if k not in known_handlers)!r}. " + f"A handler invocation must be the only key in its dict." + ) + for key, value in mapping.items(): current_path = f"{path}.{key}" if path else key if key in known_handlers: diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index 3c74c42cf..bb0778122 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -119,8 +119,14 @@ def string_to_number(data: dict, field_path: str) -> int | float | None: return float(s) -class HandlerError(Exception): - """Raised when a handler function raises, carrying the handler name for attribution.""" +class HandlerError(ValueError): + """Raised when a handler function raises, carrying the handler name for attribution. + + Extends ValueError so that existing ``except ValueError`` handlers around + ``transform_from_mapping``, ``dump_with_mapping``, and ``validate_with_mapping`` + continue to work after this class was introduced. Callers that want handler-level + attribution can catch ``HandlerError`` specifically (it is more derived). + """ def __init__(self, handler: str, cause: Exception) -> None: super().__init__(str(cause)) diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py index a8213dac4..b0353ada1 100644 --- a/lib/python-sdk/tests/extensions/test_transforms.py +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -88,6 +88,15 @@ def test_structural_error_raises_with_path(): build_transforms({"funding": {"amount": [1, 2]}}, {}) +def test_handler_with_sibling_keys_raises(): + """build_transforms raises when a handler key has siblings in the same dict.""" + with pytest.raises(ValueError, match="sibling keys"): + build_transforms({"title": {"field": "x", "extra": "literal"}}, {}) + # Nested occurrence is also caught, and the path is reported + with pytest.raises(ValueError, match="nested.title"): + build_transforms({"nested": {"title": {"field": "x", "extra": "literal"}}}, {}) + + # --- to_common transform --- @@ -138,8 +147,12 @@ def boom(data, _arg): ) result = to_common(SOURCE_DATA) assert len(result.errors) == 1 - assert isinstance(result.errors[0], PluginError) - assert "handler exploded" in str(result.errors[0]) + err = result.errors[0] + assert isinstance(err, PluginError) + assert "handler exploded" in str(err) + assert err.handler == "boom" + assert isinstance(err.cause, RuntimeError) + assert str(err.cause) == "handler exploded" # --- model_validate via common_model --- diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 57cad3070..8f826c03e 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -320,6 +320,17 @@ def test_string_to_number_invalid_raises(input_data): assert isinstance(exc_info.value.cause, ValueError) +def test_handler_error_is_value_error(): + """HandlerError is a subclass of ValueError for backward compat with existing callers.""" + err = HandlerError("myHandler", ValueError("bad input")) + assert isinstance(err, ValueError) + # Callers catching ValueError continue to work; callers wanting attribution catch HandlerError + with pytest.raises(ValueError): + transform_from_mapping( + {"bad": "not-a-number"}, {"x": {"stringToNumber": "bad"}} + ) + + def test_pydantic_model_instance_is_normalized(): """transform_from_mapping accepts a Pydantic model instance and extracts fields correctly.""" From bea786de187f9a500da07acba0e34e14c6cc9a07 Mon Sep 17 00:00:00 2001 From: jcrichlake <145698165+jcrichlake@users.noreply.github.com> Date: Tue, 12 May 2026 13:14:55 -0400 Subject: [PATCH 10/10] Update website/src/content/docs/governance/adr/0022-plugin-framework.mdx Co-authored-by: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> --- .../src/content/docs/governance/adr/0022-plugin-framework.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 607396b38..7bd9cef37 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -202,7 +202,7 @@ function mergeExtensions( // Handler signature matches ADR-0017 runtime conventions. type Handler = (value: unknown, context: unknown) => unknown; -/// Utility: generates toCommon and fromCommon functions from separate declarative +// Utility: generates toCommon and fromCommon functions from separate declarative // mapping objects (ADR-0017 format). Using this utility is optional — plugin authors // may provide plain hand-written functions instead. Mappings are validated at call // time (see Decision #7); the optional `handlers` argument registers custom handler