diff --git a/lib/python-sdk/.coverage b/lib/python-sdk/.coverage deleted file mode 100644 index 9ac35a7e..00000000 Binary files a/lib/python-sdk/.coverage and /dev/null differ diff --git a/lib/python-sdk/.gitignore b/lib/python-sdk/.gitignore index 4c00284b..a4db3a2c 100644 --- a/lib/python-sdk/.gitignore +++ b/lib/python-sdk/.gitignore @@ -9,6 +9,7 @@ __pycache__/ *.py[cod] *$py.class .pytest_cache/ +.coverage #Generated Schema objects generated/ \ No newline at end of file diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 0a3786f8..33e8ebd2 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -44,12 +44,12 @@ Here are some key concepts that are used to define custom fields and plugins tha | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | | **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | -| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | -| **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | +| **`SchemaExtensions`** | A legacy TypedDict mapping extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec`. Still accepted by `with_custom_fields()`. For plugins, declare custom fields inside `ObjectSchemasInput.custom_fields` instead. | +| **`Plugin`** | A dataclass assembled by the code generator. `.schemas` is a container object where each attribute (e.g. `.schemas.Opportunity`) is an `ObjectSchemas` instance providing the model class (`.common`), transform callables (`.to_common`, `.from_common`), and native type (`.native`). `.extensions` holds the serializable extension declarations. | | **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | -| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict and returns a `TransformResult`. | +| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict **or a Pydantic model instance** and returns a `TransformResult`. | | **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | -| **`ObjectSchemasInput`**| Bundles a `to_common` and `from_common` callable for a single object type. Passed to `define_plugin()` via the `transform_schemas` parameter. | +| **`ObjectSchemasInput`**| Bundles `custom_fields`, `to_common`, and `from_common` for a single object type. Passed to `define_plugin()` via the `schemas` parameter. | @@ -123,40 +123,33 @@ The following is an example `cg_config.py` file, which you pass to the build ste ```python -from common_grants_sdk import define_plugin, merge_extensions -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType -# Extensions that might come from a shared HHS package -hhs_extensions: SchemaExtensions = { - "Opportunity": { - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - }, -} - -# Extensions specific to this project -local_extensions: SchemaExtensions = { - "Opportunity": { - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - }, -} - config = define_plugin( - merge_extensions([hhs_extensions, local_extensions], on_conflict="error"), + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "programArea": CustomFieldSpec( + field_type=CustomFieldType.STRING, + description="HHS program area code (e.g. 'CFDA-93.243')", + ), + "legacyGrantId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Numeric ID from the legacy grants management system", + ), + "eligibilityTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", + ), + "awardCeiling": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + description="Maximum award amount in USD", + ), + } + ) + } ) ``` @@ -215,7 +208,7 @@ api_response = { # Use the model returned via opportunity_extensions # --------------------------------------------------------------------------- -opp = opportunity_extensions.schemas.Opportunity.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) ``` @@ -270,36 +263,26 @@ A plugin is a Python class that contains extension specs and generated schemas ```python -T = TypeVar("T") - -@dataclass(frozen=True) -class Plugin(Generic[T]): - """Runtime plugin container with both extension specs and generated schemas.""" - - extensions: SchemaExtensions - schemas: T -``` - - -```python -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType -# Extensions specific to this project -local_extensions: SchemaExtensions = { - "Opportunity": { - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - }, -} - -config = define_plugin(local_extensions) +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "eligibilityTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", + ), + "awardCeiling": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + description="Maximum award amount in USD", + ), + } + ) + } +) ``` After running the build step the imported extension object will have 2 fields to use. @@ -393,7 +376,7 @@ After installing the plugin (e.g. `poetry add opportunity-extensions`): ```python from opportunity_extensions import opportunity_extensions -opp = opportunity_extensions.schemas.Opportunity.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) print(opp.custom_fields.program_area.value) # typed as str print(opp.custom_fields.legacy_grant_id.value) # typed as int ``` @@ -414,32 +397,34 @@ Before publishing a new version of your plugin: ### Combining Plugins -Use `merge_extensions()` to combine field specs from multiple sources before passing them to `define_plugin()`: +Custom fields from multiple logical sources are combined by declaring them all inside a single `ObjectSchemasInput.custom_fields` dict. Because the dict is plain Python, there is no special merge utility needed — just add the keys side by side: ```python -from common_grants_sdk import define_plugin, merge_extensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput +from common_grants_sdk.schemas.pydantic import CustomFieldType config = define_plugin( - merge_extensions([shared_extensions, local_extensions], on_conflict="error"), + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + # fields from a shared HHS package + "programArea": CustomFieldSpec(field_type=CustomFieldType.STRING), + "legacyGrantId": CustomFieldSpec(field_type=CustomFieldType.INTEGER), + # fields specific to this project + "eligibilityTypes": CustomFieldSpec(field_type=CustomFieldType.ARRAY), + "awardCeiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER), + } + ) + } ) ``` -`on_conflict` controls what happens when two sources declare the same field key on the same model: - -| Strategy | Behaviour | -|---|---| -| `"error"` (default) | Raises `ValueError` — safest, forces explicit resolution | -| `"first_wins"` | Keeps the definition from the first source in the list | -| `"last_wins"` | Overwrites with the definition from the last source | - -Prefer unique, namespaced field names so `"error"` is never triggered. - -> [!Note] -> The `"first_wins"` and `"last_wins"` strategies resolve conflicts at runtime but the merged result loses the specific field types of the overridden definitions. For full static type safety, use the default `"error"` strategy with non-overlapping, namespaced field names. +`merge_extensions()` is still available for merging `PluginExtensions` objects that carry declarative `mappings` (ADR-0017 transform configs). It no longer merges `custom_fields`. #### Verify type inference before publishing -After building your package, import the plugin in a test file and confirm that `.extensions` keys and `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. +After building your package, import the plugin in a test file and confirm that `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. ## Bidirectional Transforms @@ -447,7 +432,7 @@ Plugins can define bidirectional mappings between a source system's native data ### Defining transforms -Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `transform_schemas`: +Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `schemas`: ```python from common_grants_sdk.extensions import ( @@ -494,22 +479,20 @@ to_common, from_common = build_transforms( ) plugin = define_plugin( - extensions={ - "Opportunity": { - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Unique identifier in legacy database", - ), - } - }, meta=PluginExtensionsMeta( name="my-system", version="0.1.0", source_system="my-system.example.gov", capabilities=["customFields", "transforms"], ), - transform_schemas={ + schemas={ "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Unique identifier in legacy database", + ), + }, to_common=to_common, from_common=from_common, ) @@ -519,6 +502,37 @@ plugin = define_plugin( Both directions must be provided explicitly. `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. +#### Hand-written callables + +`build_transforms()` is optional. You can supply any plain Python callable to `ObjectSchemasInput` as long as it matches the expected signature: + +```python +def to_common(native_data: dict) -> TransformResult: + ... + +def from_common(cg_data: dict) -> TransformResult: + ... + +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + to_common=to_common, + from_common=from_common, + ) + }, +) +``` + +The key requirement when porting existing transform code is that **both callables must return `TransformResult`**. The type annotation enforces this, but it is easy to miss when wrapping a function that previously returned a plain dict. Wrap the return value like so: + +```python +from common_grants_sdk.extensions.types import TransformResult + +def to_common(native_data: dict) -> TransformResult: + result = my_existing_transform(native_data) # returns a plain dict + return TransformResult(result=result, errors=[]) +``` + ### Mapping format A mapping dict describes how to build an output object from a source dict. Each leaf node is either a literal value or a single-key dict that invokes a named handler. @@ -552,13 +566,13 @@ Custom handlers are merged with the defaults; they cannot override built-in hand ### Using transforms -The compiled callables are stored on the plugin's `transform_schemas` dict, keyed by object name. Each callable takes a data dict and returns a `TransformResult`: +The compiled callables are stored on the plugin's `schemas` object, accessible by attribute name. Each callable takes a data dict (or a Pydantic model instance) and returns a `TransformResult`: ```python -opp_transforms = plugin.transform_schemas["Opportunity"] +opp_schemas = plugin.schemas.Opportunity # Source system → CommonGrants -result = opp_transforms.to_common(native_data) +result = opp_schemas.to_common(native_data) if result.errors: for err in result.errors: print(f"[{err.path}] {err}") @@ -566,12 +580,15 @@ else: cg_data = result.result # CommonGrants → source system -result = opp_transforms.from_common(cg_data) +result = opp_schemas.from_common(cg_data) native_data = result.result ``` `TransformResult.errors` is always a list (empty on success). A non-empty errors list means the transform encountered a problem but still returned a partial result in `result`. +> [!IMPORTANT] +> When `common_model` is set on `build_transforms()`, `to_common` returns a validated Pydantic model instance in `result.result`. That instance can be passed directly to `from_common`. In that case, field paths in `from_common_mapping` must use the model's **camelCase alias names** (e.g. `"status.value"`, `"funding.minAwardAmount.amount"`), not Python snake_case attribute names. This matches the camelCase convention used throughout CommonGrants field paths. + See `examples/transforms.py` for a complete working example with roundtrip verification. @@ -586,11 +603,11 @@ from plugins.opportunity_extensions import opportunity_extensions client = Client(base_url="https://api.example.gov") # Get a single opportunity with typed custom fields -opp = client.opportunities.get(opp_id, schema=opportunity_extensions.schemas.Opportunity) +opp = client.opportunities.get(opp_id, schema=opportunity_extensions.schemas.Opportunity.common) print(opp.custom_fields.program_area.value) # typed as str # List with the same schema -response = client.opportunities.list(schema=opportunity_extensions.schemas.Opportunity) +response = client.opportunities.list(schema=opportunity_extensions.schemas.Opportunity.common) for opp in response.items: print(opp.custom_fields.legacy_grant_id.value) # typed as int @@ -598,7 +615,7 @@ for opp in response.items: results = client.opportunities.search( search="health", status=["open"], - schema=opportunity_extensions.schemas.Opportunity, + schema=opportunity_extensions.schemas.Opportunity.common, ) ``` @@ -614,17 +631,22 @@ The generator converts `camelCase` keys to `snake_case` Python attribute names a ```python # cg_config.py -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType -extensions: SchemaExtensions = { - "Opportunity": { - "legacyGrantId": CustomFieldSpec( # camelCase key — matches the JSON - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - }, -} +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyGrantId": CustomFieldSpec( # camelCase key — matches the JSON + field_type=CustomFieldType.INTEGER, + description="Numeric ID from the legacy grants management system", + ), + } + ) + } +) ``` ```python @@ -635,7 +657,7 @@ api_response = { }, } -opp = my_plugin.schemas.Opportunity.model_validate(api_response) +opp = my_plugin.schemas.Opportunity.common.model_validate(api_response) opp.custom_fields.legacy_grant_id.value # 98765, typed as int ``` @@ -646,7 +668,7 @@ Additional field naming guidelines: ### Keep plugins focused -A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, use `merge_extensions()` to compose separate plugins rather than bundling everything into one. +A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, declare them all in one `ObjectSchemasInput.custom_fields` dict — Python dict literals compose cleanly without a special merge utility. ### Type safety @@ -655,22 +677,27 @@ A plugin should represent a single logical concern (one agency's fields, one int ```python from pydantic import BaseModel - from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions + from common_grants_sdk import define_plugin + from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType class LegacyRef(BaseModel): system: str id: int - extensions: SchemaExtensions = { - "Opportunity": { - "legacyRef": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=LegacyRef, - description="Reference to the opportunity in the legacy system", - ), - }, - } + config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyRef": CustomFieldSpec( + field_type=CustomFieldType.OBJECT, + value=LegacyRef, + description="Reference to the opportunity in the legacy system", + ), + } + ) + } + ) ``` > [!NOTE] @@ -681,9 +708,10 @@ A plugin should represent a single logical concern (one agency's fields, one int When you define Pydantic models for complex `value` fields, export them as named exports from your package. Downstream consumers may need these types for use with `get_custom_field_value()`: ```python -# __init__.py of a plugin package +# cg_config.py of a plugin package from pydantic import BaseModel -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType # Export value types so consumers can reference them directly @@ -691,15 +719,19 @@ class ProgramAreaValue(BaseModel): code: str name: str -extensions: SchemaExtensions = { - "Opportunity": { - "programArea": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=ProgramAreaValue, - description="The HHS program area for this opportunity", - ), - }, -} +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "programArea": CustomFieldSpec( + field_type=CustomFieldType.OBJECT, + value=ProgramAreaValue, + description="The HHS program area for this opportunity", + ), + } + ) + } +) ``` This allows consumers to use `get_custom_field_value()` with the same type the plugin uses for validation: @@ -707,7 +739,7 @@ This allows consumers to use `get_custom_field_value()` with the same type the p ```python from commongrants_hhs_plugin import hhs_plugin, ProgramAreaValue -opp = hhs_plugin.schemas.Opportunity.model_validate(api_response) +opp = hhs_plugin.schemas.Opportunity.common.model_validate(api_response) # Extract the value with full type safety using the exported type area = opp.get_custom_field_value("programArea", ProgramAreaValue) diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 7a9ef118..8589e1fa 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,10 +1,9 @@ """Public extension APIs for the CommonGrants Python SDK.""" -from .plugin import Plugin, PluginConfig, define_plugin +from .plugin import Plugin, PluginConfig, define_plugin, inject_transforms from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions, merge_extensions from .transforms import build_transforms from .types import ( - ClientConfig, Handler, ObjectMappings, ObjectSchemas, @@ -25,11 +24,11 @@ "PluginConfig", "SchemaExtensions", "define_plugin", + "inject_transforms", "merge_extensions", # New: build_transforms "build_transforms", # New: ADR-0022 types - "ClientConfig", "Handler", "ObjectMappings", "ObjectSchemas", diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py index 9657a84a..a259f678 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ b/lib/python-sdk/common_grants_sdk/extensions/generate.py @@ -7,15 +7,15 @@ import keyword import re from pathlib import Path -from typing import Iterable, cast +from typing import Any, Iterable from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.utils.json import snake from .plugin import PluginConfig -from .specs import CustomFieldSpec, SchemaExtensions +from .specs import CustomFieldSpec # Maps extensible model names to the SDK base class they extend in generated code. -# Add an entry here (and to SchemaExtensions) when a new model gains customFields support. +# Add an entry here when a new model gains customFields support. MODEL_BASE_CLASS: dict[str, str] = { "Opportunity": "OpportunityBase", } @@ -33,7 +33,7 @@ } -def _load_config(config_path: Path) -> PluginConfig: +def _load_config(config_path: Path) -> PluginConfig[Any]: """Load and validate a plugin config file, returning the PluginConfig object. Uses importlib to load cg_config.py as an isolated module so it doesn't @@ -59,13 +59,29 @@ def _load_config(config_path: Path) -> PluginConfig: spec.loader.exec_module(module) config = getattr(module, "config", None) - if config is None or not hasattr(config, "extensions"): + if not isinstance(config, PluginConfig): raise RuntimeError( 'Plugin config must expose a "config" variable created by define_plugin()' ) return config +def _extract_custom_fields( + config: PluginConfig[Any], +) -> dict[str, dict[str, CustomFieldSpec]]: + """Extract custom field specs from config.schemas into the flat shape used by generators. + + Returns an empty dict if config.schemas is None or has no schemas with custom_fields. + """ + if config.schemas is None: + return {} + return { + obj: schema.custom_fields + for obj, schema in config.schemas.items() + if schema.custom_fields is not None + } + + def _normalize_identifier(name: str) -> str: """Convert an arbitrary string into a valid Python identifier. @@ -173,7 +189,9 @@ def _annotation_for_spec(spec: CustomFieldSpec, resolved_type: CustomFieldType) return rendered or "Any" -def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: +def _collect_extra_imports( + custom_fields: dict[str, dict[str, CustomFieldSpec]], +) -> list[str]: """Collect import lines needed for external types used as ``spec.value``. Walks all specs and returns one ``import`` line per distinct external type. @@ -186,7 +204,7 @@ def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: types are imported using their ``__module__`` path directly. Args: - extensions: The merged ``SchemaExtensions`` from the plugin config. + custom_fields: The flat custom fields mapping extracted from the plugin config. Returns: A deduplicated list of import-statement strings in the order they were @@ -195,7 +213,7 @@ def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: seen: set[tuple[str, str]] = set() imports: list[str] = [] - for fields in cast(dict[str, dict[str, CustomFieldSpec]], extensions).values(): + for fields in custom_fields.values(): for spec in fields.values(): if not isinstance(spec.value, type): continue @@ -217,7 +235,9 @@ def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: return imports -def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: +def _model_blocks( + custom_fields: dict[str, dict[str, CustomFieldSpec]], +) -> Iterable[str]: """Yield source-code blocks for every model defined in the extensions mapping. For each model, yields three blocks in dependency order: @@ -228,7 +248,7 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: the container. Args: - extensions: The merged ``SchemaExtensions`` from the plugin config. + custom_fields: The flat custom fields mapping extracted from the plugin config. Yields: Source-code strings to be joined and written into ``schemas.py``. @@ -236,9 +256,7 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: Raises: ValueError: If a model name is not present in ``MODEL_BASE_CLASS``. """ - for model_name, fields in cast( - dict[str, dict[str, CustomFieldSpec]], extensions - ).items(): + for model_name, fields in custom_fields.items(): if model_name not in MODEL_BASE_CLASS: raise ValueError( f'Generator does not support model "{model_name}". ' @@ -257,7 +275,7 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: spec=spec, resolved_type=resolved_type ) # Use spec.name as the runtime display name if provided, otherwise fall back - # to the field key (the dict key in SchemaExtensions). + # to the field key (the dict key in ObjectSchemasInput.custom_fields). field_name_default = spec.name or field_key # repr() produces a quoted string literal safe to embed directly in source code. description_default = repr(spec.description) if spec.description else "None" @@ -316,16 +334,25 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: ) -def _render_schemas_py(extensions: SchemaExtensions) -> str: +def _render_schemas_py( + custom_fields: dict[str, dict[str, CustomFieldSpec]], + mappings_only_objs: set[str] | None = None, +) -> str: """Render the full source of the generated ``schemas.py`` file. Produces a self-contained module containing typed ``CustomField`` subclasses, a ``CustomFields`` container, and an extended model class for each entry in - ``extensions``. Also emits a ``_Schemas`` container object (attribute access + ``custom_fields``. Also emits a ``_Schemas`` container object (attribute access rather than dict lookup) and a module-level ``schemas`` instance. + For objects that only have ``mappings`` (no ``custom_fields``), the ``_Schemas`` + object will expose the base SDK model class directly (e.g. ``schemas.Opportunity`` + will be ``OpportunityBase``). + Args: - extensions: The merged ``SchemaExtensions`` from the plugin config. + custom_fields: The flat custom fields mapping extracted from the plugin config. + mappings_only_objs: Set of object names that have mappings but no custom_fields. + These will be exposed on ``_Schemas`` as their base SDK class. Returns: A string of valid Python source code ready to be written to disk. @@ -334,13 +361,34 @@ def _render_schemas_py(extensions: SchemaExtensions) -> str: # plugin.schemas.Opportunity rather than plugin.schemas["Opportunity"]. # The dynamic __init__ assignment is necessary because model names aren't # known until generation time, so a static class body can't be used. - model_names = list(extensions.keys()) - blocks = "\n\n\n".join(_model_blocks(extensions)) - schema_assignments = "\n".join( - [f" self.{name} = {name}" for name in model_names] or [" pass"] - ) - all_exports = ", ".join([f'"{name}"' for name in model_names] + ['"schemas"']) - extra_imports = _collect_extra_imports(extensions) + model_names = list(custom_fields.keys()) + blocks = "\n\n\n".join(_model_blocks(custom_fields)) + mappings_only: set[str] = mappings_only_objs or set() + + # Build schema assignments: each attribute is an ObjectSchemas instance so + # callers get a unified interface (plugin.schemas.Opportunity.common for the + # model class, .to_common/.from_common for transforms). + # to_common/from_common default to None here; root __init__.py injects the + # real callables for any object that has transforms configured. + assignments: list[str] = [ + f" self.{name} = ObjectSchemas(native=dict, common={name}, to_common=None, from_common=None)" + for name in model_names + ] + for obj in sorted(mappings_only): + if obj not in MODEL_BASE_CLASS: + raise ValueError( + f'Generator does not support model "{obj}". ' + f"Supported models: {sorted(MODEL_BASE_CLASS)}" + ) + base_class = MODEL_BASE_CLASS[obj] + assignments.append( + f" self.{obj} = ObjectSchemas(native=dict, common={base_class}, to_common=None, from_common=None)" + ) + + schema_assignments = "\n".join(assignments or [" pass"]) + all_names = model_names + sorted(mappings_only) + all_exports = ", ".join([f'"{name}"' for name in all_names] + ['"schemas"']) + extra_imports = _collect_extra_imports(custom_fields) extra_import_lines = ["", *extra_imports] if extra_imports else [] return "\n".join( @@ -353,13 +401,12 @@ def _render_schemas_py(extensions: SchemaExtensions) -> str: "", "from pydantic import ConfigDict, Field", "", + "from common_grants_sdk.extensions.types import ObjectSchemas", "from common_grants_sdk.schemas.pydantic.base import CommonGrantsBaseModel", "from common_grants_sdk.schemas.pydantic.fields import CustomField, CustomFieldType", "from common_grants_sdk.schemas.pydantic.models import OpportunityBase", *extra_import_lines, - "", - blocks, - "", + *([blocks, ""] if blocks else []), "class _Schemas:", " def __init__(self) -> None:", schema_assignments, @@ -393,34 +440,92 @@ def _render_generated_init_py() -> str: ) -def _render_plugin_init_py(plugin_variable_name: str) -> str: - """Render the source of the plugin directory's root ``__init__.py`` file. +def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) -> str: + """Render the source of the plugin directory's root __init__.py file. - The generated file imports ``config`` directly from ``cg_config.py`` and - exports a ``Plugin`` instance named after the plugin directory alongside the - ``schemas`` object. + Emits a fully compiled Plugin instance. Transform callables are injected + into the _Schemas object (from generated/schemas.py) before Plugin is + constructed, so plugin.schemas.Opportunity.to_common etc. are populated. + """ + # Collect the sets of objects needing transform injection. + # Only count schemas that have explicit callable transforms, not those with custom_fields only. + explicit_objs: set[str] = ( + { + obj + for obj, s in config.schemas.items() + if s.to_common is not None or s.from_common is not None + } + if config.schemas + else set() + ) + mappings_objs: set[str] = ( + {obj for obj, s in config.extensions.schemas.items() if s.mappings is not None} + if config.extensions and config.extensions.schemas + else set() + ) - Args: - plugin_variable_name: A valid Python identifier derived from the plugin - directory name (e.g. ``"opportunity_extensions"``). + needs_build_transforms = bool(mappings_objs - explicit_objs) - Returns: - A string of valid Python source code ready to be written to disk. - """ - return "\n".join( - [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from __future__ import annotations", - "", - "from common_grants_sdk.extensions import Plugin", - "from .cg_config import config", - "from .generated import schemas", - "", - f"{plugin_variable_name} = Plugin(", - " extensions=config.extensions,", - " schemas=schemas,", + # Build pre-plugin lines: inject transforms into the _Schemas object before + # constructing Plugin. schemas.py initialises each ObjectSchemas with + # to_common=None/from_common=None; we mutate those attrs here. + inject_lines: list[str] = [] + + # Mappings-only objects: call build_transforms() then inject results. + for obj in sorted(mappings_objs - explicit_objs): + inject_lines += [ + f"_{obj}_to_common, _{obj}_from_common = build_transforms(", + f' to_common_mapping=config.extensions.schemas["{obj}"].mappings.to_common,', + f' from_common_mapping=config.extensions.schemas["{obj}"].mappings.from_common,', + f" common_model=schemas.{obj}.common,", ")", + f"schemas.{obj}.to_common = _{obj}_to_common", + f"schemas.{obj}.from_common = _{obj}_from_common", + "", + ] + + # Explicit schemas: single inject_transforms() call handles all objects. + # Reassigning the return value preserves the concrete _Schemas type for mypy. + if explicit_objs: + inject_lines.append("schemas = inject_transforms(config, schemas)") + inject_lines.append("") + + imports = [ + "# This file is auto-generated. Do not edit it manually — it will be overwritten", + "# the next time `python -m common_grants_sdk.extensions.generate` is run.", + "from __future__ import annotations", + "", + ] + + sdk_imports = {"Plugin"} + if explicit_objs: + sdk_imports.add("inject_transforms") + if needs_build_transforms: + sdk_imports.add("build_transforms") + imports.append( + f"from common_grants_sdk.extensions import {', '.join(sorted(sdk_imports))}" + ) + imports += [ + "from .cg_config import config", + "from .generated import schemas", + "", + ] + + plugin_lines = [ + f"{plugin_variable_name} = Plugin(", + " schemas=schemas,", + " extensions=config.extensions,", + " meta=config.meta,", + ")", + ] + + pre_plugin = inject_lines # may be empty + + return "\n".join( + imports + + pre_plugin + + plugin_lines + + [ "", f'__all__ = ["{plugin_variable_name}", "schemas"]', "", @@ -452,6 +557,59 @@ def generate_plugin(plugin_dir: Path) -> Path: raise FileNotFoundError(f"Could not find config file: {config_path}") config = _load_config(config_path) + custom_fields = _extract_custom_fields(config) + + # Determine objects that have mappings but no custom_fields — these need a + # pass-through entry in _Schemas pointing at the base SDK model class. + explicit_cf_objs: set[str] = set(custom_fields.keys()) + mappings_only_objs: set[str] = ( + { + obj + for obj, s in config.extensions.schemas.items() + if s.mappings is not None and obj not in explicit_cf_objs + } + if config.extensions and config.extensions.schemas + else set() + ) + # Third bucket: objects in config.schemas with explicit transforms but no + # custom_fields and no extensions.schemas entry. They also need a pass-through + # entry in _Schemas or schemas. won't exist at import time. + transforms_only_objs: set[str] = ( + set(config.schemas.keys()) - explicit_cf_objs - mappings_only_objs + if config.schemas + else set() + ) + + # Validate that auto-generated transform objects have both mapping directions. + # Auto-generated objects: have mappings in extensions.schemas but no explicit + # to_common/from_common in config.schemas. + if config.extensions and config.extensions.schemas: + explicit_schema_objs: set[str] = ( + { + obj + for obj, s in config.schemas.items() + if s.to_common is not None or s.from_common is not None + } + if config.schemas + else set() + ) + ext_schemas = config.extensions.schemas + for obj, schema in ext_schemas.items(): + if schema.mappings is None or obj in explicit_schema_objs: + continue + if schema.mappings.to_common is None: + raise ValueError( + f'Plugin object "{obj}": mappings.to_common is required when ' + f"auto-generating transforms. Either provide a to_common mapping " + f"or pass an explicit to_common callable via schemas['{obj}']." + ) + if schema.mappings.from_common is None: + raise ValueError( + f'Plugin object "{obj}": mappings.from_common is required when ' + f"auto-generating transforms. Either provide a from_common mapping " + f"or pass an explicit from_common callable via schemas['{obj}']." + ) + generated_dir = plugin_dir / "generated" generated_dir.mkdir(parents=True, exist_ok=True) @@ -459,12 +617,17 @@ def generate_plugin(plugin_dir: Path) -> Path: init_generated_py = generated_dir / "__init__.py" root_init_py = plugin_dir / "__init__.py" - schemas_py.write_text(_render_schemas_py(config.extensions), encoding="utf-8") + schemas_py.write_text( + _render_schemas_py( + custom_fields, mappings_only_objs=mappings_only_objs | transforms_only_objs + ), + encoding="utf-8", + ) init_generated_py.write_text(_render_generated_init_py(), encoding="utf-8") plugin_variable_name = _normalize_identifier(plugin_dir.name) root_init_py.write_text( - _render_plugin_init_py(plugin_variable_name), encoding="utf-8" + _render_plugin_init_py(plugin_variable_name, config), encoding="utf-8" ) return generated_dir diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index eab8cb8f..7f8c037f 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -3,79 +3,146 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Callable, Generic, TypeVar +from typing import Any, Generic, TypeVar, overload -from .specs import SchemaExtensions -from .types import ClientConfig, ObjectSchemas, ObjectSchemasInput, PluginExtensionsMeta +from .types import ( + PluginExtensions, + PluginExtensionsMeta, +) T = TypeVar("T") +TSchemas = TypeVar("TSchemas") +_TSchemasContainer = TypeVar("_TSchemasContainer") @dataclass(frozen=True) -class PluginConfig: - """Build-time plugin config discoverable by the code generator. +class PluginConfig(Generic[TSchemas]): + """Build-time plugin config produced by define_plugin() and consumed by generate.py. - extensions: custom field declarations (read by generate.py — do not rename). - meta: optional plugin identity and capability declaration. - transform_schemas: optional bidirectional transform callables per object. - Stored as ObjectSchemasInput (not compiled to ObjectSchemas) in the PoC. - Full compilation with model_validate wrapping is a TODO for the real SDK. + Generic on TSchemas so the precise type of the schemas dict is preserved — e.g. + PluginConfig[dict[str, ObjectSchemasInput[MyNative, MyCg]]] — rather than being + widened to ObjectSchemasInput[Any, Any] at the storage boundary. - TODO (full SDK): add get_client, filters. + Stores inputs as-is — no compilation occurs at define_plugin() call time. + generate.py compiles this into a fully resolved Plugin by: + - Injecting the generated Pydantic model class as the common schema for each + ObjectSchemasInput entry (schemas[obj].native + common → ObjectSchemas). + - Auto-generating build_transforms() calls for any object that has + extensions.schemas[obj].mappings but no explicit schemas[obj] entry. + + All fields are optional so adopters can start with only what they need. """ - extensions: SchemaExtensions + extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None - transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None + schemas: TSchemas | None = None @dataclass class Plugin(Generic[T]): - """Runtime plugin container with extension specs and generated schemas. - - extensions: SchemaExtensions used by generate.py (do not rename or reorder — - the generated __init__.py constructs Plugin(extensions=..., schemas=...)). - schemas: generated _Schemas object (typed Pydantic model classes from generate.py). - NOTE: there is a naming collision: ADR-0022 also calls its runtime transform - dict "schemas". These are different concepts sharing the same name — a design - question to resolve in the full SDK (see Design Finding #1 in the spec). - transform_schemas: ADR-0022 runtime transform dict; named distinctly from - `schemas` to avoid collision with the generated schemas field in the PoC. - - TODO (full SDK): memoize get_client. + """Runtime plugin container assembled by generate.py after code generation. + + schemas: the _Schemas object from generated/schemas.py. Each attribute is an + ObjectSchemas instance providing unified access to the model class and + transforms for that object: + plugin.schemas.Opportunity.common → the Pydantic model class (includes + any custom fields declared by the plugin) + plugin.schemas.Opportunity.to_common → transform callable (or None) + plugin.schemas.Opportunity.from_common → transform callable (or None) + plugin.schemas.Opportunity.native → the source system's type (or dict) """ - extensions: SchemaExtensions - schemas: T # generated _Schemas object — keep as positional for generate.py compat + schemas: T + extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None - get_client: Callable[[ClientConfig], Any] | None = None # TODO: memoize - # PoC stores ObjectSchemasInput here (no compilation yet); full SDK will store - # ObjectSchemas after model_validate wrapping. Annotated as the union so both - # the current PoC usage and the future compiled form are type-safe. - transform_schemas: ( - dict[str, ObjectSchemasInput[Any, Any] | ObjectSchemas[Any, Any]] | None - ) = None - filters: dict[str, dict[str, Any]] | None = None + + +@overload +def define_plugin( + meta: PluginExtensionsMeta | None = ..., + extensions: PluginExtensions | None = ..., + schemas: None = ..., +) -> PluginConfig[None]: ... + + +@overload +def define_plugin( + meta: PluginExtensionsMeta | None = ..., + extensions: PluginExtensions | None = ..., + schemas: TSchemas = ..., +) -> PluginConfig[TSchemas]: ... def define_plugin( - extensions: SchemaExtensions, meta: PluginExtensionsMeta | None = None, - transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, - # TODO (full SDK): get_client, filters -) -> PluginConfig: - """Create a PluginConfig object consumed by the code generator. - - Backward-compatible: existing callers passing only `extensions` are unaffected. - New params are stored as-is — no compilation occurs in the PoC. - - TODO (full SDK): - - Auto-generate transforms from extensions.schemas[obj].mappings when no - explicit to_common/from_common is supplied. - - Wrap transform output with model_validate. + extensions: PluginExtensions | None = None, + schemas: Any = None, +) -> PluginConfig[Any]: + """Create a PluginConfig consumed by the code generator. + + No compilation occurs here — inputs are stored as-is. The code generator + (generate.py) compiles ObjectSchemasInput → ObjectSchemas by injecting + the common model from the generated schemas. + + The return type is generic on the schemas argument: passing a typed dict + (e.g. {"Opportunity": ObjectSchemasInput[MyNative, MyCg](...) }) preserves + those per-object generics on the returned PluginConfig rather than widening + them to Any. """ return PluginConfig( extensions=extensions, meta=meta, - transform_schemas=transform_schemas, + schemas=schemas, ) + + +def inject_transforms( + config: PluginConfig[Any], schemas: _TSchemasContainer +) -> _TSchemasContainer: + """Wire transform callables from plugin config into the generated schemas container. + + Called by the generated plugin __init__.py to inject to_common/from_common + callables (and the native type) from cg_config into the ObjectSchemas instances + produced by the code generator. + + Iterates over all entries in config.schemas that have at least one callable, + validates that both directions are present, then sets the attributes on the + matching schemas container attribute (e.g. schemas.Opportunity). + + Returns the same schemas container (mutated in place) so callers can write + ``schemas = inject_transforms(config, schemas)`` and retain the concrete + generated type rather than widening to Any. + + Args: + config: The PluginConfig produced by define_plugin(). + schemas: The generated _Schemas container from generated/schemas.py. + + Returns: + The same schemas container, with transform callables injected. + + Raises: + ValueError: If a schema with any callable is missing its counterpart, + or if the object name is not found in the schemas container. + """ + if not config.schemas: + return schemas + for obj_name, schema_input in config.schemas.items(): + if schema_input.to_common is None and schema_input.from_common is None: + continue + obj_schemas = getattr(schemas, obj_name, None) + if obj_schemas is None: + raise ValueError( + f"Plugin object {obj_name!r}: not found in generated schemas" + ) + if schema_input.to_common is None: + raise ValueError( + f"Plugin object {obj_name!r}: to_common callable is required" + ) + if schema_input.from_common is None: + raise ValueError( + f"Plugin object {obj_name!r}: from_common callable is required" + ) + obj_schemas.native = schema_input.native or dict + obj_schemas.to_common = schema_input.to_common + obj_schemas.from_common = schema_input.from_common + return schemas diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index 6cf9a74b..d6ef9785 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -1,10 +1,18 @@ """Extension types and utilities for SDK schema customization.""" +from __future__ import annotations + from dataclasses import dataclass -from typing import Any, Literal, Optional, TypedDict, cast +from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict from ..schemas.pydantic.fields.custom import CustomFieldType +if TYPE_CHECKING: + from .types import ( + PluginExtensions, + PluginExtensionsMeta, + ) + ConflictStrategy = Literal["error", "first_wins", "last_wins"] @@ -19,75 +27,109 @@ class CustomFieldSpec: class SchemaExtensions(TypedDict, total=False): - """Maps extensible model names to custom field specifications - - This class is the definitive list of base model names that support custom fields. + """Maps extensible model names to custom field specifications. - Add schemas here if they support `custom_fields` extensions - - - Args: - total: Determines if all keys are required or not, defaults to False which means no keys are required. + Retained for callers that still use the flat TypedDict form directly. + merge_extensions now accepts PluginExtensions instead. """ Opportunity: dict[str, CustomFieldSpec] -def _merge_fields( - model_name: str, - model_fields: dict[str, CustomFieldSpec], - source_fields: dict[str, CustomFieldSpec], +def _merge_meta( + current: PluginExtensionsMeta | None, + incoming: PluginExtensionsMeta, on_conflict: ConflictStrategy, -) -> None: - """Merge source_fields into model_fields in place, applying the conflict strategy.""" - for field_name, spec in source_fields.items(): - if field_name in model_fields: +) -> PluginExtensionsMeta: + """Merge incoming meta into current, respecting on_conflict for non-None field collisions.""" + from .types import PluginExtensionsMeta as _PluginExtensionsMeta + + if current is None: + return incoming + + merged: dict[str, Any] = {} + for field_name in ("name", "version", "source_system", "capabilities"): + current_val = getattr(current, field_name) + incoming_val = getattr(incoming, field_name) + + if incoming_val is None: + merged[field_name] = current_val + elif current_val is None: + merged[field_name] = incoming_val + else: + # Both have non-None values — apply conflict strategy if on_conflict == "error": raise ValueError( - f'merge_extensions: duplicate field "{field_name}" on model "{model_name}"' + f'merge_extensions: duplicate meta field "{field_name}" ' + f"(existing: {current_val!r}, incoming: {incoming_val!r})" ) - if on_conflict == "first_wins": - continue - model_fields[field_name] = spec + merged[field_name] = ( + current_val if on_conflict == "first_wins" else incoming_val + ) - -def _merge_source( - result: dict[str, dict[str, CustomFieldSpec]], - source: SchemaExtensions, - on_conflict: ConflictStrategy, -) -> None: - """Merge a single source into result in place.""" - for model_name, source_fields in cast( - dict[str, dict[str, CustomFieldSpec]], source - ).items(): - model_fields = result.setdefault(model_name, {}) - _merge_fields(model_name, model_fields, source_fields, on_conflict) + return _PluginExtensionsMeta( + name=merged["name"], + version=merged["version"], + sourceSystem=merged["source_system"], + capabilities=merged["capabilities"], + ) def merge_extensions( - sources: list[SchemaExtensions], on_conflict: ConflictStrategy = "error" -) -> SchemaExtensions: - """Merge multiple extension sources into one schema extension mapping. + sources: list[PluginExtensions], on_conflict: ConflictStrategy = "error" +) -> PluginExtensions: + """Merge multiple PluginExtensions into one. Args: - sources: Ordered list of extension mappings to merge. - on_conflict: Duplicate field strategy per model. - - ``"error"``: raise on duplicate field name. - - ``"first_wins"``: keep first seen definition. - - ``"last_wins"``: overwrite with latest definition. + sources: Ordered list of PluginExtensions to merge. + on_conflict: Strategy for duplicate field names per object. + - "error": raise on first duplicate (default). + - "first_wins": keep first seen value. + - "last_wins": overwrite with latest value. """ + from .types import ObjectMappings as _ObjectMappings + from .types import PluginExtensions as _PluginExtensions + from .types import PluginExtensionsSchema as _PluginExtensionsSchema + if on_conflict not in {"error", "first_wins", "last_wins"}: raise ValueError( 'merge_extensions: on_conflict must be "error", "first_wins", or "last_wins"' ) if len(sources) == 0: - return {} + return _PluginExtensions() if len(sources) == 1: return sources[0] - result: dict[str, dict[str, CustomFieldSpec]] = {} - for source in sources: - _merge_source(result, source, on_conflict) + # Accumulate into plain dicts; construct PluginExtensions once at the end. + merged_mappings: dict[str, _ObjectMappings] = {} + merged_meta: PluginExtensionsMeta | None = None - return cast(SchemaExtensions, result) + for source in sources: + if source.schemas: + for obj, src_schema in source.schemas.items(): + if src_schema.mappings: + if obj in merged_mappings: + if on_conflict == "error": + raise ValueError( + f'merge_extensions: duplicate mappings for object "{obj}"' + ) + if on_conflict == "first_wins": + continue + merged_mappings[obj] = src_schema.mappings + if source.meta: + merged_meta = _merge_meta(merged_meta, source.meta, on_conflict) + + return _PluginExtensions( + meta=merged_meta, + schemas=( + { + obj: _PluginExtensionsSchema( + mappings=merged_mappings.get(obj), + ) + for obj in merged_mappings + } + if merged_mappings + else None + ), + ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index c2aaacdd..f204e312 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -10,7 +10,7 @@ from __future__ import annotations -from typing import Any, Callable +from typing import Any, Callable, TypeVar, overload from pydantic import BaseModel, ValidationError @@ -22,6 +22,8 @@ from .types import Handler, PluginError, TransformResult +TCommon = TypeVar("TCommon", bound=BaseModel) + def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> None: """Walk the mapping tree and raise ValueError on structural malformation. @@ -68,6 +70,30 @@ def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> _validate_mapping(value, known_handlers, current_path) +@overload +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = ..., + common_model: None = ..., +) -> tuple[ + Callable[[Any], TransformResult[Any]], + Callable[[Any], TransformResult[Any]], +]: ... + + +@overload +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = ..., + common_model: type[TCommon] = ..., +) -> tuple[ + Callable[[Any], TransformResult[TCommon | dict[str, Any]]], + Callable[[Any], TransformResult[dict[str, Any]]], +]: ... + + def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], @@ -159,6 +185,9 @@ def to_common(native: Any) -> TransformResult[Any]: for e in exc.errors() ] return TransformResult(result=result, errors=errors) + except Exception as exc: + error = PluginError(str(exc), path=None, source_value=result, cause=exc) + return TransformResult(result=result, errors=[error]) def from_common(common: Any) -> TransformResult[Any]: try: diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 8be07358..eccacc51 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -18,7 +18,6 @@ # Type aliases Handler = Callable[[Any, Any], Any] -ClientConfig = dict[str, Any] class PluginError(Exception): @@ -89,9 +88,8 @@ class PluginExtensionsMeta(BaseModel): class PluginExtensionsSchema(BaseModel): - """Per-object config inside extensions.schemas. + """Per-object config inside extensions.schemas. Holds declarative mappings only. - custom_fields: custom field declarations (merged by merge_extensions). mappings: optional ADR-0017 declarative mappings. When present and no explicit to_common / from_common is supplied in schemas[obj], define_plugin() will auto-invoke build_transforms() on these (TODO — ADR-0022 Decision #6). @@ -99,9 +97,6 @@ class PluginExtensionsSchema(BaseModel): model_config = ConfigDict(populate_by_name=True) - custom_fields: dict[str, CustomFieldSpec] | None = Field( - default=None, alias="customFields" - ) mappings: ObjectMappings | None = None @@ -125,6 +120,9 @@ class ObjectSchemasInput(Generic[TNative, TCommon]): hand-written or generated via build_transforms(). native defaults to dict[str, Any] if omitted. + custom_fields declares any extra fields this object exposes beyond the base + CommonGrants schema. The code generator reads these and emits typed subclasses. + common is intentionally absent here. It is injected by define_plugin() during compilation from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes produced by the code generator. Plugin authors never set it directly — @@ -132,20 +130,29 @@ class ObjectSchemasInput(Generic[TNative, TCommon]): """ native: type[TNative] | None = None + custom_fields: dict[str, CustomFieldSpec] | None = None to_common: Callable[[TNative], TransformResult[TCommon]] | None = None from_common: Callable[[TCommon], TransformResult[TNative]] | None = None @dataclass class ObjectSchemas(Generic[TNative, TCommon]): - """Runtime compiled type produced by define_plugin() — not provided directly by authors. - - In the PoC, define_plugin() stores ObjectSchemasInput as-is; full compilation - (adding common from the base CG model, wrapping with model_validate) is a TODO - for the real SDK (ADR-0022 Decision #7). + """Runtime compiled schema container for a single object (ADR-0022). + + Bundles the type information and transform callables for one schema object + (e.g. Opportunity). Accessed via attribute lookup on the plugin's schemas + container: plugin.schemas.Opportunity. + + native: The source system's Python type (defaults to dict when not specified). + common: The CommonGrants-format Pydantic model class produced by the generator. + If the plugin declares custom_fields, this is a generated subclass of + the base CG model (e.g. OpportunityBase) with those fields already + baked in as typed attributes. + to_common: Transforms native_data → TransformResult[common] (None if not configured). + from_common: Transforms common_data → TransformResult[native] (None if not configured). """ native: type[TNative] common: type[TCommon] - to_common: Callable[[TNative], TransformResult[TCommon]] - from_common: Callable[[TCommon], TransformResult[TNative]] + to_common: Callable[[TNative], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TNative]] | None = None diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index d876ad40..05e5ba6c 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -1,5 +1,5 @@ from typing import Optional, Any, Type, TypeVar -from pydantic import BaseModel, Field, create_model, ConfigDict +from pydantic import BaseModel, Field, create_model, ConfigDict, model_validator from ..schemas.pydantic.fields import CustomField, CustomFieldType from ..schemas.pydantic.base import CommonGrantsBaseModel from common_grants_sdk.utils.json import snake @@ -37,10 +37,29 @@ def add_custom_fields( # Accumulate all field definitions field_defs: dict[str, Any] = create_custom_field_schema(name=name, fields=fields) - # Unknown keys ignored for now. - # TODO: switch extra="allow" + add validator to parse extras into CustomField class _CustomFieldsBase(CommonGrantsBaseModel): - model_config = ConfigDict(populate_by_name=True, extra="ignore") + model_config = ConfigDict(populate_by_name=True, extra="allow") + + @model_validator(mode="before") + @classmethod + def _parse_extra_as_custom_fields(cls, values: Any) -> Any: + """Wrap unknown dict-valued keys as CustomField instances before validation.""" + if not isinstance(values, dict): + return values + # Build the set of known keys: both Python attr names and their aliases + known: set[str] = set() + if hasattr(cls, "model_fields"): + for attr, field_info in cls.model_fields.items(): + known.add(attr) + alias = getattr(field_info, "alias", None) + if alias: + known.add(alias) + for key, val in list(values.items()): + if key not in known and isinstance(val, dict) and "fieldType" in val: + # Inject the key as the name if not already present + enriched = {"name": key, **val} + values[key] = CustomField.model_validate(enriched) + return values # Create container with ALL accumulated field definitions, # this will be used when we recreate the base pydantic model with the diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index bb077812..c36fed6e 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -7,6 +7,8 @@ from typing import Any, Callable +from pydantic import BaseModel + handle_func = Callable[[dict, Any], Any] @@ -146,7 +148,7 @@ def __init__(self, handler: str, cause: Exception) -> None: def transform_from_mapping( - data: dict, + data: Any, mapping: dict, depth: int = 0, max_depth: int = 500, @@ -204,9 +206,10 @@ def transform_from_mapping( # Normalize Pydantic model instances to plain dicts so that field path # extraction works regardless of whether the caller passes a raw dict or a # validated model (e.g. the output of to_common with common_model set). + # by_alias=True ensures camelCase alias keys are used (matching CG mapping paths). # mode="json" matches the convention used by CommonGrantsBaseModel.dump_with_mapping. - if hasattr(data, "model_dump"): - data = data.model_dump(mode="json") + if isinstance(data, BaseModel): + data = data.model_dump(mode="json", by_alias=True) # Check for maximum depth # This is a sanity check to prevent stack overflow from deeply nested mappings diff --git a/lib/python-sdk/examples/plugin_custom_fields.py b/lib/python-sdk/examples/plugin_custom_fields.py index 41c364c1..89d4a21e 100644 --- a/lib/python-sdk/examples/plugin_custom_fields.py +++ b/lib/python-sdk/examples/plugin_custom_fields.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from plugins.opportunity_extensions import opportunity_extensions # noqa: E402 +from plugins.opportunity_extensions.cg_config import config as opp_config # noqa: E402 # --------------------------------------------------------------------------- # Sample API payload containing our four custom fields @@ -56,7 +57,7 @@ # Use the model returned via opportunity_extensions # --------------------------------------------------------------------------- -opp = opportunity_extensions.schemas.Opportunity.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) assert opp.custom_fields is not None assert opp.custom_fields.program_area is not None @@ -76,6 +77,9 @@ # The plugin also exposes the original extension specs # --------------------------------------------------------------------------- -print("Registered extensions:") -for field_name, spec in opportunity_extensions.extensions["Opportunity"].items(): +print("Registered custom fields:") +assert opp_config.schemas is not None +_opp_custom_fields = opp_config.schemas["Opportunity"].custom_fields +assert _opp_custom_fields is not None +for field_name, spec in _opp_custom_fields.items(): print(f" {field_name}: {spec.field_type} — {spec.description}") diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py index 5ae38fea..ae59f443 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/__init__.py +++ b/lib/python-sdk/examples/plugins/grants_gov/__init__.py @@ -2,13 +2,16 @@ # the next time `python -m common_grants_sdk.extensions.generate` is run. from __future__ import annotations -from common_grants_sdk.extensions import Plugin +from common_grants_sdk.extensions import Plugin, inject_transforms from .cg_config import config from .generated import schemas +schemas = inject_transforms(config, schemas) + grants_gov = Plugin( - extensions=config.extensions, schemas=schemas, + extensions=config.extensions, + meta=config.meta, ) __all__ = ["grants_gov", "schemas"] diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py index b3f28b5e..4000d845 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -75,6 +75,14 @@ }, }, }, + "customFields": { + "legacyIdStr": { + "value": {"numberToString": "data.opportunity_id"}, + }, + "priorityScore": { + "value": {"stringToNumber": "data.priority_score_str"}, + }, + }, }, # from_common: CommonGrants Opportunity → grants.gov native from_common_mapping={ @@ -97,6 +105,9 @@ "forecasted_post_date": {"field": "keyDates.appOpens.date"}, "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, }, + "priority_score_str": { + "numberToString": "customFields.priorityScore.value" + }, } }, ) @@ -105,39 +116,44 @@ # Plugin config # --------------------------------------------------------------------------- -plugin = define_plugin( - # extensions: SchemaExtensions — dict[str, dict[str, CustomFieldSpec]] - extensions={ - "Opportunity": { - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - name="Legacy ID", - description="Unique identifier in legacy database", - ), - "agencyName": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Agency", - description="Agency hosting the opportunity", - ), - "applicantTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - name="Applicant types", - description="Types of applicants eligible to apply", - ), - } - }, +config = define_plugin( meta=PluginExtensionsMeta( name="grants-gov", version="0.1.0", sourceSystem="grants.gov", capabilities=["customFields", "transforms"], ), - transform_schemas={ + schemas={ "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + name="Legacy ID", + description="Unique identifier in legacy database", + ), + "legacyIdStr": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Legacy ID (string)", + description="Legacy ID coerced to a string via numberToString", + ), + "agencyName": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Agency", + description="Agency hosting the opportunity", + ), + "applicantTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + name="Applicant types", + description="Types of applicants eligible to apply", + ), + "priorityScore": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + name="Priority score", + description="Numeric priority score coerced from a string via stringToNumber", + ), + }, to_common=to_common, from_common=from_common, ) }, ) - -config = plugin diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py index 3e1f6a8a..736eb239 100644 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py +++ b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py @@ -7,8 +7,9 @@ from .generated import schemas opportunity_extensions = Plugin( - extensions=config.extensions, schemas=schemas, + extensions=config.extensions, + meta=config.meta, ) __all__ = ["opportunity_extensions", "schemas"] diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py index a81c7c10..a926e7e9 100644 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py +++ b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py @@ -1,47 +1,39 @@ -"""Plugin config for the opportunity_extensions example. - -Defines four custom fields for the Opportunity model and registers them -with the CommonGrants SDK plugin framework. - -To generate the typed Pydantic models, run this command from the lib/python-sdk directory: - - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions - -This will emit generated/ and __init__.py alongside this file. """ +Plugin configuration for opportunity extensions. -from common_grants_sdk import define_plugin, merge_extensions -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions -from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType +Defines custom field extensions for the Opportunity schema: +- HHS-specific fields (programArea, legacyGrantId) +- Local fields (eligibilityTypes, awardCeiling) +""" -# Extensions that might come from a shared HHS package -hhs_extensions: SchemaExtensions = { - "Opportunity": { - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - }, -} +from typing import Any -# Extensions specific to this project -local_extensions: SchemaExtensions = { - "Opportunity": { - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - }, -} +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput +from common_grants_sdk.extensions.plugin import PluginConfig +from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType -config = define_plugin( - merge_extensions([hhs_extensions, local_extensions], on_conflict="error"), +config: PluginConfig[Any] = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "programArea": CustomFieldSpec( + field_type=CustomFieldType.STRING, + description="HHS program area code (e.g. 'CFDA-93.243')", + ), + "legacyGrantId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Numeric ID from the legacy grants management system", + ), + "eligibilityTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + description="Types of organizations eligible to apply", + ), + "awardCeiling": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + description="Maximum award amount in USD", + ), + } + ) + } ) diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index 03052105..fc1dac7a 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -23,7 +23,7 @@ # adds the script's directory (examples/) to sys.path. Import from there using # the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only # works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. -from plugins.grants_gov.cg_config import plugin +from plugins.grants_gov import grants_gov as plugin from plugins.grants_gov.generated.schemas import Opportunity from common_grants_sdk.extensions import build_transforms @@ -44,6 +44,7 @@ "opportunity_status": "posted", "opportunity_title": "Research into conservation techniques", "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "priority_score_str": "75", "summary": { "applicant_types": ["state_governments"], "archive_date": "2025-05-01", @@ -149,8 +150,7 @@ def _section(title: str) -> None: def main() -> None: - assert plugin.transform_schemas is not None - opp = plugin.transform_schemas["Opportunity"] + opp = plugin.schemas.Opportunity _section("SOURCE DATA (grants.gov format)") print(json.dumps(SOURCE_DATA, indent=2)) @@ -211,6 +211,11 @@ def main() -> None: .get("summary", {}) .get("award_ceiling"), ), + ( + "priority_score_str", + SOURCE_DATA["data"]["priority_score_str"], + native_result.result.get("data", {}).get("priority_score_str"), + ), ] all_pass = True for field, original, roundtripped in checks: @@ -239,6 +244,7 @@ def main() -> None: print(f" [path={err.path}] {err}") else: print("Validation: PASS — result is a typed Opportunity instance") + assert isinstance(custom_cg.result, Opportunity) opp_instance = custom_cg.result print(f"\n title: {opp_instance.title}") print(f" id: {opp_instance.id}") diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index 8fcadc79..b8947a47 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -1,23 +1,28 @@ -"""Tests for expanded plugin.py — backward compat + new optional fields.""" +"""Tests for plugin.py — PluginExtensions-based API.""" from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin -from common_grants_sdk.extensions.specs import SchemaExtensions from common_grants_sdk.extensions.types import ( ObjectSchemasInput, + PluginExtensions, PluginExtensionsMeta, + PluginExtensionsSchema, TransformResult, ) -EXTENSIONS: SchemaExtensions = {} # minimal valid extensions - -def test_define_plugin_backward_compat(): - """define_plugin(extensions=...) still returns PluginConfig with all optional fields None.""" - config = define_plugin(extensions=EXTENSIONS) +def test_define_plugin_no_args(): + """define_plugin() with no args returns PluginConfig with all fields None.""" + config = define_plugin() assert isinstance(config, PluginConfig) - assert config.extensions is EXTENSIONS + assert config.extensions is None assert config.meta is None - assert config.transform_schemas is None + assert config.schemas is None + + +def test_define_plugin_with_extensions(): + ext = PluginExtensions() + config = define_plugin(extensions=ext) + assert config.extensions is ext def test_define_plugin_with_meta_and_schemas(): @@ -31,43 +36,48 @@ def passthrough(x): to_common=passthrough, from_common=passthrough ) } - config = define_plugin(extensions=EXTENSIONS, meta=meta, transform_schemas=schemas) + config = define_plugin(meta=meta, schemas=schemas) assert config.meta is meta assert config.meta.name == "test" - assert config.transform_schemas is schemas - - -def test_plugin_fields(): - """Plugin accepts all fields; optional ones default to None.""" - base = Plugin(extensions=EXTENSIONS, schemas=object()) - assert base.meta is None - assert base.get_client is None - assert base.transform_schemas is None - assert base.filters is None - - meta = PluginExtensionsMeta(name="p", source_system="s") - schemas = {"Opportunity": object()} - full = Plugin( - extensions=EXTENSIONS, schemas=object(), meta=meta, transform_schemas=schemas - ) - assert full.meta is meta - assert full.transform_schemas is schemas + assert config.schemas is schemas -def test_transform_schemas_callable_roundtrip(): - """The demo calls config.transform_schemas["Opportunity"].to_common(data).""" +def test_define_plugin_schemas_callable_roundtrip(): + """config.schemas["Opportunity"].to_common(data) works.""" def always_transformed(_x): return TransformResult(result={"transformed": True}, errors=[]) config = define_plugin( - extensions=EXTENSIONS, - transform_schemas={ + schemas={ "Opportunity": ObjectSchemasInput( to_common=always_transformed, from_common=always_transformed ) }, ) - result = config.transform_schemas["Opportunity"].to_common({"raw": "data"}) + result = config.schemas["Opportunity"].to_common({"raw": "data"}) assert result.result == {"transformed": True} assert result.errors == [] + + +def test_plugin_fields_default_to_none(): + """Plugin.schemas holds the container; meta/extensions default to None.""" + base = Plugin(schemas=object()) + assert base.meta is None + assert base.extensions is None + + +def test_plugin_fields_populated(): + meta = PluginExtensionsMeta(name="p", source_system="s") + ext = PluginExtensions(schemas={"Opportunity": PluginExtensionsSchema()}) + full = Plugin(schemas=object(), extensions=ext, meta=meta) + assert full.meta is meta + assert full.extensions is ext + + +def test_plugin_schemas_is_attribute_container(): + """Plugin.schemas holds the _Schemas object (no generated_schemas field).""" + s = object() + p = Plugin(schemas=s) + assert p.schemas is s + assert not hasattr(p, "generated_schemas") diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py index b0353ada..320a3b28 100644 --- a/lib/python-sdk/tests/extensions/test_transforms.py +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -1,5 +1,7 @@ """Tests for build_transforms() in common_grants_sdk.extensions.transforms.""" +from typing import Any + import pytest from pydantic import BaseModel from common_grants_sdk.extensions.transforms import build_transforms @@ -23,7 +25,7 @@ "title": {"field": "data.opportunity_title"}, "status": { "value": { - "switch": { + "match": { "field": "data.opportunity_status", "case": { "posted": "open", @@ -47,7 +49,7 @@ "data": { "opportunity_title": {"field": "title"}, "opportunity_status": { - "switch": { + "match": { "field": "status.value", "case": { "open": "posted", @@ -67,7 +69,7 @@ # --- Call-time validation --- -@pytest.mark.parametrize("name", ["field", "switch"]) +@pytest.mark.parametrize("name", ["field", "match", "switch"]) def test_handler_collision_raises(name): """build_transforms raises if custom handler shadows a default handler name.""" with pytest.raises(ValueError, match="collide with defaults"): @@ -194,6 +196,30 @@ def test_common_model_validation_failure(): assert result.result["title"] == "Research into conservation techniques" +def test_common_model_non_validation_error_is_caught() -> None: + """Non-ValidationError exceptions from model_validate surface as PluginError (errors-as-values contract).""" + + class _BrokenModel(BaseModel): + title: str + + @classmethod + def model_validate(cls, obj: Any, **kwargs: Any) -> "_BrokenModel": + raise TypeError("misconfigured root validator") + + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_BrokenModel, + ) + result = to_common(SOURCE_DATA) + assert len(result.errors) == 1 + assert isinstance(result.errors[0], PluginError) + assert "misconfigured root validator" in str(result.errors[0]) + # raw transformed dict is preserved so the caller can inspect it + assert isinstance(result.result, dict) + assert result.result["title"] == "Research into conservation techniques" + + def test_custom_handler_registered_per_call(): """Custom handlers apply only to the call they are registered on.""" diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index f44a3a86..7627a099 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -1,6 +1,5 @@ """Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" -import pytest from common_grants_sdk.extensions.specs import CustomFieldSpec from common_grants_sdk.extensions.types import ( ObjectMappings, @@ -81,7 +80,6 @@ def test_plugin_extensions_meta(): def test_plugin_extensions_schema(): - assert PluginExtensionsSchema().custom_fields is None assert PluginExtensionsSchema().mappings is None s = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) assert s.mappings.to_common == {"a": "b"} @@ -93,10 +91,9 @@ def test_plugin_extensions_schema(): def test_plugin_extensions(): assert PluginExtensions().meta is None assert PluginExtensions().schemas is None - spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) - schema = PluginExtensionsSchema(customFields={"legacyId": spec}) + schema = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) ext = PluginExtensions(schemas={"Opportunity": schema}) - assert ext.schemas["Opportunity"].custom_fields == {"legacyId": spec} + assert ext.schemas["Opportunity"].mappings is not None # --- ObjectSchemasInput --- @@ -104,13 +101,18 @@ def test_plugin_extensions(): def test_object_schemas_input(): assert ObjectSchemasInput().native is None + assert ObjectSchemasInput().custom_fields is None assert ObjectSchemasInput().to_common is None + spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) + inp = ObjectSchemasInput(custom_fields={"legacyId": spec}) + assert inp.custom_fields == {"legacyId": spec} + def passthrough(x): return TransformResult(result=x, errors=[]) - inp = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) - assert inp.to_common is passthrough + inp2 = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) + assert inp2.to_common is passthrough # --- ObjectSchemas --- @@ -126,7 +128,7 @@ def passthrough(x): assert schemas.native is dict assert schemas.common is dict - with pytest.raises(TypeError): - ObjectSchemas( - native=dict, common=dict, to_common=passthrough - ) # missing from_common + # to_common and from_common are optional — omitting them is valid + minimal = ObjectSchemas(native=dict, common=dict) + assert minimal.to_common is None + assert minimal.from_common is None diff --git a/lib/python-sdk/tests/schemas/test_plugin_registry.py b/lib/python-sdk/tests/schemas/test_plugin_registry.py index c32ce034..6d4ae5b0 100644 --- a/lib/python-sdk/tests/schemas/test_plugin_registry.py +++ b/lib/python-sdk/tests/schemas/test_plugin_registry.py @@ -1,9 +1,12 @@ """Tests for OpportunityBase.with_custom_fields().""" +from typing import Any + import pytest -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk.extensions import CustomFieldSpec from common_grants_sdk.extensions import Plugin +from common_grants_sdk.extensions.types import ObjectSchemas from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.schemas.pydantic.models.opp_base import OpportunityBase @@ -15,27 +18,25 @@ class _Schemas: """Minimal schemas container that mirrors the generated _Schemas class.""" - def __init__(self, **models): - for name, model in models.items(): - setattr(self, name, model) + Opportunity: ObjectSchemas[Any, Any] def _make_plugin( field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity" ) -> "Plugin[_Schemas]": - """Build a Plugin whose schemas.Opportunity is produced by with_custom_fields().""" + """Build a Plugin whose schemas.Opportunity.common is produced by with_custom_fields().""" extended = OpportunityBase.with_custom_fields( custom_fields=field_specs, model_name=model_name, ) - extensions: SchemaExtensions = {"Opportunity": field_specs} - return Plugin(extensions=extensions, schemas=_Schemas(Opportunity=extended)) + s = _Schemas() + s.Opportunity = ObjectSchemas(native=dict, common=extended) + return Plugin(schemas=s) def _make_plugin_without_opportunity() -> "Plugin[_Schemas]": - """Build a Plugin that has no Opportunity schema (only Application).""" - extensions: SchemaExtensions = {} - return Plugin(extensions=extensions, schemas=_Schemas()) + """Build a Plugin that has no Opportunity schema.""" + return Plugin(schemas=_Schemas()) # --------------------------------------------------------------------------- @@ -84,7 +85,7 @@ def sample_payload() -> dict: def test_plugin_schema_is_subclass_of_opportunity_base(simple_plugin): - Opportunity = simple_plugin.schemas.Opportunity + Opportunity = simple_plugin.schemas.Opportunity.common assert Opportunity is not OpportunityBase assert issubclass(Opportunity, OpportunityBase) @@ -101,7 +102,10 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): {"award_ceiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER)} ) - assert simple_plugin.schemas.Opportunity is not second_plugin.schemas.Opportunity + assert ( + simple_plugin.schemas.Opportunity.common + is not second_plugin.schemas.Opportunity.common + ) # --------------------------------------------------------------------------- @@ -112,7 +116,7 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( simple_plugin, sample_payload ): - Opportunity = simple_plugin.schemas.Opportunity + Opportunity = simple_plugin.schemas.Opportunity.common opp = Opportunity.model_validate(sample_payload) @@ -123,7 +127,7 @@ def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( def test_plugin_schema_validates_custom_fields(simple_plugin, sample_payload): - Opportunity = simple_plugin.schemas.Opportunity + Opportunity = simple_plugin.schemas.Opportunity.common opp = Opportunity.model_validate(sample_payload) diff --git a/lib/python-sdk/tests/utils/test_custom_fields.py b/lib/python-sdk/tests/utils/test_custom_fields.py index b0e1e2c0..dd9f10e6 100644 --- a/lib/python-sdk/tests/utils/test_custom_fields.py +++ b/lib/python-sdk/tests/utils/test_custom_fields.py @@ -470,3 +470,29 @@ def test_empty_description_becomes_none(self): } ) assert opp.custom_fields.legacy_id.description is None + + +class TestUnknownCustomFieldsPreserved: + """Unknown customFields keys should survive round-trip (extra='allow' + validator).""" + + def test_unknown_key_survives_round_trip(self): + fields = { + "legacyId": CustomFieldSpec(field_type=CustomFieldType.INTEGER, value=int), + } + Opportunity = OpportunityBase.with_custom_fields( + custom_fields=fields, model_name="Opportunity" + ) + payload = { + **BASE_OPP, + "customFields": { + "legacyId": {"fieldType": "integer", "value": 42}, + "unknownField": {"fieldType": "string", "value": "preserved"}, + }, + } + + opp = Opportunity.model_validate(payload) + + assert opp.custom_fields is not None + assert opp.custom_fields.legacy_id.value == 42 + # unknown field should be present in model_extra (extra="allow" behaviour) + assert "unknownField" in opp.custom_fields.model_extra diff --git a/lib/python-sdk/tests/utils/test_merge_extensions.py b/lib/python-sdk/tests/utils/test_merge_extensions.py index 935595e2..945b1f83 100644 --- a/lib/python-sdk/tests/utils/test_merge_extensions.py +++ b/lib/python-sdk/tests/utils/test_merge_extensions.py @@ -1,134 +1,140 @@ import pytest -from common_grants_sdk.extensions import ( - CustomFieldSpec, - SchemaExtensions, - merge_extensions, +from common_grants_sdk.extensions import merge_extensions +from common_grants_sdk.extensions.types import ( + ObjectMappings, + PluginExtensions, + PluginExtensionsMeta, + PluginExtensionsSchema, ) -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -def test_merge_disjoint_extensions() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of eligible organizations", - ) - } - } - source_two: SchemaExtensions = { - "Opportunity": { - "priority_score": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Internal ranking score", +def test_merge_empty_returns_empty_plugin_extensions() -> None: + result = merge_extensions([]) + assert isinstance(result, PluginExtensions) + assert result.schemas is None + assert result.meta is None + + +def test_merge_single_source_passthrough() -> None: + source = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) ) } - } + ) + merged = merge_extensions([source]) + assert merged is source - merged = merge_extensions([source_one, source_two]) - assert "eligibility_type" in merged["Opportunity"] - assert "priority_score" in merged["Opportunity"] +def test_merge_meta_raises_on_duplicate_name_by_default() -> None: + source_one = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-a")) + source_two = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-b")) + with pytest.raises(ValueError, match="duplicate"): + merge_extensions([source_one, source_two]) -def test_merge_raises_on_duplicate_field_by_default() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.ARRAY) - } - } - source_two: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.STRING) - } - } - with pytest.raises( - ValueError, - match='duplicate field "eligibility_type" on model "Opportunity"', - ): - merge_extensions([source_one, source_two]) +def test_merge_meta_last_wins() -> None: + source_one = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-a")) + source_two = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-b")) + merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + assert merged.meta is not None + assert merged.meta.name == "plugin-b" -def test_merge_last_wins_on_duplicate_field() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="First", + +def test_merge_raises_on_duplicate_mappings_by_default() -> None: + source_one = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) ) } - } - source_two: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Last", + ) + source_two = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "other"}}, fromCommon={} + ) ) } - } + ) - merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + with pytest.raises(ValueError, match='duplicate mappings for object "Opportunity"'): + merge_extensions([source_one, source_two]) - assert ( - merged["Opportunity"]["eligibility_type"].field_type == CustomFieldType.STRING - ) - assert merged["Opportunity"]["eligibility_type"].description == "Last" +def test_merge_meta_none_fields_do_not_overwrite() -> None: + """A None field in the second source does not erase a value from the first.""" + source_one = PluginExtensions( + meta=PluginExtensionsMeta(name="plugin-a", version="1.0") + ) + source_two = PluginExtensions(meta=PluginExtensionsMeta(name=None, version="2.0")) -def test_merge_first_wins_on_duplicate_field() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="First", + merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + assert merged.meta is not None + assert merged.meta.name == "plugin-a" # preserved from source_one + assert merged.meta.version == "2.0" # overwritten by source_two + + +def test_merge_mappings_first_wins() -> None: + source_one = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) ) } - } - source_two: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Last", + ) + source_two = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "other"}}, fromCommon={} + ) ) } - } + ) merged = merge_extensions([source_one, source_two], on_conflict="first_wins") - - assert merged["Opportunity"]["eligibility_type"].field_type == CustomFieldType.ARRAY - assert merged["Opportunity"]["eligibility_type"].description == "First" - - -def test_merge_empty_inputs_returns_empty_mapping() -> None: - assert merge_extensions([]) == {} - - -def test_merge_single_source_passthrough() -> None: - source: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.ARRAY) - } + assert merged.schemas is not None + assert merged.schemas["Opportunity"].mappings is not None + assert merged.schemas["Opportunity"].mappings.to_common == { + "title": {"field": "name"} } - merged = merge_extensions([source]) - assert merged is source - - -def test_merge_overlapping_model_keys_without_field_conflicts() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.ARRAY) +def test_merge_mappings_last_wins() -> None: + source_one = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) + ) } - } - source_two: SchemaExtensions = { - "Opportunity": { - "funding_track": CustomFieldSpec(field_type=CustomFieldType.STRING) + ) + source_two = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "other"}}, fromCommon={} + ) + ) } - } - - merged = merge_extensions([source_one, source_two]) + ) - assert set(merged["Opportunity"].keys()) == {"eligibility_type", "funding_track"} + merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + assert merged.schemas is not None + assert merged.schemas["Opportunity"].mappings is not None + assert merged.schemas["Opportunity"].mappings.to_common == { + "title": {"field": "other"} + } diff --git a/lib/python-sdk/tests/utils/test_plugin_generator.py b/lib/python-sdk/tests/utils/test_plugin_generator.py index 212f824c..a182475b 100644 --- a/lib/python-sdk/tests/utils/test_plugin_generator.py +++ b/lib/python-sdk/tests/utils/test_plugin_generator.py @@ -10,7 +10,7 @@ import pytest -from common_grants_sdk import merge_extensions, define_plugin +from common_grants_sdk import define_plugin from common_grants_sdk.extensions import CustomFieldSpec from common_grants_sdk.extensions import PluginConfig from common_grants_sdk.schemas.pydantic.fields import CustomFieldType @@ -28,28 +28,25 @@ def _env_with_sdk_pythonpath() -> dict[str, str]: return env -def test_define_plugin_returns_config_with_extensions(): - extensions = { - "Opportunity": { - "program_area": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Grant category", - ) - } +def test_define_plugin_returns_config_with_schemas(): + from common_grants_sdk.extensions.types import ObjectSchemasInput + + schemas = { + "Opportunity": ObjectSchemasInput( + custom_fields={ + "program_area": CustomFieldSpec( + field_type=CustomFieldType.STRING, + description="Grant category", + ) + } + ) } - config = define_plugin(extensions) + config = define_plugin(schemas=schemas) assert isinstance(config, PluginConfig) - assert config.extensions == extensions - - -def test_merge_extensions_merges_extensions(): - one = {"Opportunity": {"program_area": CustomFieldSpec(field_type="string")}} - two = {"Opportunity": {"eligibility_type": CustomFieldSpec(field_type="array")}} - - merged = merge_extensions([one, two], on_conflict="error") - - assert set(merged["Opportunity"].keys()) == {"program_area", "eligibility_type"} + assert config.schemas is schemas + assert config.schemas["Opportunity"].custom_fields is not None + assert "program_area" in config.schemas["Opportunity"].custom_fields def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): @@ -62,23 +59,26 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): (plugin_dir / "cg_config.py").write_text( "\n".join( [ - "from common_grants_sdk import merge_extensions, define_plugin", - "from common_grants_sdk.extensions import SchemaExtensions, CustomFieldSpec", + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", - "local_extensions: SchemaExtensions = {", - ' "Opportunity": {', - ' "program_area": CustomFieldSpec(', - ' field_type="string",', - ' description="Program area",', - " ),", - ' "eligibility_type": CustomFieldSpec(', - ' field_type="array",', - ' description="Types of eligible organizations",', - " ),", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "program_area": CustomFieldSpec(', + ' field_type="string",', + ' description="Program area",', + " ),", + ' "eligibility_type": CustomFieldSpec(', + ' field_type="array",', + ' description="Types of eligible organizations",', + " ),", + " },", + " )", " },", - "}", - "", - "config = define_plugin(merge_extensions([local_extensions], on_conflict='error'))", + ")", "", ] ), @@ -105,7 +105,7 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): try: combined_module = importlib.import_module("plugins.combined") combined = getattr(combined_module, "combined") - opp_model = combined.schemas.Opportunity + opp_model = combined.schemas.Opportunity.common type_hints = get_type_hints(opp_model, include_extras=False) assert "custom_fields" in type_hints @@ -139,7 +139,7 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): "nonprofit", "city_government", ] - assert "Opportunity" in combined.extensions + assert combined.schemas.Opportunity.common is opp_model finally: sys.path.remove(str(tmp_path)) @@ -155,19 +155,24 @@ def test_generate_emits_import_for_pydantic_model_in_cg_config(tmp_path: Path): "from pydantic import BaseModel", "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", "class AgentInfo(BaseModel):", " name: str", " email: str", "", - "config = define_plugin({", - ' "Opportunity": {', - ' "point_of_contact": CustomFieldSpec(', - ' field_type="object",', - " value=AgentInfo,", - " ),", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "point_of_contact": CustomFieldSpec(', + ' field_type="object",', + " value=AgentInfo,", + " ),", + " },", + " )", " },", - "})", + ")", "", ] ), @@ -201,15 +206,20 @@ def test_generate_emits_import_for_external_module_type(tmp_path: Path): "from datetime import datetime", "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", - "config = define_plugin({", - ' "Opportunity": {', - ' "deadline": CustomFieldSpec(', - ' field_type="string",', - " value=datetime,", - " ),", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "deadline": CustomFieldSpec(', + ' field_type="string",', + " value=datetime,", + " ),", + " },", + " )", " },", - "})", + ")", "", ] ), @@ -232,6 +242,70 @@ def test_generate_emits_import_for_external_module_type(tmp_path: Path): assert "value: Optional[datetime]" in schemas_src +def test_generate_auto_builds_transforms_from_mappings(tmp_path): + """When cg_config has extensions.schemas[obj].mappings but no explicit schemas[obj], + the generated __init__.py calls build_transforms() automatically.""" + plugin_dir = tmp_path / "plugins" / "auto_transform" + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text("", encoding="utf-8") + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import PluginExtensions, PluginExtensionsSchema, ObjectMappings", + "", + "config = define_plugin(", + " extensions=PluginExtensions(", + " schemas={", + ' "Opportunity": PluginExtensionsSchema(', + " mappings=ObjectMappings(", + ' to_common={"title": {"field": "data.title"}},', + " from_common={},", + " ),", + " )", + " }", + " ),", + ")", + "", + ] + ), + encoding="utf-8", + ) + + from common_grants_sdk.extensions.generate import generate_plugin + + generate_plugin(plugin_dir) + + init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") + assert "build_transforms" in init_content + assert 'config.extensions.schemas["Opportunity"].mappings.to_common' in init_content + assert "_Opportunity_to_common" in init_content + assert "common_model=schemas.Opportunity.common" in init_content + + # Load the generated plugin and verify schemas are populated + import importlib + import sys + + # Remove any stale 'plugins' package from previous tests before inserting our path. + for key in list(sys.modules.keys()): + if key == "plugins" or key.startswith("plugins."): + del sys.modules[key] + + sys.path.insert(0, str(tmp_path)) + try: + mod = importlib.import_module("plugins.auto_transform") + plugin = getattr(mod, "auto_transform") + assert hasattr(plugin.schemas, "Opportunity") + assert plugin.schemas.Opportunity.to_common is not None + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + for key in list(sys.modules.keys()): + if key == "plugins" or key.startswith("plugins."): + del sys.modules[key] + + @pytest.mark.skipif(shutil.which("pyright") is None, reason="pyright is not installed") def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): plugins_dir = tmp_path / "plugins" @@ -244,14 +318,18 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): "\n".join( [ "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import SchemaExtensions, CustomFieldSpec", + "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", - "extensions: SchemaExtensions = {", - ' "Opportunity": {', - ' "eligibility_type": CustomFieldSpec(field_type="array"),', + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "eligibility_type": CustomFieldSpec(field_type="array"),', + " },", + " )", " },", - "}", - "config = define_plugin(extensions)", + ")", "", ] ), @@ -286,7 +364,7 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): ' "customFields": {"eligibility_type": {"fieldType": "array", "value": ["a"]}},', "}", "", - "opp = combined.schemas.Opportunity.model_validate(payload)", + "opp = combined.schemas.Opportunity.common.model_validate(payload)", "if opp.custom_fields is not None and opp.custom_fields.eligibility_type is not None:", " values = opp.custom_fields.eligibility_type.value", " reveal_type(values)", @@ -308,3 +386,173 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): ) assert pyright.returncode == 0, pyright.stdout + "\n" + pyright.stderr assert 'Type of "values" is "list[Any] | None"' in pyright.stdout + + +def test_generate_explicit_transforms(tmp_path): + """When cg_config has config.schemas with explicit to_common/from_common, + the generated __init__.py emits ObjectSchemas with the supplied callables.""" + from common_grants_sdk.extensions.generate import generate_plugin + + plugin_dir = tmp_path / "plugins" / "explicit_tf" + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text("", encoding="utf-8") + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import ObjectSchemasInput, TransformResult", + "from common_grants_sdk.extensions import CustomFieldSpec", + "", + "def _to_common(native):", + " return TransformResult(result={'title': native.get('name', '')}, errors=[])", + "", + "def _from_common(common):", + " return TransformResult(result={'name': common.get('title', '')}, errors=[])", + "", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + ' custom_fields={"legacyId": CustomFieldSpec(field_type="integer")},', + " to_common=_to_common,", + " from_common=_from_common,", + " )", + " },", + ")", + "", + ] + ), + encoding="utf-8", + ) + + generate_plugin(plugin_dir) + + init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") + # Explicit transforms use inject_transforms(), not per-object boilerplate + assert "build_transforms" not in init_content + assert "schemas = inject_transforms(config, schemas)" in init_content + # No per-object assignment lines + assert 'config.schemas["Opportunity"].to_common' not in init_content + assert 'config.schemas["Opportunity"].from_common' not in init_content + # ObjectSchemas is no longer constructed in __init__.py (only in schemas.py) + assert "ObjectSchemas" not in init_content + + # Load and verify the plugin works end-to-end + sys.path.insert(0, str(tmp_path)) + try: + mod = importlib.import_module("plugins.explicit_tf") + plugin = getattr(mod, "explicit_tf") + assert hasattr(plugin.schemas, "Opportunity") + opp_schemas = plugin.schemas.Opportunity + result = opp_schemas.to_common({"name": "Test Grant"}) + assert result.result == {"title": "Test Grant"} + assert result.errors == [] + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + for key in list(sys.modules.keys()): + if "explicit_tf" in key or key == "plugins": + del sys.modules[key] + + +def test_generate_transforms_only_no_custom_fields(tmp_path): + """Regression: config.schemas with only explicit transforms (no custom_fields, no extensions) + must produce a _Schemas entry for the object so inject_transforms() can access it at import. + """ + from common_grants_sdk.extensions.generate import generate_plugin + + plugin_dir = tmp_path / "plugins" / "transforms_only" + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text("", encoding="utf-8") + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import ObjectSchemasInput, TransformResult", + "", + "def _to_common(native):", + " return TransformResult(result={'title': native.get('name', '')}, errors=[])", + "", + "def _from_common(common):", + " return TransformResult(result={'name': common.get('title', '')}, errors=[])", + "", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " to_common=_to_common,", + " from_common=_from_common,", + " )", + " },", + ")", + "", + ] + ), + encoding="utf-8", + ) + + generate_plugin(plugin_dir) + + # schemas.py must assign self.Opportunity using the base SDK class + schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") + assert "self.Opportunity = ObjectSchemas" in schemas_src + assert "OpportunityBase" in schemas_src + + # __init__.py must use inject_transforms (not build_transforms) + init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") + assert "inject_transforms" in init_content + assert "build_transforms" not in init_content + + # Importing must not raise AttributeError / ValueError + sys.path.insert(0, str(tmp_path)) + try: + mod = importlib.import_module("plugins.transforms_only") + plugin = getattr(mod, "transforms_only") + assert hasattr(plugin.schemas, "Opportunity") + result = plugin.schemas.Opportunity.to_common({"name": "Test Grant"}) + assert result.result == {"title": "Test Grant"} + assert result.errors == [] + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + for key in list(sys.modules.keys()): + if "transforms_only" in key or key == "plugins": + del sys.modules[key] + + +def test_generate_raises_on_missing_mapping_direction(tmp_path): + """generate_plugin raises ValueError when a mappings-only object is missing + one of its mapping directions (to_common or from_common is None).""" + from common_grants_sdk.extensions.generate import generate_plugin + + plugin_dir = tmp_path / "bad_plugin" + plugin_dir.mkdir(parents=True) + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import (", + " ObjectMappings, PluginExtensions, PluginExtensionsSchema,", + ")", + "", + "config = define_plugin(", + " extensions=PluginExtensions(", + " schemas={", + ' "Opportunity": PluginExtensionsSchema(', + " mappings=ObjectMappings(", + ' to_common={"title": {"field": "data.title"}},', + " # from_common intentionally omitted (None)", + " ),", + " )", + " }", + " ),", + ")", + "", + ] + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="from_common.*required"): + generate_plugin(plugin_dir) diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 8f826c03..797cd511 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -1,5 +1,5 @@ import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, @@ -331,27 +331,46 @@ def test_handler_error_is_value_error(): ) -def test_pydantic_model_instance_is_normalized(): - """transform_from_mapping accepts a Pydantic model instance and extracts fields correctly.""" +def test_pydantic_model_instance_is_normalized() -> None: + """transform_from_mapping accepts a Pydantic model and uses camelCase alias keys.""" class Inner(BaseModel): + model_config = ConfigDict(populate_by_name=True) value: str class Source(BaseModel): + model_config = ConfigDict(populate_by_name=True) title: str - nested: Inner + nested_item: Inner = Field(alias="nestedItem") - model = Source(title="hello", nested=Inner(value="world")) + model = Source(title="hello", nestedItem=Inner(value="world")) result = transform_from_mapping( model, { "out_title": {"field": "title"}, - "out_value": {"field": "nested.value"}, + "out_value": {"field": "nestedItem.value"}, # camelCase alias path }, ) assert result == {"out_title": "hello", "out_value": "world"} +def test_pydantic_model_alias_not_snake_case() -> None: + """Field paths must use camelCase aliases, not snake_case attribute names.""" + + class Source(BaseModel): + model_config = ConfigDict(populate_by_name=True) + award_floor: int = Field(alias="awardFloor") + + model = Source(awardFloor=10000) + # camelCase alias path resolves correctly + result = transform_from_mapping(model, {"amount": {"field": "awardFloor"}}) + assert result == {"amount": 10000} + + # snake_case attribute name does NOT resolve (returns None) + result_snake = transform_from_mapping(model, {"amount": {"field": "award_floor"}}) + assert result_snake == {"amount": None} + + def test_deeply_nested(input_data): """ Test transformation with deeply nested structures. diff --git a/website/.cspell.json b/website/.cspell.json index 0eaf1b96..ad88e0eb 100644 --- a/website/.cspell.json +++ b/website/.cspell.json @@ -100,7 +100,8 @@ "apidevtools", "allof", "pypi", - "Pypi" + "Pypi", + "functools" ], "import": [] } diff --git a/website/src/content/docs/governance/adr/0017-mapping-format.md b/website/src/content/docs/governance/adr/0017-mapping-format.md index a4da1b04..45745117 100644 --- a/website/src/content/docs/governance/adr/0017-mapping-format.md +++ b/website/src/content/docs/governance/adr/0017-mapping-format.md @@ -31,7 +31,7 @@ For example, the following mapping: { "mappings": { "data": { - "title": "data.opportunity_title", + "title": { "field": "data.opportunity_title" }, "funding": { "minAwardAmount": { "amount": { @@ -89,6 +89,25 @@ Into the following output format: } ``` +### Field path convention + +Bare string values in a mapping are treated as **literals** by the transform +engine — they are returned as-is, not interpreted as field paths. + +To extract a value from the source data, use the `field` handler: + +```json +{ "title": { "field": "data.opportunity_title" } } +``` + +Not: + +```json +{ "title": "data.opportunity_title" } +``` + +The second form sets `title` to the literal string `"data.opportunity_title"`. + ### Example Here's a more complex example of the proposed mapping format. The following examples also serve as the input and target output for each option below. @@ -203,7 +222,7 @@ And we want to translate this data into the following format: { "mappings": { "data": { - "title": "data.opportunity_title", + "title": { "field": "data.opportunity_title" }, "status": { "value": { "match": { @@ -236,12 +255,12 @@ And we want to translate this data into the following format: }, "keyDates": { "appOpens": { - "date": "data.summary.forecasted_post_date", + "date": { "field": "data.summary.forecasted_post_date" }, "name": { "const": "Open Date" }, "description": { "const": "Applications begin being accepted" } }, "appDeadline": { - "date": "data.summary.forecasted_close_date", + "date": { "field": "data.summary.forecasted_close_date" }, "name": { "const": "Application Deadline" }, "description": { "const": "Final submission deadline for all grant applications" @@ -249,7 +268,7 @@ And we want to translate this data into the following format: }, "otherDates": { "forecastedAwardDate": { - "date": "data.summary.forecasted_award_date", + "date": { "field": "data.summary.forecasted_award_date" }, "name": { "const": "Forecasted award date" }, "description": { "const": "When we expect to announce awards for this opportunity." @@ -259,19 +278,19 @@ And we want to translate this data into the following format: }, "customFields": { "legacyId": { - "value": "data.opportunity_id", + "value": { "field": "data.opportunity_id" }, "name": { "const": "Legacy ID" }, "type": { "const": "number" }, "description": { "const": "Unique identifier in legacy database" } }, "agencyName": { - "value": "data.agency_name", + "value": { "field": "data.agency_name" }, "name": { "const": "Agency" }, "type": { "const": "string" }, "description": { "const": "Agency hosting the opportunity" } }, "applicantTypes": { - "value": "data.summary.applicant_types", + "value": { "field": "data.summary.applicant_types" }, "name": { "const": "Applicant types" }, "type": { "const": "array" }, "description": { "const": "Types of applicants eligible to apply" } @@ -346,7 +365,7 @@ JSON mapping is best if: { "mappings": { "data": { - "title": "data.opportunity_title", + "title": { "field": "data.opportunity_title" }, "status": { "value": { "match": { @@ -379,12 +398,12 @@ JSON mapping is best if: }, "keyDates": { "appOpens": { - "date": "data.summary.forecasted_post_date", + "date": { "field": "data.summary.forecasted_post_date" }, "name": { "const": "Open Date" }, "description": { "const": "Applications begin being accepted" } }, "appDeadline": { - "date": "data.summary.forecasted_close_date", + "date": { "field": "data.summary.forecasted_close_date" }, "name": { "const": "Application Deadline" }, "description": { "const": "Final submission deadline for all grant applications" @@ -392,7 +411,7 @@ JSON mapping is best if: }, "otherDates": { "forecastedAwardDate": { - "date": "data.summary.forecasted_award_date", + "date": { "field": "data.summary.forecasted_award_date" }, "name": { "const": "Forecasted award date" }, "description": { "const": "When we expect to announce awards for this opportunity." @@ -402,19 +421,19 @@ JSON mapping is best if: }, "customFields": { "legacyId": { - "value": "data.opportunity_id", + "value": { "field": "data.opportunity_id" }, "name": { "const": "Legacy ID" }, "type": { "const": "number" }, "description": { "const": "Unique identifier in legacy database" } }, "agencyName": { - "value": "data.agency_name", + "value": { "field": "data.agency_name" }, "name": { "const": "Agency" }, "type": { "const": "string" }, "description": { "const": "Agency hosting the opportunity" } }, "applicantTypes": { - "value": "data.summary.applicant_types", + "value": { "field": "data.summary.applicant_types" }, "name": { "const": "Applicant types" }, "type": { "const": "array" }, "description": { "const": "Types of applicants eligible to apply" } diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 52b33b74..151f2025 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -40,7 +40,7 @@ We decided to: 5. **Make all top-level Plugin fields optional** so adopters can publish a plugin that provides only the features they need — for example, custom fields only — and expand to include transforms, client config, or additional schemas incrementally over time. -6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.`, `definePlugin()` invokes `buildTransforms()` automatically. Mappings for each direction are author-provided — `buildTransforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. +6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.`, the SDK invokes `buildTransforms()` automatically. In TypeScript this happens inside `definePlugin()`; in Python it happens inside the code generator (`generate.py`) at generation time, emitting a `build_transforms()` call into the generated `__init__.py`. Both mapping directions must be provided explicitly — `buildTransforms()` / `build_transforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. 7. **`toCommon` / `fromCommon` return a `TransformResult` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. In the current PoC, this validation is opt-in at the `buildTransforms()` call site via the `commonModel` / `common_model` parameter — when supplied, validation runs inside `toCommon` against the fully extended generated schema. In the full SDK, `definePlugin()` will additionally inject validation when auto-generating transforms from `extensions.schemas..mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. @@ -51,12 +51,18 @@ We decided to: The resulting Plugin shape: ``` -plugin.meta // name, version, sourceSystem, capabilities -plugin.getClient // (config: ClientConfig) => Client; memoized by definePlugin() -plugin.extensions // serializable; used by mergeExtensions() -plugin.schemas. = { native, common, toCommon, fromCommon } +plugin.meta // name, version, sourceSystem, capabilities +plugin.get_client // (config: ClientConfig) => Client; memoized by the code generator +plugin.extensions // serializable; used by merge_extensions() +plugin.schemas. // ObjectSchemas instance — unified access to model class and transforms +plugin.schemas..common // generated Pydantic model class (includes any declared custom fields) +plugin.schemas..native // source system type (defaults to dict) +plugin.schemas..to_common // callable: native → TransformResult[common] (None if not configured) +plugin.schemas..from_common // callable: common → TransformResult[native] (None if not configured) ``` +**Python note:** In the Python SDK, `define_plugin()` returns a `PluginConfig` (build-time input) rather than a fully compiled `Plugin`. The code generator (`generate.py`) compiles `PluginConfig → Plugin` by injecting the generated model classes as the `common` schema, wrapping `get_client` with `functools.lru_cache`, and auto-generating `build_transforms()` calls for any objects that have `extensions.schemas[obj].mappings` but no explicit `to_common`/`from_common` in `schemas[obj]`. This split is necessary because `cg_config.py` cannot import from `generated/` — it is the input to code generation. In the Python SDK, `custom_fields` is declared on `ObjectSchemasInput` (inside `schemas`) rather than on `PluginExtensionsSchema` (inside `extensions`). + ### Example interface @@ -264,21 +270,27 @@ class CustomFieldSpec: name: str = "" # optional; dict key is used as the display name fallback description: str = "" -# Runtime type — produced by define_plugin(), not provided directly by plugin authors +# Runtime schema container — assembled by the code generator, not provided directly by authors. +# Accessed via plugin.schemas. (attribute access, not dict lookup). +# common includes any custom fields declared by the plugin (it is a generated subclass of the +# base CG model, e.g. OpportunityBase, with typed custom_fields baked in). @dataclass class ObjectSchemas(Generic[TNative, TCommon]): - native: type[TNative] # expects a Pydantic BaseModel subclass - common: type[TCommon] # expects a Pydantic BaseModel subclass - to_common: Callable[[TNative], TransformResult[TCommon]] - from_common: Callable[[TCommon], TransformResult[TNative]] + native: type[TNative] # source system type; defaults to dict + common: type[TCommon] # generated Pydantic model class (includes declared custom fields) + to_common: Callable[[TNative], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TNative]] | None = None # Input type — provided by plugin authors inside define_plugin(schemas=...) # common is intentionally absent: cg_config.py cannot import from generated/ since # it is the input to generation. define_plugin() injects common during compilation # from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. +# custom_fields declares extra fields beyond the base CG schema; the code generator +# reads these and emits typed subclasses. @dataclass class ObjectSchemasInput(Generic[TNative, TCommon]): native: type[TNative] | None = None # defaults to dict[str, Any] if omitted + custom_fields: dict[str, CustomFieldSpec] | None = None to_common: Callable[[TNative], TransformResult[TCommon]] | None = None from_common: Callable[[TCommon], TransformResult[TNative]] | None = None @@ -292,18 +304,9 @@ class CustomFilterSpec: PluginCapability = Literal['customFields', 'customFilters', 'transforms', 'client'] -class PluginMeta(BaseModel): - model_config = ConfigDict(populate_by_name=True) - - name: str - version: str | None = None # optional; if omitted, define_plugin() infers it from the package's pyproject.toml / importlib.metadata - source_system: str = Field(alias='sourceSystem') - capabilities: list[PluginCapability] | None = None - -# Equivalent to TypeScript's Partial. Defined as a separate model -# rather than reusing PluginMeta because Pydantic does not have a built-in Partial. -# Note: if PluginMeta gains new required fields, this class must be updated manually. -# Drift can be caught with: assert PluginMeta.model_fields.keys() == PluginExtensionsMeta.model_fields.keys() +# PluginExtensionsMeta is used for both Plugin.meta (top-level plugin identity) and +# PluginExtensions.meta (serializable meta in the extensions JSON object). All fields +# are optional at the type level; plugins should populate name and source_system. class PluginExtensionsMeta(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -320,9 +323,9 @@ class ObjectMappings(BaseModel): from_common: dict[str, Any] | None = Field(default=None, alias='fromCommon') # CommonGrants → native class PluginExtensionsSchema(BaseModel): + """Per-object config inside extensions.schemas. Holds declarative mappings only.""" model_config = ConfigDict(populate_by_name=True) - custom_fields: dict[str, CustomFieldSpec] | None = Field(default=None, alias='customFields') # Optional declarative mappings. When present and no explicit to_common / from_common # is supplied in schemas[obj], define_plugin() auto-invokes build_transforms() on these. mappings: ObjectMappings | None = None @@ -333,44 +336,64 @@ class PluginExtensions(BaseModel): ClientConfig = dict[str, Any] # plugin authors define their own keys (auth, base_url, timeout, etc.) -# No `filters` field on Plugin — handled by the Client returned by get_client() (see Decision #2). +# Runtime plugin container — assembled by the code generator (generate.py) from +# the generated model classes and a compiled PluginConfig. Plugin authors do not +# construct this directly; it is emitted into the plugin's __init__.py. +# +# schemas: the _Schemas object from generated/schemas.py. Each attribute is an +# ObjectSchemas instance providing unified access to the model class and transforms: +# plugin.schemas.Opportunity.common → Pydantic model class (with custom fields) +# plugin.schemas.Opportunity.to_common → transform callable (or None) +# plugin.schemas.Opportunity.from_common → transform callable (or None) +# plugin.schemas.Opportunity.native → source system type (or dict) @dataclass -class Plugin: - meta: PluginMeta | None = None - get_client: Callable[[ClientConfig], Client] | None = None +class Plugin(Generic[T]): + schemas: T extensions: PluginExtensions | None = None - schemas: dict[ExtensibleSchemaName, ObjectSchemas[Any, Any]] | None = None + meta: PluginExtensionsMeta | None = None + get_client: Callable[[ClientConfig], Any] | None = None + filters: dict[str, Any] | None = None + +# Build-time config — produced by define_plugin(), consumed by generate.py. +# +# Compilation from PluginConfig → Plugin (injecting the common model classes from +# generated/, wrapping get_client with functools.lru_cache) happens inside +# generate.py at code-generation time, not at define_plugin() call time. This split +# is necessary in Python because cg_config.py cannot import from generated/ — it is +# the input to code generation. +@dataclass(frozen=True) +class PluginConfig: + extensions: PluginExtensions | None = None + meta: PluginExtensionsMeta | None = None + schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None + get_client: Callable[[ClientConfig], Any] | None = None + filters: dict[str, Any] | None = None # All params are optional — adopters can start with only what they need and expand # incrementally. Unlike TypeScript, Python supports named optional params at the # function root, so no DefinePluginOptions wrapper object is needed. # -# define_plugin compiles inputs into a Plugin by: +# define_plugin stores inputs as-is in a PluginConfig. The code generator +# (generate.py) then compiles PluginConfig → Plugin by: # - extending the base CommonGrants model with any declared custom_fields → common -# - native defaults to dict[str, Any] if omitted (extensions is JSON-safe; runtime -# Pydantic models cannot be included) -# - wrapping get_client with memoization so the same Client instance is returned -# for equivalent configs automatically +# - native defaults to dict[str, Any] if omitted +# - wrapping get_client with functools.lru_cache for memoization # # to_common / from_common may be plain hand-written callables, generated via -# build_transforms() and passed in schemas, or auto-generated by define_plugin() -# itself — when extensions.schemas[obj].mappings is declared and schemas[obj] provides -# no explicit transform, define_plugin() invokes build_transforms() internally. -# All transforms return TransformResult[T]; define_plugin() validates the result field -# at runtime with model_validate and appends any validation failures to the errors -# list rather than raising (see Decision #7). +# build_transforms() and passed in schemas, or auto-generated by the code generator +# itself — when extensions.schemas[obj].mappings is declared and schemas[obj] has no +# to_common/from_common, generate.py invokes build_transforms() in the emitted code. def define_plugin( - meta: PluginMeta | None = None, - get_client: Callable[[ClientConfig], Client] | None = None, + meta: PluginExtensionsMeta | None = None, + get_client: Callable[[ClientConfig], Any] | None = None, extensions: PluginExtensions | None = None, - schemas: dict[ExtensibleSchemaName, ObjectSchemasInput[Any, Any]] | None = None, - filters: dict[ExtensibleSchemaName, dict[str, CustomFilterSpec]] | None = None, -) -> Plugin: ... + schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, + filters: dict[str, Any] | None = None, +) -> PluginConfig: ... -# Exact signature shape is provisional pending SDK pin in #744. def merge_extensions( - *extensions: PluginExtensions, - on_conflict: Literal["error", "firstWins", "lastWins"] = "error", + sources: list[PluginExtensions], + on_conflict: Literal["error", "first_wins", "last_wins"] = "error", ) -> PluginExtensions: ... # Handler signature matches ADR-0017 runtime conventions. @@ -531,7 +554,7 @@ to_common, from_common = build_transforms( ) plugin = define_plugin( - meta=PluginMeta(name='grants-gov-plugin', version='1.0.0', source_system='grants.gov'), # source_system serializes as 'sourceSystem' in JSON + meta=PluginExtensionsMeta(name='grants-gov-plugin', version='1.0.0', source_system='grants.gov'), # source_system serializes as 'sourceSystem' in JSON # define_plugin memoizes get_client — the same Client is returned for equivalent configs. get_client=lambda config: Client(config=Config( base_url=config.get('base_url', 'https://api.grants.gov'), @@ -540,19 +563,14 @@ plugin = define_plugin( page_size=config.get('page_size', 100), list_items_limit=config.get('list_items_limit', 1000), )), - extensions=PluginExtensions( - schemas={ - 'Opportunity': PluginExtensionsSchema( - custom_fields={ - 'programArea': CustomFieldSpec(field_type=CustomFieldType.STRING, description='HHS program area code'), - 'legacyGrantId': CustomFieldSpec(field_type=CustomFieldType.INTEGER, description='Numeric ID from legacy system'), - }, - ), - }, - ), + # Python SDK: custom_fields lives on ObjectSchemasInput, not PluginExtensionsSchema schemas={ 'Opportunity': ObjectSchemasInput( native=GrantsGovOpportunity, + custom_fields={ + 'programArea': CustomFieldSpec(field_type=CustomFieldType.STRING, description='HHS program area code'), + 'legacyGrantId': CustomFieldSpec(field_type=CustomFieldType.INTEGER, description='Numeric ID from legacy system'), + }, to_common=to_common, from_common=from_common, ), @@ -561,7 +579,7 @@ plugin = define_plugin( # Combine extensions from multiple packages before constructing the plugin -merged = merge_extensions(base_extensions, grants_gov_extensions) +merged = merge_extensions([base_extensions, grants_gov_extensions]) merged_plugin = define_plugin(extensions=merged) # Calling get_client() with a config dict — memoized, so repeated calls return the same instance @@ -623,7 +641,7 @@ client = grants_gov_plugin.get_client({ # Use the compiled schemas to transform native data into CommonGrants shape. # to_common / from_common return TransformResult[T] = {result, errors} — consumers # apply their own strict-vs-lenient rule for what counts as success. -to_common = grants_gov_plugin.schemas['Opportunity'].to_common +to_common = grants_gov_plugin.schemas.Opportunity.to_common outcome = to_common(raw_grants_gov_data) if not outcome.errors: use(outcome.result) # strict: treat any error (including handler warnings) as failure @@ -647,7 +665,7 @@ print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms", " ### Consequences - **Positive consequences** - - Client stays singular — `getClient()` is memoized by `definePlugin()`, so one source system always produces one `Client` instance regardless of how many times `getClient()` is called + - Client stays singular — `getClient()` / `get_client()` is memoized (in TypeScript by `definePlugin()`; in Python by the code generator wrapping it with `functools.lru_cache`), so one source system always produces one `Client` instance regardless of how many times it is called - Top-level surface (`meta`, `client`, `schemas`, `extensions`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only - Dependency injection works along functional lines: pass `getClient`, pass `Schemas`, pass `Extensions` as coherent units without needing to reassemble from per-object branches - `mergeExtensions()` / `merge_extensions()` operates on flat, serializable data at the root, not on deeply nested per-object branches