From 21c0c8e64b99a9f280ae101186513e1b7adf5908 Mon Sep 17 00:00:00 2001 From: jcrichlake <145698165+jcrichlake@users.noreply.github.com> Date: Wed, 13 May 2026 15:39:49 -0400 Subject: [PATCH 01/51] [Issue #799] Transform PoC (#810) * Issue #799 Transform PoC * Fixing plugin in makefile * Adding custom handler example * Updating transform contract * Fixing make:checks errors * Reducing duplicate tests and updating makefile * Fixing pr comments. * Adding ADR fixes * PR comments rd 2 * Update website/src/content/docs/governance/adr/0022-plugin-framework.mdx Co-authored-by: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> --------- Co-authored-by: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> --- lib/python-sdk/.coverage | Bin 0 -> 53248 bytes lib/python-sdk/Makefile | 2 +- .../common_grants_sdk/extensions/README.md | 152 +++++++++- .../common_grants_sdk/extensions/__init__.py | 29 ++ .../common_grants_sdk/extensions/generate.py | 10 +- .../common_grants_sdk/extensions/plugin.py | 69 ++++- .../extensions/transforms.py | 182 ++++++++++++ .../common_grants_sdk/extensions/types.py | 151 ++++++++++ .../common_grants_sdk/utils/transformation.py | 94 +++++- lib/python-sdk/examples/README.md | 100 +++++++ .../examples/plugins/grants_gov/__init__.py | 14 + .../examples/plugins/grants_gov/cg_config.py | 143 +++++++++ lib/python-sdk/examples/transforms.py | 281 ++++++++++++++++++ lib/python-sdk/tests/extensions/__init__.py | 0 .../tests/extensions/test_plugin.py | 73 +++++ .../tests/extensions/test_transforms.py | 213 +++++++++++++ lib/python-sdk/tests/extensions/test_types.py | 132 ++++++++ .../tests/utils/test_transformation.py | 125 +++++++- .../governance/adr/0022-plugin-framework.mdx | 26 +- 19 files changed, 1771 insertions(+), 25 deletions(-) create mode 100644 lib/python-sdk/.coverage create mode 100644 lib/python-sdk/common_grants_sdk/extensions/transforms.py create mode 100644 lib/python-sdk/common_grants_sdk/extensions/types.py create mode 100644 lib/python-sdk/examples/plugins/grants_gov/__init__.py create mode 100644 lib/python-sdk/examples/plugins/grants_gov/cg_config.py create mode 100644 lib/python-sdk/examples/transforms.py create mode 100644 lib/python-sdk/tests/extensions/__init__.py create mode 100644 lib/python-sdk/tests/extensions/test_plugin.py create mode 100644 lib/python-sdk/tests/extensions/test_transforms.py create mode 100644 lib/python-sdk/tests/extensions/test_types.py diff --git a/lib/python-sdk/.coverage b/lib/python-sdk/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..9ac35a7ec410f45428a8dc61ec83f94552ae0181 GIT binary patch literal 53248 zcmeI)%WoS+90%}SJGE=a)&^CT6@`+yKpHiUYgOWTfB-pER4Ro+4@kJ<^*G*kAMWlt zO%4d$B2_}_v7Au<9}bn^!Vz()=!qK=QhVU{+u8M_jjLYbsPJ9c&U$yApLzL}{ru7e zCsv{xgr1DWIb+E%P2&S048thVy-N3NE7HzF_JsbL6Z^AvOUCBif2!7(Mx}7qu)eNd zwCa_gs&6d*TJcIhEZ#55l1(SDK>z{}fWZG-V0fugtSv8_x4wyG+f{KWZ58Ur>PMe# zZ(ZINm$yE=ur2g4ak?yMtJlSr2!orVuR_svTqPX8MeTvz)WNq{uBSnb97041WF9iKyp(`oc|5vr~Vm2ay^ zm*TY3Sud~O{)MWnZ{$4rt49eW#i; zBj*;?u?D_#W1vQjzahpkHUP!D*0u~2*GglSN(RD014mtJqRiO?ba+PB;q$Zm4UdxPPQs(ebw$ukh789R zx^bvCPmY;Jh9N^9Xd*}hQb~`rw{c;q45IA@?diUj^q5G|0HL`c3)UU@HV>7E>tF}F zDwI2FGd-)Pg`@6B)ODJrLT!6>R@WJIfmU<0TOKYKinWy$^HwS8MrmuMe~MFGC~L_B zrqS$_r(`q;vx($+lVlcSJWFO{Bt1jbsTT{it(95TNyD|8>CW+CaiLf{dD0x}5yA5k z-;YgIf{%=cz*7ssix_`eM+rc1C`Y%6Cx+kwayOxwGTU>aWzf6NJ8(*~JdD8Es>lH7_CVvPo6L__qcG_yq0 zvpi#E>2LPQgj4loM-O*yl!+8?_S7@U>DB0C8LQawl;9vqVUmt@Q%1rL6`iEDbbT%V zyl%j))oSu1ze_&OFXe`-X0dkWj5!>ngHg{jt%w$NPiDnVpXR1I$x|oC{pBJ}xb$jg zvY)Ws1P4FWIq>ABTd=JavQX)3o$@-r!8fhP2Hmhh00Izz00bZa0SG_<0uX=z1R(I= z6Udu6v&irNbJib*^%p&0g8&2|009U<00Izz00bZa0SG|gr4*>-@~5oq7anJq%>2r7 z`Wt|^Hk;=*&z7lHIqQ*OJ+l6ODJzIVLjVF0fB*y_009U<00Izz00bZ~5~$=)nc2qx zxpID`lzs}JfB%2au#EUFyN$^4`mPG=J9KrRII8! zEDW!d=JNg@*Z-Sq!y+>XKmY;|fB*y_009U<00Izzz`OkNhA2 z0SG_<0uX=z1Rwwb2tWV=5SWKRK3^^A@Bbef*28(^h&&(w0SG_<0uX=z1Rwwb2tWV= z5I8h}Wpkladh^zAKUc)>e;Qq{@Zh`q=7V2eeY^OY{{H{5VLd*y+>tN@AOHafKmY;| zfB*y_009U<00KuRus|OQB;WrVOGj4-jNtEr7fs009U<00Izz00bZa0SG_< I0*5c~FLLaI$^ZZW literal 0 HcmV?d00001 diff --git a/lib/python-sdk/Makefile b/lib/python-sdk/Makefile index ba7ff350d..bd9d4614e 100644 --- a/lib/python-sdk/Makefile +++ b/lib/python-sdk/Makefile @@ -28,4 +28,4 @@ check-types: plugins checks: check-format check-lint check-types plugins: - $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions + $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions examples/plugins/grants_gov diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 9083276ce..0a3786f8f 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -22,6 +22,10 @@ The `common-grants/sdk/extensions` module contains the utilities for working wit - [Defining a plugin](#defining-a-plugin) - [Publishing a plugin](#publishing-a-plugin) - [Combining Plugins](#combining-plugins) +- [Bidirectional Transforms](#bidirectional-transforms) + - [Defining transforms](#defining-transforms) + - [Mapping format](#mapping-format) + - [Using transforms](#using-transforms) - [Using plugins with the API client](#using-plugins-with-the-api-client) - [Best practices](#best-practices) - [Field naming](#field-naming) @@ -37,11 +41,15 @@ The `common-grants/sdk/extensions` module contains the utilities for working wit Here are some key concepts that are used to define custom fields and plugins that extend base schemas from the CommonGrants protocol. | Concept | Description | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | -| **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | -| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | -| **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | +| **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | +| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | +| **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | +| **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | +| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict and returns a `TransformResult`. | +| **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | +| **`ObjectSchemasInput`**| Bundles a `to_common` and `from_common` callable for a single object type. Passed to `define_plugin()` via the `transform_schemas` parameter. | @@ -433,6 +441,140 @@ Prefer unique, namespaced field names so `"error"` is never triggered. After building your package, import the plugin in a test file and confirm that `.extensions` keys and `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. +## Bidirectional Transforms + +Plugins can define bidirectional mappings between a source system's native data format and the CommonGrants format. These transforms are authored as plain Python dicts and compiled into callable functions by `build_transforms()`. + +### Defining transforms + +Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `transform_schemas`: + +```python +from common_grants_sdk.extensions import ( + CustomFieldSpec, + ObjectSchemasInput, + PluginExtensionsMeta, + build_transforms, + define_plugin, +) +from common_grants_sdk.schemas.pydantic.fields import CustomFieldType + +to_common, from_common = build_transforms( + to_common_mapping={ + "title": {"field": "data.opportunity_title"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": {"posted": "open", "archived": "closed", "forecasted": "forecasted"}, + "default": "custom", + } + }, + "description": {"const": "The opportunity is currently accepting applications"}, + }, + "funding": { + "minAwardAmount": { + "amount": {"field": "data.summary.award_floor"}, + "currency": {"const": "USD"}, + }, + }, + }, + from_common_mapping={ + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "match": { + "field": "status.value", + "case": {"open": "posted", "closed": "archived", "forecasted": "forecasted"}, + "default": "custom", + } + }, + } + }, +) + +plugin = define_plugin( + extensions={ + "Opportunity": { + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Unique identifier in legacy database", + ), + } + }, + meta=PluginExtensionsMeta( + name="my-system", + version="0.1.0", + source_system="my-system.example.gov", + capabilities=["customFields", "transforms"], + ), + transform_schemas={ + "Opportunity": ObjectSchemasInput( + to_common=to_common, + from_common=from_common, + ) + }, +) +``` + +Both directions must be provided explicitly. `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. + +### Mapping format + +A mapping dict describes how to build an output object from a source dict. Each leaf node is either a literal value or a single-key dict that invokes a named handler. + +| Handler | Syntax | Description | +|---|---|---| +| `const` | `{"const": "USD"}` | Returns a fixed literal value, ignoring source data | +| `field` | `{"field": "data.summary.award_floor"}` | Extracts a value using a dot-notation path | +| `match` | `{"match": {"field": "...", "case": {...}, "default": "..."}}` | Case-based lookup on a field value (canonical ADR name) | +| `switch` | `{"switch": {...}}` | Alias for `match`, kept for backward compatibility | +| `numberToString` | `{"numberToString": "data.summary.award_floor"}` | Extracts a numeric value and coerces it to a string | +| `stringToNumber` | `{"stringToNumber": "some.string.field"}` | Extracts a string and coerces it to `int` or `float` | + +Bare non-dict values (strings, numbers, booleans) in a mapping are treated as literals and passed through unchanged. Use `{"const": ...}` when you want a literal value inside a dict node that might otherwise be mistaken for a field name. + +You can also register custom handlers by passing a `handlers` dict to `build_transforms()`: + +```python +def handle_upper(data, field_path): + val = get_from_path(data, field_path) + return val.upper() if isinstance(val, str) else val + +to_common, from_common = build_transforms( + to_common_mapping={"title": {"upper": "data.opportunity_title"}}, + from_common_mapping={...}, + handlers={"upper": handle_upper}, +) +``` + +Custom handlers are merged with the defaults; they cannot override built-in handler names. + +### Using transforms + +The compiled callables are stored on the plugin's `transform_schemas` dict, keyed by object name. Each callable takes a data dict and returns a `TransformResult`: + +```python +opp_transforms = plugin.transform_schemas["Opportunity"] + +# Source system → CommonGrants +result = opp_transforms.to_common(native_data) +if result.errors: + for err in result.errors: + print(f"[{err.path}] {err}") +else: + cg_data = result.result + +# CommonGrants → source system +result = opp_transforms.from_common(cg_data) +native_data = result.result +``` + +`TransformResult.errors` is always a list (empty on success). A non-empty errors list means the transform encountered a problem but still returned a partial result in `result`. + +See `examples/transforms.py` for a complete working example with roundtrip verification. + + ## Using plugins with the API client Pass a plugin's extended schema to the API client via the `schema` parameter. The client uses it to hydrate API responses into fully typed models. The `schema` parameter accepts any `Type[OpportunityBase]` subclass. diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 87a089683..7a9ef1185 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -2,8 +2,23 @@ from .plugin import Plugin, PluginConfig, define_plugin from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions, merge_extensions +from .transforms import build_transforms +from .types import ( + ClientConfig, + Handler, + ObjectMappings, + ObjectSchemas, + ObjectSchemasInput, + PluginCapability, + PluginError, + PluginExtensions, + PluginExtensionsMeta, + PluginExtensionsSchema, + TransformResult, +) __all__ = [ + # Existing exports (unchanged) "ConflictStrategy", "CustomFieldSpec", "Plugin", @@ -11,4 +26,18 @@ "SchemaExtensions", "define_plugin", "merge_extensions", + # New: build_transforms + "build_transforms", + # New: ADR-0022 types + "ClientConfig", + "Handler", + "ObjectMappings", + "ObjectSchemas", + "ObjectSchemasInput", + "PluginCapability", + "PluginError", + "PluginExtensions", + "PluginExtensionsMeta", + "PluginExtensionsSchema", + "TransformResult", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py index 40eba082b..9657a84a1 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ b/lib/python-sdk/common_grants_sdk/extensions/generate.py @@ -487,13 +487,15 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "--plugin", - default=".", - help="Path to plugin directory containing cg_config.py (default: current directory)", + nargs="+", + default=["."], + help="One or more plugin directories containing cg_config.py (default: current directory)", ) args = parser.parse_args(argv) - generated_dir = generate_plugin(Path(args.plugin)) - print(f"Generated plugin schemas at {generated_dir}") + for plugin_path in args.plugin: + generated_dir = generate_plugin(Path(plugin_path)) + print(f"Generated plugin schemas at {generated_dir}") return 0 diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 7b4670f34..eab8cb8f4 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -1,28 +1,81 @@ """Plugin configuration and composition APIs.""" +from __future__ import annotations + from dataclasses import dataclass -from typing import Generic, TypeVar +from typing import Any, Callable, Generic, TypeVar from .specs import SchemaExtensions +from .types import ClientConfig, ObjectSchemas, ObjectSchemasInput, PluginExtensionsMeta T = TypeVar("T") @dataclass(frozen=True) class PluginConfig: - """Build-time plugin config discoverable by the generator.""" + """Build-time plugin config discoverable by the code generator. + + extensions: custom field declarations (read by generate.py — do not rename). + meta: optional plugin identity and capability declaration. + transform_schemas: optional bidirectional transform callables per object. + Stored as ObjectSchemasInput (not compiled to ObjectSchemas) in the PoC. + Full compilation with model_validate wrapping is a TODO for the real SDK. + + TODO (full SDK): add get_client, filters. + """ extensions: SchemaExtensions + meta: PluginExtensionsMeta | None = None + transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None -@dataclass(frozen=True) +@dataclass class Plugin(Generic[T]): - """Runtime plugin container with both extension specs and generated schemas.""" + """Runtime plugin container with extension specs and generated schemas. + + extensions: SchemaExtensions used by generate.py (do not rename or reorder — + the generated __init__.py constructs Plugin(extensions=..., schemas=...)). + schemas: generated _Schemas object (typed Pydantic model classes from generate.py). + NOTE: there is a naming collision: ADR-0022 also calls its runtime transform + dict "schemas". These are different concepts sharing the same name — a design + question to resolve in the full SDK (see Design Finding #1 in the spec). + transform_schemas: ADR-0022 runtime transform dict; named distinctly from + `schemas` to avoid collision with the generated schemas field in the PoC. + + TODO (full SDK): memoize get_client. + """ extensions: SchemaExtensions - schemas: T + schemas: T # generated _Schemas object — keep as positional for generate.py compat + meta: PluginExtensionsMeta | None = None + get_client: Callable[[ClientConfig], Any] | None = None # TODO: memoize + # PoC stores ObjectSchemasInput here (no compilation yet); full SDK will store + # ObjectSchemas after model_validate wrapping. Annotated as the union so both + # the current PoC usage and the future compiled form are type-safe. + transform_schemas: ( + dict[str, ObjectSchemasInput[Any, Any] | ObjectSchemas[Any, Any]] | None + ) = None + filters: dict[str, dict[str, Any]] | None = None + + +def define_plugin( + extensions: SchemaExtensions, + meta: PluginExtensionsMeta | None = None, + transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, + # TODO (full SDK): get_client, filters +) -> PluginConfig: + """Create a PluginConfig object consumed by the code generator. + Backward-compatible: existing callers passing only `extensions` are unaffected. + New params are stored as-is — no compilation occurs in the PoC. -def define_plugin(extensions: SchemaExtensions) -> PluginConfig: - """Create a plugin config object consumed by the code generator.""" - return PluginConfig(extensions=extensions) + TODO (full SDK): + - Auto-generate transforms from extensions.schemas[obj].mappings when no + explicit to_common/from_common is supplied. + - Wrap transform output with model_validate. + """ + return PluginConfig( + extensions=extensions, + meta=meta, + transform_schemas=transform_schemas, + ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py new file mode 100644 index 000000000..c2aaacdd4 --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -0,0 +1,182 @@ +"""build_transforms() — generates to_common/from_common callables from mapping dicts. + +Using this utility is optional — plugin authors may provide plain hand-written +callables instead. + +Mappings are validated at call time. Custom handler names are +registered per call only; name collisions with defaults raise at call time +rather than silently shadowing them. +""" + +from __future__ import annotations + +from typing import Any, Callable + +from pydantic import BaseModel, ValidationError + +from common_grants_sdk.utils.transformation import ( + DEFAULT_HANDLERS, + HandlerError, + transform_from_mapping, +) + +from .types import Handler, PluginError, TransformResult + + +def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> None: + """Walk the mapping tree and raise ValueError on structural malformation. + + For each dict node: + - If a key is a known handler, the node must contain ONLY that handler key. + The corresponding value is a runtime-only handler argument and is NOT + recursed into. + - All other keys are output field names (always valid); their values are + recursed into. + + Raises ValueError if any node is not a dict, string, number, boolean, or None + (e.g. a list where a scalar or dict is expected), or if a handler key appears + alongside sibling keys in the same dict (ambiguous — handler invocations must + be the sole key in their dict). + + Note: this function cannot detect intended-but-unknown handler invocations + because unknown keys are indistinguishable from output field names at static + analysis time. That detection is deferred to the full SDK. + """ + if mapping is None or isinstance(mapping, (str, int, float, bool)): + return # primitives and None are valid literals + + if not isinstance(mapping, dict): + raise ValueError( + f"Invalid mapping node at '{path}': expected dict, str, number, or bool, " + f"got {type(mapping).__name__}" + ) + + handler_keys = [k for k in mapping if k in known_handlers] + if handler_keys and len(mapping) > 1: + label = f" at '{path}'" if path else "" + raise ValueError( + f"Invalid mapping node{label}: handler key {handler_keys[0]!r} " + f"cannot have sibling keys {sorted(k for k in mapping if k not in known_handlers)!r}. " + f"A handler invocation must be the only key in its dict." + ) + + for key, value in mapping.items(): + current_path = f"{path}.{key}" if path else key + if key in known_handlers: + # Handler invocation — argument is runtime-only, do not recurse + continue + _validate_mapping(value, known_handlers, current_path) + + +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = None, + common_model: type[BaseModel] | None = None, +) -> tuple[ + Callable[[Any], TransformResult[Any]], + Callable[[Any], TransformResult[Any]], +]: + """Generate to_common and from_common callables from mapping dicts. + + Args: + to_common_mapping: mapping from native source → CommonGrants. + from_common_mapping: mapping from CommonGrants → native source. + handlers: Optional additional handlers registered for this call only. + Keys must not collide with DEFAULT_HANDLERS (raises ValueError if they do). + common_model: Optional Pydantic model class to validate the to_common output + against. Must be the fully extended generated model class (e.g. the + generated Opportunity from generated/schemas.py), NOT the base class + (e.g. OpportunityBase). Passing a base class will silently weaken + validation — custom_fields will only be checked against + dict[str, CustomField] rather than the typed container produced by the + plugin's custom field declarations. When provided, model_validate is + called on the transform result and any ValidationErrors are appended to + TransformResult.errors rather than raised. + + Note on result shape: when common_model is set, TransformResult.result + holds the validated Pydantic instance on success, or the raw transformed + dict on ValidationError (so callers can inspect the malformed data + alongside the errors). This is intentional — check TransformResult.errors + before consuming TransformResult.result. + + Returns: + A (to_common, from_common) tuple. Each callable accepts a dict and returns + TransformResult[Any]. Failures surface as PluginError entries in + TransformResult.errors rather than being raised. + + Raises: + ValueError: At call time if handler names collide with defaults, + or if either mapping has structural malformation. + + TODO (full SDK): + - Validate field-path resolvability at call time (requires sample data or + schema introspection). + """ + # Custom handler names must not shadow defaults + if handlers: + collisions = set(handlers) & set(DEFAULT_HANDLERS) + if collisions: + raise ValueError( + f"build_transforms: handler names collide with defaults: {sorted(collisions)}" + ) + + merged = {**DEFAULT_HANDLERS, **(handlers or {})} + known = set(merged) + + # Validate mapping structure at call time + _validate_mapping(to_common_mapping, known) + _validate_mapping(from_common_mapping, known) + + def to_common(native: Any) -> TransformResult[Any]: + try: + result = transform_from_mapping(native, to_common_mapping, handlers=merged) + except HandlerError as exc: + error = PluginError( + str(exc.cause), + path=None, + handler=exc.handler, + source_value=native, + cause=exc.cause, + ) + return TransformResult(result={}, errors=[error]) + except Exception as exc: + error = PluginError(str(exc), path=None, source_value=native, cause=exc) + return TransformResult(result={}, errors=[error]) + + if common_model is None: + return TransformResult(result=result, errors=[]) + + try: + validated = common_model.model_validate(result) + return TransformResult(result=validated, errors=[]) + except ValidationError as exc: + errors = [ + PluginError( + e["msg"], + path=".".join(str(loc) for loc in e["loc"]), + ) + for e in exc.errors() + ] + return TransformResult(result=result, errors=errors) + + def from_common(common: Any) -> TransformResult[Any]: + try: + result = transform_from_mapping( + common, from_common_mapping, handlers=merged + ) + return TransformResult(result=result, errors=[]) + except HandlerError as exc: + error = PluginError( + str(exc.cause), + path=None, + handler=exc.handler, + source_value=common, + cause=exc.cause, + ) + return TransformResult(result={}, errors=[error]) + except Exception as exc: + error = PluginError(str(exc), path=None, source_value=common, cause=exc) + return TransformResult(result={}, errors=[error]) + + return to_common, from_common diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py new file mode 100644 index 000000000..8be07358a --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -0,0 +1,151 @@ +"""Plugin framework types for the CommonGrants Python SDK.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Generic, Literal, TypeVar + +from pydantic import BaseModel, ConfigDict, Field + +from .specs import CustomFieldSpec + +TNative = TypeVar("TNative") +TCommon = TypeVar("TCommon") +T = TypeVar("T") + +# Capability enum — Literal rather than StrEnum to stay JSON-safe. +PluginCapability = Literal["customFields", "customFilters", "transforms", "client"] + +# Type aliases +Handler = Callable[[Any, Any], Any] +ClientConfig = dict[str, Any] + + +class PluginError(Exception): + """Structured transformation error per ADR-0022 Decision #9. + + Carries field path, handler name, source value, and underlying cause so + consumers can reason about failures programmatically without parsing error text. + + Note: source_value may contain PII when transforming applicant data. + Adopters are responsible for redacting it before logging or re-raising. + The SDK does not redact by default. + """ + + def __init__( + self, + message: str, + *, + path: str | None = None, + handler: str | None = None, + source_value: Any = None, + cause: BaseException | None = None, + ) -> None: + super().__init__(message) + self.path = path + self.handler = handler + self.source_value = source_value + self.cause = cause + + +@dataclass +class TransformResult(Generic[T]): + """Unconditional return shape for to_common / from_common (ADR-0022 Decision #7). + + result: the transformed value (may be partial on error). + errors: aggregated PluginErrors; empty on full success. + + Consumers apply their own strict-vs-lenient rule for what counts as success: + - Strict: treat any non-empty errors as failure. + - Lenient: use result despite warnings; inspect errors for context. + """ + + result: T + errors: list[PluginError] + + +class ObjectMappings(BaseModel): + """ADR-0017 mapping dicts for a single object, stored in the serializable extensions config. + + Each direction is author-provided — build_transforms() does not invert one into + the other because many-to-one handlers like switch are not reversible (Decision #6). + """ + + model_config = ConfigDict(populate_by_name=True) + + to_common: dict[str, Any] | None = Field(default=None, alias="toCommon") + from_common: dict[str, Any] | None = Field(default=None, alias="fromCommon") + + +class PluginExtensionsMeta(BaseModel): + """Plugin identity and capability declaration. All fields are optional.""" + + model_config = ConfigDict(populate_by_name=True) + + name: str | None = None + version: str | None = None + source_system: str | None = Field(default=None, alias="sourceSystem") + capabilities: list[PluginCapability] | None = None + + +class PluginExtensionsSchema(BaseModel): + """Per-object config inside extensions.schemas. + + custom_fields: custom field declarations (merged by merge_extensions). + mappings: optional ADR-0017 declarative mappings. When present and no explicit + to_common / from_common is supplied in schemas[obj], define_plugin() will + auto-invoke build_transforms() on these (TODO — ADR-0022 Decision #6). + """ + + model_config = ConfigDict(populate_by_name=True) + + custom_fields: dict[str, CustomFieldSpec] | None = Field( + default=None, alias="customFields" + ) + mappings: ObjectMappings | None = None + + +class PluginExtensions(BaseModel): + """Serializable portion of plugin config — safe to store as JSON. + + Used by merge_extensions() to combine declarations from multiple plugin packages. + """ + + model_config = ConfigDict(populate_by_name=True) + + meta: PluginExtensionsMeta | None = None + schemas: dict[str, PluginExtensionsSchema] | None = None + + +@dataclass +class ObjectSchemasInput(Generic[TNative, TCommon]): + """Input type provided by plugin authors inside define_plugin(schemas=...). + + Plugin authors supply to_common and from_common as plain callables — either + hand-written or generated via build_transforms(). native defaults to + dict[str, Any] if omitted. + + common is intentionally absent here. It is injected by define_plugin() during + compilation from ObjectSchemasInput → ObjectSchemas, resolved from the generated + model classes produced by the code generator. Plugin authors never set it directly — + cg_config.py cannot import from generated/ (it is the input to generation). + """ + + native: type[TNative] | None = None + to_common: Callable[[TNative], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TNative]] | None = None + + +@dataclass +class ObjectSchemas(Generic[TNative, TCommon]): + """Runtime compiled type produced by define_plugin() — not provided directly by authors. + + In the PoC, define_plugin() stores ObjectSchemasInput as-is; full compilation + (adding common from the base CG model, wrapping with model_validate) is a TODO + for the real SDK (ADR-0022 Decision #7). + """ + + native: type[TNative] + common: type[TCommon] + to_common: Callable[[TNative], TransformResult[TCommon]] + from_common: Callable[[TCommon], TransformResult[TNative]] diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index 10153a71e..bb0778122 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -64,10 +64,84 @@ def switch_on_value(data: dict, switch_spec: dict) -> Any: return lookup.get(val, switch_spec.get("default")) +def const_value(_data: dict, value: Any) -> Any: + """ + Handles a const transformation by returning a fixed literal value. + + Args: + _data: The source data dictionary (unused) + value: The constant value to return + + Returns: + The constant value exactly as specified + """ + return value + + +def number_to_string(data: dict, field_path: str) -> str | None: + """ + Handles a numberToString transformation by extracting a numeric value and coercing it to a string. + + Args: + data: The source data dictionary + field_path: A dot-separated string representing the path to the numeric value + + Returns: + The value at the specified path converted to a string, or None if the path doesn't exist + """ + val = get_from_path(data, field_path) + return str(val) if val is not None else None + + +def string_to_number(data: dict, field_path: str) -> int | float | None: + """ + Handles a stringToNumber transformation by extracting a string value and coercing it to a number. + + Attempts integer conversion first; falls back to float for decimal strings. + + Args: + data: The source data dictionary + field_path: A dot-separated string representing the path to the string value + + Returns: + The value at the specified path converted to int or float, or None if the path doesn't exist + + Raises: + ValueError: If the extracted value cannot be converted to a number + """ + val = get_from_path(data, field_path) + if val is None: + return None + s = str(val) + try: + return int(s) + except ValueError: + return float(s) + + +class HandlerError(ValueError): + """Raised when a handler function raises, carrying the handler name for attribution. + + Extends ValueError so that existing ``except ValueError`` handlers around + ``transform_from_mapping``, ``dump_with_mapping``, and ``validate_with_mapping`` + continue to work after this class was introduced. Callers that want handler-level + attribution can catch ``HandlerError`` specifically (it is more derived). + """ + + def __init__(self, handler: str, cause: Exception) -> None: + super().__init__(str(cause)) + self.handler = handler + self.cause = cause + + # Registry for handlers DEFAULT_HANDLERS: dict[str, handle_func] = { + "const": const_value, "field": pluck_field_value, - "switch": switch_on_value, + "match": switch_on_value, # ADR-0017 canonical name + "numberToString": number_to_string, + "stringToNumber": string_to_number, + "switch": switch_on_value, # alias kept for backward compatibility } @@ -83,8 +157,12 @@ def transform_from_mapping( The mapping supports both literal values and transformations keyed by the following reserved words: + - `const`: Returns a fixed literal value regardless of input data - `field`: Extracts a value from the data using a dot-notation path - - `switch`: Performs a case-based lookup based on a field value + - `match`: Performs a case-based lookup based on a field value (canonical) + - `numberToString`: Extracts a numeric value and coerces it to a string + - `stringToNumber`: Extracts a string value and coerces it to int or float + - `switch`: Alias for `match` (kept for backward compatibility) Args: data: The source data dictionary to transform @@ -123,6 +201,13 @@ def transform_from_mapping( } ``` """ + # Normalize Pydantic model instances to plain dicts so that field path + # extraction works regardless of whether the caller passes a raw dict or a + # validated model (e.g. the output of to_common with common_model set). + # mode="json" matches the convention used by CommonGrantsBaseModel.dump_with_mapping. + if hasattr(data, "model_dump"): + data = data.model_dump(mode="json") + # Check for maximum depth # This is a sanity check to prevent stack overflow from deeply nested mappings # which may be a concern when running this function on third-party mappings @@ -150,7 +235,10 @@ def transform_node(node: Any, depth: int) -> Any: # Returns: `extract_field_value(data, "opportunity_status")` if k in handlers: handler_func = handlers[k] - return handler_func(data, v) + try: + return handler_func(data, v) + except Exception as exc: + raise HandlerError(k, exc) from exc # Otherwise, preserve the dictionary structure and # recursively apply the transformation to each value. diff --git a/lib/python-sdk/examples/README.md b/lib/python-sdk/examples/README.md index d624cffa5..96b7bbc67 100644 --- a/lib/python-sdk/examples/README.md +++ b/lib/python-sdk/examples/README.md @@ -119,6 +119,106 @@ None ``` +# Bidirectional transforms example + +This example demonstrates the plugin transform framework: mapping source system data (grants.gov format) to the CommonGrants format and back again, with a roundtrip consistency check. No API server is required — the script runs entirely offline using sample data defined in the file itself. + +**Step 1:** Generate the typed models for the grants.gov plugin (only needed once, or after changing `cg_config.py`): + +```bash +cd lib/python-sdk +poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov +``` + +Or generate all example plugins at once with: + +```bash +make plugins +``` + +**Step 2:** Run the example: + +```bash +poetry run python examples/transforms.py +``` + +**Output Example:** +``` +============================================================ +SOURCE DATA (grants.gov format) +============================================================ +{ + "data": { + "agency_name": "Department of Examples", + "created_at": "2025-01-15T09:00:00Z", + "last_modified_at": "2025-04-01T12:30:00Z", + "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", + "opportunity_id": 12345, + "opportunity_number": "ABC-123-XYZ-001", + "opportunity_status": "posted", + "opportunity_title": "Research into conservation techniques", + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "summary": { + "applicant_types": ["state_governments"], + "archive_date": "2025-05-01", + "award_ceiling": 100000, + "award_floor": 10000, + "forecasted_award_date": "2025-09-01", + "forecasted_close_date": "2025-07-15", + "forecasted_post_date": "2025-05-01" + } + } +} + +============================================================ +to_common: grants.gov → CommonGrants +============================================================ +Errors: none + +Result: +{ + "title": "Research into conservation techniques", + "status": { "value": "open", "description": "The opportunity is currently accepting applications" }, + "funding": { + "minAwardAmount": { "amount": 10000, "currency": "USD" }, + "maxAwardAmount": { "amount": 100000, "currency": "USD" } + }, + ... +} + +============================================================ +from_common: CommonGrants → grants.gov +============================================================ +Errors: none + +Result: +{ + "data": { + "opportunity_title": "Research into conservation techniques", + "opportunity_status": "posted", + "summary": { + "award_floor": 10000, + "award_ceiling": 100000, + "forecasted_post_date": "2025-05-01", + "forecasted_close_date": "2025-07-15" + } + } +} + +============================================================ +ROUNDTRIP CHECK +============================================================ + [PASS] title: 'Research into conservation techniques' -> 'Research into conservation techniques' + [PASS] status: 'posted' -> 'posted' + [PASS] award_floor: 10000 -> 10000 + [PASS] award_ceiling: 100000 -> 100000 + +Roundtrip result: ALL PASS +``` + +The transform mappings live in `examples/plugins/grants_gov/cg_config.py`. See the [extensions README](../common_grants_sdk/extensions/README.md#bidirectional-transforms) for a full explanation of the mapping format. + + # Plugin framework example This example uses the plugin framework to define four typed custom fields, generate static Pydantic models, and validate an API payload. diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py new file mode 100644 index 000000000..5ae38feac --- /dev/null +++ b/lib/python-sdk/examples/plugins/grants_gov/__init__.py @@ -0,0 +1,14 @@ +# This file is auto-generated. Do not edit it manually — it will be overwritten +# the next time `python -m common_grants_sdk.extensions.generate` is run. +from __future__ import annotations + +from common_grants_sdk.extensions import Plugin +from .cg_config import config +from .generated import schemas + +grants_gov = Plugin( + extensions=config.extensions, + schemas=schemas, +) + +__all__ = ["grants_gov", "schemas"] diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py new file mode 100644 index 000000000..b3f28b5e3 --- /dev/null +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -0,0 +1,143 @@ +"""Grants.gov sample plugin — bidirectional transform PoC. + +Demonstrates the plugin framework shape using the grants.gov scenario. + +Usage (from lib/python-sdk/): + poetry run python examples/transforms.py + +Code generation (generates typed custom-field schemas): + poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov +""" + +from common_grants_sdk.extensions import ( + CustomFieldSpec, + ObjectSchemasInput, + PluginExtensionsMeta, + build_transforms, + define_plugin, +) +from common_grants_sdk.schemas.pydantic.fields import CustomFieldType + +# --------------------------------------------------------------------------- +# Bidirectional transforms +# +# Both directions are author-provided — build_transforms() does not invert +# one into the other because many-to-one handlers like switch are not +# reversible. +# +# Convention: field extraction uses {"field": "dot.notation.path"} — bare +# string values are treated as literals by transform_from_mapping(), not +# as field paths. See Design Finding #2 in the spec for the open question +# about which convention is canonical. +# --------------------------------------------------------------------------- + +to_common, from_common = build_transforms( + # to_common: grants.gov native → CommonGrants Opportunity + to_common_mapping={ + "title": {"field": "data.opportunity_title"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": { + "forecasted": "forecasted", + "posted": "open", + "archived": "closed", + }, + "default": "custom", + } + }, + "description": { + "const": "The opportunity is currently accepting applications" + }, + }, + "funding": { + "minAwardAmount": { + "amount": {"field": "data.summary.award_floor"}, + "currency": {"const": "USD"}, + }, + "maxAwardAmount": { + "amount": {"field": "data.summary.award_ceiling"}, + "currency": {"const": "USD"}, + }, + }, + "keyDates": { + "appOpens": { + "name": {"const": "Open Date"}, + "date": {"field": "data.summary.forecasted_post_date"}, + "description": {"const": "Applications begin being accepted"}, + }, + "appDeadline": { + "name": {"const": "Application Deadline"}, + "date": {"field": "data.summary.forecasted_close_date"}, + "description": { + "const": "Final submission deadline for all grant applications" + }, + }, + }, + }, + # from_common: CommonGrants Opportunity → grants.gov native + from_common_mapping={ + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "match": { + "field": "status.value", + "case": { + "open": "posted", + "closed": "archived", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "summary": { + "award_floor": {"field": "funding.minAwardAmount.amount"}, + "award_ceiling": {"field": "funding.maxAwardAmount.amount"}, + "forecasted_post_date": {"field": "keyDates.appOpens.date"}, + "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, + }, + } + }, +) + +# --------------------------------------------------------------------------- +# Plugin config +# --------------------------------------------------------------------------- + +plugin = define_plugin( + # extensions: SchemaExtensions — dict[str, dict[str, CustomFieldSpec]] + extensions={ + "Opportunity": { + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + name="Legacy ID", + description="Unique identifier in legacy database", + ), + "agencyName": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Agency", + description="Agency hosting the opportunity", + ), + "applicantTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + name="Applicant types", + description="Types of applicants eligible to apply", + ), + } + }, + meta=PluginExtensionsMeta( + name="grants-gov", + version="0.1.0", + sourceSystem="grants.gov", + capabilities=["customFields", "transforms"], + ), + transform_schemas={ + "Opportunity": ObjectSchemasInput( + to_common=to_common, + from_common=from_common, + ) + }, +) + +config = plugin diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py new file mode 100644 index 000000000..030521058 --- /dev/null +++ b/lib/python-sdk/examples/transforms.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Bidirectional transform PoC — plugin transformation interface. + +Demonstrates source (grants.gov) → CommonGrants and CommonGrants → source +bidirectional transformations using the grants.gov sample plugin. + +Requires generated schemas (examples/plugins/grants_gov/generated/). +Generate them first (from lib/python-sdk/): + poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov +Or run all plugins at once: + make plugins + +Then run (from lib/python-sdk/): + poetry run python examples/transforms.py +""" + +from __future__ import annotations + +import json +from typing import Any + +# When run as `poetry run python examples/transforms.py`, Python automatically +# adds the script's directory (examples/) to sys.path. Import from there using +# the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only +# works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. +from plugins.grants_gov.cg_config import plugin +from plugins.grants_gov.generated.schemas import Opportunity + +from common_grants_sdk.extensions import build_transforms +from common_grants_sdk.utils.transformation import get_from_path + +# --------------------------------------------------------------------------- +# Sample grants.gov source data +# --------------------------------------------------------------------------- + +SOURCE_DATA: dict[str, Any] = { + "data": { + "agency_name": "Department of Examples", + "created_at": "2025-01-15T09:00:00Z", + "last_modified_at": "2025-04-01T12:30:00Z", + "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", + "opportunity_id": 12345, + "opportunity_number": "ABC-123-XYZ-001", + "opportunity_status": "posted", + "opportunity_title": "Research into conservation techniques", + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "summary": { + "applicant_types": ["state_governments"], + "archive_date": "2025-05-01", + "award_ceiling": 100000, + "award_floor": 10000, + "forecasted_award_date": "2025-09-01", + "forecasted_close_date": "2025-07-15", + "forecasted_post_date": "2025-05-01", + }, + } +} + + +# --------------------------------------------------------------------------- +# Custom handlers: join_fields and split_field +# +# join_fields concatenates multiple source field values with a configurable +# separator. Mapping spec: {"join": {"fields": ["a.b", "c.d"], "sep": " — "}} +# +# split_field is the inverse: it splits a single field on a separator and +# returns the element at the given index. +# Mapping spec: {"split": {"field": "label", "sep": " — ", "index": 0}} +# --------------------------------------------------------------------------- + + +def join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + """Custom handler that joins multiple field values with a separator.""" + sep = spec.get("sep", " ") + parts = [get_from_path(data, path) for path in spec.get("fields", [])] + values = [str(p) for p in parts if p is not None] + return sep.join(values) if values else None + + +def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + """Custom handler that splits a field value and returns the element at index.""" + value = get_from_path(data, spec.get("field", "")) + if value is None: + return None + parts = str(value).split(spec.get("sep", " ")) + index = spec.get("index", 0) + return parts[index] if index < len(parts) else None + + +# Transform that uses the custom handlers and validates output against the generated +# Opportunity model. common_model=Opportunity (from generated/schemas.py) ensures +# model_validate runs against the extended class with typed custom fields +# (legacyId, agencyName, applicantTypes), not just the base OpportunityBase. +to_common_with_custom, from_common_with_custom = build_transforms( + to_common_mapping={ + "id": {"field": "data.opportunity_uuid"}, + "title": {"field": "data.opportunity_title"}, + "description": {"field": "data.opportunity_description"}, + "createdAt": {"field": "data.created_at"}, + "lastModifiedAt": {"field": "data.last_modified_at"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": { + "posted": "open", + "archived": "closed", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + }, + "label": { + "join": { + "fields": ["data.opportunity_number", "data.opportunity_title"], + "sep": " — ", + } + }, + "customFields": { + "legacyId": { + "value": {"field": "data.opportunity_id"}, + }, + "agencyName": { + "value": {"field": "data.agency_name"}, + }, + "applicantTypes": { + "value": {"field": "data.summary.applicant_types"}, + }, + }, + }, + from_common_mapping={ + "data": { + # label is produced by the join handler above but gets dropped by + # model_validate (it is not a CG field), so from_common maps directly + # from the standard CG title field instead. + "opportunity_title": {"field": "title"}, + } + }, + handlers={"join": join_fields, "split": split_field}, + common_model=Opportunity, +) + + +def _section(title: str) -> None: + print(f"\n{'=' * 60}") + print(title) + print("=" * 60) + + +def main() -> None: + assert plugin.transform_schemas is not None + opp = plugin.transform_schemas["Opportunity"] + + _section("SOURCE DATA (grants.gov format)") + print(json.dumps(SOURCE_DATA, indent=2)) + + # --- to_common: grants.gov → CommonGrants --- + assert opp.to_common is not None + cg_result = opp.to_common(SOURCE_DATA) + + _section("to_common: grants.gov → CommonGrants") + if cg_result.errors: + print(f"ERRORS ({len(cg_result.errors)}):") + for err in cg_result.errors: + print(f" [path={err.path}] {err}") + else: + print("Errors: none") + print("\nResult:") + print(json.dumps(cg_result.result, indent=2)) + + # --- from_common: CommonGrants → grants.gov --- + assert opp.from_common is not None + native_result = opp.from_common(cg_result.result) + + _section("from_common: CommonGrants → grants.gov") + if native_result.errors: + print(f"ERRORS ({len(native_result.errors)}):") + for err in native_result.errors: + print(f" [path={err.path}] {err}") + else: + print("Errors: none") + print("\nResult:") + print(json.dumps(native_result.result, indent=2)) + + # --- Roundtrip comparison --- + # Note: SOURCE_DATA contains fields not covered by the mappings (agency_name, + # opportunity_id, etc.). Those fields are intentionally absent from the roundtrip + # output — the mapping layer is selective by design. + _section("ROUNDTRIP CHECK") + checks = [ + ( + "title", + SOURCE_DATA["data"]["opportunity_title"], + native_result.result.get("data", {}).get("opportunity_title"), + ), + ( + "status", + SOURCE_DATA["data"]["opportunity_status"], + native_result.result.get("data", {}).get("opportunity_status"), + ), + ( + "award_floor", + SOURCE_DATA["data"]["summary"]["award_floor"], + native_result.result.get("data", {}).get("summary", {}).get("award_floor"), + ), + ( + "award_ceiling", + SOURCE_DATA["data"]["summary"]["award_ceiling"], + native_result.result.get("data", {}) + .get("summary", {}) + .get("award_ceiling"), + ), + ] + all_pass = True + for field, original, roundtripped in checks: + ok = original == roundtripped + if not ok: + all_pass = False + status = "PASS" if ok else "FAIL" + print(f" [{status}] {field}: {original!r} -> {roundtripped!r}") + + print( + f"\nRoundtrip result ({len(checks)} mapped fields checked; unmapped fields dropped by design): {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}" + ) + + # --- Custom handler + model_validate demo --- + _section( + "CUSTOM HANDLER + MODEL VALIDATE DEMO (join / split / extended Opportunity)" + ) + print("Custom handlers: join, split") + print("common_model: generated Opportunity (with typed customFields)\n") + + custom_cg = to_common_with_custom(SOURCE_DATA) + + if custom_cg.errors: + print(f"ERRORS ({len(custom_cg.errors)}):") + for err in custom_cg.errors: + print(f" [path={err.path}] {err}") + else: + print("Validation: PASS — result is a typed Opportunity instance") + opp_instance = custom_cg.result + print(f"\n title: {opp_instance.title}") + print(f" id: {opp_instance.id}") + print(f" status: {opp_instance.status.value}") + if opp_instance.custom_fields: + cf = opp_instance.custom_fields + print("\n customFields (typed):") + if cf.legacy_id: + print( + f" legacyId.value: {cf.legacy_id.value!r} ({type(cf.legacy_id.value).__name__})" + ) + if cf.agency_name: + print( + f" agencyName.value: {cf.agency_name.value!r} ({type(cf.agency_name.value).__name__})" + ) + if cf.applicant_types: + print( + f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})" + ) + + custom_native = from_common_with_custom( + custom_cg.result if not custom_cg.errors else {} + ) + orig_title = SOURCE_DATA["data"]["opportunity_title"] + rt_title = custom_native.result.get("data", {}).get("opportunity_title") + print( + f"\n [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}" + ) + + # --- Plugin metadata --- + _section("PLUGIN METADATA") + assert plugin.meta is not None + print(f"name: {plugin.meta.name}") + print(f"version: {plugin.meta.version}") + print(f"sourceSystem: {plugin.meta.source_system}") + print(f"capabilities: {plugin.meta.capabilities}") + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/tests/extensions/__init__.py b/lib/python-sdk/tests/extensions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py new file mode 100644 index 000000000..8fcadc799 --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -0,0 +1,73 @@ +"""Tests for expanded plugin.py — backward compat + new optional fields.""" + +from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin +from common_grants_sdk.extensions.specs import SchemaExtensions +from common_grants_sdk.extensions.types import ( + ObjectSchemasInput, + PluginExtensionsMeta, + TransformResult, +) + +EXTENSIONS: SchemaExtensions = {} # minimal valid extensions + + +def test_define_plugin_backward_compat(): + """define_plugin(extensions=...) still returns PluginConfig with all optional fields None.""" + config = define_plugin(extensions=EXTENSIONS) + assert isinstance(config, PluginConfig) + assert config.extensions is EXTENSIONS + assert config.meta is None + assert config.transform_schemas is None + + +def test_define_plugin_with_meta_and_schemas(): + meta = PluginExtensionsMeta(name="test", source_system="test-system") + + def passthrough(x): + return TransformResult(result=x, errors=[]) + + schemas = { + "Opportunity": ObjectSchemasInput( + to_common=passthrough, from_common=passthrough + ) + } + config = define_plugin(extensions=EXTENSIONS, meta=meta, transform_schemas=schemas) + assert config.meta is meta + assert config.meta.name == "test" + assert config.transform_schemas is schemas + + +def test_plugin_fields(): + """Plugin accepts all fields; optional ones default to None.""" + base = Plugin(extensions=EXTENSIONS, schemas=object()) + assert base.meta is None + assert base.get_client is None + assert base.transform_schemas is None + assert base.filters is None + + meta = PluginExtensionsMeta(name="p", source_system="s") + schemas = {"Opportunity": object()} + full = Plugin( + extensions=EXTENSIONS, schemas=object(), meta=meta, transform_schemas=schemas + ) + assert full.meta is meta + assert full.transform_schemas is schemas + + +def test_transform_schemas_callable_roundtrip(): + """The demo calls config.transform_schemas["Opportunity"].to_common(data).""" + + def always_transformed(_x): + return TransformResult(result={"transformed": True}, errors=[]) + + config = define_plugin( + extensions=EXTENSIONS, + transform_schemas={ + "Opportunity": ObjectSchemasInput( + to_common=always_transformed, from_common=always_transformed + ) + }, + ) + result = config.transform_schemas["Opportunity"].to_common({"raw": "data"}) + assert result.result == {"transformed": True} + assert result.errors == [] diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py new file mode 100644 index 000000000..b0353ada1 --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -0,0 +1,213 @@ +"""Tests for build_transforms() in common_grants_sdk.extensions.transforms.""" + +import pytest +from pydantic import BaseModel +from common_grants_sdk.extensions.transforms import build_transforms +from common_grants_sdk.extensions.types import PluginError, TransformResult + +# Shared source data matching the ADR-0017 grants.gov example +SOURCE_DATA = { + "data": { + "opportunity_title": "Research into conservation techniques", + "opportunity_status": "posted", + "summary": { + "award_floor": 10000, + "award_ceiling": 100000, + "forecasted_post_date": "2025-05-01", + "forecasted_close_date": "2025-07-15", + }, + } +} + +TO_COMMON_MAPPING = { + "title": {"field": "data.opportunity_title"}, + "status": { + "value": { + "switch": { + "field": "data.opportunity_status", + "case": { + "posted": "open", + "archived": "closed", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "description": "The opportunity is currently accepting applications", + }, + "funding": { + "minAwardAmount": { + "amount": {"field": "data.summary.award_floor"}, + "currency": "USD", + }, + }, +} + +FROM_COMMON_MAPPING = { + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "switch": { + "field": "status.value", + "case": { + "open": "posted", + "closed": "archived", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "summary": { + "award_floor": {"field": "funding.minAwardAmount.amount"}, + }, + } +} + + +# --- Call-time validation --- + + +@pytest.mark.parametrize("name", ["field", "switch"]) +def test_handler_collision_raises(name): + """build_transforms raises if custom handler shadows a default handler name.""" + with pytest.raises(ValueError, match="collide with defaults"): + build_transforms( + TO_COMMON_MAPPING, + FROM_COMMON_MAPPING, + handlers={name: lambda d, v: v}, + ) + + +def test_structural_error_raises_with_path(): + """build_transforms raises on list nodes and includes the field path.""" + with pytest.raises(ValueError, match="Invalid mapping node"): + build_transforms( + {"title": ["should", "not", "be", "a", "list"]}, FROM_COMMON_MAPPING + ) + with pytest.raises(ValueError, match="funding.amount"): + build_transforms({"funding": {"amount": [1, 2]}}, {}) + + +def test_handler_with_sibling_keys_raises(): + """build_transforms raises when a handler key has siblings in the same dict.""" + with pytest.raises(ValueError, match="sibling keys"): + build_transforms({"title": {"field": "x", "extra": "literal"}}, {}) + # Nested occurrence is also caught, and the path is reported + with pytest.raises(ValueError, match="nested.title"): + build_transforms({"nested": {"title": {"field": "x", "extra": "literal"}}}, {}) + + +# --- to_common transform --- + + +def test_to_common(): + to_common, _ = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + result = to_common(SOURCE_DATA) + assert isinstance(result, TransformResult) + assert result.errors == [] + assert result.result["title"] == "Research into conservation techniques" + assert result.result["status"]["value"] == "open" + assert ( + result.result["status"]["description"] + == "The opportunity is currently accepting applications" + ) + assert result.result["funding"]["minAwardAmount"]["amount"] == 10000 + assert result.result["funding"]["minAwardAmount"]["currency"] == "USD" + + +# --- from_common roundtrip --- + + +def test_from_common_roundtrip(): + """Status roundtrip: posted → open → posted.""" + to_common, from_common = build_transforms(TO_COMMON_MAPPING, FROM_COMMON_MAPPING) + native = from_common(to_common(SOURCE_DATA).result) + assert isinstance(native, TransformResult) + assert native.errors == [] + assert ( + native.result["data"]["opportunity_title"] + == "Research into conservation techniques" + ) + assert native.result["data"]["opportunity_status"] == "posted" + + +# --- Error surfacing --- + + +def test_exception_surfaces_as_plugin_error_not_raised(): + """Exceptions inside handlers surface as PluginError, not raised.""" + + def boom(data, _arg): + raise RuntimeError("handler exploded") + + to_common, _ = build_transforms( + {"title": {"boom": "anything"}}, + {}, + handlers={"boom": boom}, + ) + result = to_common(SOURCE_DATA) + assert len(result.errors) == 1 + err = result.errors[0] + assert isinstance(err, PluginError) + assert "handler exploded" in str(err) + assert err.handler == "boom" + assert isinstance(err.cause, RuntimeError) + assert str(err.cause) == "handler exploded" + + +# --- model_validate via common_model --- + + +class _TitleModel(BaseModel): + title: str + + +class _StrictModel(BaseModel): + title: str + required_field: str # always missing from SOURCE_DATA transform output + + +def test_common_model_validates_result(): + """When common_model is provided, result is a model instance on success.""" + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_TitleModel, + ) + result = to_common(SOURCE_DATA) + assert result.errors == [] + assert isinstance(result.result, _TitleModel) + assert result.result.title == "Research into conservation techniques" + + +def test_common_model_validation_failure(): + """ValidationError surfaces as PluginError entries; raw dict is still returned.""" + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_StrictModel, + ) + result = to_common(SOURCE_DATA) + assert len(result.errors) >= 1 + assert all(isinstance(e, PluginError) for e in result.errors) + assert any("required_field" in (e.path or "") for e in result.errors) + assert result.result["title"] == "Research into conservation techniques" + + +def test_custom_handler_registered_per_call(): + """Custom handlers apply only to the call they are registered on.""" + + def handle_upper(data, path): + parts = path.split(".") + val = data + for part in parts: + val = val.get(part) if isinstance(val, dict) else None + return str(val).upper() if val is not None else None + + to_common, _ = build_transforms( + {"title": {"upper": "data.opportunity_title"}}, + {}, + handlers={"upper": handle_upper}, + ) + result = to_common(SOURCE_DATA) + assert result.result["title"] == "RESEARCH INTO CONSERVATION TECHNIQUES" diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py new file mode 100644 index 000000000..f44a3a86b --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -0,0 +1,132 @@ +"""Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" + +import pytest +from common_grants_sdk.extensions.specs import CustomFieldSpec +from common_grants_sdk.extensions.types import ( + ObjectMappings, + ObjectSchemas, + ObjectSchemasInput, + PluginError, + PluginExtensions, + PluginExtensionsMeta, + PluginExtensionsSchema, + TransformResult, +) +from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType + +# --- PluginError --- + + +def test_plugin_error_is_exception_with_defaults(): + err = PluginError("something went wrong") + assert isinstance(err, Exception) + assert str(err) == "something went wrong" + assert err.path is None + assert err.handler is None + assert err.source_value is None + assert err.cause is None + + +def test_plugin_error_structured_fields(): + cause = ValueError("root cause") + err = PluginError( + "msg", + path="status.value", + handler="switch", + source_value={"x": 1}, + cause=cause, + ) + assert err.path == "status.value" + assert err.handler == "switch" + assert err.source_value == {"x": 1} + assert err.cause is cause + + +# --- TransformResult --- + + +def test_transform_result(): + ok = TransformResult(result={"title": "hello"}, errors=[]) + assert ok.result == {"title": "hello"} + assert ok.errors == [] + + err = PluginError("bad") + partial = TransformResult(result={}, errors=[err]) + assert len(partial.errors) == 1 + assert partial.errors[0] is err + + +# --- ObjectMappings --- + + +def test_object_mappings(): + assert ObjectMappings().to_common is None + assert ObjectMappings().from_common is None + m = ObjectMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) + assert m.to_common == {"title": "x"} + assert m.from_common == {"x": "title"} + + +# --- PluginExtensionsMeta --- + + +def test_plugin_extensions_meta(): + assert PluginExtensionsMeta().name is None + assert PluginExtensionsMeta().source_system is None + m = PluginExtensionsMeta(sourceSystem="grants.gov") + assert m.source_system == "grants.gov" + + +# --- PluginExtensionsSchema --- + + +def test_plugin_extensions_schema(): + assert PluginExtensionsSchema().custom_fields is None + assert PluginExtensionsSchema().mappings is None + s = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) + assert s.mappings.to_common == {"a": "b"} + + +# --- PluginExtensions --- + + +def test_plugin_extensions(): + assert PluginExtensions().meta is None + assert PluginExtensions().schemas is None + spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) + schema = PluginExtensionsSchema(customFields={"legacyId": spec}) + ext = PluginExtensions(schemas={"Opportunity": schema}) + assert ext.schemas["Opportunity"].custom_fields == {"legacyId": spec} + + +# --- ObjectSchemasInput --- + + +def test_object_schemas_input(): + assert ObjectSchemasInput().native is None + assert ObjectSchemasInput().to_common is None + + def passthrough(x): + return TransformResult(result=x, errors=[]) + + inp = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) + assert inp.to_common is passthrough + + +# --- ObjectSchemas --- + + +def test_object_schemas(): + def passthrough(x): + return TransformResult(result=x, errors=[]) + + schemas = ObjectSchemas( + native=dict, common=dict, to_common=passthrough, from_common=passthrough + ) + assert schemas.native is dict + assert schemas.common is dict + + with pytest.raises(TypeError): + ObjectSchemas( + native=dict, common=dict, to_common=passthrough + ) # missing from_common diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 5365e665f..8f826c03e 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -1,7 +1,9 @@ import pytest +from pydantic import BaseModel from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, + HandlerError, transform_from_mapping, ) @@ -179,13 +181,15 @@ def test_extend_with_concat(input_data): - The handler works with both field values and constants """ - # Patch in a concat handler for this test def handle_concat(data, concat_spec): return "".join( str(transform_from_mapping(data, part)) for part in concat_spec["parts"] ) - DEFAULT_HANDLERS["concat"] = handle_concat + handlers = { + **DEFAULT_HANDLERS, + "concat": handle_concat, + } mapping = { "opportunity_code": { @@ -198,7 +202,7 @@ def handle_concat(data, concat_spec): } } } - result = transform_from_mapping(input_data, mapping) + result = transform_from_mapping(input_data, mapping, handlers=handlers) assert result == {"opportunity_code": "ABC-123-XYZ-001-12345"} @@ -233,6 +237,121 @@ def handle_type(data, type_spec): assert result == {"id_str": "12345"} +def test_const_string(input_data): + """Test const handler returns a fixed string value.""" + mapping = {"currency": {"const": "USD"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"currency": "USD"} + + +def test_const_number(input_data): + """Test const handler returns a fixed numeric value.""" + mapping = {"version": {"const": 1}} + result = transform_from_mapping(input_data, mapping) + assert result == {"version": 1} + + +def test_const_ignores_source_data(input_data): + """Test const handler is independent of source data.""" + mapping = {"x": {"const": "fixed"}} + result = transform_from_mapping({}, mapping) + assert result == {"x": "fixed"} + + +def test_match_key_alias(input_data): + """Test match key (ADR-0017 canonical name) works identically to switch.""" + mapping = { + "status": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } + } + } + result = transform_from_mapping(input_data, mapping) + assert result == {"status": "open"} + + +def test_number_to_string(input_data): + """Test numberToString handler coerces a numeric field to a string.""" + mapping = {"floor_str": {"numberToString": "summary.award_floor"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"floor_str": "10000"} + + +def test_number_to_string_missing_field(input_data): + """Test numberToString returns None when the field path does not exist.""" + mapping = {"x": {"numberToString": "nonexistent.path"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"x": None} + + +def test_string_to_number_integer(input_data): + """Test stringToNumber handler coerces a string integer field to int.""" + data = {**input_data, "amount_str": "50000"} + result = transform_from_mapping(data, {"amount": {"stringToNumber": "amount_str"}}) + assert result == {"amount": 50000} + assert isinstance(result["amount"], int) + + +def test_string_to_number_float(input_data): + """Test stringToNumber handler coerces a decimal string to float.""" + data = {**input_data, "rate_str": "3.14"} + result = transform_from_mapping(data, {"rate": {"stringToNumber": "rate_str"}}) + assert result == {"rate": 3.14} + assert isinstance(result["rate"], float) + + +def test_string_to_number_missing_field(input_data): + """Test stringToNumber returns None when the field path does not exist.""" + mapping = {"x": {"stringToNumber": "nonexistent.path"}} + result = transform_from_mapping(input_data, mapping) + assert result == {"x": None} + + +def test_string_to_number_invalid_raises(input_data): + """Test stringToNumber raises HandlerError (wrapping ValueError) for non-numeric strings.""" + + data = {**input_data, "bad": "not-a-number"} + with pytest.raises(HandlerError) as exc_info: + transform_from_mapping(data, {"x": {"stringToNumber": "bad"}}) + assert exc_info.value.handler == "stringToNumber" + assert isinstance(exc_info.value.cause, ValueError) + + +def test_handler_error_is_value_error(): + """HandlerError is a subclass of ValueError for backward compat with existing callers.""" + err = HandlerError("myHandler", ValueError("bad input")) + assert isinstance(err, ValueError) + # Callers catching ValueError continue to work; callers wanting attribution catch HandlerError + with pytest.raises(ValueError): + transform_from_mapping( + {"bad": "not-a-number"}, {"x": {"stringToNumber": "bad"}} + ) + + +def test_pydantic_model_instance_is_normalized(): + """transform_from_mapping accepts a Pydantic model instance and extracts fields correctly.""" + + class Inner(BaseModel): + value: str + + class Source(BaseModel): + title: str + nested: Inner + + model = Source(title="hello", nested=Inner(value="world")) + result = transform_from_mapping( + model, + { + "out_title": {"field": "title"}, + "out_value": {"field": "nested.value"}, + }, + ) + assert result == {"out_title": "hello", "out_value": "world"} + + def test_deeply_nested(input_data): """ Test transformation with deeply nested structures. diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index c649b2f7b..52b33b74c 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -42,7 +42,7 @@ We decided to: 6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.`, `definePlugin()` invokes `buildTransforms()` automatically. Mappings for each direction are author-provided — `buildTransforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. -7. **`toCommon` / `fromCommon` return a `TransformResult` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. `definePlugin()` wraps the underlying transform output with runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`); validation failures surface as entries in `errors` rather than thrown exceptions. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. +7. **`toCommon` / `fromCommon` return a `TransformResult` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. In the current PoC, this validation is opt-in at the `buildTransforms()` call site via the `commonModel` / `common_model` parameter — when supplied, validation runs inside `toCommon` against the fully extended generated schema. In the full SDK, `definePlugin()` will additionally inject validation when auto-generating transforms from `extensions.schemas..mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. 8. **Custom handlers are registered per utility call, not globally.** `buildTransforms()` accepts an optional `handlers` argument for registering additional handler names. Per-call scoping keeps behavior explicit and testable; name collisions with the default set raise at `buildTransforms()` call time rather than silently shadowing them. Handler-name lookup must not resolve inherited attributes, because mapping JSON can be reconstituted from untrusted sources via `mergeExtensions()`. @@ -102,6 +102,9 @@ interface ObjectSchemas { } // Input type — provided by plugin authors inside DefinePluginOptions.schemas +// common is intentionally absent: the plugin config file cannot import from generated/ +// since it is the input to generation. definePlugin() injects common during compilation +// from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. interface ObjectSchemasInput { native?: ZodType; // defaults to Record if omitted toCommon?: (native: TNative) => TransformResult; @@ -203,10 +206,19 @@ type Handler = (value: unknown, context: unknown) => unknown; // may provide plain hand-written functions instead. Mappings are validated at call // time (see Decision #7); the optional `handlers` argument registers custom handler // names for this call only (see Decision #8). +// When commonModel is provided, toCommon calls commonModel.parse (Zod) on its output +// and appends any validation errors to TransformResult.errors rather than throwing. +// commonModel must be the fully extended generated schema (e.g. the generated +// Opportunity with typed customFields), not the base schema — passing a base schema +// silently weakens validation of typed custom fields. +// The underlying mapping runtime normalizes model/schema instances to plain objects +// at the entry point, so fromCommon can receive the validated output of toCommon +// and field paths still resolve correctly. function buildTransforms( toCommonMapping: Record, // ADR-0017 mapping from native → CommonGrants fromCommonMapping: Record, // ADR-0017 mapping from CommonGrants → native handlers?: Record, + commonModel?: ZodType, // must be the generated extended schema, not the base ): { toCommon: (native: TNative) => TransformResult; fromCommon: (common: TCommon) => TransformResult; @@ -261,6 +273,9 @@ class ObjectSchemas(Generic[TNative, TCommon]): from_common: Callable[[TCommon], TransformResult[TNative]] # Input type — provided by plugin authors inside define_plugin(schemas=...) +# common is intentionally absent: cg_config.py cannot import from generated/ since +# it is the input to generation. define_plugin() injects common during compilation +# from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. @dataclass class ObjectSchemasInput(Generic[TNative, TCommon]): native: type[TNative] | None = None # defaults to dict[str, Any] if omitted @@ -366,10 +381,19 @@ Handler = Callable[[Any, Any], Any] # may provide plain hand-written callables instead. Mappings are validated at call # time (see Decision #7); the optional `handlers` argument registers custom handler # names for this call only (see Decision #8). +# When common_model is provided, to_common calls model_validate on its output and +# appends any ValidationErrors to TransformResult.errors rather than raising. +# common_model must be the fully extended generated model class (e.g. +# generated/schemas.py's Opportunity), not the base class — passing a base class +# silently weakens validation of typed custom fields. +# transform_from_mapping normalizes Pydantic model instances to plain dicts via +# model_dump(mode="json") at the entry point, so from_common can receive the +# validated model output of to_common and field paths still resolve correctly. def build_transforms( to_common_mapping: dict[str, Any], # ADR-0017 mapping from native → CommonGrants from_common_mapping: dict[str, Any], # ADR-0017 mapping from CommonGrants → native handlers: dict[str, Handler] | None = None, + common_model: type[BaseModel] | None = None, # must be the generated extended model, not the base ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], From a1ccced04334244933e0c9789ac6865ce4fdedae Mon Sep 17 00:00:00 2001 From: jcrichlake <145698165+jcrichlake@users.noreply.github.com> Date: Fri, 29 May 2026 14:00:42 -0400 Subject: [PATCH 02/51] 757 extend define plugin (#838) * First pass on Transform completion * Fixing bugs and updating ADR * Fixing cspell * Fixing dependency * Removing stale comment * Refactoring * Fixing test * doc string chagnes * Updating examples to include more mapping types * Removing getClient and filters * Fixing Switch statement and adding by_alias flag * Putting custom fields under the same node as other per object properties * Updating to preserve mypy typing * Updating transforms.py from using Any to generics leveraging schemas to preserve type hinting. * Cleaning up __init__.py * Addressing missing branch * Adding transform custom field validation * Fix safety version conflict in poetry-audit-plugin CI steps Pin safety<3.3.0 via poetry's own venv pip to resolve import error introduced in safety 3.3.0. Remove unused audit plugin install from ci-lib-pysdk.yml which has no poetry audit step. * Updating docstring --- .../workflows/ci-example-california-api.yml | 4 +- .../workflows/ci-example-pennsylvania-api.yml | 4 +- .github/workflows/ci-lib-pysdk.yml | 3 - .github/workflows/ci-template-fast-api.yml | 4 +- lib/python-sdk/.coverage | Bin 53248 -> 0 bytes lib/python-sdk/.gitignore | 1 + .../common_grants_sdk/extensions/README.md | 311 ++++++++------- .../common_grants_sdk/extensions/__init__.py | 5 +- .../common_grants_sdk/extensions/generate.py | 269 ++++++++++--- .../common_grants_sdk/extensions/plugin.py | 165 +++++--- .../common_grants_sdk/extensions/specs.py | 134 ++++--- .../extensions/transforms.py | 83 +++- .../common_grants_sdk/extensions/types.py | 33 +- .../common_grants_sdk/utils/custom_fields.py | 27 +- .../common_grants_sdk/utils/transformation.py | 9 +- .../examples/plugin_custom_fields.py | 10 +- .../examples/plugins/grants_gov/__init__.py | 7 +- .../examples/plugins/grants_gov/cg_config.py | 64 +-- .../opportunity_extensions/__init__.py | 3 +- .../opportunity_extensions/cg_config.py | 74 ++-- lib/python-sdk/examples/transforms.py | 12 +- .../tests/extensions/test_plugin.py | 76 ++-- .../tests/extensions/test_transforms.py | 101 ++++- lib/python-sdk/tests/extensions/test_types.py | 32 +- .../tests/schemas/test_plugin_registry.py | 32 +- .../tests/utils/test_custom_fields.py | 26 ++ .../tests/utils/test_merge_extensions.py | 206 +++++----- .../tests/utils/test_plugin_generator.py | 364 +++++++++++++++--- .../tests/utils/test_transformation.py | 31 +- website/.cspell.json | 3 +- .../governance/adr/0017-mapping-format.md | 49 ++- .../governance/adr/0022-plugin-framework.mdx | 140 ++++--- 32 files changed, 1597 insertions(+), 685 deletions(-) delete mode 100644 lib/python-sdk/.coverage diff --git a/.github/workflows/ci-example-california-api.yml b/.github/workflows/ci-example-california-api.yml index ba327d6c4..6c0bb5bea 100644 --- a/.github/workflows/ci-example-california-api.yml +++ b/.github/workflows/ci-example-california-api.yml @@ -27,7 +27,9 @@ jobs: uses: Gr1N/setup-poetry@v9 - name: Install poetry-audit-plugin - run: poetry self add poetry-audit-plugin + run: | + poetry self add poetry-audit-plugin + $HOME/.local/share/pypoetry/venv/bin/pip install "safety<3.3.0" - name: Install Node.js uses: actions/setup-node@v6 diff --git a/.github/workflows/ci-example-pennsylvania-api.yml b/.github/workflows/ci-example-pennsylvania-api.yml index faf6aab15..5392e8663 100644 --- a/.github/workflows/ci-example-pennsylvania-api.yml +++ b/.github/workflows/ci-example-pennsylvania-api.yml @@ -27,7 +27,9 @@ jobs: uses: Gr1N/setup-poetry@v9 - name: Install poetry-audit-plugin - run: poetry self add poetry-audit-plugin + run: | + poetry self add poetry-audit-plugin + $HOME/.local/share/pypoetry/venv/bin/pip install "safety<3.3.0" - name: Install Node.js uses: actions/setup-node@v6 diff --git a/.github/workflows/ci-lib-pysdk.yml b/.github/workflows/ci-lib-pysdk.yml index 7e45e8208..cf86b389b 100644 --- a/.github/workflows/ci-lib-pysdk.yml +++ b/.github/workflows/ci-lib-pysdk.yml @@ -24,9 +24,6 @@ jobs: - name: Install Poetry uses: Gr1N/setup-poetry@v9 - - name: Install poetry-audit-plugin - run: poetry self add poetry-audit-plugin - - name: Install dependencies run: make install diff --git a/.github/workflows/ci-template-fast-api.yml b/.github/workflows/ci-template-fast-api.yml index 4a4b6fb1f..ddd56340e 100644 --- a/.github/workflows/ci-template-fast-api.yml +++ b/.github/workflows/ci-template-fast-api.yml @@ -28,7 +28,9 @@ jobs: uses: Gr1N/setup-poetry@v9 - name: Install poetry-audit-plugin - run: poetry self add poetry-audit-plugin + run: | + poetry self add poetry-audit-plugin + $HOME/.local/share/pypoetry/venv/bin/pip install "safety<3.3.0" - name: Install Node.js uses: actions/setup-node@v6 diff --git a/lib/python-sdk/.coverage b/lib/python-sdk/.coverage deleted file mode 100644 index 9ac35a7ec410f45428a8dc61ec83f94552ae0181..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53248 zcmeI)%WoS+90%}SJGE=a)&^CT6@`+yKpHiUYgOWTfB-pER4Ro+4@kJ<^*G*kAMWlt zO%4d$B2_}_v7Au<9}bn^!Vz()=!qK=QhVU{+u8M_jjLYbsPJ9c&U$yApLzL}{ru7e zCsv{xgr1DWIb+E%P2&S048thVy-N3NE7HzF_JsbL6Z^AvOUCBif2!7(Mx}7qu)eNd zwCa_gs&6d*TJcIhEZ#55l1(SDK>z{}fWZG-V0fugtSv8_x4wyG+f{KWZ58Ur>PMe# zZ(ZINm$yE=ur2g4ak?yMtJlSr2!orVuR_svTqPX8MeTvz)WNq{uBSnb97041WF9iKyp(`oc|5vr~Vm2ay^ zm*TY3Sud~O{)MWnZ{$4rt49eW#i; zBj*;?u?D_#W1vQjzahpkHUP!D*0u~2*GglSN(RD014mtJqRiO?ba+PB;q$Zm4UdxPPQs(ebw$ukh789R zx^bvCPmY;Jh9N^9Xd*}hQb~`rw{c;q45IA@?diUj^q5G|0HL`c3)UU@HV>7E>tF}F zDwI2FGd-)Pg`@6B)ODJrLT!6>R@WJIfmU<0TOKYKinWy$^HwS8MrmuMe~MFGC~L_B zrqS$_r(`q;vx($+lVlcSJWFO{Bt1jbsTT{it(95TNyD|8>CW+CaiLf{dD0x}5yA5k z-;YgIf{%=cz*7ssix_`eM+rc1C`Y%6Cx+kwayOxwGTU>aWzf6NJ8(*~JdD8Es>lH7_CVvPo6L__qcG_yq0 zvpi#E>2LPQgj4loM-O*yl!+8?_S7@U>DB0C8LQawl;9vqVUmt@Q%1rL6`iEDbbT%V zyl%j))oSu1ze_&OFXe`-X0dkWj5!>ngHg{jt%w$NPiDnVpXR1I$x|oC{pBJ}xb$jg zvY)Ws1P4FWIq>ABTd=JavQX)3o$@-r!8fhP2Hmhh00Izz00bZa0SG_<0uX=z1R(I= z6Udu6v&irNbJib*^%p&0g8&2|009U<00Izz00bZa0SG|gr4*>-@~5oq7anJq%>2r7 z`Wt|^Hk;=*&z7lHIqQ*OJ+l6ODJzIVLjVF0fB*y_009U<00Izz00bZ~5~$=)nc2qx zxpID`lzs}JfB%2au#EUFyN$^4`mPG=J9KrRII8! zEDW!d=JNg@*Z-Sq!y+>XKmY;|fB*y_009U<00Izzz`OkNhA2 z0SG_<0uX=z1Rwwb2tWV=5SWKRK3^^A@Bbef*28(^h&&(w0SG_<0uX=z1Rwwb2tWV= z5I8h}Wpkladh^zAKUc)>e;Qq{@Zh`q=7V2eeY^OY{{H{5VLd*y+>tN@AOHafKmY;| zfB*y_009U<00KuRus|OQB;WrVOGj4-jNtEr7fs009U<00Izz00bZa0SG_< I0*5c~FLLaI$^ZZW diff --git a/lib/python-sdk/.gitignore b/lib/python-sdk/.gitignore index 4c00284b0..a4db3a2c8 100644 --- a/lib/python-sdk/.gitignore +++ b/lib/python-sdk/.gitignore @@ -9,6 +9,7 @@ __pycache__/ *.py[cod] *$py.class .pytest_cache/ +.coverage #Generated Schema objects generated/ \ No newline at end of file diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 0a3786f8f..0f0fa92ca 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -44,12 +44,12 @@ Here are some key concepts that are used to define custom fields and plugins tha | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | | **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | -| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec` objects. This is the shape that `define_plugin()` and `with_custom_fields()` accept. | -| **`Plugin`** | A dataclass with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Pydantic models with typed `customFields` applied). Created by `define_plugin()`. | +| **`SchemaExtensions`** | A legacy TypedDict mapping extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec`. Still accepted by `with_custom_fields()`. For plugins, declare custom fields inside `ObjectSchemasInput.custom_fields` instead. | +| **`Plugin`** | A dataclass assembled by the code generator. `.schemas` is a container object where each attribute (e.g. `.schemas.Opportunity`) is an `ObjectSchemas` instance providing the model class (`.common`), transform callables (`.to_common`, `.from_common`), and native type (`.native`). `.extensions` holds the serializable extension declarations. | | **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | -| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict and returns a `TransformResult`. | +| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict **or a Pydantic model instance** and returns a `TransformResult`. | | **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | -| **`ObjectSchemasInput`**| Bundles a `to_common` and `from_common` callable for a single object type. Passed to `define_plugin()` via the `transform_schemas` parameter. | +| **`ObjectSchemasInput`**| Bundles `custom_fields`, `to_common`, and `from_common` for a single object type. Passed to `define_plugin()` via the `schemas` parameter. | @@ -123,40 +123,33 @@ The following is an example `cg_config.py` file, which you pass to the build ste ```python -from common_grants_sdk import define_plugin, merge_extensions -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType -# Extensions that might come from a shared HHS package -hhs_extensions: SchemaExtensions = { - "Opportunity": { - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - }, -} - -# Extensions specific to this project -local_extensions: SchemaExtensions = { - "Opportunity": { - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - }, -} - config = define_plugin( - merge_extensions([hhs_extensions, local_extensions], on_conflict="error"), + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "programArea": CustomFieldSpec( + field_type=CustomFieldType.STRING, + description="HHS program area code (e.g. 'CFDA-93.243')", + ), + "legacyGrantId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Numeric ID from the legacy grants management system", + ), + "eligibilityTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", + ), + "awardCeiling": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + description="Maximum award amount in USD", + ), + } + ) + } ) ``` @@ -215,7 +208,7 @@ api_response = { # Use the model returned via opportunity_extensions # --------------------------------------------------------------------------- -opp = opportunity_extensions.schemas.Opportunity.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) ``` @@ -270,36 +263,26 @@ A plugin is a Python class that contains extension specs and generated schemas ```python -T = TypeVar("T") - -@dataclass(frozen=True) -class Plugin(Generic[T]): - """Runtime plugin container with both extension specs and generated schemas.""" - - extensions: SchemaExtensions - schemas: T -``` - - -```python -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType -# Extensions specific to this project -local_extensions: SchemaExtensions = { - "Opportunity": { - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - }, -} - -config = define_plugin(local_extensions) +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "eligibilityTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", + ), + "awardCeiling": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + description="Maximum award amount in USD", + ), + } + ) + } +) ``` After running the build step the imported extension object will have 2 fields to use. @@ -393,7 +376,7 @@ After installing the plugin (e.g. `poetry add opportunity-extensions`): ```python from opportunity_extensions import opportunity_extensions -opp = opportunity_extensions.schemas.Opportunity.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) print(opp.custom_fields.program_area.value) # typed as str print(opp.custom_fields.legacy_grant_id.value) # typed as int ``` @@ -414,32 +397,34 @@ Before publishing a new version of your plugin: ### Combining Plugins -Use `merge_extensions()` to combine field specs from multiple sources before passing them to `define_plugin()`: +Custom fields from multiple logical sources are combined by declaring them all inside a single `ObjectSchemasInput.custom_fields` dict. Because the dict is plain Python, there is no special merge utility needed — just add the keys side by side: ```python -from common_grants_sdk import define_plugin, merge_extensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput +from common_grants_sdk.schemas.pydantic import CustomFieldType config = define_plugin( - merge_extensions([shared_extensions, local_extensions], on_conflict="error"), + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + # fields from a shared HHS package + "programArea": CustomFieldSpec(field_type=CustomFieldType.STRING), + "legacyGrantId": CustomFieldSpec(field_type=CustomFieldType.INTEGER), + # fields specific to this project + "eligibilityTypes": CustomFieldSpec(field_type=CustomFieldType.ARRAY), + "awardCeiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER), + } + ) + } ) ``` -`on_conflict` controls what happens when two sources declare the same field key on the same model: - -| Strategy | Behaviour | -|---|---| -| `"error"` (default) | Raises `ValueError` — safest, forces explicit resolution | -| `"first_wins"` | Keeps the definition from the first source in the list | -| `"last_wins"` | Overwrites with the definition from the last source | - -Prefer unique, namespaced field names so `"error"` is never triggered. - -> [!Note] -> The `"first_wins"` and `"last_wins"` strategies resolve conflicts at runtime but the merged result loses the specific field types of the overridden definitions. For full static type safety, use the default `"error"` strategy with non-overlapping, namespaced field names. +`merge_extensions()` is still available for merging `PluginExtensions` objects that carry declarative `mappings` (ADR-0017 transform configs). It no longer merges `custom_fields`. #### Verify type inference before publishing -After building your package, import the plugin in a test file and confirm that `.extensions` keys and `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. +After building your package, import the plugin in a test file and confirm that `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. ## Bidirectional Transforms @@ -447,7 +432,7 @@ Plugins can define bidirectional mappings between a source system's native data ### Defining transforms -Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `transform_schemas`: +Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `schemas`: ```python from common_grants_sdk.extensions import ( @@ -494,22 +479,20 @@ to_common, from_common = build_transforms( ) plugin = define_plugin( - extensions={ - "Opportunity": { - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Unique identifier in legacy database", - ), - } - }, meta=PluginExtensionsMeta( name="my-system", version="0.1.0", source_system="my-system.example.gov", capabilities=["customFields", "transforms"], ), - transform_schemas={ + schemas={ "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Unique identifier in legacy database", + ), + }, to_common=to_common, from_common=from_common, ) @@ -519,6 +502,37 @@ plugin = define_plugin( Both directions must be provided explicitly. `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. +#### Hand-written callables + +`build_transforms()` is optional. You can supply any plain Python callable to `ObjectSchemasInput` as long as it matches the expected signature: + +```python +def to_common(native_data: dict) -> TransformResult: + ... + +def from_common(cg_data: dict) -> TransformResult: + ... + +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + to_common=to_common, + from_common=from_common, + ) + }, +) +``` + +The key requirement when porting existing transform code is that **both callables must return `TransformResult`**. The type annotation enforces this, but it is easy to miss when wrapping a function that previously returned a plain dict. Wrap the return value like so: + +```python +from common_grants_sdk.extensions.types import TransformResult + +def to_common(native_data: dict) -> TransformResult: + result = my_existing_transform(native_data) # returns a plain dict + return TransformResult(result=result, errors=[]) +``` + ### Mapping format A mapping dict describes how to build an output object from a source dict. Each leaf node is either a literal value or a single-key dict that invokes a named handler. @@ -550,15 +564,32 @@ to_common, from_common = build_transforms( Custom handlers are merged with the defaults; they cannot override built-in handler names. +Handlers are scoped to the `build_transforms()` call they are registered on — they do not affect other calls: + +```python +# Only the first pair of callables knows about "upper" +to_common_with_upper, _ = build_transforms( + to_common_mapping={"title": {"upper": "data.opportunity_title"}}, + from_common_mapping={}, + handlers={"upper": handle_upper}, +) + +# This call has no knowledge of "upper" — using it in the mapping would fail at call time +to_common_plain, _ = build_transforms( + to_common_mapping={"title": {"field": "data.opportunity_title"}}, + from_common_mapping={}, +) +``` + ### Using transforms -The compiled callables are stored on the plugin's `transform_schemas` dict, keyed by object name. Each callable takes a data dict and returns a `TransformResult`: +The compiled callables are stored on the plugin's `schemas` object, accessible by attribute name. Each callable takes a data dict (or a Pydantic model instance) and returns a `TransformResult`: ```python -opp_transforms = plugin.transform_schemas["Opportunity"] +opp_schemas = plugin.schemas.Opportunity # Source system → CommonGrants -result = opp_transforms.to_common(native_data) +result = opp_schemas.to_common(native_data) if result.errors: for err in result.errors: print(f"[{err.path}] {err}") @@ -566,12 +597,15 @@ else: cg_data = result.result # CommonGrants → source system -result = opp_transforms.from_common(cg_data) +result = opp_schemas.from_common(cg_data) native_data = result.result ``` `TransformResult.errors` is always a list (empty on success). A non-empty errors list means the transform encountered a problem but still returned a partial result in `result`. +> [!IMPORTANT] +> When `common_model` is set on `build_transforms()`, `to_common` returns a validated Pydantic model instance in `result.result`. That instance can be passed directly to `from_common`. In that case, field paths in `from_common_mapping` must use the model's **camelCase alias names** (e.g. `"status.value"`, `"funding.minAwardAmount.amount"`), not Python snake_case attribute names. This matches the camelCase convention used throughout CommonGrants field paths. + See `examples/transforms.py` for a complete working example with roundtrip verification. @@ -586,11 +620,11 @@ from plugins.opportunity_extensions import opportunity_extensions client = Client(base_url="https://api.example.gov") # Get a single opportunity with typed custom fields -opp = client.opportunities.get(opp_id, schema=opportunity_extensions.schemas.Opportunity) +opp = client.opportunities.get(opp_id, schema=opportunity_extensions.schemas.Opportunity.common) print(opp.custom_fields.program_area.value) # typed as str # List with the same schema -response = client.opportunities.list(schema=opportunity_extensions.schemas.Opportunity) +response = client.opportunities.list(schema=opportunity_extensions.schemas.Opportunity.common) for opp in response.items: print(opp.custom_fields.legacy_grant_id.value) # typed as int @@ -598,7 +632,7 @@ for opp in response.items: results = client.opportunities.search( search="health", status=["open"], - schema=opportunity_extensions.schemas.Opportunity, + schema=opportunity_extensions.schemas.Opportunity.common, ) ``` @@ -614,17 +648,22 @@ The generator converts `camelCase` keys to `snake_case` Python attribute names a ```python # cg_config.py -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType -extensions: SchemaExtensions = { - "Opportunity": { - "legacyGrantId": CustomFieldSpec( # camelCase key — matches the JSON - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - }, -} +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyGrantId": CustomFieldSpec( # camelCase key — matches the JSON + field_type=CustomFieldType.INTEGER, + description="Numeric ID from the legacy grants management system", + ), + } + ) + } +) ``` ```python @@ -635,7 +674,7 @@ api_response = { }, } -opp = my_plugin.schemas.Opportunity.model_validate(api_response) +opp = my_plugin.schemas.Opportunity.common.model_validate(api_response) opp.custom_fields.legacy_grant_id.value # 98765, typed as int ``` @@ -646,7 +685,7 @@ Additional field naming guidelines: ### Keep plugins focused -A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, use `merge_extensions()` to compose separate plugins rather than bundling everything into one. +A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, declare them all in one `ObjectSchemasInput.custom_fields` dict — Python dict literals compose cleanly without a special merge utility. ### Type safety @@ -655,22 +694,27 @@ A plugin should represent a single logical concern (one agency's fields, one int ```python from pydantic import BaseModel - from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions + from common_grants_sdk import define_plugin + from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType class LegacyRef(BaseModel): system: str id: int - extensions: SchemaExtensions = { - "Opportunity": { - "legacyRef": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=LegacyRef, - description="Reference to the opportunity in the legacy system", - ), - }, - } + config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyRef": CustomFieldSpec( + field_type=CustomFieldType.OBJECT, + value=LegacyRef, + description="Reference to the opportunity in the legacy system", + ), + } + ) + } + ) ``` > [!NOTE] @@ -681,9 +725,10 @@ A plugin should represent a single logical concern (one agency's fields, one int When you define Pydantic models for complex `value` fields, export them as named exports from your package. Downstream consumers may need these types for use with `get_custom_field_value()`: ```python -# __init__.py of a plugin package +# cg_config.py of a plugin package from pydantic import BaseModel -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput from common_grants_sdk.schemas.pydantic import CustomFieldType # Export value types so consumers can reference them directly @@ -691,15 +736,19 @@ class ProgramAreaValue(BaseModel): code: str name: str -extensions: SchemaExtensions = { - "Opportunity": { - "programArea": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=ProgramAreaValue, - description="The HHS program area for this opportunity", - ), - }, -} +config = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "programArea": CustomFieldSpec( + field_type=CustomFieldType.OBJECT, + value=ProgramAreaValue, + description="The HHS program area for this opportunity", + ), + } + ) + } +) ``` This allows consumers to use `get_custom_field_value()` with the same type the plugin uses for validation: @@ -707,7 +756,7 @@ This allows consumers to use `get_custom_field_value()` with the same type the p ```python from commongrants_hhs_plugin import hhs_plugin, ProgramAreaValue -opp = hhs_plugin.schemas.Opportunity.model_validate(api_response) +opp = hhs_plugin.schemas.Opportunity.common.model_validate(api_response) # Extract the value with full type safety using the exported type area = opp.get_custom_field_value("programArea", ProgramAreaValue) diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 7a9ef1185..8589e1fa8 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,10 +1,9 @@ """Public extension APIs for the CommonGrants Python SDK.""" -from .plugin import Plugin, PluginConfig, define_plugin +from .plugin import Plugin, PluginConfig, define_plugin, inject_transforms from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions, merge_extensions from .transforms import build_transforms from .types import ( - ClientConfig, Handler, ObjectMappings, ObjectSchemas, @@ -25,11 +24,11 @@ "PluginConfig", "SchemaExtensions", "define_plugin", + "inject_transforms", "merge_extensions", # New: build_transforms "build_transforms", # New: ADR-0022 types - "ClientConfig", "Handler", "ObjectMappings", "ObjectSchemas", diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py index 9657a84a1..a259f6788 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ b/lib/python-sdk/common_grants_sdk/extensions/generate.py @@ -7,15 +7,15 @@ import keyword import re from pathlib import Path -from typing import Iterable, cast +from typing import Any, Iterable from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.utils.json import snake from .plugin import PluginConfig -from .specs import CustomFieldSpec, SchemaExtensions +from .specs import CustomFieldSpec # Maps extensible model names to the SDK base class they extend in generated code. -# Add an entry here (and to SchemaExtensions) when a new model gains customFields support. +# Add an entry here when a new model gains customFields support. MODEL_BASE_CLASS: dict[str, str] = { "Opportunity": "OpportunityBase", } @@ -33,7 +33,7 @@ } -def _load_config(config_path: Path) -> PluginConfig: +def _load_config(config_path: Path) -> PluginConfig[Any]: """Load and validate a plugin config file, returning the PluginConfig object. Uses importlib to load cg_config.py as an isolated module so it doesn't @@ -59,13 +59,29 @@ def _load_config(config_path: Path) -> PluginConfig: spec.loader.exec_module(module) config = getattr(module, "config", None) - if config is None or not hasattr(config, "extensions"): + if not isinstance(config, PluginConfig): raise RuntimeError( 'Plugin config must expose a "config" variable created by define_plugin()' ) return config +def _extract_custom_fields( + config: PluginConfig[Any], +) -> dict[str, dict[str, CustomFieldSpec]]: + """Extract custom field specs from config.schemas into the flat shape used by generators. + + Returns an empty dict if config.schemas is None or has no schemas with custom_fields. + """ + if config.schemas is None: + return {} + return { + obj: schema.custom_fields + for obj, schema in config.schemas.items() + if schema.custom_fields is not None + } + + def _normalize_identifier(name: str) -> str: """Convert an arbitrary string into a valid Python identifier. @@ -173,7 +189,9 @@ def _annotation_for_spec(spec: CustomFieldSpec, resolved_type: CustomFieldType) return rendered or "Any" -def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: +def _collect_extra_imports( + custom_fields: dict[str, dict[str, CustomFieldSpec]], +) -> list[str]: """Collect import lines needed for external types used as ``spec.value``. Walks all specs and returns one ``import`` line per distinct external type. @@ -186,7 +204,7 @@ def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: types are imported using their ``__module__`` path directly. Args: - extensions: The merged ``SchemaExtensions`` from the plugin config. + custom_fields: The flat custom fields mapping extracted from the plugin config. Returns: A deduplicated list of import-statement strings in the order they were @@ -195,7 +213,7 @@ def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: seen: set[tuple[str, str]] = set() imports: list[str] = [] - for fields in cast(dict[str, dict[str, CustomFieldSpec]], extensions).values(): + for fields in custom_fields.values(): for spec in fields.values(): if not isinstance(spec.value, type): continue @@ -217,7 +235,9 @@ def _collect_extra_imports(extensions: SchemaExtensions) -> list[str]: return imports -def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: +def _model_blocks( + custom_fields: dict[str, dict[str, CustomFieldSpec]], +) -> Iterable[str]: """Yield source-code blocks for every model defined in the extensions mapping. For each model, yields three blocks in dependency order: @@ -228,7 +248,7 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: the container. Args: - extensions: The merged ``SchemaExtensions`` from the plugin config. + custom_fields: The flat custom fields mapping extracted from the plugin config. Yields: Source-code strings to be joined and written into ``schemas.py``. @@ -236,9 +256,7 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: Raises: ValueError: If a model name is not present in ``MODEL_BASE_CLASS``. """ - for model_name, fields in cast( - dict[str, dict[str, CustomFieldSpec]], extensions - ).items(): + for model_name, fields in custom_fields.items(): if model_name not in MODEL_BASE_CLASS: raise ValueError( f'Generator does not support model "{model_name}". ' @@ -257,7 +275,7 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: spec=spec, resolved_type=resolved_type ) # Use spec.name as the runtime display name if provided, otherwise fall back - # to the field key (the dict key in SchemaExtensions). + # to the field key (the dict key in ObjectSchemasInput.custom_fields). field_name_default = spec.name or field_key # repr() produces a quoted string literal safe to embed directly in source code. description_default = repr(spec.description) if spec.description else "None" @@ -316,16 +334,25 @@ def _model_blocks(extensions: SchemaExtensions) -> Iterable[str]: ) -def _render_schemas_py(extensions: SchemaExtensions) -> str: +def _render_schemas_py( + custom_fields: dict[str, dict[str, CustomFieldSpec]], + mappings_only_objs: set[str] | None = None, +) -> str: """Render the full source of the generated ``schemas.py`` file. Produces a self-contained module containing typed ``CustomField`` subclasses, a ``CustomFields`` container, and an extended model class for each entry in - ``extensions``. Also emits a ``_Schemas`` container object (attribute access + ``custom_fields``. Also emits a ``_Schemas`` container object (attribute access rather than dict lookup) and a module-level ``schemas`` instance. + For objects that only have ``mappings`` (no ``custom_fields``), the ``_Schemas`` + object will expose the base SDK model class directly (e.g. ``schemas.Opportunity`` + will be ``OpportunityBase``). + Args: - extensions: The merged ``SchemaExtensions`` from the plugin config. + custom_fields: The flat custom fields mapping extracted from the plugin config. + mappings_only_objs: Set of object names that have mappings but no custom_fields. + These will be exposed on ``_Schemas`` as their base SDK class. Returns: A string of valid Python source code ready to be written to disk. @@ -334,13 +361,34 @@ def _render_schemas_py(extensions: SchemaExtensions) -> str: # plugin.schemas.Opportunity rather than plugin.schemas["Opportunity"]. # The dynamic __init__ assignment is necessary because model names aren't # known until generation time, so a static class body can't be used. - model_names = list(extensions.keys()) - blocks = "\n\n\n".join(_model_blocks(extensions)) - schema_assignments = "\n".join( - [f" self.{name} = {name}" for name in model_names] or [" pass"] - ) - all_exports = ", ".join([f'"{name}"' for name in model_names] + ['"schemas"']) - extra_imports = _collect_extra_imports(extensions) + model_names = list(custom_fields.keys()) + blocks = "\n\n\n".join(_model_blocks(custom_fields)) + mappings_only: set[str] = mappings_only_objs or set() + + # Build schema assignments: each attribute is an ObjectSchemas instance so + # callers get a unified interface (plugin.schemas.Opportunity.common for the + # model class, .to_common/.from_common for transforms). + # to_common/from_common default to None here; root __init__.py injects the + # real callables for any object that has transforms configured. + assignments: list[str] = [ + f" self.{name} = ObjectSchemas(native=dict, common={name}, to_common=None, from_common=None)" + for name in model_names + ] + for obj in sorted(mappings_only): + if obj not in MODEL_BASE_CLASS: + raise ValueError( + f'Generator does not support model "{obj}". ' + f"Supported models: {sorted(MODEL_BASE_CLASS)}" + ) + base_class = MODEL_BASE_CLASS[obj] + assignments.append( + f" self.{obj} = ObjectSchemas(native=dict, common={base_class}, to_common=None, from_common=None)" + ) + + schema_assignments = "\n".join(assignments or [" pass"]) + all_names = model_names + sorted(mappings_only) + all_exports = ", ".join([f'"{name}"' for name in all_names] + ['"schemas"']) + extra_imports = _collect_extra_imports(custom_fields) extra_import_lines = ["", *extra_imports] if extra_imports else [] return "\n".join( @@ -353,13 +401,12 @@ def _render_schemas_py(extensions: SchemaExtensions) -> str: "", "from pydantic import ConfigDict, Field", "", + "from common_grants_sdk.extensions.types import ObjectSchemas", "from common_grants_sdk.schemas.pydantic.base import CommonGrantsBaseModel", "from common_grants_sdk.schemas.pydantic.fields import CustomField, CustomFieldType", "from common_grants_sdk.schemas.pydantic.models import OpportunityBase", *extra_import_lines, - "", - blocks, - "", + *([blocks, ""] if blocks else []), "class _Schemas:", " def __init__(self) -> None:", schema_assignments, @@ -393,34 +440,92 @@ def _render_generated_init_py() -> str: ) -def _render_plugin_init_py(plugin_variable_name: str) -> str: - """Render the source of the plugin directory's root ``__init__.py`` file. +def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) -> str: + """Render the source of the plugin directory's root __init__.py file. - The generated file imports ``config`` directly from ``cg_config.py`` and - exports a ``Plugin`` instance named after the plugin directory alongside the - ``schemas`` object. + Emits a fully compiled Plugin instance. Transform callables are injected + into the _Schemas object (from generated/schemas.py) before Plugin is + constructed, so plugin.schemas.Opportunity.to_common etc. are populated. + """ + # Collect the sets of objects needing transform injection. + # Only count schemas that have explicit callable transforms, not those with custom_fields only. + explicit_objs: set[str] = ( + { + obj + for obj, s in config.schemas.items() + if s.to_common is not None or s.from_common is not None + } + if config.schemas + else set() + ) + mappings_objs: set[str] = ( + {obj for obj, s in config.extensions.schemas.items() if s.mappings is not None} + if config.extensions and config.extensions.schemas + else set() + ) - Args: - plugin_variable_name: A valid Python identifier derived from the plugin - directory name (e.g. ``"opportunity_extensions"``). + needs_build_transforms = bool(mappings_objs - explicit_objs) - Returns: - A string of valid Python source code ready to be written to disk. - """ - return "\n".join( - [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from __future__ import annotations", - "", - "from common_grants_sdk.extensions import Plugin", - "from .cg_config import config", - "from .generated import schemas", - "", - f"{plugin_variable_name} = Plugin(", - " extensions=config.extensions,", - " schemas=schemas,", + # Build pre-plugin lines: inject transforms into the _Schemas object before + # constructing Plugin. schemas.py initialises each ObjectSchemas with + # to_common=None/from_common=None; we mutate those attrs here. + inject_lines: list[str] = [] + + # Mappings-only objects: call build_transforms() then inject results. + for obj in sorted(mappings_objs - explicit_objs): + inject_lines += [ + f"_{obj}_to_common, _{obj}_from_common = build_transforms(", + f' to_common_mapping=config.extensions.schemas["{obj}"].mappings.to_common,', + f' from_common_mapping=config.extensions.schemas["{obj}"].mappings.from_common,', + f" common_model=schemas.{obj}.common,", ")", + f"schemas.{obj}.to_common = _{obj}_to_common", + f"schemas.{obj}.from_common = _{obj}_from_common", + "", + ] + + # Explicit schemas: single inject_transforms() call handles all objects. + # Reassigning the return value preserves the concrete _Schemas type for mypy. + if explicit_objs: + inject_lines.append("schemas = inject_transforms(config, schemas)") + inject_lines.append("") + + imports = [ + "# This file is auto-generated. Do not edit it manually — it will be overwritten", + "# the next time `python -m common_grants_sdk.extensions.generate` is run.", + "from __future__ import annotations", + "", + ] + + sdk_imports = {"Plugin"} + if explicit_objs: + sdk_imports.add("inject_transforms") + if needs_build_transforms: + sdk_imports.add("build_transforms") + imports.append( + f"from common_grants_sdk.extensions import {', '.join(sorted(sdk_imports))}" + ) + imports += [ + "from .cg_config import config", + "from .generated import schemas", + "", + ] + + plugin_lines = [ + f"{plugin_variable_name} = Plugin(", + " schemas=schemas,", + " extensions=config.extensions,", + " meta=config.meta,", + ")", + ] + + pre_plugin = inject_lines # may be empty + + return "\n".join( + imports + + pre_plugin + + plugin_lines + + [ "", f'__all__ = ["{plugin_variable_name}", "schemas"]', "", @@ -452,6 +557,59 @@ def generate_plugin(plugin_dir: Path) -> Path: raise FileNotFoundError(f"Could not find config file: {config_path}") config = _load_config(config_path) + custom_fields = _extract_custom_fields(config) + + # Determine objects that have mappings but no custom_fields — these need a + # pass-through entry in _Schemas pointing at the base SDK model class. + explicit_cf_objs: set[str] = set(custom_fields.keys()) + mappings_only_objs: set[str] = ( + { + obj + for obj, s in config.extensions.schemas.items() + if s.mappings is not None and obj not in explicit_cf_objs + } + if config.extensions and config.extensions.schemas + else set() + ) + # Third bucket: objects in config.schemas with explicit transforms but no + # custom_fields and no extensions.schemas entry. They also need a pass-through + # entry in _Schemas or schemas. won't exist at import time. + transforms_only_objs: set[str] = ( + set(config.schemas.keys()) - explicit_cf_objs - mappings_only_objs + if config.schemas + else set() + ) + + # Validate that auto-generated transform objects have both mapping directions. + # Auto-generated objects: have mappings in extensions.schemas but no explicit + # to_common/from_common in config.schemas. + if config.extensions and config.extensions.schemas: + explicit_schema_objs: set[str] = ( + { + obj + for obj, s in config.schemas.items() + if s.to_common is not None or s.from_common is not None + } + if config.schemas + else set() + ) + ext_schemas = config.extensions.schemas + for obj, schema in ext_schemas.items(): + if schema.mappings is None or obj in explicit_schema_objs: + continue + if schema.mappings.to_common is None: + raise ValueError( + f'Plugin object "{obj}": mappings.to_common is required when ' + f"auto-generating transforms. Either provide a to_common mapping " + f"or pass an explicit to_common callable via schemas['{obj}']." + ) + if schema.mappings.from_common is None: + raise ValueError( + f'Plugin object "{obj}": mappings.from_common is required when ' + f"auto-generating transforms. Either provide a from_common mapping " + f"or pass an explicit from_common callable via schemas['{obj}']." + ) + generated_dir = plugin_dir / "generated" generated_dir.mkdir(parents=True, exist_ok=True) @@ -459,12 +617,17 @@ def generate_plugin(plugin_dir: Path) -> Path: init_generated_py = generated_dir / "__init__.py" root_init_py = plugin_dir / "__init__.py" - schemas_py.write_text(_render_schemas_py(config.extensions), encoding="utf-8") + schemas_py.write_text( + _render_schemas_py( + custom_fields, mappings_only_objs=mappings_only_objs | transforms_only_objs + ), + encoding="utf-8", + ) init_generated_py.write_text(_render_generated_init_py(), encoding="utf-8") plugin_variable_name = _normalize_identifier(plugin_dir.name) root_init_py.write_text( - _render_plugin_init_py(plugin_variable_name), encoding="utf-8" + _render_plugin_init_py(plugin_variable_name, config), encoding="utf-8" ) return generated_dir diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index eab8cb8f4..7f8c037fa 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -3,79 +3,146 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Callable, Generic, TypeVar +from typing import Any, Generic, TypeVar, overload -from .specs import SchemaExtensions -from .types import ClientConfig, ObjectSchemas, ObjectSchemasInput, PluginExtensionsMeta +from .types import ( + PluginExtensions, + PluginExtensionsMeta, +) T = TypeVar("T") +TSchemas = TypeVar("TSchemas") +_TSchemasContainer = TypeVar("_TSchemasContainer") @dataclass(frozen=True) -class PluginConfig: - """Build-time plugin config discoverable by the code generator. +class PluginConfig(Generic[TSchemas]): + """Build-time plugin config produced by define_plugin() and consumed by generate.py. - extensions: custom field declarations (read by generate.py — do not rename). - meta: optional plugin identity and capability declaration. - transform_schemas: optional bidirectional transform callables per object. - Stored as ObjectSchemasInput (not compiled to ObjectSchemas) in the PoC. - Full compilation with model_validate wrapping is a TODO for the real SDK. + Generic on TSchemas so the precise type of the schemas dict is preserved — e.g. + PluginConfig[dict[str, ObjectSchemasInput[MyNative, MyCg]]] — rather than being + widened to ObjectSchemasInput[Any, Any] at the storage boundary. - TODO (full SDK): add get_client, filters. + Stores inputs as-is — no compilation occurs at define_plugin() call time. + generate.py compiles this into a fully resolved Plugin by: + - Injecting the generated Pydantic model class as the common schema for each + ObjectSchemasInput entry (schemas[obj].native + common → ObjectSchemas). + - Auto-generating build_transforms() calls for any object that has + extensions.schemas[obj].mappings but no explicit schemas[obj] entry. + + All fields are optional so adopters can start with only what they need. """ - extensions: SchemaExtensions + extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None - transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None + schemas: TSchemas | None = None @dataclass class Plugin(Generic[T]): - """Runtime plugin container with extension specs and generated schemas. - - extensions: SchemaExtensions used by generate.py (do not rename or reorder — - the generated __init__.py constructs Plugin(extensions=..., schemas=...)). - schemas: generated _Schemas object (typed Pydantic model classes from generate.py). - NOTE: there is a naming collision: ADR-0022 also calls its runtime transform - dict "schemas". These are different concepts sharing the same name — a design - question to resolve in the full SDK (see Design Finding #1 in the spec). - transform_schemas: ADR-0022 runtime transform dict; named distinctly from - `schemas` to avoid collision with the generated schemas field in the PoC. - - TODO (full SDK): memoize get_client. + """Runtime plugin container assembled by generate.py after code generation. + + schemas: the _Schemas object from generated/schemas.py. Each attribute is an + ObjectSchemas instance providing unified access to the model class and + transforms for that object: + plugin.schemas.Opportunity.common → the Pydantic model class (includes + any custom fields declared by the plugin) + plugin.schemas.Opportunity.to_common → transform callable (or None) + plugin.schemas.Opportunity.from_common → transform callable (or None) + plugin.schemas.Opportunity.native → the source system's type (or dict) """ - extensions: SchemaExtensions - schemas: T # generated _Schemas object — keep as positional for generate.py compat + schemas: T + extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None - get_client: Callable[[ClientConfig], Any] | None = None # TODO: memoize - # PoC stores ObjectSchemasInput here (no compilation yet); full SDK will store - # ObjectSchemas after model_validate wrapping. Annotated as the union so both - # the current PoC usage and the future compiled form are type-safe. - transform_schemas: ( - dict[str, ObjectSchemasInput[Any, Any] | ObjectSchemas[Any, Any]] | None - ) = None - filters: dict[str, dict[str, Any]] | None = None + + +@overload +def define_plugin( + meta: PluginExtensionsMeta | None = ..., + extensions: PluginExtensions | None = ..., + schemas: None = ..., +) -> PluginConfig[None]: ... + + +@overload +def define_plugin( + meta: PluginExtensionsMeta | None = ..., + extensions: PluginExtensions | None = ..., + schemas: TSchemas = ..., +) -> PluginConfig[TSchemas]: ... def define_plugin( - extensions: SchemaExtensions, meta: PluginExtensionsMeta | None = None, - transform_schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, - # TODO (full SDK): get_client, filters -) -> PluginConfig: - """Create a PluginConfig object consumed by the code generator. - - Backward-compatible: existing callers passing only `extensions` are unaffected. - New params are stored as-is — no compilation occurs in the PoC. - - TODO (full SDK): - - Auto-generate transforms from extensions.schemas[obj].mappings when no - explicit to_common/from_common is supplied. - - Wrap transform output with model_validate. + extensions: PluginExtensions | None = None, + schemas: Any = None, +) -> PluginConfig[Any]: + """Create a PluginConfig consumed by the code generator. + + No compilation occurs here — inputs are stored as-is. The code generator + (generate.py) compiles ObjectSchemasInput → ObjectSchemas by injecting + the common model from the generated schemas. + + The return type is generic on the schemas argument: passing a typed dict + (e.g. {"Opportunity": ObjectSchemasInput[MyNative, MyCg](...) }) preserves + those per-object generics on the returned PluginConfig rather than widening + them to Any. """ return PluginConfig( extensions=extensions, meta=meta, - transform_schemas=transform_schemas, + schemas=schemas, ) + + +def inject_transforms( + config: PluginConfig[Any], schemas: _TSchemasContainer +) -> _TSchemasContainer: + """Wire transform callables from plugin config into the generated schemas container. + + Called by the generated plugin __init__.py to inject to_common/from_common + callables (and the native type) from cg_config into the ObjectSchemas instances + produced by the code generator. + + Iterates over all entries in config.schemas that have at least one callable, + validates that both directions are present, then sets the attributes on the + matching schemas container attribute (e.g. schemas.Opportunity). + + Returns the same schemas container (mutated in place) so callers can write + ``schemas = inject_transforms(config, schemas)`` and retain the concrete + generated type rather than widening to Any. + + Args: + config: The PluginConfig produced by define_plugin(). + schemas: The generated _Schemas container from generated/schemas.py. + + Returns: + The same schemas container, with transform callables injected. + + Raises: + ValueError: If a schema with any callable is missing its counterpart, + or if the object name is not found in the schemas container. + """ + if not config.schemas: + return schemas + for obj_name, schema_input in config.schemas.items(): + if schema_input.to_common is None and schema_input.from_common is None: + continue + obj_schemas = getattr(schemas, obj_name, None) + if obj_schemas is None: + raise ValueError( + f"Plugin object {obj_name!r}: not found in generated schemas" + ) + if schema_input.to_common is None: + raise ValueError( + f"Plugin object {obj_name!r}: to_common callable is required" + ) + if schema_input.from_common is None: + raise ValueError( + f"Plugin object {obj_name!r}: from_common callable is required" + ) + obj_schemas.native = schema_input.native or dict + obj_schemas.to_common = schema_input.to_common + obj_schemas.from_common = schema_input.from_common + return schemas diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index 6cf9a74bd..d6ef97852 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -1,10 +1,18 @@ """Extension types and utilities for SDK schema customization.""" +from __future__ import annotations + from dataclasses import dataclass -from typing import Any, Literal, Optional, TypedDict, cast +from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict from ..schemas.pydantic.fields.custom import CustomFieldType +if TYPE_CHECKING: + from .types import ( + PluginExtensions, + PluginExtensionsMeta, + ) + ConflictStrategy = Literal["error", "first_wins", "last_wins"] @@ -19,75 +27,109 @@ class CustomFieldSpec: class SchemaExtensions(TypedDict, total=False): - """Maps extensible model names to custom field specifications - - This class is the definitive list of base model names that support custom fields. + """Maps extensible model names to custom field specifications. - Add schemas here if they support `custom_fields` extensions - - - Args: - total: Determines if all keys are required or not, defaults to False which means no keys are required. + Retained for callers that still use the flat TypedDict form directly. + merge_extensions now accepts PluginExtensions instead. """ Opportunity: dict[str, CustomFieldSpec] -def _merge_fields( - model_name: str, - model_fields: dict[str, CustomFieldSpec], - source_fields: dict[str, CustomFieldSpec], +def _merge_meta( + current: PluginExtensionsMeta | None, + incoming: PluginExtensionsMeta, on_conflict: ConflictStrategy, -) -> None: - """Merge source_fields into model_fields in place, applying the conflict strategy.""" - for field_name, spec in source_fields.items(): - if field_name in model_fields: +) -> PluginExtensionsMeta: + """Merge incoming meta into current, respecting on_conflict for non-None field collisions.""" + from .types import PluginExtensionsMeta as _PluginExtensionsMeta + + if current is None: + return incoming + + merged: dict[str, Any] = {} + for field_name in ("name", "version", "source_system", "capabilities"): + current_val = getattr(current, field_name) + incoming_val = getattr(incoming, field_name) + + if incoming_val is None: + merged[field_name] = current_val + elif current_val is None: + merged[field_name] = incoming_val + else: + # Both have non-None values — apply conflict strategy if on_conflict == "error": raise ValueError( - f'merge_extensions: duplicate field "{field_name}" on model "{model_name}"' + f'merge_extensions: duplicate meta field "{field_name}" ' + f"(existing: {current_val!r}, incoming: {incoming_val!r})" ) - if on_conflict == "first_wins": - continue - model_fields[field_name] = spec + merged[field_name] = ( + current_val if on_conflict == "first_wins" else incoming_val + ) - -def _merge_source( - result: dict[str, dict[str, CustomFieldSpec]], - source: SchemaExtensions, - on_conflict: ConflictStrategy, -) -> None: - """Merge a single source into result in place.""" - for model_name, source_fields in cast( - dict[str, dict[str, CustomFieldSpec]], source - ).items(): - model_fields = result.setdefault(model_name, {}) - _merge_fields(model_name, model_fields, source_fields, on_conflict) + return _PluginExtensionsMeta( + name=merged["name"], + version=merged["version"], + sourceSystem=merged["source_system"], + capabilities=merged["capabilities"], + ) def merge_extensions( - sources: list[SchemaExtensions], on_conflict: ConflictStrategy = "error" -) -> SchemaExtensions: - """Merge multiple extension sources into one schema extension mapping. + sources: list[PluginExtensions], on_conflict: ConflictStrategy = "error" +) -> PluginExtensions: + """Merge multiple PluginExtensions into one. Args: - sources: Ordered list of extension mappings to merge. - on_conflict: Duplicate field strategy per model. - - ``"error"``: raise on duplicate field name. - - ``"first_wins"``: keep first seen definition. - - ``"last_wins"``: overwrite with latest definition. + sources: Ordered list of PluginExtensions to merge. + on_conflict: Strategy for duplicate field names per object. + - "error": raise on first duplicate (default). + - "first_wins": keep first seen value. + - "last_wins": overwrite with latest value. """ + from .types import ObjectMappings as _ObjectMappings + from .types import PluginExtensions as _PluginExtensions + from .types import PluginExtensionsSchema as _PluginExtensionsSchema + if on_conflict not in {"error", "first_wins", "last_wins"}: raise ValueError( 'merge_extensions: on_conflict must be "error", "first_wins", or "last_wins"' ) if len(sources) == 0: - return {} + return _PluginExtensions() if len(sources) == 1: return sources[0] - result: dict[str, dict[str, CustomFieldSpec]] = {} - for source in sources: - _merge_source(result, source, on_conflict) + # Accumulate into plain dicts; construct PluginExtensions once at the end. + merged_mappings: dict[str, _ObjectMappings] = {} + merged_meta: PluginExtensionsMeta | None = None - return cast(SchemaExtensions, result) + for source in sources: + if source.schemas: + for obj, src_schema in source.schemas.items(): + if src_schema.mappings: + if obj in merged_mappings: + if on_conflict == "error": + raise ValueError( + f'merge_extensions: duplicate mappings for object "{obj}"' + ) + if on_conflict == "first_wins": + continue + merged_mappings[obj] = src_schema.mappings + if source.meta: + merged_meta = _merge_meta(merged_meta, source.meta, on_conflict) + + return _PluginExtensions( + meta=merged_meta, + schemas=( + { + obj: _PluginExtensionsSchema( + mappings=merged_mappings.get(obj), + ) + for obj in merged_mappings + } + if merged_mappings + else None + ), + ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index c2aaacdd4..1ff5570c8 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -10,7 +10,7 @@ from __future__ import annotations -from typing import Any, Callable +from typing import Any, Callable, TypeVar, overload from pydantic import BaseModel, ValidationError @@ -22,6 +22,39 @@ from .types import Handler, PluginError, TransformResult +TCommon = TypeVar("TCommon", bound=BaseModel) + + +def _validate_output_paths( + mapping: dict[str, Any], + model: type[BaseModel], + known_handlers: set[str], + direction: str = "to_common", +) -> None: + """Validate that top-level output keys in mapping are valid fields on model. + + Called when common_model is supplied to build_transforms(). Custom fields + declared by the plugin appear as regular model fields on the generated common + model and are therefore treated as valid output paths automatically. + + Raises ValueError if any top-level key is not a field name or alias on model. + """ + valid_names: set[str] = set(model.model_fields.keys()) + for field_info in model.model_fields.values(): + if field_info.alias: + valid_names.add(field_info.alias) + + # Top-level handler invocations (rare but structurally valid) are not output keys + output_keys = {k for k in mapping if k not in known_handlers} + invalid = output_keys - valid_names + if invalid: + noun = "field" if len(invalid) == 1 else "fields" + raise ValueError( + f"build_transforms ({direction}_mapping): unknown output {noun} " + f"{sorted(invalid)!r} for model {model.__name__}. " + f"Declare them as custom_fields in ObjectSchemasInput or check the field name." + ) + def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> None: """Walk the mapping tree and raise ValueError on structural malformation. @@ -68,6 +101,30 @@ def _validate_mapping(mapping: Any, known_handlers: set[str], path: str = "") -> _validate_mapping(value, known_handlers, current_path) +@overload +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = ..., + common_model: None = ..., +) -> tuple[ + Callable[[Any], TransformResult[Any]], + Callable[[Any], TransformResult[Any]], +]: ... + + +@overload +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = ..., + common_model: type[TCommon] = ..., +) -> tuple[ + Callable[[Any], TransformResult[TCommon | dict[str, Any]]], + Callable[[Any], TransformResult[dict[str, Any]]], +]: ... + + def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], @@ -84,6 +141,23 @@ def build_transforms( from_common_mapping: mapping from CommonGrants → native source. handlers: Optional additional handlers registered for this call only. Keys must not collide with DEFAULT_HANDLERS (raises ValueError if they do). + Each ``build_transforms()`` call gets its own isolated handler registry — + handlers passed here are not visible to any other call. + + Example:: + + def handle_upper(data, path): + parts = path.split(".") + val = data + for p in parts: + val = val.get(p) if isinstance(val, dict) else None + return str(val).upper() if val is not None else None + + to_common, _ = build_transforms( + {"title": {"upper": "data.opportunity_title"}}, + {}, + handlers={"upper": handle_upper}, + ) common_model: Optional Pydantic model class to validate the to_common output against. Must be the fully extended generated model class (e.g. the generated Opportunity from generated/schemas.py), NOT the base class @@ -128,6 +202,10 @@ def build_transforms( _validate_mapping(to_common_mapping, known) _validate_mapping(from_common_mapping, known) + # When common_model is provided, validate that to_common output keys are real fields + if common_model is not None: + _validate_output_paths(to_common_mapping, common_model, known, "to_common") + def to_common(native: Any) -> TransformResult[Any]: try: result = transform_from_mapping(native, to_common_mapping, handlers=merged) @@ -159,6 +237,9 @@ def to_common(native: Any) -> TransformResult[Any]: for e in exc.errors() ] return TransformResult(result=result, errors=errors) + except Exception as exc: + error = PluginError(str(exc), path=None, source_value=result, cause=exc) + return TransformResult(result=result, errors=[error]) def from_common(common: Any) -> TransformResult[Any]: try: diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 8be07358a..eccacc519 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -18,7 +18,6 @@ # Type aliases Handler = Callable[[Any, Any], Any] -ClientConfig = dict[str, Any] class PluginError(Exception): @@ -89,9 +88,8 @@ class PluginExtensionsMeta(BaseModel): class PluginExtensionsSchema(BaseModel): - """Per-object config inside extensions.schemas. + """Per-object config inside extensions.schemas. Holds declarative mappings only. - custom_fields: custom field declarations (merged by merge_extensions). mappings: optional ADR-0017 declarative mappings. When present and no explicit to_common / from_common is supplied in schemas[obj], define_plugin() will auto-invoke build_transforms() on these (TODO — ADR-0022 Decision #6). @@ -99,9 +97,6 @@ class PluginExtensionsSchema(BaseModel): model_config = ConfigDict(populate_by_name=True) - custom_fields: dict[str, CustomFieldSpec] | None = Field( - default=None, alias="customFields" - ) mappings: ObjectMappings | None = None @@ -125,6 +120,9 @@ class ObjectSchemasInput(Generic[TNative, TCommon]): hand-written or generated via build_transforms(). native defaults to dict[str, Any] if omitted. + custom_fields declares any extra fields this object exposes beyond the base + CommonGrants schema. The code generator reads these and emits typed subclasses. + common is intentionally absent here. It is injected by define_plugin() during compilation from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes produced by the code generator. Plugin authors never set it directly — @@ -132,20 +130,29 @@ class ObjectSchemasInput(Generic[TNative, TCommon]): """ native: type[TNative] | None = None + custom_fields: dict[str, CustomFieldSpec] | None = None to_common: Callable[[TNative], TransformResult[TCommon]] | None = None from_common: Callable[[TCommon], TransformResult[TNative]] | None = None @dataclass class ObjectSchemas(Generic[TNative, TCommon]): - """Runtime compiled type produced by define_plugin() — not provided directly by authors. - - In the PoC, define_plugin() stores ObjectSchemasInput as-is; full compilation - (adding common from the base CG model, wrapping with model_validate) is a TODO - for the real SDK (ADR-0022 Decision #7). + """Runtime compiled schema container for a single object (ADR-0022). + + Bundles the type information and transform callables for one schema object + (e.g. Opportunity). Accessed via attribute lookup on the plugin's schemas + container: plugin.schemas.Opportunity. + + native: The source system's Python type (defaults to dict when not specified). + common: The CommonGrants-format Pydantic model class produced by the generator. + If the plugin declares custom_fields, this is a generated subclass of + the base CG model (e.g. OpportunityBase) with those fields already + baked in as typed attributes. + to_common: Transforms native_data → TransformResult[common] (None if not configured). + from_common: Transforms common_data → TransformResult[native] (None if not configured). """ native: type[TNative] common: type[TCommon] - to_common: Callable[[TNative], TransformResult[TCommon]] - from_common: Callable[[TCommon], TransformResult[TNative]] + to_common: Callable[[TNative], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TNative]] | None = None diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index d876ad406..05e5ba6c6 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -1,5 +1,5 @@ from typing import Optional, Any, Type, TypeVar -from pydantic import BaseModel, Field, create_model, ConfigDict +from pydantic import BaseModel, Field, create_model, ConfigDict, model_validator from ..schemas.pydantic.fields import CustomField, CustomFieldType from ..schemas.pydantic.base import CommonGrantsBaseModel from common_grants_sdk.utils.json import snake @@ -37,10 +37,29 @@ def add_custom_fields( # Accumulate all field definitions field_defs: dict[str, Any] = create_custom_field_schema(name=name, fields=fields) - # Unknown keys ignored for now. - # TODO: switch extra="allow" + add validator to parse extras into CustomField class _CustomFieldsBase(CommonGrantsBaseModel): - model_config = ConfigDict(populate_by_name=True, extra="ignore") + model_config = ConfigDict(populate_by_name=True, extra="allow") + + @model_validator(mode="before") + @classmethod + def _parse_extra_as_custom_fields(cls, values: Any) -> Any: + """Wrap unknown dict-valued keys as CustomField instances before validation.""" + if not isinstance(values, dict): + return values + # Build the set of known keys: both Python attr names and their aliases + known: set[str] = set() + if hasattr(cls, "model_fields"): + for attr, field_info in cls.model_fields.items(): + known.add(attr) + alias = getattr(field_info, "alias", None) + if alias: + known.add(alias) + for key, val in list(values.items()): + if key not in known and isinstance(val, dict) and "fieldType" in val: + # Inject the key as the name if not already present + enriched = {"name": key, **val} + values[key] = CustomField.model_validate(enriched) + return values # Create container with ALL accumulated field definitions, # this will be used when we recreate the base pydantic model with the diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index bb0778122..c36fed6eb 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -7,6 +7,8 @@ from typing import Any, Callable +from pydantic import BaseModel + handle_func = Callable[[dict, Any], Any] @@ -146,7 +148,7 @@ def __init__(self, handler: str, cause: Exception) -> None: def transform_from_mapping( - data: dict, + data: Any, mapping: dict, depth: int = 0, max_depth: int = 500, @@ -204,9 +206,10 @@ def transform_from_mapping( # Normalize Pydantic model instances to plain dicts so that field path # extraction works regardless of whether the caller passes a raw dict or a # validated model (e.g. the output of to_common with common_model set). + # by_alias=True ensures camelCase alias keys are used (matching CG mapping paths). # mode="json" matches the convention used by CommonGrantsBaseModel.dump_with_mapping. - if hasattr(data, "model_dump"): - data = data.model_dump(mode="json") + if isinstance(data, BaseModel): + data = data.model_dump(mode="json", by_alias=True) # Check for maximum depth # This is a sanity check to prevent stack overflow from deeply nested mappings diff --git a/lib/python-sdk/examples/plugin_custom_fields.py b/lib/python-sdk/examples/plugin_custom_fields.py index 41c364c1a..89d4a21ee 100644 --- a/lib/python-sdk/examples/plugin_custom_fields.py +++ b/lib/python-sdk/examples/plugin_custom_fields.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from plugins.opportunity_extensions import opportunity_extensions # noqa: E402 +from plugins.opportunity_extensions.cg_config import config as opp_config # noqa: E402 # --------------------------------------------------------------------------- # Sample API payload containing our four custom fields @@ -56,7 +57,7 @@ # Use the model returned via opportunity_extensions # --------------------------------------------------------------------------- -opp = opportunity_extensions.schemas.Opportunity.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) assert opp.custom_fields is not None assert opp.custom_fields.program_area is not None @@ -76,6 +77,9 @@ # The plugin also exposes the original extension specs # --------------------------------------------------------------------------- -print("Registered extensions:") -for field_name, spec in opportunity_extensions.extensions["Opportunity"].items(): +print("Registered custom fields:") +assert opp_config.schemas is not None +_opp_custom_fields = opp_config.schemas["Opportunity"].custom_fields +assert _opp_custom_fields is not None +for field_name, spec in _opp_custom_fields.items(): print(f" {field_name}: {spec.field_type} — {spec.description}") diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py index 5ae38feac..ae59f443f 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/__init__.py +++ b/lib/python-sdk/examples/plugins/grants_gov/__init__.py @@ -2,13 +2,16 @@ # the next time `python -m common_grants_sdk.extensions.generate` is run. from __future__ import annotations -from common_grants_sdk.extensions import Plugin +from common_grants_sdk.extensions import Plugin, inject_transforms from .cg_config import config from .generated import schemas +schemas = inject_transforms(config, schemas) + grants_gov = Plugin( - extensions=config.extensions, schemas=schemas, + extensions=config.extensions, + meta=config.meta, ) __all__ = ["grants_gov", "schemas"] diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py index b3f28b5e3..4000d845d 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -75,6 +75,14 @@ }, }, }, + "customFields": { + "legacyIdStr": { + "value": {"numberToString": "data.opportunity_id"}, + }, + "priorityScore": { + "value": {"stringToNumber": "data.priority_score_str"}, + }, + }, }, # from_common: CommonGrants Opportunity → grants.gov native from_common_mapping={ @@ -97,6 +105,9 @@ "forecasted_post_date": {"field": "keyDates.appOpens.date"}, "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, }, + "priority_score_str": { + "numberToString": "customFields.priorityScore.value" + }, } }, ) @@ -105,39 +116,44 @@ # Plugin config # --------------------------------------------------------------------------- -plugin = define_plugin( - # extensions: SchemaExtensions — dict[str, dict[str, CustomFieldSpec]] - extensions={ - "Opportunity": { - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - name="Legacy ID", - description="Unique identifier in legacy database", - ), - "agencyName": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Agency", - description="Agency hosting the opportunity", - ), - "applicantTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - name="Applicant types", - description="Types of applicants eligible to apply", - ), - } - }, +config = define_plugin( meta=PluginExtensionsMeta( name="grants-gov", version="0.1.0", sourceSystem="grants.gov", capabilities=["customFields", "transforms"], ), - transform_schemas={ + schemas={ "Opportunity": ObjectSchemasInput( + custom_fields={ + "legacyId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + name="Legacy ID", + description="Unique identifier in legacy database", + ), + "legacyIdStr": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Legacy ID (string)", + description="Legacy ID coerced to a string via numberToString", + ), + "agencyName": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Agency", + description="Agency hosting the opportunity", + ), + "applicantTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + name="Applicant types", + description="Types of applicants eligible to apply", + ), + "priorityScore": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + name="Priority score", + description="Numeric priority score coerced from a string via stringToNumber", + ), + }, to_common=to_common, from_common=from_common, ) }, ) - -config = plugin diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py index 3e1f6a8ac..736eb2390 100644 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py +++ b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py @@ -7,8 +7,9 @@ from .generated import schemas opportunity_extensions = Plugin( - extensions=config.extensions, schemas=schemas, + extensions=config.extensions, + meta=config.meta, ) __all__ = ["opportunity_extensions", "schemas"] diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py index a81c7c10c..a926e7e9f 100644 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py +++ b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py @@ -1,47 +1,39 @@ -"""Plugin config for the opportunity_extensions example. - -Defines four custom fields for the Opportunity model and registers them -with the CommonGrants SDK plugin framework. - -To generate the typed Pydantic models, run this command from the lib/python-sdk directory: - - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions - -This will emit generated/ and __init__.py alongside this file. """ +Plugin configuration for opportunity extensions. -from common_grants_sdk import define_plugin, merge_extensions -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions -from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType +Defines custom field extensions for the Opportunity schema: +- HHS-specific fields (programArea, legacyGrantId) +- Local fields (eligibilityTypes, awardCeiling) +""" -# Extensions that might come from a shared HHS package -hhs_extensions: SchemaExtensions = { - "Opportunity": { - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - }, -} +from typing import Any -# Extensions specific to this project -local_extensions: SchemaExtensions = { - "Opportunity": { - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - }, -} +from common_grants_sdk import define_plugin +from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput +from common_grants_sdk.extensions.plugin import PluginConfig +from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType -config = define_plugin( - merge_extensions([hhs_extensions, local_extensions], on_conflict="error"), +config: PluginConfig[Any] = define_plugin( + schemas={ + "Opportunity": ObjectSchemasInput( + custom_fields={ + "programArea": CustomFieldSpec( + field_type=CustomFieldType.STRING, + description="HHS program area code (e.g. 'CFDA-93.243')", + ), + "legacyGrantId": CustomFieldSpec( + field_type=CustomFieldType.INTEGER, + description="Numeric ID from the legacy grants management system", + ), + "eligibilityTypes": CustomFieldSpec( + field_type=CustomFieldType.ARRAY, + description="Types of organizations eligible to apply", + ), + "awardCeiling": CustomFieldSpec( + field_type=CustomFieldType.NUMBER, + description="Maximum award amount in USD", + ), + } + ) + } ) diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index 030521058..fc1dac7a3 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -23,7 +23,7 @@ # adds the script's directory (examples/) to sys.path. Import from there using # the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only # works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. -from plugins.grants_gov.cg_config import plugin +from plugins.grants_gov import grants_gov as plugin from plugins.grants_gov.generated.schemas import Opportunity from common_grants_sdk.extensions import build_transforms @@ -44,6 +44,7 @@ "opportunity_status": "posted", "opportunity_title": "Research into conservation techniques", "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "priority_score_str": "75", "summary": { "applicant_types": ["state_governments"], "archive_date": "2025-05-01", @@ -149,8 +150,7 @@ def _section(title: str) -> None: def main() -> None: - assert plugin.transform_schemas is not None - opp = plugin.transform_schemas["Opportunity"] + opp = plugin.schemas.Opportunity _section("SOURCE DATA (grants.gov format)") print(json.dumps(SOURCE_DATA, indent=2)) @@ -211,6 +211,11 @@ def main() -> None: .get("summary", {}) .get("award_ceiling"), ), + ( + "priority_score_str", + SOURCE_DATA["data"]["priority_score_str"], + native_result.result.get("data", {}).get("priority_score_str"), + ), ] all_pass = True for field, original, roundtripped in checks: @@ -239,6 +244,7 @@ def main() -> None: print(f" [path={err.path}] {err}") else: print("Validation: PASS — result is a typed Opportunity instance") + assert isinstance(custom_cg.result, Opportunity) opp_instance = custom_cg.result print(f"\n title: {opp_instance.title}") print(f" id: {opp_instance.id}") diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index 8fcadc799..b8947a47c 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -1,23 +1,28 @@ -"""Tests for expanded plugin.py — backward compat + new optional fields.""" +"""Tests for plugin.py — PluginExtensions-based API.""" from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin -from common_grants_sdk.extensions.specs import SchemaExtensions from common_grants_sdk.extensions.types import ( ObjectSchemasInput, + PluginExtensions, PluginExtensionsMeta, + PluginExtensionsSchema, TransformResult, ) -EXTENSIONS: SchemaExtensions = {} # minimal valid extensions - -def test_define_plugin_backward_compat(): - """define_plugin(extensions=...) still returns PluginConfig with all optional fields None.""" - config = define_plugin(extensions=EXTENSIONS) +def test_define_plugin_no_args(): + """define_plugin() with no args returns PluginConfig with all fields None.""" + config = define_plugin() assert isinstance(config, PluginConfig) - assert config.extensions is EXTENSIONS + assert config.extensions is None assert config.meta is None - assert config.transform_schemas is None + assert config.schemas is None + + +def test_define_plugin_with_extensions(): + ext = PluginExtensions() + config = define_plugin(extensions=ext) + assert config.extensions is ext def test_define_plugin_with_meta_and_schemas(): @@ -31,43 +36,48 @@ def passthrough(x): to_common=passthrough, from_common=passthrough ) } - config = define_plugin(extensions=EXTENSIONS, meta=meta, transform_schemas=schemas) + config = define_plugin(meta=meta, schemas=schemas) assert config.meta is meta assert config.meta.name == "test" - assert config.transform_schemas is schemas - - -def test_plugin_fields(): - """Plugin accepts all fields; optional ones default to None.""" - base = Plugin(extensions=EXTENSIONS, schemas=object()) - assert base.meta is None - assert base.get_client is None - assert base.transform_schemas is None - assert base.filters is None - - meta = PluginExtensionsMeta(name="p", source_system="s") - schemas = {"Opportunity": object()} - full = Plugin( - extensions=EXTENSIONS, schemas=object(), meta=meta, transform_schemas=schemas - ) - assert full.meta is meta - assert full.transform_schemas is schemas + assert config.schemas is schemas -def test_transform_schemas_callable_roundtrip(): - """The demo calls config.transform_schemas["Opportunity"].to_common(data).""" +def test_define_plugin_schemas_callable_roundtrip(): + """config.schemas["Opportunity"].to_common(data) works.""" def always_transformed(_x): return TransformResult(result={"transformed": True}, errors=[]) config = define_plugin( - extensions=EXTENSIONS, - transform_schemas={ + schemas={ "Opportunity": ObjectSchemasInput( to_common=always_transformed, from_common=always_transformed ) }, ) - result = config.transform_schemas["Opportunity"].to_common({"raw": "data"}) + result = config.schemas["Opportunity"].to_common({"raw": "data"}) assert result.result == {"transformed": True} assert result.errors == [] + + +def test_plugin_fields_default_to_none(): + """Plugin.schemas holds the container; meta/extensions default to None.""" + base = Plugin(schemas=object()) + assert base.meta is None + assert base.extensions is None + + +def test_plugin_fields_populated(): + meta = PluginExtensionsMeta(name="p", source_system="s") + ext = PluginExtensions(schemas={"Opportunity": PluginExtensionsSchema()}) + full = Plugin(schemas=object(), extensions=ext, meta=meta) + assert full.meta is meta + assert full.extensions is ext + + +def test_plugin_schemas_is_attribute_container(): + """Plugin.schemas holds the _Schemas object (no generated_schemas field).""" + s = object() + p = Plugin(schemas=s) + assert p.schemas is s + assert not hasattr(p, "generated_schemas") diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py index b0353ada1..e049b7021 100644 --- a/lib/python-sdk/tests/extensions/test_transforms.py +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -1,10 +1,24 @@ """Tests for build_transforms() in common_grants_sdk.extensions.transforms.""" +from typing import Any + import pytest from pydantic import BaseModel from common_grants_sdk.extensions.transforms import build_transforms from common_grants_sdk.extensions.types import PluginError, TransformResult + +# Shared model fixtures for output-path validation tests +class _CommonModel(BaseModel): + title: str + status: dict[str, Any] | None = None + + +class _CommonModelWithCustomFields(BaseModel): + title: str + custom_fields: dict[str, Any] | None = None + + # Shared source data matching the ADR-0017 grants.gov example SOURCE_DATA = { "data": { @@ -23,7 +37,7 @@ "title": {"field": "data.opportunity_title"}, "status": { "value": { - "switch": { + "match": { "field": "data.opportunity_status", "case": { "posted": "open", @@ -47,7 +61,7 @@ "data": { "opportunity_title": {"field": "title"}, "opportunity_status": { - "switch": { + "match": { "field": "status.value", "case": { "open": "posted", @@ -67,7 +81,7 @@ # --- Call-time validation --- -@pytest.mark.parametrize("name", ["field", "switch"]) +@pytest.mark.parametrize("name", ["field", "match", "switch"]) def test_handler_collision_raises(name): """build_transforms raises if custom handler shadows a default handler name.""" with pytest.raises(ValueError, match="collide with defaults"): @@ -194,6 +208,30 @@ def test_common_model_validation_failure(): assert result.result["title"] == "Research into conservation techniques" +def test_common_model_non_validation_error_is_caught() -> None: + """Non-ValidationError exceptions from model_validate surface as PluginError (errors-as-values contract).""" + + class _BrokenModel(BaseModel): + title: str + + @classmethod + def model_validate(cls, obj: Any, **kwargs: Any) -> "_BrokenModel": + raise TypeError("misconfigured root validator") + + to_common, _ = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {}, + common_model=_BrokenModel, + ) + result = to_common(SOURCE_DATA) + assert len(result.errors) == 1 + assert isinstance(result.errors[0], PluginError) + assert "misconfigured root validator" in str(result.errors[0]) + # raw transformed dict is preserved so the caller can inspect it + assert isinstance(result.result, dict) + assert result.result["title"] == "Research into conservation techniques" + + def test_custom_handler_registered_per_call(): """Custom handlers apply only to the call they are registered on.""" @@ -211,3 +249,60 @@ def handle_upper(data, path): ) result = to_common(SOURCE_DATA) assert result.result["title"] == "RESEARCH INTO CONSERVATION TECHNIQUES" + + +# --- Output field-path validation (when common_model is provided) --- + + +def test_unknown_output_key_raises_when_model_provided(): + """build_transforms raises at call time if a to_common output key is not on the model.""" + with pytest.raises(ValueError, match="unknown_xyz"): + build_transforms( + {"title": {"field": "data.opportunity_title"}, "unknown_xyz": "literal"}, + {}, + common_model=_CommonModel, + ) + + +def test_custom_fields_key_is_valid_output_path(): + """custom_fields is accepted as a top-level output key when the model declares it.""" + to_common, _ = build_transforms( + { + "title": {"field": "data.opportunity_title"}, + "custom_fields": {"legacyId": {"field": "data.opportunity_title"}}, + }, + {}, + common_model=_CommonModelWithCustomFields, + ) + result = to_common(SOURCE_DATA) + assert result.errors == [] + assert isinstance(result.result, _CommonModelWithCustomFields) + assert result.result.title == "Research into conservation techniques" + assert ( + result.result.custom_fields["legacyId"] + == "Research into conservation techniques" + ) + + +def test_output_path_validation_only_applies_to_to_common(): + """from_common output keys are not validated against common_model (they target native format).""" + # from_common_mapping has keys that are NOT on _CommonModel — that's fine + to_common, from_common = build_transforms( + {"title": {"field": "data.opportunity_title"}}, + {"data": {"opportunity_title": {"field": "title"}}}, + common_model=_CommonModel, + ) + result = from_common({"title": "hello"}) + assert result.errors == [] + assert result.result["data"]["opportunity_title"] == "hello" + + +def test_no_output_validation_without_common_model(): + """Without common_model, unknown output keys are allowed (no schema to validate against).""" + to_common, _ = build_transforms( + {"any_key_is_fine": {"field": "data.opportunity_title"}}, + {}, + ) + result = to_common(SOURCE_DATA) + assert result.errors == [] + assert result.result["any_key_is_fine"] == "Research into conservation techniques" diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index f44a3a86b..9a6e90250 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -1,6 +1,5 @@ """Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" -import pytest from common_grants_sdk.extensions.specs import CustomFieldSpec from common_grants_sdk.extensions.types import ( ObjectMappings, @@ -42,6 +41,14 @@ def test_plugin_error_structured_fields(): assert err.cause is cause +def test_plugin_error_source_value_excluded_from_str_and_repr(): + """source_value must not appear in str(err) or repr(err) — PII defence per ADR-0022 Decision #9.""" + sensitive = {"ssn": "123-45-6789"} + err = PluginError("transform failed", source_value=sensitive) + assert "123-45-6789" not in str(err) + assert "123-45-6789" not in repr(err) + + # --- TransformResult --- @@ -81,7 +88,6 @@ def test_plugin_extensions_meta(): def test_plugin_extensions_schema(): - assert PluginExtensionsSchema().custom_fields is None assert PluginExtensionsSchema().mappings is None s = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) assert s.mappings.to_common == {"a": "b"} @@ -93,10 +99,9 @@ def test_plugin_extensions_schema(): def test_plugin_extensions(): assert PluginExtensions().meta is None assert PluginExtensions().schemas is None - spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) - schema = PluginExtensionsSchema(customFields={"legacyId": spec}) + schema = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) ext = PluginExtensions(schemas={"Opportunity": schema}) - assert ext.schemas["Opportunity"].custom_fields == {"legacyId": spec} + assert ext.schemas["Opportunity"].mappings is not None # --- ObjectSchemasInput --- @@ -104,13 +109,18 @@ def test_plugin_extensions(): def test_object_schemas_input(): assert ObjectSchemasInput().native is None + assert ObjectSchemasInput().custom_fields is None assert ObjectSchemasInput().to_common is None + spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) + inp = ObjectSchemasInput(custom_fields={"legacyId": spec}) + assert inp.custom_fields == {"legacyId": spec} + def passthrough(x): return TransformResult(result=x, errors=[]) - inp = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) - assert inp.to_common is passthrough + inp2 = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) + assert inp2.to_common is passthrough # --- ObjectSchemas --- @@ -126,7 +136,7 @@ def passthrough(x): assert schemas.native is dict assert schemas.common is dict - with pytest.raises(TypeError): - ObjectSchemas( - native=dict, common=dict, to_common=passthrough - ) # missing from_common + # to_common and from_common are optional — omitting them is valid + minimal = ObjectSchemas(native=dict, common=dict) + assert minimal.to_common is None + assert minimal.from_common is None diff --git a/lib/python-sdk/tests/schemas/test_plugin_registry.py b/lib/python-sdk/tests/schemas/test_plugin_registry.py index c32ce0343..6d4ae5b0b 100644 --- a/lib/python-sdk/tests/schemas/test_plugin_registry.py +++ b/lib/python-sdk/tests/schemas/test_plugin_registry.py @@ -1,9 +1,12 @@ """Tests for OpportunityBase.with_custom_fields().""" +from typing import Any + import pytest -from common_grants_sdk.extensions import CustomFieldSpec, SchemaExtensions +from common_grants_sdk.extensions import CustomFieldSpec from common_grants_sdk.extensions import Plugin +from common_grants_sdk.extensions.types import ObjectSchemas from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.schemas.pydantic.models.opp_base import OpportunityBase @@ -15,27 +18,25 @@ class _Schemas: """Minimal schemas container that mirrors the generated _Schemas class.""" - def __init__(self, **models): - for name, model in models.items(): - setattr(self, name, model) + Opportunity: ObjectSchemas[Any, Any] def _make_plugin( field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity" ) -> "Plugin[_Schemas]": - """Build a Plugin whose schemas.Opportunity is produced by with_custom_fields().""" + """Build a Plugin whose schemas.Opportunity.common is produced by with_custom_fields().""" extended = OpportunityBase.with_custom_fields( custom_fields=field_specs, model_name=model_name, ) - extensions: SchemaExtensions = {"Opportunity": field_specs} - return Plugin(extensions=extensions, schemas=_Schemas(Opportunity=extended)) + s = _Schemas() + s.Opportunity = ObjectSchemas(native=dict, common=extended) + return Plugin(schemas=s) def _make_plugin_without_opportunity() -> "Plugin[_Schemas]": - """Build a Plugin that has no Opportunity schema (only Application).""" - extensions: SchemaExtensions = {} - return Plugin(extensions=extensions, schemas=_Schemas()) + """Build a Plugin that has no Opportunity schema.""" + return Plugin(schemas=_Schemas()) # --------------------------------------------------------------------------- @@ -84,7 +85,7 @@ def sample_payload() -> dict: def test_plugin_schema_is_subclass_of_opportunity_base(simple_plugin): - Opportunity = simple_plugin.schemas.Opportunity + Opportunity = simple_plugin.schemas.Opportunity.common assert Opportunity is not OpportunityBase assert issubclass(Opportunity, OpportunityBase) @@ -101,7 +102,10 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): {"award_ceiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER)} ) - assert simple_plugin.schemas.Opportunity is not second_plugin.schemas.Opportunity + assert ( + simple_plugin.schemas.Opportunity.common + is not second_plugin.schemas.Opportunity.common + ) # --------------------------------------------------------------------------- @@ -112,7 +116,7 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( simple_plugin, sample_payload ): - Opportunity = simple_plugin.schemas.Opportunity + Opportunity = simple_plugin.schemas.Opportunity.common opp = Opportunity.model_validate(sample_payload) @@ -123,7 +127,7 @@ def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( def test_plugin_schema_validates_custom_fields(simple_plugin, sample_payload): - Opportunity = simple_plugin.schemas.Opportunity + Opportunity = simple_plugin.schemas.Opportunity.common opp = Opportunity.model_validate(sample_payload) diff --git a/lib/python-sdk/tests/utils/test_custom_fields.py b/lib/python-sdk/tests/utils/test_custom_fields.py index b0e1e2c03..dd9f10e62 100644 --- a/lib/python-sdk/tests/utils/test_custom_fields.py +++ b/lib/python-sdk/tests/utils/test_custom_fields.py @@ -470,3 +470,29 @@ def test_empty_description_becomes_none(self): } ) assert opp.custom_fields.legacy_id.description is None + + +class TestUnknownCustomFieldsPreserved: + """Unknown customFields keys should survive round-trip (extra='allow' + validator).""" + + def test_unknown_key_survives_round_trip(self): + fields = { + "legacyId": CustomFieldSpec(field_type=CustomFieldType.INTEGER, value=int), + } + Opportunity = OpportunityBase.with_custom_fields( + custom_fields=fields, model_name="Opportunity" + ) + payload = { + **BASE_OPP, + "customFields": { + "legacyId": {"fieldType": "integer", "value": 42}, + "unknownField": {"fieldType": "string", "value": "preserved"}, + }, + } + + opp = Opportunity.model_validate(payload) + + assert opp.custom_fields is not None + assert opp.custom_fields.legacy_id.value == 42 + # unknown field should be present in model_extra (extra="allow" behaviour) + assert "unknownField" in opp.custom_fields.model_extra diff --git a/lib/python-sdk/tests/utils/test_merge_extensions.py b/lib/python-sdk/tests/utils/test_merge_extensions.py index 935595e20..945b1f837 100644 --- a/lib/python-sdk/tests/utils/test_merge_extensions.py +++ b/lib/python-sdk/tests/utils/test_merge_extensions.py @@ -1,134 +1,140 @@ import pytest -from common_grants_sdk.extensions import ( - CustomFieldSpec, - SchemaExtensions, - merge_extensions, +from common_grants_sdk.extensions import merge_extensions +from common_grants_sdk.extensions.types import ( + ObjectMappings, + PluginExtensions, + PluginExtensionsMeta, + PluginExtensionsSchema, ) -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -def test_merge_disjoint_extensions() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of eligible organizations", - ) - } - } - source_two: SchemaExtensions = { - "Opportunity": { - "priority_score": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Internal ranking score", +def test_merge_empty_returns_empty_plugin_extensions() -> None: + result = merge_extensions([]) + assert isinstance(result, PluginExtensions) + assert result.schemas is None + assert result.meta is None + + +def test_merge_single_source_passthrough() -> None: + source = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) ) } - } + ) + merged = merge_extensions([source]) + assert merged is source - merged = merge_extensions([source_one, source_two]) - assert "eligibility_type" in merged["Opportunity"] - assert "priority_score" in merged["Opportunity"] +def test_merge_meta_raises_on_duplicate_name_by_default() -> None: + source_one = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-a")) + source_two = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-b")) + with pytest.raises(ValueError, match="duplicate"): + merge_extensions([source_one, source_two]) -def test_merge_raises_on_duplicate_field_by_default() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.ARRAY) - } - } - source_two: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.STRING) - } - } - with pytest.raises( - ValueError, - match='duplicate field "eligibility_type" on model "Opportunity"', - ): - merge_extensions([source_one, source_two]) +def test_merge_meta_last_wins() -> None: + source_one = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-a")) + source_two = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-b")) + merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + assert merged.meta is not None + assert merged.meta.name == "plugin-b" -def test_merge_last_wins_on_duplicate_field() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="First", + +def test_merge_raises_on_duplicate_mappings_by_default() -> None: + source_one = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) ) } - } - source_two: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Last", + ) + source_two = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "other"}}, fromCommon={} + ) ) } - } + ) - merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + with pytest.raises(ValueError, match='duplicate mappings for object "Opportunity"'): + merge_extensions([source_one, source_two]) - assert ( - merged["Opportunity"]["eligibility_type"].field_type == CustomFieldType.STRING - ) - assert merged["Opportunity"]["eligibility_type"].description == "Last" +def test_merge_meta_none_fields_do_not_overwrite() -> None: + """A None field in the second source does not erase a value from the first.""" + source_one = PluginExtensions( + meta=PluginExtensionsMeta(name="plugin-a", version="1.0") + ) + source_two = PluginExtensions(meta=PluginExtensionsMeta(name=None, version="2.0")) -def test_merge_first_wins_on_duplicate_field() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="First", + merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + assert merged.meta is not None + assert merged.meta.name == "plugin-a" # preserved from source_one + assert merged.meta.version == "2.0" # overwritten by source_two + + +def test_merge_mappings_first_wins() -> None: + source_one = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) ) } - } - source_two: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Last", + ) + source_two = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "other"}}, fromCommon={} + ) ) } - } + ) merged = merge_extensions([source_one, source_two], on_conflict="first_wins") - - assert merged["Opportunity"]["eligibility_type"].field_type == CustomFieldType.ARRAY - assert merged["Opportunity"]["eligibility_type"].description == "First" - - -def test_merge_empty_inputs_returns_empty_mapping() -> None: - assert merge_extensions([]) == {} - - -def test_merge_single_source_passthrough() -> None: - source: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.ARRAY) - } + assert merged.schemas is not None + assert merged.schemas["Opportunity"].mappings is not None + assert merged.schemas["Opportunity"].mappings.to_common == { + "title": {"field": "name"} } - merged = merge_extensions([source]) - assert merged is source - - -def test_merge_overlapping_model_keys_without_field_conflicts() -> None: - source_one: SchemaExtensions = { - "Opportunity": { - "eligibility_type": CustomFieldSpec(field_type=CustomFieldType.ARRAY) +def test_merge_mappings_last_wins() -> None: + source_one = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "name"}}, fromCommon={} + ) + ) } - } - source_two: SchemaExtensions = { - "Opportunity": { - "funding_track": CustomFieldSpec(field_type=CustomFieldType.STRING) + ) + source_two = PluginExtensions( + schemas={ + "Opportunity": PluginExtensionsSchema( + mappings=ObjectMappings( + toCommon={"title": {"field": "other"}}, fromCommon={} + ) + ) } - } - - merged = merge_extensions([source_one, source_two]) + ) - assert set(merged["Opportunity"].keys()) == {"eligibility_type", "funding_track"} + merged = merge_extensions([source_one, source_two], on_conflict="last_wins") + assert merged.schemas is not None + assert merged.schemas["Opportunity"].mappings is not None + assert merged.schemas["Opportunity"].mappings.to_common == { + "title": {"field": "other"} + } diff --git a/lib/python-sdk/tests/utils/test_plugin_generator.py b/lib/python-sdk/tests/utils/test_plugin_generator.py index 212f824cd..a182475b0 100644 --- a/lib/python-sdk/tests/utils/test_plugin_generator.py +++ b/lib/python-sdk/tests/utils/test_plugin_generator.py @@ -10,7 +10,7 @@ import pytest -from common_grants_sdk import merge_extensions, define_plugin +from common_grants_sdk import define_plugin from common_grants_sdk.extensions import CustomFieldSpec from common_grants_sdk.extensions import PluginConfig from common_grants_sdk.schemas.pydantic.fields import CustomFieldType @@ -28,28 +28,25 @@ def _env_with_sdk_pythonpath() -> dict[str, str]: return env -def test_define_plugin_returns_config_with_extensions(): - extensions = { - "Opportunity": { - "program_area": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Grant category", - ) - } +def test_define_plugin_returns_config_with_schemas(): + from common_grants_sdk.extensions.types import ObjectSchemasInput + + schemas = { + "Opportunity": ObjectSchemasInput( + custom_fields={ + "program_area": CustomFieldSpec( + field_type=CustomFieldType.STRING, + description="Grant category", + ) + } + ) } - config = define_plugin(extensions) + config = define_plugin(schemas=schemas) assert isinstance(config, PluginConfig) - assert config.extensions == extensions - - -def test_merge_extensions_merges_extensions(): - one = {"Opportunity": {"program_area": CustomFieldSpec(field_type="string")}} - two = {"Opportunity": {"eligibility_type": CustomFieldSpec(field_type="array")}} - - merged = merge_extensions([one, two], on_conflict="error") - - assert set(merged["Opportunity"].keys()) == {"program_area", "eligibility_type"} + assert config.schemas is schemas + assert config.schemas["Opportunity"].custom_fields is not None + assert "program_area" in config.schemas["Opportunity"].custom_fields def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): @@ -62,23 +59,26 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): (plugin_dir / "cg_config.py").write_text( "\n".join( [ - "from common_grants_sdk import merge_extensions, define_plugin", - "from common_grants_sdk.extensions import SchemaExtensions, CustomFieldSpec", + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", - "local_extensions: SchemaExtensions = {", - ' "Opportunity": {', - ' "program_area": CustomFieldSpec(', - ' field_type="string",', - ' description="Program area",', - " ),", - ' "eligibility_type": CustomFieldSpec(', - ' field_type="array",', - ' description="Types of eligible organizations",', - " ),", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "program_area": CustomFieldSpec(', + ' field_type="string",', + ' description="Program area",', + " ),", + ' "eligibility_type": CustomFieldSpec(', + ' field_type="array",', + ' description="Types of eligible organizations",', + " ),", + " },", + " )", " },", - "}", - "", - "config = define_plugin(merge_extensions([local_extensions], on_conflict='error'))", + ")", "", ] ), @@ -105,7 +105,7 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): try: combined_module = importlib.import_module("plugins.combined") combined = getattr(combined_module, "combined") - opp_model = combined.schemas.Opportunity + opp_model = combined.schemas.Opportunity.common type_hints = get_type_hints(opp_model, include_extras=False) assert "custom_fields" in type_hints @@ -139,7 +139,7 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): "nonprofit", "city_government", ] - assert "Opportunity" in combined.extensions + assert combined.schemas.Opportunity.common is opp_model finally: sys.path.remove(str(tmp_path)) @@ -155,19 +155,24 @@ def test_generate_emits_import_for_pydantic_model_in_cg_config(tmp_path: Path): "from pydantic import BaseModel", "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", "class AgentInfo(BaseModel):", " name: str", " email: str", "", - "config = define_plugin({", - ' "Opportunity": {', - ' "point_of_contact": CustomFieldSpec(', - ' field_type="object",', - " value=AgentInfo,", - " ),", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "point_of_contact": CustomFieldSpec(', + ' field_type="object",', + " value=AgentInfo,", + " ),", + " },", + " )", " },", - "})", + ")", "", ] ), @@ -201,15 +206,20 @@ def test_generate_emits_import_for_external_module_type(tmp_path: Path): "from datetime import datetime", "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", - "config = define_plugin({", - ' "Opportunity": {', - ' "deadline": CustomFieldSpec(', - ' field_type="string",', - " value=datetime,", - " ),", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "deadline": CustomFieldSpec(', + ' field_type="string",', + " value=datetime,", + " ),", + " },", + " )", " },", - "})", + ")", "", ] ), @@ -232,6 +242,70 @@ def test_generate_emits_import_for_external_module_type(tmp_path: Path): assert "value: Optional[datetime]" in schemas_src +def test_generate_auto_builds_transforms_from_mappings(tmp_path): + """When cg_config has extensions.schemas[obj].mappings but no explicit schemas[obj], + the generated __init__.py calls build_transforms() automatically.""" + plugin_dir = tmp_path / "plugins" / "auto_transform" + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text("", encoding="utf-8") + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import PluginExtensions, PluginExtensionsSchema, ObjectMappings", + "", + "config = define_plugin(", + " extensions=PluginExtensions(", + " schemas={", + ' "Opportunity": PluginExtensionsSchema(', + " mappings=ObjectMappings(", + ' to_common={"title": {"field": "data.title"}},', + " from_common={},", + " ),", + " )", + " }", + " ),", + ")", + "", + ] + ), + encoding="utf-8", + ) + + from common_grants_sdk.extensions.generate import generate_plugin + + generate_plugin(plugin_dir) + + init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") + assert "build_transforms" in init_content + assert 'config.extensions.schemas["Opportunity"].mappings.to_common' in init_content + assert "_Opportunity_to_common" in init_content + assert "common_model=schemas.Opportunity.common" in init_content + + # Load the generated plugin and verify schemas are populated + import importlib + import sys + + # Remove any stale 'plugins' package from previous tests before inserting our path. + for key in list(sys.modules.keys()): + if key == "plugins" or key.startswith("plugins."): + del sys.modules[key] + + sys.path.insert(0, str(tmp_path)) + try: + mod = importlib.import_module("plugins.auto_transform") + plugin = getattr(mod, "auto_transform") + assert hasattr(plugin.schemas, "Opportunity") + assert plugin.schemas.Opportunity.to_common is not None + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + for key in list(sys.modules.keys()): + if key == "plugins" or key.startswith("plugins."): + del sys.modules[key] + + @pytest.mark.skipif(shutil.which("pyright") is None, reason="pyright is not installed") def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): plugins_dir = tmp_path / "plugins" @@ -244,14 +318,18 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): "\n".join( [ "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import SchemaExtensions, CustomFieldSpec", + "from common_grants_sdk.extensions import CustomFieldSpec", + "from common_grants_sdk.extensions.types import ObjectSchemasInput", "", - "extensions: SchemaExtensions = {", - ' "Opportunity": {', - ' "eligibility_type": CustomFieldSpec(field_type="array"),', + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " custom_fields={", + ' "eligibility_type": CustomFieldSpec(field_type="array"),', + " },", + " )", " },", - "}", - "config = define_plugin(extensions)", + ")", "", ] ), @@ -286,7 +364,7 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): ' "customFields": {"eligibility_type": {"fieldType": "array", "value": ["a"]}},', "}", "", - "opp = combined.schemas.Opportunity.model_validate(payload)", + "opp = combined.schemas.Opportunity.common.model_validate(payload)", "if opp.custom_fields is not None and opp.custom_fields.eligibility_type is not None:", " values = opp.custom_fields.eligibility_type.value", " reveal_type(values)", @@ -308,3 +386,173 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): ) assert pyright.returncode == 0, pyright.stdout + "\n" + pyright.stderr assert 'Type of "values" is "list[Any] | None"' in pyright.stdout + + +def test_generate_explicit_transforms(tmp_path): + """When cg_config has config.schemas with explicit to_common/from_common, + the generated __init__.py emits ObjectSchemas with the supplied callables.""" + from common_grants_sdk.extensions.generate import generate_plugin + + plugin_dir = tmp_path / "plugins" / "explicit_tf" + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text("", encoding="utf-8") + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import ObjectSchemasInput, TransformResult", + "from common_grants_sdk.extensions import CustomFieldSpec", + "", + "def _to_common(native):", + " return TransformResult(result={'title': native.get('name', '')}, errors=[])", + "", + "def _from_common(common):", + " return TransformResult(result={'name': common.get('title', '')}, errors=[])", + "", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + ' custom_fields={"legacyId": CustomFieldSpec(field_type="integer")},', + " to_common=_to_common,", + " from_common=_from_common,", + " )", + " },", + ")", + "", + ] + ), + encoding="utf-8", + ) + + generate_plugin(plugin_dir) + + init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") + # Explicit transforms use inject_transforms(), not per-object boilerplate + assert "build_transforms" not in init_content + assert "schemas = inject_transforms(config, schemas)" in init_content + # No per-object assignment lines + assert 'config.schemas["Opportunity"].to_common' not in init_content + assert 'config.schemas["Opportunity"].from_common' not in init_content + # ObjectSchemas is no longer constructed in __init__.py (only in schemas.py) + assert "ObjectSchemas" not in init_content + + # Load and verify the plugin works end-to-end + sys.path.insert(0, str(tmp_path)) + try: + mod = importlib.import_module("plugins.explicit_tf") + plugin = getattr(mod, "explicit_tf") + assert hasattr(plugin.schemas, "Opportunity") + opp_schemas = plugin.schemas.Opportunity + result = opp_schemas.to_common({"name": "Test Grant"}) + assert result.result == {"title": "Test Grant"} + assert result.errors == [] + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + for key in list(sys.modules.keys()): + if "explicit_tf" in key or key == "plugins": + del sys.modules[key] + + +def test_generate_transforms_only_no_custom_fields(tmp_path): + """Regression: config.schemas with only explicit transforms (no custom_fields, no extensions) + must produce a _Schemas entry for the object so inject_transforms() can access it at import. + """ + from common_grants_sdk.extensions.generate import generate_plugin + + plugin_dir = tmp_path / "plugins" / "transforms_only" + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text("", encoding="utf-8") + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import ObjectSchemasInput, TransformResult", + "", + "def _to_common(native):", + " return TransformResult(result={'title': native.get('name', '')}, errors=[])", + "", + "def _from_common(common):", + " return TransformResult(result={'name': common.get('title', '')}, errors=[])", + "", + "config = define_plugin(", + " schemas={", + ' "Opportunity": ObjectSchemasInput(', + " to_common=_to_common,", + " from_common=_from_common,", + " )", + " },", + ")", + "", + ] + ), + encoding="utf-8", + ) + + generate_plugin(plugin_dir) + + # schemas.py must assign self.Opportunity using the base SDK class + schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") + assert "self.Opportunity = ObjectSchemas" in schemas_src + assert "OpportunityBase" in schemas_src + + # __init__.py must use inject_transforms (not build_transforms) + init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") + assert "inject_transforms" in init_content + assert "build_transforms" not in init_content + + # Importing must not raise AttributeError / ValueError + sys.path.insert(0, str(tmp_path)) + try: + mod = importlib.import_module("plugins.transforms_only") + plugin = getattr(mod, "transforms_only") + assert hasattr(plugin.schemas, "Opportunity") + result = plugin.schemas.Opportunity.to_common({"name": "Test Grant"}) + assert result.result == {"title": "Test Grant"} + assert result.errors == [] + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + for key in list(sys.modules.keys()): + if "transforms_only" in key or key == "plugins": + del sys.modules[key] + + +def test_generate_raises_on_missing_mapping_direction(tmp_path): + """generate_plugin raises ValueError when a mappings-only object is missing + one of its mapping directions (to_common or from_common is None).""" + from common_grants_sdk.extensions.generate import generate_plugin + + plugin_dir = tmp_path / "bad_plugin" + plugin_dir.mkdir(parents=True) + + (plugin_dir / "cg_config.py").write_text( + "\n".join( + [ + "from common_grants_sdk import define_plugin", + "from common_grants_sdk.extensions.types import (", + " ObjectMappings, PluginExtensions, PluginExtensionsSchema,", + ")", + "", + "config = define_plugin(", + " extensions=PluginExtensions(", + " schemas={", + ' "Opportunity": PluginExtensionsSchema(', + " mappings=ObjectMappings(", + ' to_common={"title": {"field": "data.title"}},', + " # from_common intentionally omitted (None)", + " ),", + " )", + " }", + " ),", + ")", + "", + ] + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="from_common.*required"): + generate_plugin(plugin_dir) diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 8f826c03e..797cd511a 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -1,5 +1,5 @@ import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field from common_grants_sdk.utils.transformation import ( DEFAULT_HANDLERS, @@ -331,27 +331,46 @@ def test_handler_error_is_value_error(): ) -def test_pydantic_model_instance_is_normalized(): - """transform_from_mapping accepts a Pydantic model instance and extracts fields correctly.""" +def test_pydantic_model_instance_is_normalized() -> None: + """transform_from_mapping accepts a Pydantic model and uses camelCase alias keys.""" class Inner(BaseModel): + model_config = ConfigDict(populate_by_name=True) value: str class Source(BaseModel): + model_config = ConfigDict(populate_by_name=True) title: str - nested: Inner + nested_item: Inner = Field(alias="nestedItem") - model = Source(title="hello", nested=Inner(value="world")) + model = Source(title="hello", nestedItem=Inner(value="world")) result = transform_from_mapping( model, { "out_title": {"field": "title"}, - "out_value": {"field": "nested.value"}, + "out_value": {"field": "nestedItem.value"}, # camelCase alias path }, ) assert result == {"out_title": "hello", "out_value": "world"} +def test_pydantic_model_alias_not_snake_case() -> None: + """Field paths must use camelCase aliases, not snake_case attribute names.""" + + class Source(BaseModel): + model_config = ConfigDict(populate_by_name=True) + award_floor: int = Field(alias="awardFloor") + + model = Source(awardFloor=10000) + # camelCase alias path resolves correctly + result = transform_from_mapping(model, {"amount": {"field": "awardFloor"}}) + assert result == {"amount": 10000} + + # snake_case attribute name does NOT resolve (returns None) + result_snake = transform_from_mapping(model, {"amount": {"field": "award_floor"}}) + assert result_snake == {"amount": None} + + def test_deeply_nested(input_data): """ Test transformation with deeply nested structures. diff --git a/website/.cspell.json b/website/.cspell.json index 0eaf1b96a..ad88e0eb9 100644 --- a/website/.cspell.json +++ b/website/.cspell.json @@ -100,7 +100,8 @@ "apidevtools", "allof", "pypi", - "Pypi" + "Pypi", + "functools" ], "import": [] } diff --git a/website/src/content/docs/governance/adr/0017-mapping-format.md b/website/src/content/docs/governance/adr/0017-mapping-format.md index a4da1b044..457451177 100644 --- a/website/src/content/docs/governance/adr/0017-mapping-format.md +++ b/website/src/content/docs/governance/adr/0017-mapping-format.md @@ -31,7 +31,7 @@ For example, the following mapping: { "mappings": { "data": { - "title": "data.opportunity_title", + "title": { "field": "data.opportunity_title" }, "funding": { "minAwardAmount": { "amount": { @@ -89,6 +89,25 @@ Into the following output format: } ``` +### Field path convention + +Bare string values in a mapping are treated as **literals** by the transform +engine — they are returned as-is, not interpreted as field paths. + +To extract a value from the source data, use the `field` handler: + +```json +{ "title": { "field": "data.opportunity_title" } } +``` + +Not: + +```json +{ "title": "data.opportunity_title" } +``` + +The second form sets `title` to the literal string `"data.opportunity_title"`. + ### Example Here's a more complex example of the proposed mapping format. The following examples also serve as the input and target output for each option below. @@ -203,7 +222,7 @@ And we want to translate this data into the following format: { "mappings": { "data": { - "title": "data.opportunity_title", + "title": { "field": "data.opportunity_title" }, "status": { "value": { "match": { @@ -236,12 +255,12 @@ And we want to translate this data into the following format: }, "keyDates": { "appOpens": { - "date": "data.summary.forecasted_post_date", + "date": { "field": "data.summary.forecasted_post_date" }, "name": { "const": "Open Date" }, "description": { "const": "Applications begin being accepted" } }, "appDeadline": { - "date": "data.summary.forecasted_close_date", + "date": { "field": "data.summary.forecasted_close_date" }, "name": { "const": "Application Deadline" }, "description": { "const": "Final submission deadline for all grant applications" @@ -249,7 +268,7 @@ And we want to translate this data into the following format: }, "otherDates": { "forecastedAwardDate": { - "date": "data.summary.forecasted_award_date", + "date": { "field": "data.summary.forecasted_award_date" }, "name": { "const": "Forecasted award date" }, "description": { "const": "When we expect to announce awards for this opportunity." @@ -259,19 +278,19 @@ And we want to translate this data into the following format: }, "customFields": { "legacyId": { - "value": "data.opportunity_id", + "value": { "field": "data.opportunity_id" }, "name": { "const": "Legacy ID" }, "type": { "const": "number" }, "description": { "const": "Unique identifier in legacy database" } }, "agencyName": { - "value": "data.agency_name", + "value": { "field": "data.agency_name" }, "name": { "const": "Agency" }, "type": { "const": "string" }, "description": { "const": "Agency hosting the opportunity" } }, "applicantTypes": { - "value": "data.summary.applicant_types", + "value": { "field": "data.summary.applicant_types" }, "name": { "const": "Applicant types" }, "type": { "const": "array" }, "description": { "const": "Types of applicants eligible to apply" } @@ -346,7 +365,7 @@ JSON mapping is best if: { "mappings": { "data": { - "title": "data.opportunity_title", + "title": { "field": "data.opportunity_title" }, "status": { "value": { "match": { @@ -379,12 +398,12 @@ JSON mapping is best if: }, "keyDates": { "appOpens": { - "date": "data.summary.forecasted_post_date", + "date": { "field": "data.summary.forecasted_post_date" }, "name": { "const": "Open Date" }, "description": { "const": "Applications begin being accepted" } }, "appDeadline": { - "date": "data.summary.forecasted_close_date", + "date": { "field": "data.summary.forecasted_close_date" }, "name": { "const": "Application Deadline" }, "description": { "const": "Final submission deadline for all grant applications" @@ -392,7 +411,7 @@ JSON mapping is best if: }, "otherDates": { "forecastedAwardDate": { - "date": "data.summary.forecasted_award_date", + "date": { "field": "data.summary.forecasted_award_date" }, "name": { "const": "Forecasted award date" }, "description": { "const": "When we expect to announce awards for this opportunity." @@ -402,19 +421,19 @@ JSON mapping is best if: }, "customFields": { "legacyId": { - "value": "data.opportunity_id", + "value": { "field": "data.opportunity_id" }, "name": { "const": "Legacy ID" }, "type": { "const": "number" }, "description": { "const": "Unique identifier in legacy database" } }, "agencyName": { - "value": "data.agency_name", + "value": { "field": "data.agency_name" }, "name": { "const": "Agency" }, "type": { "const": "string" }, "description": { "const": "Agency hosting the opportunity" } }, "applicantTypes": { - "value": "data.summary.applicant_types", + "value": { "field": "data.summary.applicant_types" }, "name": { "const": "Applicant types" }, "type": { "const": "array" }, "description": { "const": "Types of applicants eligible to apply" } diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 52b33b74c..151f20255 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -40,7 +40,7 @@ We decided to: 5. **Make all top-level Plugin fields optional** so adopters can publish a plugin that provides only the features they need — for example, custom fields only — and expand to include transforms, client config, or additional schemas incrementally over time. -6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.`, `definePlugin()` invokes `buildTransforms()` automatically. Mappings for each direction are author-provided — `buildTransforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. +6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.`, the SDK invokes `buildTransforms()` automatically. In TypeScript this happens inside `definePlugin()`; in Python it happens inside the code generator (`generate.py`) at generation time, emitting a `build_transforms()` call into the generated `__init__.py`. Both mapping directions must be provided explicitly — `buildTransforms()` / `build_transforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. 7. **`toCommon` / `fromCommon` return a `TransformResult` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. In the current PoC, this validation is opt-in at the `buildTransforms()` call site via the `commonModel` / `common_model` parameter — when supplied, validation runs inside `toCommon` against the fully extended generated schema. In the full SDK, `definePlugin()` will additionally inject validation when auto-generating transforms from `extensions.schemas..mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. @@ -51,12 +51,18 @@ We decided to: The resulting Plugin shape: ``` -plugin.meta // name, version, sourceSystem, capabilities -plugin.getClient // (config: ClientConfig) => Client; memoized by definePlugin() -plugin.extensions // serializable; used by mergeExtensions() -plugin.schemas. = { native, common, toCommon, fromCommon } +plugin.meta // name, version, sourceSystem, capabilities +plugin.get_client // (config: ClientConfig) => Client; memoized by the code generator +plugin.extensions // serializable; used by merge_extensions() +plugin.schemas. // ObjectSchemas instance — unified access to model class and transforms +plugin.schemas..common // generated Pydantic model class (includes any declared custom fields) +plugin.schemas..native // source system type (defaults to dict) +plugin.schemas..to_common // callable: native → TransformResult[common] (None if not configured) +plugin.schemas..from_common // callable: common → TransformResult[native] (None if not configured) ``` +**Python note:** In the Python SDK, `define_plugin()` returns a `PluginConfig` (build-time input) rather than a fully compiled `Plugin`. The code generator (`generate.py`) compiles `PluginConfig → Plugin` by injecting the generated model classes as the `common` schema, wrapping `get_client` with `functools.lru_cache`, and auto-generating `build_transforms()` calls for any objects that have `extensions.schemas[obj].mappings` but no explicit `to_common`/`from_common` in `schemas[obj]`. This split is necessary because `cg_config.py` cannot import from `generated/` — it is the input to code generation. In the Python SDK, `custom_fields` is declared on `ObjectSchemasInput` (inside `schemas`) rather than on `PluginExtensionsSchema` (inside `extensions`). + ### Example interface @@ -264,21 +270,27 @@ class CustomFieldSpec: name: str = "" # optional; dict key is used as the display name fallback description: str = "" -# Runtime type — produced by define_plugin(), not provided directly by plugin authors +# Runtime schema container — assembled by the code generator, not provided directly by authors. +# Accessed via plugin.schemas. (attribute access, not dict lookup). +# common includes any custom fields declared by the plugin (it is a generated subclass of the +# base CG model, e.g. OpportunityBase, with typed custom_fields baked in). @dataclass class ObjectSchemas(Generic[TNative, TCommon]): - native: type[TNative] # expects a Pydantic BaseModel subclass - common: type[TCommon] # expects a Pydantic BaseModel subclass - to_common: Callable[[TNative], TransformResult[TCommon]] - from_common: Callable[[TCommon], TransformResult[TNative]] + native: type[TNative] # source system type; defaults to dict + common: type[TCommon] # generated Pydantic model class (includes declared custom fields) + to_common: Callable[[TNative], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TNative]] | None = None # Input type — provided by plugin authors inside define_plugin(schemas=...) # common is intentionally absent: cg_config.py cannot import from generated/ since # it is the input to generation. define_plugin() injects common during compilation # from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. +# custom_fields declares extra fields beyond the base CG schema; the code generator +# reads these and emits typed subclasses. @dataclass class ObjectSchemasInput(Generic[TNative, TCommon]): native: type[TNative] | None = None # defaults to dict[str, Any] if omitted + custom_fields: dict[str, CustomFieldSpec] | None = None to_common: Callable[[TNative], TransformResult[TCommon]] | None = None from_common: Callable[[TCommon], TransformResult[TNative]] | None = None @@ -292,18 +304,9 @@ class CustomFilterSpec: PluginCapability = Literal['customFields', 'customFilters', 'transforms', 'client'] -class PluginMeta(BaseModel): - model_config = ConfigDict(populate_by_name=True) - - name: str - version: str | None = None # optional; if omitted, define_plugin() infers it from the package's pyproject.toml / importlib.metadata - source_system: str = Field(alias='sourceSystem') - capabilities: list[PluginCapability] | None = None - -# Equivalent to TypeScript's Partial. Defined as a separate model -# rather than reusing PluginMeta because Pydantic does not have a built-in Partial. -# Note: if PluginMeta gains new required fields, this class must be updated manually. -# Drift can be caught with: assert PluginMeta.model_fields.keys() == PluginExtensionsMeta.model_fields.keys() +# PluginExtensionsMeta is used for both Plugin.meta (top-level plugin identity) and +# PluginExtensions.meta (serializable meta in the extensions JSON object). All fields +# are optional at the type level; plugins should populate name and source_system. class PluginExtensionsMeta(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -320,9 +323,9 @@ class ObjectMappings(BaseModel): from_common: dict[str, Any] | None = Field(default=None, alias='fromCommon') # CommonGrants → native class PluginExtensionsSchema(BaseModel): + """Per-object config inside extensions.schemas. Holds declarative mappings only.""" model_config = ConfigDict(populate_by_name=True) - custom_fields: dict[str, CustomFieldSpec] | None = Field(default=None, alias='customFields') # Optional declarative mappings. When present and no explicit to_common / from_common # is supplied in schemas[obj], define_plugin() auto-invokes build_transforms() on these. mappings: ObjectMappings | None = None @@ -333,44 +336,64 @@ class PluginExtensions(BaseModel): ClientConfig = dict[str, Any] # plugin authors define their own keys (auth, base_url, timeout, etc.) -# No `filters` field on Plugin — handled by the Client returned by get_client() (see Decision #2). +# Runtime plugin container — assembled by the code generator (generate.py) from +# the generated model classes and a compiled PluginConfig. Plugin authors do not +# construct this directly; it is emitted into the plugin's __init__.py. +# +# schemas: the _Schemas object from generated/schemas.py. Each attribute is an +# ObjectSchemas instance providing unified access to the model class and transforms: +# plugin.schemas.Opportunity.common → Pydantic model class (with custom fields) +# plugin.schemas.Opportunity.to_common → transform callable (or None) +# plugin.schemas.Opportunity.from_common → transform callable (or None) +# plugin.schemas.Opportunity.native → source system type (or dict) @dataclass -class Plugin: - meta: PluginMeta | None = None - get_client: Callable[[ClientConfig], Client] | None = None +class Plugin(Generic[T]): + schemas: T extensions: PluginExtensions | None = None - schemas: dict[ExtensibleSchemaName, ObjectSchemas[Any, Any]] | None = None + meta: PluginExtensionsMeta | None = None + get_client: Callable[[ClientConfig], Any] | None = None + filters: dict[str, Any] | None = None + +# Build-time config — produced by define_plugin(), consumed by generate.py. +# +# Compilation from PluginConfig → Plugin (injecting the common model classes from +# generated/, wrapping get_client with functools.lru_cache) happens inside +# generate.py at code-generation time, not at define_plugin() call time. This split +# is necessary in Python because cg_config.py cannot import from generated/ — it is +# the input to code generation. +@dataclass(frozen=True) +class PluginConfig: + extensions: PluginExtensions | None = None + meta: PluginExtensionsMeta | None = None + schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None + get_client: Callable[[ClientConfig], Any] | None = None + filters: dict[str, Any] | None = None # All params are optional — adopters can start with only what they need and expand # incrementally. Unlike TypeScript, Python supports named optional params at the # function root, so no DefinePluginOptions wrapper object is needed. # -# define_plugin compiles inputs into a Plugin by: +# define_plugin stores inputs as-is in a PluginConfig. The code generator +# (generate.py) then compiles PluginConfig → Plugin by: # - extending the base CommonGrants model with any declared custom_fields → common -# - native defaults to dict[str, Any] if omitted (extensions is JSON-safe; runtime -# Pydantic models cannot be included) -# - wrapping get_client with memoization so the same Client instance is returned -# for equivalent configs automatically +# - native defaults to dict[str, Any] if omitted +# - wrapping get_client with functools.lru_cache for memoization # # to_common / from_common may be plain hand-written callables, generated via -# build_transforms() and passed in schemas, or auto-generated by define_plugin() -# itself — when extensions.schemas[obj].mappings is declared and schemas[obj] provides -# no explicit transform, define_plugin() invokes build_transforms() internally. -# All transforms return TransformResult[T]; define_plugin() validates the result field -# at runtime with model_validate and appends any validation failures to the errors -# list rather than raising (see Decision #7). +# build_transforms() and passed in schemas, or auto-generated by the code generator +# itself — when extensions.schemas[obj].mappings is declared and schemas[obj] has no +# to_common/from_common, generate.py invokes build_transforms() in the emitted code. def define_plugin( - meta: PluginMeta | None = None, - get_client: Callable[[ClientConfig], Client] | None = None, + meta: PluginExtensionsMeta | None = None, + get_client: Callable[[ClientConfig], Any] | None = None, extensions: PluginExtensions | None = None, - schemas: dict[ExtensibleSchemaName, ObjectSchemasInput[Any, Any]] | None = None, - filters: dict[ExtensibleSchemaName, dict[str, CustomFilterSpec]] | None = None, -) -> Plugin: ... + schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, + filters: dict[str, Any] | None = None, +) -> PluginConfig: ... -# Exact signature shape is provisional pending SDK pin in #744. def merge_extensions( - *extensions: PluginExtensions, - on_conflict: Literal["error", "firstWins", "lastWins"] = "error", + sources: list[PluginExtensions], + on_conflict: Literal["error", "first_wins", "last_wins"] = "error", ) -> PluginExtensions: ... # Handler signature matches ADR-0017 runtime conventions. @@ -531,7 +554,7 @@ to_common, from_common = build_transforms( ) plugin = define_plugin( - meta=PluginMeta(name='grants-gov-plugin', version='1.0.0', source_system='grants.gov'), # source_system serializes as 'sourceSystem' in JSON + meta=PluginExtensionsMeta(name='grants-gov-plugin', version='1.0.0', source_system='grants.gov'), # source_system serializes as 'sourceSystem' in JSON # define_plugin memoizes get_client — the same Client is returned for equivalent configs. get_client=lambda config: Client(config=Config( base_url=config.get('base_url', 'https://api.grants.gov'), @@ -540,19 +563,14 @@ plugin = define_plugin( page_size=config.get('page_size', 100), list_items_limit=config.get('list_items_limit', 1000), )), - extensions=PluginExtensions( - schemas={ - 'Opportunity': PluginExtensionsSchema( - custom_fields={ - 'programArea': CustomFieldSpec(field_type=CustomFieldType.STRING, description='HHS program area code'), - 'legacyGrantId': CustomFieldSpec(field_type=CustomFieldType.INTEGER, description='Numeric ID from legacy system'), - }, - ), - }, - ), + # Python SDK: custom_fields lives on ObjectSchemasInput, not PluginExtensionsSchema schemas={ 'Opportunity': ObjectSchemasInput( native=GrantsGovOpportunity, + custom_fields={ + 'programArea': CustomFieldSpec(field_type=CustomFieldType.STRING, description='HHS program area code'), + 'legacyGrantId': CustomFieldSpec(field_type=CustomFieldType.INTEGER, description='Numeric ID from legacy system'), + }, to_common=to_common, from_common=from_common, ), @@ -561,7 +579,7 @@ plugin = define_plugin( # Combine extensions from multiple packages before constructing the plugin -merged = merge_extensions(base_extensions, grants_gov_extensions) +merged = merge_extensions([base_extensions, grants_gov_extensions]) merged_plugin = define_plugin(extensions=merged) # Calling get_client() with a config dict — memoized, so repeated calls return the same instance @@ -623,7 +641,7 @@ client = grants_gov_plugin.get_client({ # Use the compiled schemas to transform native data into CommonGrants shape. # to_common / from_common return TransformResult[T] = {result, errors} — consumers # apply their own strict-vs-lenient rule for what counts as success. -to_common = grants_gov_plugin.schemas['Opportunity'].to_common +to_common = grants_gov_plugin.schemas.Opportunity.to_common outcome = to_common(raw_grants_gov_data) if not outcome.errors: use(outcome.result) # strict: treat any error (including handler warnings) as failure @@ -647,7 +665,7 @@ print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms", " ### Consequences - **Positive consequences** - - Client stays singular — `getClient()` is memoized by `definePlugin()`, so one source system always produces one `Client` instance regardless of how many times `getClient()` is called + - Client stays singular — `getClient()` / `get_client()` is memoized (in TypeScript by `definePlugin()`; in Python by the code generator wrapping it with `functools.lru_cache`), so one source system always produces one `Client` instance regardless of how many times it is called - Top-level surface (`meta`, `client`, `schemas`, `extensions`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only - Dependency injection works along functional lines: pass `getClient`, pass `Schemas`, pass `Extensions` as coherent units without needing to reassemble from per-object branches - `mergeExtensions()` / `merge_extensions()` operates on flat, serializable data at the root, not on deeply nested per-object branches From 3a7894177db55614b701c91e7ed95c124dc90165 Mon Sep 17 00:00:00 2001 From: jcrichlake <145698165+jcrichlake@users.noreply.github.com> Date: Tue, 26 May 2026 13:03:45 -0400 Subject: [PATCH 03/51] [Issue #736] ADR: Nullable Optional Fields (#855) * Initial ADR * Formatting ADR * Fixing spelling --- website/.cspell.json | 3 +- .../adr/0024-optional-field-nullability.md | 123 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 website/src/content/docs/governance/adr/0024-optional-field-nullability.md diff --git a/website/.cspell.json b/website/.cspell.json index ad88e0eb9..c250d1bbc 100644 --- a/website/.cspell.json +++ b/website/.cspell.json @@ -49,7 +49,8 @@ "CFDA", "HBCUs", "TCCUs", - "widal001" + "widal001", + "categoricals" ], "ignoreWords": [ "opblock", diff --git a/website/src/content/docs/governance/adr/0024-optional-field-nullability.md b/website/src/content/docs/governance/adr/0024-optional-field-nullability.md new file mode 100644 index 000000000..f4cc54100 --- /dev/null +++ b/website/src/content/docs/governance/adr/0024-optional-field-nullability.md @@ -0,0 +1,123 @@ +--- +title: Optional field nullability in the protocol spec and SDKs +description: ADR documenting the decision on whether optional fields in the CommonGrants protocol should be nullable at the spec level. +--- + +Both the Python SDK (`Optional[X]` via Pydantic) and the TypeScript SDK (`.nullish()` via Zod) currently render optional fields as nullable. This is intentional for cross-language compatibility, since Python's `Optional[X]` requires nullability. However, the base protocol spec (TypeSpec-generated) declares these fields as optional but non-nullable. Once the `resolveAnyOf()` bug in `cg check spec` is fixed (see [#735](https://github.com/HHS/simpler-grants-protocol/issues/735)), both SDK implementations will correctly fail validation because they add `nullable` where the protocol doesn't allow it. A decision is needed on whether the base protocol should align with the SDKs, or vice versa. + +Experience from production grant data publishing systems highlights a subtlety that shapes this decision: `optional + nullable` is not sufficient on its own. Publishers need to express three distinct field states, not two: + +| State | Meaning | +| ------------------------ | ----------------------------------------------------------------------- | +| Field absent | "Not provided" — the publisher did not supply this data | +| Explicit N/A value | "Doesn't apply" — the field is intentionally irrelevant for this record | +| Field present with value | "Has a value" | + +The key insight is that the three-state model can be preserved under a nullable spec — but only if `null` and absent are given distinct, explicitly defined meanings. If `null` is defined as "doesn't apply" (an active publisher assertion) and field absence is defined as "not provided," the full semantic space is preserved for all field types, including scalars like dates where no natural sentinel value exists. This is not possible under a non-nullable spec, where scalar fields have no ergonomic way to express "doesn't apply." + +## Decision + +We've decided to make optional fields nullable in the base protocol (Option 1), with explicitly defined semantics for each field state. Field absence means "not provided." An explicit `null` means "doesn't apply" — the publisher actively asserts the field is irrelevant for this record. A present value means "has a value." The SDKs already reflect this model and require no changes. + +| Wire representation | Meaning | +| ------------------- | -------------------------------------------------------------------- | +| Field absent | "Not provided" — publisher did not supply this data | +| `null` | "Doesn't apply" — publisher actively asserts the field is irrelevant | +| Value | "Has a value" | + +- **Positive consequences** + - Preserves the full three-state semantic model for all field types, including scalars like dates where no sentinel value is practical + - Matches current behavior of both SDKs — no SDK changes required + - Publishers can actively assert "doesn't apply" on any field type using `null` + - Aligns with OpenAPI 3.1 / JSON Schema nullable conventions +- **Negative consequences** + - Requires clear documentation of the `null` vs. absent distinction — consumers must not treat them as equivalent + - Slightly increases implementation burden on consumers, who must handle both absent and `null` and interpret them differently + +### Criteria + +- **Cross-language compatibility:** The solution should work naturally for Python, TypeScript, and future SDK implementations without requiring awkward workarounds. +- **Wire-format clarity:** It should be clear whether `null` and absent/omitted carry the same semantic meaning or differ. +- **Semantic expressiveness:** Publishers should be able to distinguish "not provided" from "doesn't apply" without ambiguity. +- **Backwards compatibility:** Existing implementations should not be unnecessarily broken. +- **Alignment with OpenAPI 3.1 / JSON Schema conventions:** The approach should follow established conventions for optional and nullable fields. + +### Options considered + +- **Option 1 — Make optional fields nullable in the base protocol** +- **Option 2 — Keep optional fields non-nullable; update the SDKs** +- **Option 3 — Allow both representations (treat `null` and absent as equivalent)** + +## Evaluation + +### Side-by-side + +- ✅ Criterion met +- ❌ Criterion not met +- 🟡 Partially met or unsure + +| Criteria | Option 1 (nullable in spec) | Option 2 (non-nullable, fix SDKs) | Option 3 (allow both) | +| ---------------------------------------- | :-------------------------: | :-------------------------------: | :-------------------: | +| Cross-language compatibility | ✅ | 🟡 | ✅ | +| Wire-format clarity | ✅ | ✅ | ❌ | +| Semantic expressiveness | ✅ | 🟡 | ❌ | +| Backwards compatibility | ✅ | ❌ | ✅ | +| Alignment with OpenAPI 3.1 / JSON Schema | ✅ | ✅ | 🟡 | + +### Option 1 — Make optional fields nullable in the base protocol + +:::note[Bottom line] +Option 1 is best if: + +- we want to preserve the full three-state semantic model across all field types, including scalars like dates +- and are willing to document and enforce the `null` vs. absent distinction in the spec and consumer guidance + ::: + +Update the TypeSpec source to use nullable optionals, aligning with both SDKs. `null` and field absence are given distinct, protocol-defined meanings: `null` means "doesn't apply" and absence means "not provided." Publishers must actively send `null` to assert N/A — they cannot simply omit the field. + +- **Pros** + - Preserves the three-state semantic model for all field types, including dates and other scalars where sentinel values are impractical + - Matches current behavior of both SDKs — no SDK changes required + - Natural fit for Python (`Optional[X]`) and TypeScript (`.nullish()`) +- **Cons** + - Requires clear, enforced documentation of the `null` vs. absent distinction — without it, consumers may treat them as equivalent and lose the semantic benefit + - Slightly increases consumer implementation burden (must handle and distinguish both states) + +### Option 2 — Keep optional fields non-nullable; update the SDKs + +:::note[Bottom line] +Option 2 is best if: + +- all optional fields in the protocol are numerics or categoricals where a sentinel value is practical +- but does not work for scalar types like dates, where no natural "doesn't apply" value exists + ::: + +Both SDKs stop using `.nullish()` / `Optional[X]` and rely solely on field absence. The Python SDK would need a custom Pydantic config to omit `None` fields from serialization rather than sending `null`. Publishers use explicit sentinel values (zero for numerics, a `not_applicable` enum variant for categoricals) to assert "doesn't apply." + +- **Pros** + - `null` is never sent on the wire — no ambiguity between `null` and absent + - Works well for numerics and categoricals where a sentinel value is natural + - Aligns with OpenAPI 3.1 / JSON Schema conventions for non-nullable fields +- **Cons** + - Does not solve the "doesn't apply" problem for scalar types like dates — no ergonomic sentinel exists + - Requires changes to both SDKs and any existing implementations that send `null` + - Python's `Optional[X]` is idiomatic for "may not be present" — working around it adds complexity + +### Option 3 — Allow both representations + +:::note[Bottom line] +Option 3 is best if: + +- we want to maximize pragmatic compatibility across implementations +- but can accept a looser spec where `null` and absent are treated as semantically equivalent + ::: + +The protocol accepts either an absent field or an explicit `null`. The `cg check spec` validator is updated to treat a `type: T` schema as compatible with `type: [T, "null"]`. + +- **Pros** + - No breaking changes for any existing implementation + - Maximum flexibility for SDK authors +- **Cons** + - Collapses "not provided" and "doesn't apply" into a single ambiguous state + - Loosens the spec in a way that may hide real compatibility issues + - Makes it harder to tighten the spec or introduce meaningful N/A semantics in the future From 5cab45fa37f916ccf23be0b6bdb2f9b4622af0f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 10:52:57 -0700 Subject: [PATCH 04/51] chore(deps): bump tsx in the tooling group across 1 directory (#830) Bumps the tooling group with 1 update in the / directory: [tsx](https://github.com/privatenumber/tsx). Updates `tsx` from 4.21.0 to 4.22.3 - [Release notes](https://github.com/privatenumber/tsx/releases) - [Changelog](https://github.com/privatenumber/tsx/blob/master/release.config.cjs) - [Commits](https://github.com/privatenumber/tsx/compare/v4.21.0...v4.22.3) --- updated-dependencies: - dependency-name: tsx dependency-version: 4.21.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: tooling ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lib/ts-sdk/package.json | 2 +- pnpm-lock.yaml | 510 ++++++++++++++++++++-------------------- website/package.json | 2 +- 3 files changed, 253 insertions(+), 261 deletions(-) diff --git a/lib/ts-sdk/package.json b/lib/ts-sdk/package.json index a387ea780..d99ef8db1 100644 --- a/lib/ts-sdk/package.json +++ b/lib/ts-sdk/package.json @@ -108,7 +108,7 @@ "json-schema-faker": "^0.6.1", "prettier": "catalog:", "ts-node": "catalog:", - "tsx": "^4.21.0", + "tsx": "^4.22.3", "typescript": "catalog:", "typescript-eslint": "catalog:", "vitest": "catalog:" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79698462c..d8b9ec0f0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -116,7 +116,7 @@ importers: version: 1.10.0 '@vitest/coverage-v8': specifier: 'catalog:' - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) eslint: specifier: 'catalog:' version: 9.39.4 @@ -131,7 +131,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: 'catalog:' - version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) lib/cli: dependencies: @@ -207,7 +207,7 @@ importers: version: 0.81.0(@typespec/compiler@1.11.0(@types/node@20.19.41)) '@vitest/coverage-v8': specifier: 'catalog:' - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) eslint: specifier: 'catalog:' version: 9.39.4 @@ -219,7 +219,7 @@ importers: version: 5.5.5(eslint-config-prettier@9.1.2(eslint@9.39.4))(eslint@9.39.4)(prettier@3.8.3) eslint-plugin-vitest: specifier: 'catalog:' - version: 0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + version: 0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) prettier: specifier: 'catalog:' version: 3.8.3 @@ -240,7 +240,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: 'catalog:' - version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) lib/core: dependencies: @@ -337,7 +337,7 @@ importers: version: 0.81.0(@typespec/compiler@1.11.0(@types/node@20.19.41)) '@vitest/coverage-v8': specifier: 'catalog:' - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) ajv: specifier: ^8.20.0 version: 8.20.0 @@ -366,8 +366,8 @@ importers: specifier: 'catalog:' version: 10.9.2(@types/node@20.19.41)(typescript@5.9.3) tsx: - specifier: ^4.21.0 - version: 4.21.0 + specifier: ^4.22.3 + version: 4.22.3 typescript: specifier: 'catalog:' version: 5.9.3 @@ -376,7 +376,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: 'catalog:' - version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) website: dependencies: @@ -385,10 +385,10 @@ importers: version: 15.3.5(@types/json-schema@7.0.15) '@astrojs/react': specifier: ^5.0.5 - version: 5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.21.0)(yaml@2.8.3) + version: 5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3) '@astrojs/starlight': specifier: ^0.39.2 - version: 0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3))(typescript@5.9.3) + version: 0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) '@jsonforms/core': specifier: ^3.7.0 version: 3.7.0 @@ -409,7 +409,7 @@ importers: version: 8.20.0 astro: specifier: ^6.3.6 - version: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3) + version: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) js-yaml: specifier: ^4.1.1 version: 4.1.1 @@ -432,8 +432,8 @@ importers: specifier: ^5.32.6 version: 5.32.6(@types/react@18.3.28)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) tsx: - specifier: ^4.21.0 - version: 4.21.0 + specifier: ^4.22.3 + version: 4.22.3 typescript: specifier: 'catalog:' version: 5.9.3 @@ -509,13 +509,13 @@ importers: version: 0.14.1 starlight-links-validator: specifier: ^0.24.0 - version: 0.24.0(@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3)) + version: 0.24.0(@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) typescript-eslint: specifier: 'catalog:' version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: catalog:website - version: 4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + version: 4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) wrangler: specifier: ^4.93.1 version: 4.93.1 @@ -1104,14 +1104,14 @@ packages: cpu: [ppc64] os: [aix] - '@esbuild/aix-ppc64@0.27.4': - resolution: {integrity: sha512-cQPwL2mp2nSmHHJlCyoXgHGhbEPMrEEU5xhkcy3Hs/O7nGZqEpZ2sUtLaL9MORLtDfRvVl2/3PAuEkYZH0Ty8Q==} + '@esbuild/aix-ppc64@0.27.7': + resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/aix-ppc64@0.27.7': - resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} + '@esbuild/aix-ppc64@0.28.0': + resolution: {integrity: sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] @@ -1128,14 +1128,14 @@ packages: cpu: [arm64] os: [android] - '@esbuild/android-arm64@0.27.4': - resolution: {integrity: sha512-gdLscB7v75wRfu7QSm/zg6Rx29VLdy9eTr2t44sfTW7CxwAtQghZ4ZnqHk3/ogz7xao0QAgrkradbBzcqFPasw==} + '@esbuild/android-arm64@0.27.7': + resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm64@0.27.7': - resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} + '@esbuild/android-arm64@0.28.0': + resolution: {integrity: sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==} engines: {node: '>=18'} cpu: [arm64] os: [android] @@ -1152,14 +1152,14 @@ packages: cpu: [arm] os: [android] - '@esbuild/android-arm@0.27.4': - resolution: {integrity: sha512-X9bUgvxiC8CHAGKYufLIHGXPJWnr0OCdR0anD2e21vdvgCI8lIfqFbnoeOz7lBjdrAGUhqLZLcQo6MLhTO2DKQ==} + '@esbuild/android-arm@0.27.7': + resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-arm@0.27.7': - resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} + '@esbuild/android-arm@0.28.0': + resolution: {integrity: sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==} engines: {node: '>=18'} cpu: [arm] os: [android] @@ -1176,14 +1176,14 @@ packages: cpu: [x64] os: [android] - '@esbuild/android-x64@0.27.4': - resolution: {integrity: sha512-PzPFnBNVF292sfpfhiyiXCGSn9HZg5BcAz+ivBuSsl6Rk4ga1oEXAamhOXRFyMcjwr2DVtm40G65N3GLeH1Lvw==} + '@esbuild/android-x64@0.27.7': + resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/android-x64@0.27.7': - resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} + '@esbuild/android-x64@0.28.0': + resolution: {integrity: sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==} engines: {node: '>=18'} cpu: [x64] os: [android] @@ -1200,14 +1200,14 @@ packages: cpu: [arm64] os: [darwin] - '@esbuild/darwin-arm64@0.27.4': - resolution: {integrity: sha512-b7xaGIwdJlht8ZFCvMkpDN6uiSmnxxK56N2GDTMYPr2/gzvfdQN8rTfBsvVKmIVY/X7EM+/hJKEIbbHs9oA4tQ==} + '@esbuild/darwin-arm64@0.27.7': + resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-arm64@0.27.7': - resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} + '@esbuild/darwin-arm64@0.28.0': + resolution: {integrity: sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] @@ -1224,14 +1224,14 @@ packages: cpu: [x64] os: [darwin] - '@esbuild/darwin-x64@0.27.4': - resolution: {integrity: sha512-sR+OiKLwd15nmCdqpXMnuJ9W2kpy0KigzqScqHI3Hqwr7IXxBp3Yva+yJwoqh7rE8V77tdoheRYataNKL4QrPw==} + '@esbuild/darwin-x64@0.27.7': + resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/darwin-x64@0.27.7': - resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} + '@esbuild/darwin-x64@0.28.0': + resolution: {integrity: sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] @@ -1248,14 +1248,14 @@ packages: cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-arm64@0.27.4': - resolution: {integrity: sha512-jnfpKe+p79tCnm4GVav68A7tUFeKQwQyLgESwEAUzyxk/TJr4QdGog9sqWNcUbr/bZt/O/HXouspuQDd9JxFSw==} + '@esbuild/freebsd-arm64@0.27.7': + resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-arm64@0.27.7': - resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} + '@esbuild/freebsd-arm64@0.28.0': + resolution: {integrity: sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] @@ -1272,14 +1272,14 @@ packages: cpu: [x64] os: [freebsd] - '@esbuild/freebsd-x64@0.27.4': - resolution: {integrity: sha512-2kb4ceA/CpfUrIcTUl1wrP/9ad9Atrp5J94Lq69w7UwOMolPIGrfLSvAKJp0RTvkPPyn6CIWrNy13kyLikZRZQ==} + '@esbuild/freebsd-x64@0.27.7': + resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/freebsd-x64@0.27.7': - resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} + '@esbuild/freebsd-x64@0.28.0': + resolution: {integrity: sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] @@ -1296,14 +1296,14 @@ packages: cpu: [arm64] os: [linux] - '@esbuild/linux-arm64@0.27.4': - resolution: {integrity: sha512-7nQOttdzVGth1iz57kxg9uCz57dxQLHWxopL6mYuYthohPKEK0vU0C3O21CcBK6KDlkYVcnDXY099HcCDXd9dA==} + '@esbuild/linux-arm64@0.27.7': + resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm64@0.27.7': - resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} + '@esbuild/linux-arm64@0.28.0': + resolution: {integrity: sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==} engines: {node: '>=18'} cpu: [arm64] os: [linux] @@ -1320,14 +1320,14 @@ packages: cpu: [arm] os: [linux] - '@esbuild/linux-arm@0.27.4': - resolution: {integrity: sha512-aBYgcIxX/wd5n2ys0yESGeYMGF+pv6g0DhZr3G1ZG4jMfruU9Tl1i2Z+Wnj9/KjGz1lTLCcorqE2viePZqj4Eg==} + '@esbuild/linux-arm@0.27.7': + resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-arm@0.27.7': - resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} + '@esbuild/linux-arm@0.28.0': + resolution: {integrity: sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==} engines: {node: '>=18'} cpu: [arm] os: [linux] @@ -1344,14 +1344,14 @@ packages: cpu: [ia32] os: [linux] - '@esbuild/linux-ia32@0.27.4': - resolution: {integrity: sha512-oPtixtAIzgvzYcKBQM/qZ3R+9TEUd1aNJQu0HhGyqtx6oS7qTpvjheIWBbes4+qu1bNlo2V4cbkISr8q6gRBFA==} + '@esbuild/linux-ia32@0.27.7': + resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-ia32@0.27.7': - resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} + '@esbuild/linux-ia32@0.28.0': + resolution: {integrity: sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==} engines: {node: '>=18'} cpu: [ia32] os: [linux] @@ -1368,14 +1368,14 @@ packages: cpu: [loong64] os: [linux] - '@esbuild/linux-loong64@0.27.4': - resolution: {integrity: sha512-8mL/vh8qeCoRcFH2nM8wm5uJP+ZcVYGGayMavi8GmRJjuI3g1v6Z7Ni0JJKAJW+m0EtUuARb6Lmp4hMjzCBWzA==} + '@esbuild/linux-loong64@0.27.7': + resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-loong64@0.27.7': - resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} + '@esbuild/linux-loong64@0.28.0': + resolution: {integrity: sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] @@ -1392,14 +1392,14 @@ packages: cpu: [mips64el] os: [linux] - '@esbuild/linux-mips64el@0.27.4': - resolution: {integrity: sha512-1RdrWFFiiLIW7LQq9Q2NES+HiD4NyT8Itj9AUeCl0IVCA459WnPhREKgwrpaIfTOe+/2rdntisegiPWn/r/aAw==} + '@esbuild/linux-mips64el@0.27.7': + resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-mips64el@0.27.7': - resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} + '@esbuild/linux-mips64el@0.28.0': + resolution: {integrity: sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] @@ -1416,14 +1416,14 @@ packages: cpu: [ppc64] os: [linux] - '@esbuild/linux-ppc64@0.27.4': - resolution: {integrity: sha512-tLCwNG47l3sd9lpfyx9LAGEGItCUeRCWeAx6x2Jmbav65nAwoPXfewtAdtbtit/pJFLUWOhpv0FpS6GQAmPrHA==} + '@esbuild/linux-ppc64@0.27.7': + resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-ppc64@0.27.7': - resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} + '@esbuild/linux-ppc64@0.28.0': + resolution: {integrity: sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] @@ -1440,14 +1440,14 @@ packages: cpu: [riscv64] os: [linux] - '@esbuild/linux-riscv64@0.27.4': - resolution: {integrity: sha512-BnASypppbUWyqjd1KIpU4AUBiIhVr6YlHx/cnPgqEkNoVOhHg+YiSVxM1RLfiy4t9cAulbRGTNCKOcqHrEQLIw==} + '@esbuild/linux-riscv64@0.27.7': + resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-riscv64@0.27.7': - resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} + '@esbuild/linux-riscv64@0.28.0': + resolution: {integrity: sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] @@ -1464,14 +1464,14 @@ packages: cpu: [s390x] os: [linux] - '@esbuild/linux-s390x@0.27.4': - resolution: {integrity: sha512-+eUqgb/Z7vxVLezG8bVB9SfBie89gMueS+I0xYh2tJdw3vqA/0ImZJ2ROeWwVJN59ihBeZ7Tu92dF/5dy5FttA==} + '@esbuild/linux-s390x@0.27.7': + resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-s390x@0.27.7': - resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} + '@esbuild/linux-s390x@0.28.0': + resolution: {integrity: sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==} engines: {node: '>=18'} cpu: [s390x] os: [linux] @@ -1488,14 +1488,14 @@ packages: cpu: [x64] os: [linux] - '@esbuild/linux-x64@0.27.4': - resolution: {integrity: sha512-S5qOXrKV8BQEzJPVxAwnryi2+Iq5pB40gTEIT69BQONqR7JH1EPIcQ/Uiv9mCnn05jff9umq/5nqzxlqTOg9NA==} + '@esbuild/linux-x64@0.27.7': + resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/linux-x64@0.27.7': - resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} + '@esbuild/linux-x64@0.28.0': + resolution: {integrity: sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==} engines: {node: '>=18'} cpu: [x64] os: [linux] @@ -1512,14 +1512,14 @@ packages: cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-arm64@0.27.4': - resolution: {integrity: sha512-xHT8X4sb0GS8qTqiwzHqpY00C95DPAq7nAwX35Ie/s+LO9830hrMd3oX0ZMKLvy7vsonee73x0lmcdOVXFzd6Q==} + '@esbuild/netbsd-arm64@0.27.7': + resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-arm64@0.27.7': - resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} + '@esbuild/netbsd-arm64@0.28.0': + resolution: {integrity: sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] @@ -1536,14 +1536,14 @@ packages: cpu: [x64] os: [netbsd] - '@esbuild/netbsd-x64@0.27.4': - resolution: {integrity: sha512-RugOvOdXfdyi5Tyv40kgQnI0byv66BFgAqjdgtAKqHoZTbTF2QqfQrFwa7cHEORJf6X2ht+l9ABLMP0dnKYsgg==} + '@esbuild/netbsd-x64@0.27.7': + resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/netbsd-x64@0.27.7': - resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} + '@esbuild/netbsd-x64@0.28.0': + resolution: {integrity: sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] @@ -1560,14 +1560,14 @@ packages: cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-arm64@0.27.4': - resolution: {integrity: sha512-2MyL3IAaTX+1/qP0O1SwskwcwCoOI4kV2IBX1xYnDDqthmq5ArrW94qSIKCAuRraMgPOmG0RDTA74mzYNQA9ow==} + '@esbuild/openbsd-arm64@0.27.7': + resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-arm64@0.27.7': - resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} + '@esbuild/openbsd-arm64@0.28.0': + resolution: {integrity: sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] @@ -1584,14 +1584,14 @@ packages: cpu: [x64] os: [openbsd] - '@esbuild/openbsd-x64@0.27.4': - resolution: {integrity: sha512-u8fg/jQ5aQDfsnIV6+KwLOf1CmJnfu1ShpwqdwC0uA7ZPwFws55Ngc12vBdeUdnuWoQYx/SOQLGDcdlfXhYmXQ==} + '@esbuild/openbsd-x64@0.27.7': + resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/openbsd-x64@0.27.7': - resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} + '@esbuild/openbsd-x64@0.28.0': + resolution: {integrity: sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] @@ -1608,14 +1608,14 @@ packages: cpu: [arm64] os: [openharmony] - '@esbuild/openharmony-arm64@0.27.4': - resolution: {integrity: sha512-JkTZrl6VbyO8lDQO3yv26nNr2RM2yZzNrNHEsj9bm6dOwwu9OYN28CjzZkH57bh4w0I2F7IodpQvUAEd1mbWXg==} + '@esbuild/openharmony-arm64@0.27.7': + resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] - '@esbuild/openharmony-arm64@0.27.7': - resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} + '@esbuild/openharmony-arm64@0.28.0': + resolution: {integrity: sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] @@ -1632,14 +1632,14 @@ packages: cpu: [x64] os: [sunos] - '@esbuild/sunos-x64@0.27.4': - resolution: {integrity: sha512-/gOzgaewZJfeJTlsWhvUEmUG4tWEY2Spp5M20INYRg2ZKl9QPO3QEEgPeRtLjEWSW8FilRNacPOg8R1uaYkA6g==} + '@esbuild/sunos-x64@0.27.7': + resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/sunos-x64@0.27.7': - resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} + '@esbuild/sunos-x64@0.28.0': + resolution: {integrity: sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==} engines: {node: '>=18'} cpu: [x64] os: [sunos] @@ -1656,14 +1656,14 @@ packages: cpu: [arm64] os: [win32] - '@esbuild/win32-arm64@0.27.4': - resolution: {integrity: sha512-Z9SExBg2y32smoDQdf1HRwHRt6vAHLXcxD2uGgO/v2jK7Y718Ix4ndsbNMU/+1Qiem9OiOdaqitioZwxivhXYg==} + '@esbuild/win32-arm64@0.27.7': + resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-arm64@0.27.7': - resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} + '@esbuild/win32-arm64@0.28.0': + resolution: {integrity: sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] @@ -1680,14 +1680,14 @@ packages: cpu: [ia32] os: [win32] - '@esbuild/win32-ia32@0.27.4': - resolution: {integrity: sha512-DAyGLS0Jz5G5iixEbMHi5KdiApqHBWMGzTtMiJ72ZOLhbu/bzxgAe8Ue8CTS3n3HbIUHQz/L51yMdGMeoxXNJw==} + '@esbuild/win32-ia32@0.27.7': + resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-ia32@0.27.7': - resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} + '@esbuild/win32-ia32@0.28.0': + resolution: {integrity: sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==} engines: {node: '>=18'} cpu: [ia32] os: [win32] @@ -1704,14 +1704,14 @@ packages: cpu: [x64] os: [win32] - '@esbuild/win32-x64@0.27.4': - resolution: {integrity: sha512-+knoa0BDoeXgkNvvV1vvbZX4+hizelrkwmGJBdT17t8FNPwG2lKemmuMZlmaNQ3ws3DKKCxpb4zRZEIp3UxFCg==} + '@esbuild/win32-x64@0.27.7': + resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} engines: {node: '>=18'} cpu: [x64] os: [win32] - '@esbuild/win32-x64@0.27.7': - resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} + '@esbuild/win32-x64@0.28.0': + resolution: {integrity: sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -3919,13 +3919,13 @@ packages: engines: {node: '>=18'} hasBin: true - esbuild@0.27.4: - resolution: {integrity: sha512-Rq4vbHnYkK5fws5NF7MYTU68FPRE1ajX7heQ/8QXXWqNgqqJ/GkmmyxIzUnf2Sr/bakf8l54716CcMGHYhMrrQ==} + esbuild@0.27.7: + resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} engines: {node: '>=18'} hasBin: true - esbuild@0.27.7: - resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} + esbuild@0.28.0: + resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} engines: {node: '>=18'} hasBin: true @@ -4274,9 +4274,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - get-tsconfig@4.13.7: - resolution: {integrity: sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==} - get-tsconfig@5.0.0-beta.4: resolution: {integrity: sha512-7nF7C9fIPFEMHgEMEfgIlO9wDdZ8CyHw27rWciFZfHvHDReIiPhsYuzPRXsfvBCqFy1l8RRyyWV7QLM+ZhUJsQ==} engines: {node: '>=20.20.0'} @@ -6137,8 +6134,8 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tsx@4.21.0: - resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + tsx@4.22.3: + resolution: {integrity: sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg==} engines: {node: '>=18.0.0'} hasBin: true @@ -6929,12 +6926,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@astrojs/mdx@5.0.4(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3))': + '@astrojs/mdx@5.0.4(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@astrojs/markdown-remark': 7.1.1 '@mdx-js/mdx': 3.1.1 acorn: 8.16.0 - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3) + astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) es-module-lexer: 2.1.0 estree-util-visit: 2.0.0 hast-util-to-html: 9.0.5 @@ -6956,17 +6953,17 @@ snapshots: dependencies: prismjs: 1.30.0 - '@astrojs/react@5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.21.0)(yaml@2.8.3)': + '@astrojs/react@5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3)': dependencies: '@astrojs/internal-helpers': 0.9.1 '@types/react': 18.3.28 '@types/react-dom': 18.3.7(@types/react@18.3.28) - '@vitejs/plugin-react': 5.2.0(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + '@vitejs/plugin-react': 5.2.0(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) devalue: 5.8.1 react: 18.3.1 react-dom: 18.3.1(react@18.3.1) ultrahtml: 1.6.0 - vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) transitivePeerDependencies: - '@types/node' - jiti @@ -6987,17 +6984,17 @@ snapshots: stream-replace-string: 2.0.0 zod: 4.4.3 - '@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3))(typescript@5.9.3)': + '@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3)': dependencies: '@astrojs/markdown-remark': 7.1.1 - '@astrojs/mdx': 5.0.4(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3)) + '@astrojs/mdx': 5.0.4(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) '@astrojs/sitemap': 3.7.2 '@pagefind/default-ui': 1.5.2 '@types/hast': 3.0.4 '@types/js-yaml': 4.0.9 '@types/mdast': 4.0.4 - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3) - astro-expressive-code: 0.42.0(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3)) + astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro-expressive-code: 0.42.0(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) bcp-47: 2.1.0 hast-util-from-html: 2.0.3 hast-util-select: 6.0.4 @@ -7606,10 +7603,10 @@ snapshots: '@esbuild/aix-ppc64@0.27.3': optional: true - '@esbuild/aix-ppc64@0.27.4': + '@esbuild/aix-ppc64@0.27.7': optional: true - '@esbuild/aix-ppc64@0.27.7': + '@esbuild/aix-ppc64@0.28.0': optional: true '@esbuild/android-arm64@0.25.12': @@ -7618,10 +7615,10 @@ snapshots: '@esbuild/android-arm64@0.27.3': optional: true - '@esbuild/android-arm64@0.27.4': + '@esbuild/android-arm64@0.27.7': optional: true - '@esbuild/android-arm64@0.27.7': + '@esbuild/android-arm64@0.28.0': optional: true '@esbuild/android-arm@0.25.12': @@ -7630,10 +7627,10 @@ snapshots: '@esbuild/android-arm@0.27.3': optional: true - '@esbuild/android-arm@0.27.4': + '@esbuild/android-arm@0.27.7': optional: true - '@esbuild/android-arm@0.27.7': + '@esbuild/android-arm@0.28.0': optional: true '@esbuild/android-x64@0.25.12': @@ -7642,10 +7639,10 @@ snapshots: '@esbuild/android-x64@0.27.3': optional: true - '@esbuild/android-x64@0.27.4': + '@esbuild/android-x64@0.27.7': optional: true - '@esbuild/android-x64@0.27.7': + '@esbuild/android-x64@0.28.0': optional: true '@esbuild/darwin-arm64@0.25.12': @@ -7654,10 +7651,10 @@ snapshots: '@esbuild/darwin-arm64@0.27.3': optional: true - '@esbuild/darwin-arm64@0.27.4': + '@esbuild/darwin-arm64@0.27.7': optional: true - '@esbuild/darwin-arm64@0.27.7': + '@esbuild/darwin-arm64@0.28.0': optional: true '@esbuild/darwin-x64@0.25.12': @@ -7666,10 +7663,10 @@ snapshots: '@esbuild/darwin-x64@0.27.3': optional: true - '@esbuild/darwin-x64@0.27.4': + '@esbuild/darwin-x64@0.27.7': optional: true - '@esbuild/darwin-x64@0.27.7': + '@esbuild/darwin-x64@0.28.0': optional: true '@esbuild/freebsd-arm64@0.25.12': @@ -7678,10 +7675,10 @@ snapshots: '@esbuild/freebsd-arm64@0.27.3': optional: true - '@esbuild/freebsd-arm64@0.27.4': + '@esbuild/freebsd-arm64@0.27.7': optional: true - '@esbuild/freebsd-arm64@0.27.7': + '@esbuild/freebsd-arm64@0.28.0': optional: true '@esbuild/freebsd-x64@0.25.12': @@ -7690,10 +7687,10 @@ snapshots: '@esbuild/freebsd-x64@0.27.3': optional: true - '@esbuild/freebsd-x64@0.27.4': + '@esbuild/freebsd-x64@0.27.7': optional: true - '@esbuild/freebsd-x64@0.27.7': + '@esbuild/freebsd-x64@0.28.0': optional: true '@esbuild/linux-arm64@0.25.12': @@ -7702,10 +7699,10 @@ snapshots: '@esbuild/linux-arm64@0.27.3': optional: true - '@esbuild/linux-arm64@0.27.4': + '@esbuild/linux-arm64@0.27.7': optional: true - '@esbuild/linux-arm64@0.27.7': + '@esbuild/linux-arm64@0.28.0': optional: true '@esbuild/linux-arm@0.25.12': @@ -7714,10 +7711,10 @@ snapshots: '@esbuild/linux-arm@0.27.3': optional: true - '@esbuild/linux-arm@0.27.4': + '@esbuild/linux-arm@0.27.7': optional: true - '@esbuild/linux-arm@0.27.7': + '@esbuild/linux-arm@0.28.0': optional: true '@esbuild/linux-ia32@0.25.12': @@ -7726,10 +7723,10 @@ snapshots: '@esbuild/linux-ia32@0.27.3': optional: true - '@esbuild/linux-ia32@0.27.4': + '@esbuild/linux-ia32@0.27.7': optional: true - '@esbuild/linux-ia32@0.27.7': + '@esbuild/linux-ia32@0.28.0': optional: true '@esbuild/linux-loong64@0.25.12': @@ -7738,10 +7735,10 @@ snapshots: '@esbuild/linux-loong64@0.27.3': optional: true - '@esbuild/linux-loong64@0.27.4': + '@esbuild/linux-loong64@0.27.7': optional: true - '@esbuild/linux-loong64@0.27.7': + '@esbuild/linux-loong64@0.28.0': optional: true '@esbuild/linux-mips64el@0.25.12': @@ -7750,10 +7747,10 @@ snapshots: '@esbuild/linux-mips64el@0.27.3': optional: true - '@esbuild/linux-mips64el@0.27.4': + '@esbuild/linux-mips64el@0.27.7': optional: true - '@esbuild/linux-mips64el@0.27.7': + '@esbuild/linux-mips64el@0.28.0': optional: true '@esbuild/linux-ppc64@0.25.12': @@ -7762,10 +7759,10 @@ snapshots: '@esbuild/linux-ppc64@0.27.3': optional: true - '@esbuild/linux-ppc64@0.27.4': + '@esbuild/linux-ppc64@0.27.7': optional: true - '@esbuild/linux-ppc64@0.27.7': + '@esbuild/linux-ppc64@0.28.0': optional: true '@esbuild/linux-riscv64@0.25.12': @@ -7774,10 +7771,10 @@ snapshots: '@esbuild/linux-riscv64@0.27.3': optional: true - '@esbuild/linux-riscv64@0.27.4': + '@esbuild/linux-riscv64@0.27.7': optional: true - '@esbuild/linux-riscv64@0.27.7': + '@esbuild/linux-riscv64@0.28.0': optional: true '@esbuild/linux-s390x@0.25.12': @@ -7786,10 +7783,10 @@ snapshots: '@esbuild/linux-s390x@0.27.3': optional: true - '@esbuild/linux-s390x@0.27.4': + '@esbuild/linux-s390x@0.27.7': optional: true - '@esbuild/linux-s390x@0.27.7': + '@esbuild/linux-s390x@0.28.0': optional: true '@esbuild/linux-x64@0.25.12': @@ -7798,10 +7795,10 @@ snapshots: '@esbuild/linux-x64@0.27.3': optional: true - '@esbuild/linux-x64@0.27.4': + '@esbuild/linux-x64@0.27.7': optional: true - '@esbuild/linux-x64@0.27.7': + '@esbuild/linux-x64@0.28.0': optional: true '@esbuild/netbsd-arm64@0.25.12': @@ -7810,10 +7807,10 @@ snapshots: '@esbuild/netbsd-arm64@0.27.3': optional: true - '@esbuild/netbsd-arm64@0.27.4': + '@esbuild/netbsd-arm64@0.27.7': optional: true - '@esbuild/netbsd-arm64@0.27.7': + '@esbuild/netbsd-arm64@0.28.0': optional: true '@esbuild/netbsd-x64@0.25.12': @@ -7822,10 +7819,10 @@ snapshots: '@esbuild/netbsd-x64@0.27.3': optional: true - '@esbuild/netbsd-x64@0.27.4': + '@esbuild/netbsd-x64@0.27.7': optional: true - '@esbuild/netbsd-x64@0.27.7': + '@esbuild/netbsd-x64@0.28.0': optional: true '@esbuild/openbsd-arm64@0.25.12': @@ -7834,10 +7831,10 @@ snapshots: '@esbuild/openbsd-arm64@0.27.3': optional: true - '@esbuild/openbsd-arm64@0.27.4': + '@esbuild/openbsd-arm64@0.27.7': optional: true - '@esbuild/openbsd-arm64@0.27.7': + '@esbuild/openbsd-arm64@0.28.0': optional: true '@esbuild/openbsd-x64@0.25.12': @@ -7846,10 +7843,10 @@ snapshots: '@esbuild/openbsd-x64@0.27.3': optional: true - '@esbuild/openbsd-x64@0.27.4': + '@esbuild/openbsd-x64@0.27.7': optional: true - '@esbuild/openbsd-x64@0.27.7': + '@esbuild/openbsd-x64@0.28.0': optional: true '@esbuild/openharmony-arm64@0.25.12': @@ -7858,10 +7855,10 @@ snapshots: '@esbuild/openharmony-arm64@0.27.3': optional: true - '@esbuild/openharmony-arm64@0.27.4': + '@esbuild/openharmony-arm64@0.27.7': optional: true - '@esbuild/openharmony-arm64@0.27.7': + '@esbuild/openharmony-arm64@0.28.0': optional: true '@esbuild/sunos-x64@0.25.12': @@ -7870,10 +7867,10 @@ snapshots: '@esbuild/sunos-x64@0.27.3': optional: true - '@esbuild/sunos-x64@0.27.4': + '@esbuild/sunos-x64@0.27.7': optional: true - '@esbuild/sunos-x64@0.27.7': + '@esbuild/sunos-x64@0.28.0': optional: true '@esbuild/win32-arm64@0.25.12': @@ -7882,10 +7879,10 @@ snapshots: '@esbuild/win32-arm64@0.27.3': optional: true - '@esbuild/win32-arm64@0.27.4': + '@esbuild/win32-arm64@0.27.7': optional: true - '@esbuild/win32-arm64@0.27.7': + '@esbuild/win32-arm64@0.28.0': optional: true '@esbuild/win32-ia32@0.25.12': @@ -7894,10 +7891,10 @@ snapshots: '@esbuild/win32-ia32@0.27.3': optional: true - '@esbuild/win32-ia32@0.27.4': + '@esbuild/win32-ia32@0.27.7': optional: true - '@esbuild/win32-ia32@0.27.7': + '@esbuild/win32-ia32@0.28.0': optional: true '@esbuild/win32-x64@0.25.12': @@ -7906,10 +7903,10 @@ snapshots: '@esbuild/win32-x64@0.27.3': optional: true - '@esbuild/win32-x64@0.27.4': + '@esbuild/win32-x64@0.27.7': optional: true - '@esbuild/win32-x64@0.27.7': + '@esbuild/win32-x64@0.28.0': optional: true '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4)': @@ -9494,7 +9491,7 @@ snapshots: '@ungap/structured-clone@1.3.1': {} - '@vitejs/plugin-react@5.2.0(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3))': + '@vitejs/plugin-react@5.2.0(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@babel/core': 7.29.0 '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.29.0) @@ -9502,11 +9499,11 @@ snapshots: '@rolldown/pluginutils': 1.0.0-rc.3 '@types/babel__core': 7.20.5 react-refresh: 0.18.0 - vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3))': + '@vitest/coverage-v8@3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 @@ -9521,7 +9518,7 @@ snapshots: std-env: 3.10.0 test-exclude: 7.0.2 tinyrainbow: 2.0.0 - vitest: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vitest: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) transitivePeerDependencies: - supports-color @@ -9542,21 +9539,21 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@3.2.4(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3))': + '@vitest/mocker@3.2.4(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - '@vitest/mocker@4.1.6(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.6(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.6 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) '@vitest/pretty-format@3.2.4': dependencies: @@ -9786,12 +9783,12 @@ snapshots: transitivePeerDependencies: - supports-color - astro-expressive-code@0.42.0(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3)): + astro-expressive-code@0.42.0(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3) + astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) rehype-expressive-code: 0.42.0 - astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3): + astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3): dependencies: '@astrojs/compiler': 4.0.0 '@astrojs/internal-helpers': 0.9.1 @@ -9843,8 +9840,8 @@ snapshots: unist-util-visit: 5.1.0 unstorage: 1.17.5 vfile: 6.0.3 - vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) - vitefu: 1.1.3(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) + vitefu: 1.1.3(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) xxhash-wasm: 1.1.0 yargs-parser: 22.0.0 zod: 4.4.3 @@ -10537,35 +10534,6 @@ snapshots: '@esbuild/win32-ia32': 0.27.3 '@esbuild/win32-x64': 0.27.3 - esbuild@0.27.4: - optionalDependencies: - '@esbuild/aix-ppc64': 0.27.4 - '@esbuild/android-arm': 0.27.4 - '@esbuild/android-arm64': 0.27.4 - '@esbuild/android-x64': 0.27.4 - '@esbuild/darwin-arm64': 0.27.4 - '@esbuild/darwin-x64': 0.27.4 - '@esbuild/freebsd-arm64': 0.27.4 - '@esbuild/freebsd-x64': 0.27.4 - '@esbuild/linux-arm': 0.27.4 - '@esbuild/linux-arm64': 0.27.4 - '@esbuild/linux-ia32': 0.27.4 - '@esbuild/linux-loong64': 0.27.4 - '@esbuild/linux-mips64el': 0.27.4 - '@esbuild/linux-ppc64': 0.27.4 - '@esbuild/linux-riscv64': 0.27.4 - '@esbuild/linux-s390x': 0.27.4 - '@esbuild/linux-x64': 0.27.4 - '@esbuild/netbsd-arm64': 0.27.4 - '@esbuild/netbsd-x64': 0.27.4 - '@esbuild/openbsd-arm64': 0.27.4 - '@esbuild/openbsd-x64': 0.27.4 - '@esbuild/openharmony-arm64': 0.27.4 - '@esbuild/sunos-x64': 0.27.4 - '@esbuild/win32-arm64': 0.27.4 - '@esbuild/win32-ia32': 0.27.4 - '@esbuild/win32-x64': 0.27.4 - esbuild@0.27.7: optionalDependencies: '@esbuild/aix-ppc64': 0.27.7 @@ -10595,6 +10563,35 @@ snapshots: '@esbuild/win32-ia32': 0.27.7 '@esbuild/win32-x64': 0.27.7 + esbuild@0.28.0: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.0 + '@esbuild/android-arm': 0.28.0 + '@esbuild/android-arm64': 0.28.0 + '@esbuild/android-x64': 0.28.0 + '@esbuild/darwin-arm64': 0.28.0 + '@esbuild/darwin-x64': 0.28.0 + '@esbuild/freebsd-arm64': 0.28.0 + '@esbuild/freebsd-x64': 0.28.0 + '@esbuild/linux-arm': 0.28.0 + '@esbuild/linux-arm64': 0.28.0 + '@esbuild/linux-ia32': 0.28.0 + '@esbuild/linux-loong64': 0.28.0 + '@esbuild/linux-mips64el': 0.28.0 + '@esbuild/linux-ppc64': 0.28.0 + '@esbuild/linux-riscv64': 0.28.0 + '@esbuild/linux-s390x': 0.28.0 + '@esbuild/linux-x64': 0.28.0 + '@esbuild/netbsd-arm64': 0.28.0 + '@esbuild/netbsd-x64': 0.28.0 + '@esbuild/openbsd-arm64': 0.28.0 + '@esbuild/openbsd-x64': 0.28.0 + '@esbuild/openharmony-arm64': 0.28.0 + '@esbuild/sunos-x64': 0.28.0 + '@esbuild/win32-arm64': 0.28.0 + '@esbuild/win32-ia32': 0.28.0 + '@esbuild/win32-x64': 0.28.0 + escalade@3.2.0: {} escape-html@1.0.3: {} @@ -10635,12 +10632,12 @@ snapshots: optionalDependencies: eslint-config-prettier: 9.1.2(eslint@9.39.4) - eslint-plugin-vitest@0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)): + eslint-plugin-vitest@0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)): dependencies: '@typescript-eslint/utils': 7.18.0(eslint@9.39.4)(typescript@5.9.3) eslint: 9.39.4 optionalDependencies: - vitest: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vitest: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) transitivePeerDependencies: - supports-color - typescript @@ -10992,10 +10989,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 - get-tsconfig@4.13.7: - dependencies: - resolve-pkg-maps: 1.0.0 - get-tsconfig@5.0.0-beta.4: dependencies: resolve-pkg-maps: 1.0.0 @@ -13064,11 +13057,11 @@ snapshots: stackback@0.0.2: {} - starlight-links-validator@0.24.0(@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3)): + starlight-links-validator@0.24.0(@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - '@astrojs/starlight': 0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3))(typescript@5.9.3) + '@astrojs/starlight': 0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) '@types/picomatch': 4.0.3 - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.21.0)(yaml@2.8.3) + astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) github-slugger: 2.0.0 hast-util-from-html: 2.0.3 is-absolute-url: 5.0.0 @@ -13394,10 +13387,9 @@ snapshots: tslib@2.8.1: {} - tsx@4.21.0: + tsx@4.22.3: dependencies: - esbuild: 0.27.4 - get-tsconfig: 4.13.7 + esbuild: 0.28.0 optionalDependencies: fsevents: 2.3.3 @@ -13597,13 +13589,13 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite-node@3.2.4(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3): + vite-node@3.2.4(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) transitivePeerDependencies: - '@types/node' - jiti @@ -13618,7 +13610,7 @@ snapshots: - tsx - yaml - vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3): + vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): dependencies: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) @@ -13630,10 +13622,10 @@ snapshots: '@types/node': 20.19.41 fsevents: 2.3.3 lightningcss: 1.32.0 - tsx: 4.21.0 + tsx: 4.22.3 yaml: 2.8.3 - vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3): + vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): dependencies: esbuild: 0.27.7 fdir: 6.5.0(picomatch@4.0.4) @@ -13645,18 +13637,18 @@ snapshots: '@types/node': 20.19.41 fsevents: 2.3.3 lightningcss: 1.32.0 - tsx: 4.21.0 + tsx: 4.22.3 yaml: 2.8.3 - vitefu@1.1.3(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)): + vitefu@1.1.3(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)): optionalDependencies: - vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3): + vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + '@vitest/mocker': 3.2.4(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -13674,8 +13666,8 @@ snapshots: tinyglobby: 0.2.15 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) - vite-node: 3.2.4(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) + vite-node: 3.2.4(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.13 @@ -13694,10 +13686,10 @@ snapshots: - tsx - yaml - vitest@4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)): + vitest@4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.6 - '@vitest/mocker': 4.1.6(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.6(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.6 '@vitest/runner': 4.1.6 '@vitest/snapshot': 4.1.6 @@ -13714,7 +13706,7 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.41 diff --git a/website/package.json b/website/package.json index 810b3a553..c4596f3d9 100644 --- a/website/package.json +++ b/website/package.json @@ -51,7 +51,7 @@ "react-dom": "^18.3.1", "sharp": "^0.34.5", "swagger-ui-react": "^5.32.6", - "tsx": "^4.21.0", + "tsx": "^4.22.3", "typescript": "catalog:", "typespec-versioning-changelog": "workspace:*", "zod": "catalog:" From 55db061db29780504990de91c03fab29b0176b09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 10:53:32 -0700 Subject: [PATCH 05/51] chore(deps-dev): bump wrangler in the runtime group (#854) Bumps the runtime group with 1 update: [wrangler](https://github.com/cloudflare/workers-sdk/tree/HEAD/packages/wrangler). Updates `wrangler` from 4.93.1 to 4.94.0 - [Release notes](https://github.com/cloudflare/workers-sdk/releases) - [Commits](https://github.com/cloudflare/workers-sdk/commits/wrangler@4.94.0/packages/wrangler) --- updated-dependencies: - dependency-name: wrangler dependency-version: 4.94.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: runtime ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pnpm-lock.yaml | 112 ++++++++++++++++++++++++++++--------------- website/package.json | 2 +- 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d8b9ec0f0..f59a541a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -517,8 +517,8 @@ importers: specifier: catalog:website version: 4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) wrangler: - specifier: ^4.93.1 - version: 4.93.1 + specifier: ^4.94.0 + version: 4.94.0 packages: @@ -811,32 +811,32 @@ packages: workerd: optional: true - '@cloudflare/workerd-darwin-64@1.20260520.1': - resolution: {integrity: sha512-7ilR8QUWpFO2RdulPuYkrwRYZxi7iuX8+11G9z97bdS7wCSFuetBsgsr2ISK7l/qzCiG5zshnfIwcdlYWD01Ag==} + '@cloudflare/workerd-darwin-64@1.20260521.1': + resolution: {integrity: sha512-aiNdXmxlhwGjTSajL3I7uQPpN4lAOcXjvg5ZOlJKIywnevr798n9XCS6lvuqgniM3KjurBNWRRypMJntg/eSLg==} engines: {node: '>=16'} cpu: [x64] os: [darwin] - '@cloudflare/workerd-darwin-arm64@1.20260520.1': - resolution: {integrity: sha512-6LVIEI0Tx3hfcXiEM+eHsIQVsOz1IAAoAMMsRlrx+7YaNiuHMib7yWfyzAlZPhiAg1BgiS5oB8iDn7Ws1amG+Q==} + '@cloudflare/workerd-darwin-arm64@1.20260521.1': + resolution: {integrity: sha512-ikN8aKSi4Ak28ndOkuSO5rq6lmV6wwDQu9F9Vu6J7EkwAOth74J/Hjn4j4EuFceW/npw2Ws0Y/muzA6WKHl4TA==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] - '@cloudflare/workerd-linux-64@1.20260520.1': - resolution: {integrity: sha512-7oV9YK7o63aiHnpBeKlxOIA3nK4sKxuwhnklCwJFb0LirkSemSG1LQlVvtVInoWSYDCTCoUeGkiPsS8WsZqcEw==} + '@cloudflare/workerd-linux-64@1.20260521.1': + resolution: {integrity: sha512-D/gUhvQcG0pJr5aJl6yUoi2JxbFpjVtDq9xUJHPjfkAjL28TUVgCR/e5r8YGirepv4I1DK7ihuii9LZ2GGMJbw==} engines: {node: '>=16'} cpu: [x64] os: [linux] - '@cloudflare/workerd-linux-arm64@1.20260520.1': - resolution: {integrity: sha512-hzc/UKzw1/z+iTptBZVX7XYZTmJXsgn6RC9uKtQGUQxENxkoO1teba8qxVlKZT0AbPCQs5rg63Lsk0/eQhteqQ==} + '@cloudflare/workerd-linux-arm64@1.20260521.1': + resolution: {integrity: sha512-vhjWPIHenczegTakhRPwEmTeaavCpNqsuo3RlLCkUdU47HrwLvy/4QersGggs4+kF4Do+IE/EznCGyT40xYcLA==} engines: {node: '>=16'} cpu: [arm64] os: [linux] - '@cloudflare/workerd-windows-64@1.20260520.1': - resolution: {integrity: sha512-BjMBhXqlEPaVc68tXihFI+YcU/5T4Jmj4ELDJaEa6NuCcbUMORESgO4OJZIDS4+rC2Lkv37telOktQxEOkqY3g==} + '@cloudflare/workerd-windows-64@1.20260521.1': + resolution: {integrity: sha512-wBolYC/+lnGIEbkkPdzFtjTOWip2uQH6maeAP1ZV0kyxi5SGpsa83+wD5rH5OOle+sHE5qJMdwCKjwRwj+FKJg==} engines: {node: '>=16'} cpu: [x64] os: [win32] @@ -5067,8 +5067,8 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} - miniflare@4.20260520.0: - resolution: {integrity: sha512-krgebvYME9k7CjxiveTzx89kAMeIstfK3KfTqtzLb/4mtLMD74KHtU009h/I0CTDSVIYtXm0JzJ40OtiVRGmOA==} + miniflare@4.20260521.0: + resolution: {integrity: sha512-roRfxPq49OkuSeQsc43hRjSB1+HdHtDNKRwDEVk2hCjCBuBWxb5Wvwq88b0ULj6QVEJLN/+ZqF19M+h4VYJ/zg==} engines: {node: '>=22.0.0'} hasBin: true @@ -5721,6 +5721,26 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rosie-skills-darwin-arm64@0.6.4: + resolution: {integrity: sha512-rn1s5hqFKcxeiDEWWoFa1hdGPshR8TkwHLzy/cBavb9XJNAaUxbe3oQ78W9sQkRHAgRyzJYyk9tw68Qrdnizgg==} + cpu: [arm64] + os: [darwin] + + rosie-skills-freebsd-x64@0.6.4: + resolution: {integrity: sha512-SxCRduPBMtfjkQ+q56Yw9OLA3PyaqoALzt7kER7IDKuUVfM2O/1w8sa5xhTDiCvWkZJixnH5d5Ya6KT+/Mwcng==} + cpu: [x64] + os: [freebsd] + + rosie-skills-linux-x64@0.6.4: + resolution: {integrity: sha512-D9Y9mfu7goB0s0X59uU3hcFeUTef3VbpCIDwFMzyvJrAq3XhRACWBDMHQsHlyWdHxTXPX/ILyW65RXyrJlgqng==} + cpu: [x64] + os: [linux] + + rosie-skills@0.6.4: + resolution: {integrity: sha512-ojfhSiQRdZ2QyWbmKAHOSAUbaLYrTc5zIH7mS1jKoP8KCFSQddwVhMyFqldckTeybTfW3zNcsZzyOTzGTN1SBA==} + engines: {node: '>=18'} + hasBin: true + run-async@3.0.0: resolution: {integrity: sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==} engines: {node: '>=0.12.0'} @@ -6661,17 +6681,17 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} - workerd@1.20260520.1: - resolution: {integrity: sha512-mwW6H/NEKObeBVd0qkq91EGyOIC3TaNJBxp7kj5uChif/+qYD7nM5HE8ZYruwvEd15pRwUet+V8r21DCXCGDQQ==} + workerd@1.20260521.1: + resolution: {integrity: sha512-HzIThcZ0ZVEuzVxpY2IYZ3yssSrTjtrWXAVfmOl5rVwyqcu7aeZXGMiwrEmi9MOcC3wjy+BNv+hFrMMY5OrjQQ==} engines: {node: '>=16'} hasBin: true - wrangler@4.93.1: - resolution: {integrity: sha512-vV2GyKNWORysoJtryo45N2Tkk8wCnt/MTyW7wsevAAY+MiUArfJGvMiCb6SRB7pV5uWvEyIly1/rslehEjV32g==} + wrangler@4.94.0: + resolution: {integrity: sha512-GsNw0DomGFfeXFtKVTwn2X69UKcCxcTB0CXykjsMineJIxOeyrw7LovlHQ/3JU8KJHH7repLB+kOHvfTBA/Eew==} engines: {node: '>=22.0.0'} hasBin: true peerDependencies: - '@cloudflare/workers-types': ^4.20260520.1 + '@cloudflare/workers-types': ^4.20260521.1 peerDependenciesMeta: '@cloudflare/workers-types': optional: true @@ -7331,25 +7351,25 @@ snapshots: '@cloudflare/kv-asset-handler@0.5.0': {} - '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260520.1)': + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1)': dependencies: unenv: 2.0.0-rc.24 optionalDependencies: - workerd: 1.20260520.1 + workerd: 1.20260521.1 - '@cloudflare/workerd-darwin-64@1.20260520.1': + '@cloudflare/workerd-darwin-64@1.20260521.1': optional: true - '@cloudflare/workerd-darwin-arm64@1.20260520.1': + '@cloudflare/workerd-darwin-arm64@1.20260521.1': optional: true - '@cloudflare/workerd-linux-64@1.20260520.1': + '@cloudflare/workerd-linux-64@1.20260521.1': optional: true - '@cloudflare/workerd-linux-arm64@1.20260520.1': + '@cloudflare/workerd-linux-arm64@1.20260521.1': optional: true - '@cloudflare/workerd-windows-64@1.20260520.1': + '@cloudflare/workerd-windows-64@1.20260521.1': optional: true '@cspell/cspell-bundled-dicts@8.19.4': @@ -12113,12 +12133,12 @@ snapshots: mimic-fn@2.1.0: {} - miniflare@4.20260520.0: + miniflare@4.20260521.0: dependencies: '@cspotcode/source-map-support': 0.8.1 sharp: 0.34.5 undici: 7.24.8 - workerd: 1.20260520.1 + workerd: 1.20260521.1 ws: 8.20.1 youch: 4.1.0-beta.10 transitivePeerDependencies: @@ -12856,6 +12876,21 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.4 fsevents: 2.3.3 + rosie-skills-darwin-arm64@0.6.4: + optional: true + + rosie-skills-freebsd-x64@0.6.4: + optional: true + + rosie-skills-linux-x64@0.6.4: + optional: true + + rosie-skills@0.6.4: + optionalDependencies: + rosie-skills-darwin-arm64: 0.6.4 + rosie-skills-freebsd-x64: 0.6.4 + rosie-skills-linux-x64: 0.6.4 + run-async@3.0.0: {} run-parallel@1.2.0: @@ -13849,24 +13884,25 @@ snapshots: word-wrap@1.2.5: {} - workerd@1.20260520.1: + workerd@1.20260521.1: optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20260520.1 - '@cloudflare/workerd-darwin-arm64': 1.20260520.1 - '@cloudflare/workerd-linux-64': 1.20260520.1 - '@cloudflare/workerd-linux-arm64': 1.20260520.1 - '@cloudflare/workerd-windows-64': 1.20260520.1 + '@cloudflare/workerd-darwin-64': 1.20260521.1 + '@cloudflare/workerd-darwin-arm64': 1.20260521.1 + '@cloudflare/workerd-linux-64': 1.20260521.1 + '@cloudflare/workerd-linux-arm64': 1.20260521.1 + '@cloudflare/workerd-windows-64': 1.20260521.1 - wrangler@4.93.1: + wrangler@4.94.0: dependencies: '@cloudflare/kv-asset-handler': 0.5.0 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260520.1) + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1) blake3-wasm: 2.1.5 esbuild: 0.27.3 - miniflare: 4.20260520.0 + miniflare: 4.20260521.0 path-to-regexp: 6.3.0 + rosie-skills: 0.6.4 unenv: 2.0.0-rc.24 - workerd: 1.20260520.1 + workerd: 1.20260521.1 optionalDependencies: fsevents: 2.3.3 transitivePeerDependencies: diff --git a/website/package.json b/website/package.json index c4596f3d9..077e1b5af 100644 --- a/website/package.json +++ b/website/package.json @@ -81,7 +81,7 @@ "starlight-links-validator": "^0.24.0", "typescript-eslint": "catalog:", "vitest": "catalog:website", - "wrangler": "^4.93.1" + "wrangler": "^4.94.0" }, "overrides": { "js-yaml": "^4.1.1" From b2b8bc053018a36f8d43d16ce02c239b7fcddf51 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Tue, 26 May 2026 13:30:12 -0700 Subject: [PATCH 06/51] ci: gate audit-deps step to PRs into main for lib/* workflows (#847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lib/* CI workflows (ci-lib-ts-sdk, ci-lib-cli, ci-lib-core, ci-lib-changelog-emitter) ran the audit-deps step on every PR matching their path filters, regardless of base branch. When an advisory lives on main and can only be fixed there (e.g. brace-expansion in #842), PRs into HOLD-* batching branches failed audit with no path to action for the PR author — the fix has to land on main first, then HOLD rebases. Under the HOLD-* batching strategy, each HOLD branch eventually opens a single PR into main at the checkpoint. That PR re-runs audit, so any advisory live at merge time is gated at the actionable boundary. Auditing intermediate PRs into HOLD-* shifts noise earlier without shifting the fix earlier. Gate is base_ref == 'main' OR ref == 'refs/heads/main' so it also covers any future workflow_call invocation from a main-push context. --- .github/workflows/ci-lib-changelog-emitter.yml | 7 +++++++ .github/workflows/ci-lib-cli.yml | 7 +++++++ .github/workflows/ci-lib-core.yml | 7 +++++++ .github/workflows/ci-lib-ts-sdk.yml | 7 +++++++ 4 files changed, 28 insertions(+) diff --git a/.github/workflows/ci-lib-changelog-emitter.yml b/.github/workflows/ci-lib-changelog-emitter.yml index d2ce878c3..60a4326ee 100644 --- a/.github/workflows/ci-lib-changelog-emitter.yml +++ b/.github/workflows/ci-lib-changelog-emitter.yml @@ -38,7 +38,14 @@ jobs: run: pnpm --filter typespec-versioning-changelog run test:coverage - name: Audit dependencies + # Gated to PRs into `main` (and main-context workflow_call invocations) so + # advisories that live on `main` and can only be fixed there don't block + # PRs into HOLD-* batching branches or other non-main bases. The HOLD → main + # checkpoint PR re-runs this audit, so anything live at merge time is gated + # at the actionable boundary. + # # Calls the npm bulk advisory endpoint directly as a workaround for # pnpm audit 410 errors (pnpm/pnpm#11265). Remove this script and # revert to `pnpm --filter typespec-versioning-changelog run audit` once pnpm ships native support. + if: github.base_ref == 'main' || github.ref == 'refs/heads/main' run: node ${GITHUB_WORKSPACE}/.github/scripts/audit-deps.js --filter typespec-versioning-changelog diff --git a/.github/workflows/ci-lib-cli.yml b/.github/workflows/ci-lib-cli.yml index c5048ccb1..1345d83ac 100644 --- a/.github/workflows/ci-lib-cli.yml +++ b/.github/workflows/ci-lib-cli.yml @@ -43,7 +43,14 @@ jobs: run: pnpm --filter @common-grants/cli run build - name: Audit dependencies + # Gated to PRs into `main` (and main-context workflow_call invocations) so + # advisories that live on `main` and can only be fixed there don't block + # PRs into HOLD-* batching branches or other non-main bases. The HOLD → main + # checkpoint PR re-runs this audit, so anything live at merge time is gated + # at the actionable boundary. + # # Calls the npm bulk advisory endpoint directly as a workaround for # pnpm audit 410 errors (pnpm/pnpm#11265). Remove this script and # revert to `pnpm --filter @common-grants/cli run audit` once pnpm ships native support. + if: github.base_ref == 'main' || github.ref == 'refs/heads/main' run: node ${GITHUB_WORKSPACE}/.github/scripts/audit-deps.js --filter @common-grants/cli diff --git a/.github/workflows/ci-lib-core.yml b/.github/workflows/ci-lib-core.yml index c630e6f4d..ec3b01cc7 100644 --- a/.github/workflows/ci-lib-core.yml +++ b/.github/workflows/ci-lib-core.yml @@ -40,7 +40,14 @@ jobs: run: pnpm --filter @common-grants/core run typespec - name: Audit dependencies + # Gated to PRs into `main` (and main-context workflow_call invocations) so + # advisories that live on `main` and can only be fixed there don't block + # PRs into HOLD-* batching branches or other non-main bases. The HOLD → main + # checkpoint PR re-runs this audit, so anything live at merge time is gated + # at the actionable boundary. + # # Calls the npm bulk advisory endpoint directly as a workaround for # pnpm audit 410 errors (pnpm/pnpm#11265). Remove this script and # revert to `pnpm --filter @common-grants/core run audit` once pnpm ships native support. + if: github.base_ref == 'main' || github.ref == 'refs/heads/main' run: node ${GITHUB_WORKSPACE}/.github/scripts/audit-deps.js --filter @common-grants/core diff --git a/.github/workflows/ci-lib-ts-sdk.yml b/.github/workflows/ci-lib-ts-sdk.yml index 2641b3b4c..b634c81d7 100644 --- a/.github/workflows/ci-lib-ts-sdk.yml +++ b/.github/workflows/ci-lib-ts-sdk.yml @@ -40,7 +40,14 @@ jobs: run: pnpm --filter @common-grants/sdk run test:coverage - name: Audit dependencies + # Gated to PRs into `main` (and main-context workflow_call invocations) so + # advisories that live on `main` and can only be fixed there don't block + # PRs into HOLD-* batching branches or other non-main bases. The HOLD → main + # checkpoint PR re-runs this audit, so anything live at merge time is gated + # at the actionable boundary. + # # Calls the npm bulk advisory endpoint directly as a workaround for # pnpm audit 410 errors (pnpm/pnpm#11265). Remove this script and # revert to `pnpm --filter @common-grants/sdk run audit` once pnpm ships native support. + if: github.base_ref == 'main' || github.ref == 'refs/heads/main' run: node ${GITHUB_WORKSPACE}/.github/scripts/audit-deps.js --filter @common-grants/sdk From b74269dd433fb5b8921871789940ddc0e7f83b7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 13:49:14 -0700 Subject: [PATCH 07/51] chore(deps): bump astro (#853) Bumps the website-framework group with 1 update in the / directory: [astro](https://github.com/withastro/astro/tree/HEAD/packages/astro). Updates `astro` from 6.3.6 to 6.3.8 - [Release notes](https://github.com/withastro/astro/releases) - [Changelog](https://github.com/withastro/astro/blob/main/packages/astro/CHANGELOG.md) - [Commits](https://github.com/withastro/astro/commits/astro@6.3.8/packages/astro) --- updated-dependencies: - dependency-name: astro dependency-version: 6.3.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: website-framework ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pnpm-lock.yaml | 142 +++++++++++++++++++++++++++---------------- website/package.json | 2 +- 2 files changed, 89 insertions(+), 55 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f59a541a9..7b056f52c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -388,7 +388,7 @@ importers: version: 5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3) '@astrojs/starlight': specifier: ^0.39.2 - version: 0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) + version: 0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) '@jsonforms/core': specifier: ^3.7.0 version: 3.7.0 @@ -408,8 +408,8 @@ importers: specifier: ^8.20.0 version: 8.20.0 astro: - specifier: ^6.3.6 - version: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + specifier: ^6.3.8 + version: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) js-yaml: specifier: ^4.1.1 version: 4.1.1 @@ -509,7 +509,7 @@ importers: version: 0.14.1 starlight-links-validator: specifier: ^0.24.0 - version: 0.24.0(@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) + version: 0.24.0(@astrojs/starlight@0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) typescript-eslint: specifier: 'catalog:' version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) @@ -667,10 +667,18 @@ packages: resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} engines: {node: '>=6.9.0'} + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@7.28.5': resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/helper-validator-option@7.27.1': resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} engines: {node: '>=6.9.0'} @@ -684,8 +692,8 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - '@babel/parser@7.29.3': - resolution: {integrity: sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==} + '@babel/parser@7.29.7': + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} engines: {node: '>=6.0.0'} hasBin: true @@ -721,6 +729,10 @@ packages: resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} engines: {node: '>=6.9.0'} + '@babel/types@7.29.7': + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} + engines: {node: '>=6.9.0'} + '@bcoe/v8-coverage@1.0.2': resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} @@ -3301,8 +3313,8 @@ packages: peerDependencies: astro: ^4.0.0-beta || ^5.0.0-beta || ^3.3.0 || ^6.0.0-beta - astro@6.3.6: - resolution: {integrity: sha512-lM30gGI/iASK9Z1WQVnBBYzxVwDv8slkXbJOF7FNJdZQeBrFETpsQvYoLRupM/adt2ObP5hkYAWEeCjofoqlRw==} + astro@6.3.8: + resolution: {integrity: sha512-xH2UA8Z17IS+JaqSlSkBor7jO6gd7zXTLdmu06nKpfpDDJFbi/7KZEy3NDmWxmier+6XrCZ9Z4aitO8jhC9oiA==} engines: {node: '>=22.12.0', npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true @@ -5784,6 +5796,11 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.8.1: + resolution: {integrity: sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==} + engines: {node: '>=10'} + hasBin: true + send@0.19.2: resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==} engines: {node: '>= 0.8.0'} @@ -6057,6 +6074,10 @@ packages: resolution: {integrity: sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA==} engines: {node: '>=18'} + tinyexec@1.2.2: + resolution: {integrity: sha512-M/Q0B2cp4K7kynaT/vnED1j8TlLY+Pp7C6Wl2bl/7u/F0mUVwdyOpwomQb8JpYLitHUssAJRmLZdMCGsrx7i+g==} + engines: {node: '>=18'} + tinyglobby@0.2.15: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} @@ -6946,12 +6967,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@astrojs/mdx@5.0.4(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))': + '@astrojs/mdx@5.0.4(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@astrojs/markdown-remark': 7.1.1 '@mdx-js/mdx': 3.1.1 acorn: 8.16.0 - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) es-module-lexer: 2.1.0 estree-util-visit: 2.0.0 hast-util-to-html: 9.0.5 @@ -7004,17 +7025,17 @@ snapshots: stream-replace-string: 2.0.0 zod: 4.4.3 - '@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3)': + '@astrojs/starlight@0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3)': dependencies: '@astrojs/markdown-remark': 7.1.1 - '@astrojs/mdx': 5.0.4(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) + '@astrojs/mdx': 5.0.4(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) '@astrojs/sitemap': 3.7.2 '@pagefind/default-ui': 1.5.2 '@types/hast': 3.0.4 '@types/js-yaml': 4.0.9 '@types/mdast': 4.0.4 - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) - astro-expressive-code: 0.42.0(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) + astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro-expressive-code: 0.42.0(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) bcp-47: 2.1.0 hast-util-from-html: 2.0.3 hast-util-select: 6.0.4 @@ -7066,10 +7087,10 @@ snapshots: '@babel/helper-compilation-targets': 7.28.6 '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) '@babel/helpers': 7.29.2 - '@babel/parser': 7.29.3 + '@babel/parser': 7.29.7 '@babel/template': 7.28.6 '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 debug: 4.4.3 @@ -7081,8 +7102,8 @@ snapshots: '@babel/generator@7.29.1': dependencies: - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 @@ -7100,7 +7121,7 @@ snapshots: '@babel/helper-module-imports@7.28.6': dependencies: '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -7108,7 +7129,7 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/helper-module-imports': 7.28.6 - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 '@babel/traverse': 7.29.0 transitivePeerDependencies: - supports-color @@ -7117,22 +7138,26 @@ snapshots: '@babel/helper-string-parser@7.27.1': {} + '@babel/helper-string-parser@7.29.7': {} + '@babel/helper-validator-identifier@7.28.5': {} + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/helper-validator-option@7.27.1': {} '@babel/helpers@7.29.2': dependencies: '@babel/template': 7.28.6 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@babel/parser@7.29.2': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 - '@babel/parser@7.29.3': + '@babel/parser@7.29.7': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@babel/plugin-transform-react-jsx-self@7.27.1(@babel/core@7.29.0)': dependencies: @@ -7153,17 +7178,17 @@ snapshots: '@babel/template@7.28.6': dependencies: '@babel/code-frame': 7.29.0 - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@babel/traverse@7.29.0': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 '@babel/helper-globals': 7.28.0 - '@babel/parser': 7.29.3 + '@babel/parser': 7.29.7 '@babel/template': 7.28.6 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -7173,6 +7198,11 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 + '@babel/types@7.29.7': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + '@bcoe/v8-coverage@1.0.2': {} '@capsizecss/unpack@4.0.0': @@ -7193,7 +7223,7 @@ snapshots: outdent: 0.5.0 prettier: 2.8.8 resolve-from: 5.0.0 - semver: 7.8.0 + semver: 7.8.1 '@changesets/assemble-release-plan@6.0.10': dependencies: @@ -7202,7 +7232,7 @@ snapshots: '@changesets/should-skip-package': 0.1.2 '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 - semver: 7.8.0 + semver: 7.8.1 '@changesets/changelog-git@0.2.1': dependencies: @@ -7267,7 +7297,7 @@ snapshots: '@changesets/types': 6.1.0 '@manypkg/get-packages': 1.1.3 picocolors: 1.1.1 - semver: 7.8.0 + semver: 7.8.1 '@changesets/get-github-info@0.8.0': dependencies: @@ -9102,24 +9132,24 @@ snapshots: '@types/babel__core@7.20.5': dependencies: - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@types/babel__generator': 7.27.0 '@types/babel__template': 7.4.4 '@types/babel__traverse': 7.28.0 '@types/babel__generator@7.27.0': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@types/babel__template@7.4.4': dependencies: - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@types/babel__traverse@7.28.0': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@types/body-parser@1.19.6': dependencies: @@ -9380,7 +9410,7 @@ snapshots: globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.9 - semver: 7.8.0 + semver: 7.8.1 ts-api-utils: 1.4.3(typescript@5.9.3) optionalDependencies: typescript: 5.9.3 @@ -9395,7 +9425,7 @@ snapshots: '@typescript-eslint/visitor-keys': 8.59.3 debug: 4.4.3 minimatch: 10.2.5 - semver: 7.8.0 + semver: 7.8.1 tinyglobby: 0.2.16 ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 @@ -9799,16 +9829,16 @@ snapshots: espree: 10.4.0 fast-glob: 3.3.3 is-glob: 4.0.3 - semver: 7.8.0 + semver: 7.8.1 transitivePeerDependencies: - supports-color - astro-expressive-code@0.42.0(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): + astro-expressive-code@0.42.0(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) rehype-expressive-code: 0.42.0 - astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3): + astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3): dependencies: '@astrojs/compiler': 4.0.0 '@astrojs/internal-helpers': 0.9.1 @@ -9848,12 +9878,12 @@ snapshots: piccolore: 0.1.3 picomatch: 4.0.4 rehype: 13.0.2 - semver: 7.8.0 + semver: 7.8.1 shiki: 4.1.0 smol-toml: 1.6.1 svgo: 4.0.1 tinyclip: 0.1.12 - tinyexec: 1.1.2 + tinyexec: 1.2.2 tinyglobby: 0.2.16 ultrahtml: 1.6.0 unifont: 0.7.4 @@ -10623,7 +10653,7 @@ snapshots: eslint-compat-utils@0.6.5(eslint@9.39.4): dependencies: eslint: 9.39.4 - semver: 7.8.0 + semver: 7.8.1 eslint-config-prettier@9.1.2(eslint@9.39.4): dependencies: @@ -11631,13 +11661,13 @@ snapshots: magicast@0.5.3: dependencies: - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 source-map-js: 1.2.1 make-dir@4.0.0: dependencies: - semver: 7.8.0 + semver: 7.8.1 make-error@1.3.6: {} @@ -12923,6 +12953,8 @@ snapshots: semver@7.8.0: {} + semver@7.8.1: {} + send@0.19.2: dependencies: debug: 2.6.9 @@ -13092,11 +13124,11 @@ snapshots: stackback@0.0.2: {} - starlight-links-validator@0.24.0(@astrojs/starlight@0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): + starlight-links-validator@0.24.0(@astrojs/starlight@0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - '@astrojs/starlight': 0.39.2(astro@6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) + '@astrojs/starlight': 0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) '@types/picomatch': 4.0.3 - astro: 6.3.6(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) github-slugger: 2.0.0 hast-util-from-html: 2.0.3 is-absolute-url: 5.0.0 @@ -13332,6 +13364,8 @@ snapshots: tinyexec@1.1.2: {} + tinyexec@1.2.2: {} + tinyglobby@0.2.15: dependencies: fdir: 6.5.0(picomatch@4.0.4) @@ -13455,7 +13489,7 @@ snapshots: typescript-auto-import-cache@0.3.6: dependencies: - semver: 7.8.0 + semver: 7.8.1 typescript-eslint@8.59.3(eslint@9.39.4)(typescript@5.9.3): dependencies: @@ -13789,7 +13823,7 @@ snapshots: volar-service-typescript@0.0.70(@volar/language-service@2.4.28): dependencies: path-browserify: 1.0.1 - semver: 7.8.0 + semver: 7.8.1 typescript-auto-import-cache: 0.3.6 vscode-languageserver-textdocument: 1.0.12 vscode-nls: 5.2.0 diff --git a/website/package.json b/website/package.json index 077e1b5af..cb7052a99 100644 --- a/website/package.json +++ b/website/package.json @@ -43,7 +43,7 @@ "@types/react": "^18.3.28", "@types/react-dom": "^18.3.7", "ajv": "^8.20.0", - "astro": "^6.3.6", + "astro": "^6.3.8", "js-yaml": "^4.1.1", "json-schema-merge-allof": "^0.8.1", "openapi-sampler": "^1.7.3", From fcdb6ba37e3c5543a16c5818c3108424124e9bfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 13:52:23 -0700 Subject: [PATCH 08/51] chore(deps-dev): bump the runtime group with 2 updates (#856) Bumps the runtime group with 2 updates: [json-schema-faker](https://github.com/json-schema-faker/json-schema-faker) and [wrangler](https://github.com/cloudflare/workers-sdk/tree/HEAD/packages/wrangler). Updates `json-schema-faker` from 0.6.1 to 0.6.2 - [Release notes](https://github.com/json-schema-faker/json-schema-faker/releases) - [Commits](https://github.com/json-schema-faker/json-schema-faker/compare/v0.6.1...v0.6.2) Updates `wrangler` from 4.94.0 to 4.95.0 - [Release notes](https://github.com/cloudflare/workers-sdk/releases) - [Commits](https://github.com/cloudflare/workers-sdk/commits/HEAD/packages/wrangler) --- updated-dependencies: - dependency-name: json-schema-faker dependency-version: 0.6.2 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: runtime - dependency-name: wrangler dependency-version: 4.95.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: runtime ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lib/ts-sdk/package.json | 2 +- pnpm-lock.yaml | 86 ++++++++++++++++++++--------------------- website/package.json | 2 +- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/lib/ts-sdk/package.json b/lib/ts-sdk/package.json index d99ef8db1..f9244bbed 100644 --- a/lib/ts-sdk/package.json +++ b/lib/ts-sdk/package.json @@ -105,7 +105,7 @@ "eslint-config-prettier": "catalog:", "eslint-plugin-prettier": "catalog:", "js-yaml": "^4.1.1", - "json-schema-faker": "^0.6.1", + "json-schema-faker": "^0.6.2", "prettier": "catalog:", "ts-node": "catalog:", "tsx": "^4.22.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b056f52c..af1b7cb53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -357,8 +357,8 @@ importers: specifier: ^4.1.1 version: 4.1.1 json-schema-faker: - specifier: ^0.6.1 - version: 0.6.1 + specifier: ^0.6.2 + version: 0.6.2 prettier: specifier: 'catalog:' version: 3.8.3 @@ -517,8 +517,8 @@ importers: specifier: catalog:website version: 4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) wrangler: - specifier: ^4.94.0 - version: 4.94.0 + specifier: ^4.95.0 + version: 4.95.0 packages: @@ -823,32 +823,32 @@ packages: workerd: optional: true - '@cloudflare/workerd-darwin-64@1.20260521.1': - resolution: {integrity: sha512-aiNdXmxlhwGjTSajL3I7uQPpN4lAOcXjvg5ZOlJKIywnevr798n9XCS6lvuqgniM3KjurBNWRRypMJntg/eSLg==} + '@cloudflare/workerd-darwin-64@1.20260526.1': + resolution: {integrity: sha512-/pR3GH3gfv0PUp7DjI8v0aAIDOqFwibq4bg5xT7TZgcVdBV/cJQWckdXCMqiRtHiawLwogUX00EIOINkYJ1Zqg==} engines: {node: '>=16'} cpu: [x64] os: [darwin] - '@cloudflare/workerd-darwin-arm64@1.20260521.1': - resolution: {integrity: sha512-ikN8aKSi4Ak28ndOkuSO5rq6lmV6wwDQu9F9Vu6J7EkwAOth74J/Hjn4j4EuFceW/npw2Ws0Y/muzA6WKHl4TA==} + '@cloudflare/workerd-darwin-arm64@1.20260526.1': + resolution: {integrity: sha512-rcyu0iANYfaiezKh3Mcao1O4IIgVfQldxduiL5TZT1sP0NIeRY4YReSTrzPxNnXxSYaIqaqRHMcHbUM/ic4knA==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] - '@cloudflare/workerd-linux-64@1.20260521.1': - resolution: {integrity: sha512-D/gUhvQcG0pJr5aJl6yUoi2JxbFpjVtDq9xUJHPjfkAjL28TUVgCR/e5r8YGirepv4I1DK7ihuii9LZ2GGMJbw==} + '@cloudflare/workerd-linux-64@1.20260526.1': + resolution: {integrity: sha512-5EZAEnlLwa9oGJRo8Nd3iY5Wcd9ROGNNG90xNIGp8MEjj8v2jTn42NC47fCZKFdnLj3+S+vWEhu1x0GVJnALjA==} engines: {node: '>=16'} cpu: [x64] os: [linux] - '@cloudflare/workerd-linux-arm64@1.20260521.1': - resolution: {integrity: sha512-vhjWPIHenczegTakhRPwEmTeaavCpNqsuo3RlLCkUdU47HrwLvy/4QersGggs4+kF4Do+IE/EznCGyT40xYcLA==} + '@cloudflare/workerd-linux-arm64@1.20260526.1': + resolution: {integrity: sha512-X/YBQXeXFeCN7QTStoWrATEBc9WKl7PIqkw/dQkjyJ72gh3rkLe0+Xkzp3wO7gtxTDQMa7NPGy1W4+sdMf8q1g==} engines: {node: '>=16'} cpu: [arm64] os: [linux] - '@cloudflare/workerd-windows-64@1.20260521.1': - resolution: {integrity: sha512-wBolYC/+lnGIEbkkPdzFtjTOWip2uQH6maeAP1ZV0kyxi5SGpsa83+wD5rH5OOle+sHE5qJMdwCKjwRwj+FKJg==} + '@cloudflare/workerd-windows-64@1.20260526.1': + resolution: {integrity: sha512-R+tqpFFdcfZIljx8fIW9rj9fRTtDgfoA2yonsfAGa6e8snrmr+38mdFHtkRC0D3UyZpn/hOtmXiUBfdX2gMR7Q==} engines: {node: '>=16'} cpu: [x64] os: [win32] @@ -4668,8 +4668,8 @@ packages: json-schema-compare@0.2.2: resolution: {integrity: sha512-c4WYmDKyJXhs7WWvAWm3uIYnfyWFoIp+JEoX34rctVvEkMYCPGhXtvmFFXiffBbxfZsvQ0RNnV5H7GvDF5HCqQ==} - json-schema-faker@0.6.1: - resolution: {integrity: sha512-IMF9QGFn/j7sys1LxgYpt/+0LUozFE8R/yUav4D7NqsHnh5L0mbJZOiU724nlJ3g+ZZo6NykVtTeJLoVcOP6Xw==} + json-schema-faker@0.6.2: + resolution: {integrity: sha512-jhOV/bIUxTPM3DiKaa9YDI9UIhO6md1wBnopuIleXATBCwBxnpEfUG0KZqWJS0nTc7nGeWu6ve4vL7Sk7l2BKA==} hasBin: true json-schema-merge-allof@0.8.1: @@ -5079,8 +5079,8 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} - miniflare@4.20260521.0: - resolution: {integrity: sha512-roRfxPq49OkuSeQsc43hRjSB1+HdHtDNKRwDEVk2hCjCBuBWxb5Wvwq88b0ULj6QVEJLN/+ZqF19M+h4VYJ/zg==} + miniflare@4.20260526.0: + resolution: {integrity: sha512-JYQ7jPZZWoaaj9jWHb8Ucp6Cu2SbDVqIsAJhumqdzzLkkfq0pYkDeino/sZfW1ixJWPjv/C44zjm9gVJC2izCA==} engines: {node: '>=22.0.0'} hasBin: true @@ -6702,17 +6702,17 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} - workerd@1.20260521.1: - resolution: {integrity: sha512-HzIThcZ0ZVEuzVxpY2IYZ3yssSrTjtrWXAVfmOl5rVwyqcu7aeZXGMiwrEmi9MOcC3wjy+BNv+hFrMMY5OrjQQ==} + workerd@1.20260526.1: + resolution: {integrity: sha512-IHzymht98p10JH1zzwdCpbViAqw97HrwKl7+KfZeASFMsYSrIsAULWdPn0LRC5FTUzBpamLNyKCCKxbgXHgRHQ==} engines: {node: '>=16'} hasBin: true - wrangler@4.94.0: - resolution: {integrity: sha512-GsNw0DomGFfeXFtKVTwn2X69UKcCxcTB0CXykjsMineJIxOeyrw7LovlHQ/3JU8KJHH7repLB+kOHvfTBA/Eew==} + wrangler@4.95.0: + resolution: {integrity: sha512-vgXzFVSCdUbeCadgVXvu8fK5tzNm8T9W+7lriyGWZMx0B1+CAdr4d8JTlZszHfgjypRAHmAxb49etZGIRD9pgg==} engines: {node: '>=22.0.0'} hasBin: true peerDependencies: - '@cloudflare/workers-types': ^4.20260521.1 + '@cloudflare/workers-types': ^4.20260526.1 peerDependenciesMeta: '@cloudflare/workers-types': optional: true @@ -7381,25 +7381,25 @@ snapshots: '@cloudflare/kv-asset-handler@0.5.0': {} - '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1)': + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260526.1)': dependencies: unenv: 2.0.0-rc.24 optionalDependencies: - workerd: 1.20260521.1 + workerd: 1.20260526.1 - '@cloudflare/workerd-darwin-64@1.20260521.1': + '@cloudflare/workerd-darwin-64@1.20260526.1': optional: true - '@cloudflare/workerd-darwin-arm64@1.20260521.1': + '@cloudflare/workerd-darwin-arm64@1.20260526.1': optional: true - '@cloudflare/workerd-linux-64@1.20260521.1': + '@cloudflare/workerd-linux-64@1.20260526.1': optional: true - '@cloudflare/workerd-linux-arm64@1.20260521.1': + '@cloudflare/workerd-linux-arm64@1.20260526.1': optional: true - '@cloudflare/workerd-windows-64@1.20260521.1': + '@cloudflare/workerd-windows-64@1.20260526.1': optional: true '@cspell/cspell-bundled-dicts@8.19.4': @@ -11516,7 +11516,7 @@ snapshots: dependencies: lodash: 4.18.1 - json-schema-faker@0.6.1: {} + json-schema-faker@0.6.2: {} json-schema-merge-allof@0.8.1: dependencies: @@ -12163,12 +12163,12 @@ snapshots: mimic-fn@2.1.0: {} - miniflare@4.20260521.0: + miniflare@4.20260526.0: dependencies: '@cspotcode/source-map-support': 0.8.1 sharp: 0.34.5 undici: 7.24.8 - workerd: 1.20260521.1 + workerd: 1.20260526.1 ws: 8.20.1 youch: 4.1.0-beta.10 transitivePeerDependencies: @@ -13918,25 +13918,25 @@ snapshots: word-wrap@1.2.5: {} - workerd@1.20260521.1: + workerd@1.20260526.1: optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20260521.1 - '@cloudflare/workerd-darwin-arm64': 1.20260521.1 - '@cloudflare/workerd-linux-64': 1.20260521.1 - '@cloudflare/workerd-linux-arm64': 1.20260521.1 - '@cloudflare/workerd-windows-64': 1.20260521.1 + '@cloudflare/workerd-darwin-64': 1.20260526.1 + '@cloudflare/workerd-darwin-arm64': 1.20260526.1 + '@cloudflare/workerd-linux-64': 1.20260526.1 + '@cloudflare/workerd-linux-arm64': 1.20260526.1 + '@cloudflare/workerd-windows-64': 1.20260526.1 - wrangler@4.94.0: + wrangler@4.95.0: dependencies: '@cloudflare/kv-asset-handler': 0.5.0 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1) + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260526.1) blake3-wasm: 2.1.5 esbuild: 0.27.3 - miniflare: 4.20260521.0 + miniflare: 4.20260526.0 path-to-regexp: 6.3.0 rosie-skills: 0.6.4 unenv: 2.0.0-rc.24 - workerd: 1.20260521.1 + workerd: 1.20260526.1 optionalDependencies: fsevents: 2.3.3 transitivePeerDependencies: diff --git a/website/package.json b/website/package.json index cb7052a99..e8d101dda 100644 --- a/website/package.json +++ b/website/package.json @@ -81,7 +81,7 @@ "starlight-links-validator": "^0.24.0", "typescript-eslint": "catalog:", "vitest": "catalog:website", - "wrangler": "^4.94.0" + "wrangler": "^4.95.0" }, "overrides": { "js-yaml": "^4.1.1" From d47ee9014adcbdf4292c9aaa79edac73eb7c1c68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 14:05:11 -0700 Subject: [PATCH 09/51] chore(deps): bump openapi-sampler (#863) Bumps the runtime group with 1 update in the / directory: [openapi-sampler](https://github.com/Redocly/openapi-sampler). Updates `openapi-sampler` from 1.7.3 to 1.7.4 - [Release notes](https://github.com/Redocly/openapi-sampler/releases) - [Changelog](https://github.com/Redocly/openapi-sampler/blob/main/CHANGELOG.md) - [Commits](https://github.com/Redocly/openapi-sampler/compare/v1.7.3...v1.7.4) --- updated-dependencies: - dependency-name: openapi-sampler dependency-version: 1.7.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: runtime ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pnpm-lock.yaml | 42 +++++++++++++++++++++++++----------------- website/package.json | 2 +- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af1b7cb53..1490a58fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -417,8 +417,8 @@ importers: specifier: ^0.8.1 version: 0.8.1 openapi-sampler: - specifier: ^1.7.3 - version: 1.7.3 + specifier: ^1.7.4 + version: 1.7.4 react: specifier: ^18.3.1 version: 18.3.1 @@ -2157,8 +2157,8 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} - '@nodable/entities@2.1.0': - resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==} + '@nodable/entities@2.1.1': + resolution: {integrity: sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg==} '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} @@ -4138,11 +4138,11 @@ packages: fast-wrap-ansi@0.2.2: resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==} - fast-xml-builder@1.1.9: - resolution: {integrity: sha512-jcyKVSEX13iseJqg7n/KWw+xnu/7fdrZ333Fac54KjHDIELVCfDDJXYIm6DTJ0Su4gSzrhqiK0DzY/wZbF40mw==} + fast-xml-builder@1.2.0: + resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} - fast-xml-parser@5.7.3: - resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==} + fast-xml-parser@5.8.0: + resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true fastq@1.20.1: @@ -5238,8 +5238,8 @@ packages: resolution: {integrity: sha512-eN14VrDvl/YyGxxrkGOHkVkWEoPyhyeydOUrbvjoz8K5eIGgELASwN1eqFOJ2CTQMGCy2EntOK1KdtJ8ZMekcg==} engines: {node: '>=12.20.0'} - openapi-sampler@1.7.3: - resolution: {integrity: sha512-Qgy2+Z7xR3l7kXurtzi1PCtzAINkFKhBADBe/8cidC2fQrLUQTudLiJjQDnqJXoisWAR6zaHhC0hP6Hn5vja+g==} + openapi-sampler@1.7.4: + resolution: {integrity: sha512-CKS/rd5ucPCuEDbJnjGDXZTsuGWcmv53aCmQx7soZlPEONUGN4af0/dY5+THRFZraSEjeA78nlfzdFswC/N5SA==} openapi-server-url-templating@1.3.0: resolution: {integrity: sha512-DPlCms3KKEbjVQb0spV6Awfn6UWNheuG/+folQPzh/wUaKwuqvj8zt5gagD7qoyxtE03cIiKPgLFS3Q8Bz00uQ==} @@ -6755,6 +6755,10 @@ packages: xml-but-prettier@1.0.1: resolution: {integrity: sha512-C2CJaadHrZTqESlH03WOyw0oZTtoy2uEg6dSDF6YRg+9GnYNub53RRemLpnvtbHDFelxMx4LajiFsYeR6XJHgQ==} + xml-naming@0.1.0: + resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==} + engines: {node: '>=16.0.0'} + xml@1.0.1: resolution: {integrity: sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==} @@ -8381,7 +8385,7 @@ snapshots: '@noble/hashes@1.8.0': {} - '@nodable/entities@2.1.0': {} + '@nodable/entities@2.1.1': {} '@nodelib/fs.scandir@2.1.5': dependencies: @@ -10884,16 +10888,18 @@ snapshots: dependencies: fast-string-width: 3.0.2 - fast-xml-builder@1.1.9: + fast-xml-builder@1.2.0: dependencies: path-expression-matcher: 1.5.0 + xml-naming: 0.1.0 - fast-xml-parser@5.7.3: + fast-xml-parser@5.8.0: dependencies: - '@nodable/entities': 2.1.0 - fast-xml-builder: 1.1.9 + '@nodable/entities': 2.1.1 + fast-xml-builder: 1.2.0 path-expression-matcher: 1.5.0 strnum: 2.3.0 + xml-naming: 0.1.0 fastq@1.20.1: dependencies: @@ -12293,10 +12299,10 @@ snapshots: dependencies: apg-lite: 1.0.5 - openapi-sampler@1.7.3: + openapi-sampler@1.7.4: dependencies: '@types/json-schema': 7.0.15 - fast-xml-parser: 5.7.3 + fast-xml-parser: 5.8.0 json-pointer: 0.6.2 openapi-server-url-templating@1.3.0: @@ -13977,6 +13983,8 @@ snapshots: dependencies: repeat-string: 1.6.1 + xml-naming@0.1.0: {} + xml@1.0.1: {} xxhash-wasm@1.1.0: {} diff --git a/website/package.json b/website/package.json index e8d101dda..53c145f46 100644 --- a/website/package.json +++ b/website/package.json @@ -46,7 +46,7 @@ "astro": "^6.3.8", "js-yaml": "^4.1.1", "json-schema-merge-allof": "^0.8.1", - "openapi-sampler": "^1.7.3", + "openapi-sampler": "^1.7.4", "react": "^18.3.1", "react-dom": "^18.3.1", "sharp": "^0.34.5", From 702eac434802dcb54648268f1f83e592c380cdf4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:33:29 -0700 Subject: [PATCH 10/51] chore(deps): bump the website-framework group across 1 directory with 2 updates (#867) Bumps the website-framework group with 2 updates in the / directory: [@astrojs/react](https://github.com/withastro/astro/tree/HEAD/packages/integrations/react) and [astro](https://github.com/withastro/astro/tree/HEAD/packages/astro). Updates `@astrojs/react` from 5.0.5 to 5.0.6 - [Release notes](https://github.com/withastro/astro/releases) - [Changelog](https://github.com/withastro/astro/blob/main/packages/integrations/react/CHANGELOG.md) - [Commits](https://github.com/withastro/astro/commits/@astrojs/react@5.0.6/packages/integrations/react) Updates `astro` from 6.3.8 to 6.4.2 - [Release notes](https://github.com/withastro/astro/releases) - [Changelog](https://github.com/withastro/astro/blob/main/packages/astro/CHANGELOG.md) - [Commits](https://github.com/withastro/astro/commits/astro@6.4.2/packages/astro) --- updated-dependencies: - dependency-name: "@astrojs/react" dependency-version: 5.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: website-framework - dependency-name: astro dependency-version: 6.4.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: website-framework ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pnpm-lock.yaml | 276 +++++++++++++++++++++++-------------------- website/package.json | 4 +- 2 files changed, 147 insertions(+), 133 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1490a58fe..1930ee25c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -384,11 +384,11 @@ importers: specifier: ^15.3.5 version: 15.3.5(@types/json-schema@7.0.15) '@astrojs/react': - specifier: ^5.0.5 - version: 5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3) + specifier: ^5.0.6 + version: 5.0.6(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3) '@astrojs/starlight': specifier: ^0.39.2 - version: 0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) + version: 0.39.2(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) '@jsonforms/core': specifier: ^3.7.0 version: 3.7.0 @@ -408,8 +408,8 @@ importers: specifier: ^8.20.0 version: 8.20.0 astro: - specifier: ^6.3.8 - version: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + specifier: ^6.4.2 + version: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) js-yaml: specifier: ^4.1.1 version: 4.1.1 @@ -509,7 +509,7 @@ importers: version: 0.14.1 starlight-links-validator: specifier: ^0.24.0 - version: 0.24.0(@astrojs/starlight@0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) + version: 0.24.0(@astrojs/starlight@0.39.2(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) typescript-eslint: specifier: 'catalog:' version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) @@ -563,12 +563,12 @@ packages: '@astrojs/compiler@4.0.0': resolution: {integrity: sha512-eouss7G8ygdZqHuke033VMcVw5HTZUu+PXd/h06DGDUg/jt5btPYPqh66ENWw/mU78rBrf/oeC4oqoBwMtDMNA==} + '@astrojs/internal-helpers@0.10.0': + resolution: {integrity: sha512-Ry2R3VPeIN4uPCSA4xQc+e+vsJXkalKpEbDc07hV+a/o5Bs2N/s/uDcPJH/05L19DKh9tAy7e6JM3YZ6Cxfezw==} + '@astrojs/internal-helpers@0.9.0': resolution: {integrity: sha512-GdYkzR26re8izmyYlBqf4z2s7zNngmWLFuxw0UKiPNqHraZGS6GKWIwSHgS22RDlu2ePFJ8bzmpBcUszut/SDg==} - '@astrojs/internal-helpers@0.9.1': - resolution: {integrity: sha512-1pWuARqYom/TzuU3+0ZugsTrKlUydWKuULmDqSMTuonY+9IRDUEGKX/8PXQ1nBxRq3w85uGtd9q9SXfqEldMIQ==} - '@astrojs/language-server@2.16.7': resolution: {integrity: sha512-b64bWT74Vq/ORcSqW7TdIjjpB6hcl+Ei/lMANIUaAGlLPiYNtPTRI/j2tzvugT+LoVwfJtE2Ukq/t2OGCyEtfQ==} hasBin: true @@ -584,8 +584,8 @@ packages: '@astrojs/markdown-remark@7.1.1': resolution: {integrity: sha512-C6e9BnLGlbdv6bV8MYGeHpHxsUHrCrB4OuRLqi5LI7oiBVcBcqfUN06zpwFQdHgV48QCCrMmLpyqBr7VqC+swA==} - '@astrojs/markdown-remark@7.1.2': - resolution: {integrity: sha512-caXZ4Dc2St2dW8luEg22GlP0gupLdztCTQE4EzZOxW1pqWXz9mbeJEuHUkgDYcKWW8tjIHkydYDhWLVoxJ327Q==} + '@astrojs/markdown-remark@7.2.0': + resolution: {integrity: sha512-+YxmVQu1Bd+MFfSzjq1rOJvD9+nIOJzz5YIIhdIH01RrxRkKbyKoEgyIqP3yv51MhzMDgd79QaPv+kCVPT8vHw==} '@astrojs/mdx@5.0.4': resolution: {integrity: sha512-tSbuuYueNODiFAFaME7pjHY5lOLoxBYJi1cKd6scw9+a4ZO7C7UGdafEoVAQvOV2eO8a6RaHSAJYGVPL1w8BPA==} @@ -601,8 +601,8 @@ packages: resolution: {integrity: sha512-KTivpmnz6lDsC6o9H4+DNm2SrE/GHzw8cNAvEJwAvUT+eoaEnn/4NtbDNfRRaxaJHdp15gf+tfHAWiXR4wB3BA==} engines: {node: '>=22.12.0'} - '@astrojs/react@5.0.5': - resolution: {integrity: sha512-5jSFDqWqLdEyp7CEVD66A7AQEEuwLkCGR25NJ4FR5EjziZQqZTGc7hJOFZ97qb98BiU6vElrS70R8iI+HhufGQ==} + '@astrojs/react@5.0.6': + resolution: {integrity: sha512-3pfjmw3sUnV5WplLblDzsAKlHv9kNmlCFAw/xP/agubiZFo4d+uPXX2jynNwAfvwyI5v+Q9uIIFwyujv9jifLg==} engines: {node: '>=22.12.0'} peerDependencies: '@types/react': ^17.0.50 || ^18.0.21 || ^19.0.0 @@ -629,38 +629,42 @@ packages: resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} engines: {node: '>=6.9.0'} - '@babel/compat-data@7.29.3': - resolution: {integrity: sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg==} + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/compat-data@7.29.7': + resolution: {integrity: sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==} engines: {node: '>=6.9.0'} - '@babel/core@7.29.0': - resolution: {integrity: sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==} + '@babel/core@7.29.7': + resolution: {integrity: sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==} engines: {node: '>=6.9.0'} - '@babel/generator@7.29.1': - resolution: {integrity: sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==} + '@babel/generator@7.29.7': + resolution: {integrity: sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==} engines: {node: '>=6.9.0'} - '@babel/helper-compilation-targets@7.28.6': - resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} + '@babel/helper-compilation-targets@7.29.7': + resolution: {integrity: sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==} engines: {node: '>=6.9.0'} - '@babel/helper-globals@7.28.0': - resolution: {integrity: sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==} + '@babel/helper-globals@7.29.7': + resolution: {integrity: sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==} engines: {node: '>=6.9.0'} - '@babel/helper-module-imports@7.28.6': - resolution: {integrity: sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==} + '@babel/helper-module-imports@7.29.7': + resolution: {integrity: sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==} engines: {node: '>=6.9.0'} - '@babel/helper-module-transforms@7.28.6': - resolution: {integrity: sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==} + '@babel/helper-module-transforms@7.29.7': + resolution: {integrity: sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-plugin-utils@7.28.6': - resolution: {integrity: sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==} + '@babel/helper-plugin-utils@7.29.7': + resolution: {integrity: sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==} engines: {node: '>=6.9.0'} '@babel/helper-string-parser@7.27.1': @@ -679,12 +683,12 @@ packages: resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} engines: {node: '>=6.9.0'} - '@babel/helper-validator-option@7.27.1': - resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} + '@babel/helper-validator-option@7.29.7': + resolution: {integrity: sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==} engines: {node: '>=6.9.0'} - '@babel/helpers@7.29.2': - resolution: {integrity: sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==} + '@babel/helpers@7.29.7': + resolution: {integrity: sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==} engines: {node: '>=6.9.0'} '@babel/parser@7.29.2': @@ -697,14 +701,14 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - '@babel/plugin-transform-react-jsx-self@7.27.1': - resolution: {integrity: sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==} + '@babel/plugin-transform-react-jsx-self@7.29.7': + resolution: {integrity: sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/plugin-transform-react-jsx-source@7.27.1': - resolution: {integrity: sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==} + '@babel/plugin-transform-react-jsx-source@7.29.7': + resolution: {integrity: sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 @@ -717,12 +721,12 @@ packages: resolution: {integrity: sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==} engines: {node: '>=6.9.0'} - '@babel/template@7.28.6': - resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} + '@babel/template@7.29.7': + resolution: {integrity: sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.29.0': - resolution: {integrity: sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==} + '@babel/traverse@7.29.7': + resolution: {integrity: sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==} engines: {node: '>=6.9.0'} '@babel/types@7.29.0': @@ -3313,8 +3317,8 @@ packages: peerDependencies: astro: ^4.0.0-beta || ^5.0.0-beta || ^3.3.0 || ^6.0.0-beta - astro@6.3.8: - resolution: {integrity: sha512-xH2UA8Z17IS+JaqSlSkBor7jO6gd7zXTLdmu06nKpfpDDJFbi/7KZEy3NDmWxmier+6XrCZ9Z4aitO8jhC9oiA==} + astro@6.4.2: + resolution: {integrity: sha512-8H89CH2dKL5SCU99OCqdU9BGjmPkSJqaPurywj5XMo7eMFGUFD3vsNhdEKnEh4mK4LgGje3/QDTTSIIGst0G0Q==} engines: {node: '>=22.12.0', npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true @@ -3354,8 +3358,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - baseline-browser-mapping@2.10.29: - resolution: {integrity: sha512-Asa2krT+XTPZINCS+2QcyS8WTkObE77RwkydwF7h6DmnKqbvlalz93m/dnphUyCa6SWSP51VgtEUf2FN+gelFQ==} + baseline-browser-mapping@2.10.32: + resolution: {integrity: sha512-wbPvpyjJPC0zdfdKXxqEL3Ea+bOMD/87X4lftiJkkaBiuG6ALQy1SLmEd7BSmVCuwCQsBrCamgBoLyfFDD1EPg==} engines: {node: '>=6.0.0'} hasBin: true @@ -3437,8 +3441,8 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - caniuse-lite@1.0.30001792: - resolution: {integrity: sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw==} + caniuse-lite@1.0.30001793: + resolution: {integrity: sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==} ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -3843,8 +3847,8 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.356: - resolution: {integrity: sha512-9NgFd7m5t5MCJ5rUSjJITUXAH9mEGlrlofnMf4YEr+pz6JlP7cWmTAH+JFmbPnaSW8koVTkuW7pacORWAnA5Yw==} + electron-to-chromium@1.5.363: + resolution: {integrity: sha512-VjUKPyWzGnT1fujlkEGC/BvN70Hh70KXtAqcmniXviYlJC/ivcT+BWGPyxWVbJZLfvtKR6dqg1L7T7pgAMBtWA==} emmet@2.4.11: resolution: {integrity: sha512-23QPJB3moh/U9sT4rQzGgeyyGIrcM+GH5uVYg2C6wZIxAIJq7Ng3QLT79tl8FUwDXhyq9SusfknOrofAKqvgyQ==} @@ -4836,8 +4840,8 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.5.0: - resolution: {integrity: sha512-5YgH9UJd7wVb9hIouI2adWpgqrrICkt070Dnj8EUY1+B4B2P9eRLPAkAAo6NICA7CEhOIeBHl46u9zSNpNu7zA==} + lru-cache@11.5.1: + resolution: {integrity: sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==} engines: {node: 20 || >=22} lru-cache@5.1.1: @@ -5190,8 +5194,9 @@ packages: node-mock-http@1.0.4: resolution: {integrity: sha512-8DY+kFsDkNXy1sJglUfuODx1/opAGJGyrTuFqEoN90oRc2Vk0ZbD4K2qmKXBBEhZQzdKHIVfEJpDU8Ak2NJEvQ==} - node-releases@2.0.44: - resolution: {integrity: sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ==} + node-releases@2.0.46: + resolution: {integrity: sha512-GYVXHE2KnrzAfsAjl4uP++evGFCrAU1jta4ubEjIG7YWt/64Gqv66a30yKwWczVjA6j3bM4nBwH7Pk1JmDHaxQ==} + engines: {node: '>=18'} normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} @@ -6063,8 +6068,8 @@ packages: tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} - tinyclip@0.1.12: - resolution: {integrity: sha512-Ae3OVUqifDw0wBriIBS7yVaW44Dp6eSHQcyq4Igc7eN2TJH/2YsicswaW+J/OuMvhpDPOKEgpAZCjkb4hpoyeA==} + tinyclip@0.1.13: + resolution: {integrity: sha512-8OqlXQ35euK9+e7L68u8UwcODxkHoIkjbGsgXuARKNyQ5G6xt8nw1YPeMbxMLgCPFkToU+UEK5j05t2t8edKpQ==} engines: {node: ^16.14.0 || >= 17.3.0} tinyexec@0.3.2: @@ -6885,11 +6890,18 @@ snapshots: '@astrojs/compiler@4.0.0': {} - '@astrojs/internal-helpers@0.9.0': + '@astrojs/internal-helpers@0.10.0': dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + js-yaml: 4.1.1 picomatch: 4.0.4 + retext-smartypants: 6.2.0 + shiki: 4.1.0 + smol-toml: 1.6.1 + unified: 11.0.5 - '@astrojs/internal-helpers@0.9.1': + '@astrojs/internal-helpers@0.9.0': dependencies: picomatch: 4.0.4 @@ -6945,14 +6957,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@astrojs/markdown-remark@7.1.2': + '@astrojs/markdown-remark@7.2.0': dependencies: - '@astrojs/internal-helpers': 0.9.1 + '@astrojs/internal-helpers': 0.10.0 '@astrojs/prism': 4.0.2 github-slugger: 2.0.0 hast-util-from-html: 2.0.3 hast-util-to-text: 4.0.2 - js-yaml: 4.1.1 mdast-util-definitions: 6.0.0 rehype-raw: 7.0.0 rehype-stringify: 10.0.1 @@ -6960,9 +6971,6 @@ snapshots: remark-parse: 11.0.0 remark-rehype: 11.1.2 remark-smartypants: 3.0.2 - retext-smartypants: 6.2.0 - shiki: 4.1.0 - smol-toml: 1.6.1 unified: 11.0.5 unist-util-remove-position: 5.0.0 unist-util-visit: 5.1.0 @@ -6971,12 +6979,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@astrojs/mdx@5.0.4(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))': + '@astrojs/mdx@5.0.4(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))': dependencies: '@astrojs/markdown-remark': 7.1.1 '@mdx-js/mdx': 3.1.1 acorn: 8.16.0 - astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) es-module-lexer: 2.1.0 estree-util-visit: 2.0.0 hast-util-to-html: 9.0.5 @@ -6998,9 +7006,9 @@ snapshots: dependencies: prismjs: 1.30.0 - '@astrojs/react@5.0.5(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3)': + '@astrojs/react@5.0.6(@types/node@20.19.41)(@types/react-dom@18.3.7(@types/react@18.3.28))(@types/react@18.3.28)(lightningcss@1.32.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tsx@4.22.3)(yaml@2.8.3)': dependencies: - '@astrojs/internal-helpers': 0.9.1 + '@astrojs/internal-helpers': 0.10.0 '@types/react': 18.3.28 '@types/react-dom': 18.3.7(@types/react@18.3.28) '@vitejs/plugin-react': 5.2.0(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) @@ -7029,17 +7037,17 @@ snapshots: stream-replace-string: 2.0.0 zod: 4.4.3 - '@astrojs/starlight@0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3)': + '@astrojs/starlight@0.39.2(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3)': dependencies: '@astrojs/markdown-remark': 7.1.1 - '@astrojs/mdx': 5.0.4(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) + '@astrojs/mdx': 5.0.4(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) '@astrojs/sitemap': 3.7.2 '@pagefind/default-ui': 1.5.2 '@types/hast': 3.0.4 '@types/js-yaml': 4.0.9 '@types/mdast': 4.0.4 - astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) - astro-expressive-code: 0.42.0(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) + astro: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro-expressive-code: 0.42.0(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)) bcp-47: 2.1.0 hast-util-from-html: 2.0.3 hast-util-select: 6.0.4 @@ -7082,18 +7090,24 @@ snapshots: js-tokens: 4.0.0 picocolors: 1.1.1 - '@babel/compat-data@7.29.3': {} + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/compat-data@7.29.7': {} - '@babel/core@7.29.0': + '@babel/core@7.29.7': dependencies: - '@babel/code-frame': 7.29.0 - '@babel/generator': 7.29.1 - '@babel/helper-compilation-targets': 7.28.6 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) - '@babel/helpers': 7.29.2 + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.7 + '@babel/helper-compilation-targets': 7.29.7 + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7) + '@babel/helpers': 7.29.7 '@babel/parser': 7.29.7 - '@babel/template': 7.28.6 - '@babel/traverse': 7.29.0 + '@babel/template': 7.29.7 + '@babel/traverse': 7.29.7 '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 @@ -7104,7 +7118,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/generator@7.29.1': + '@babel/generator@7.29.7': dependencies: '@babel/parser': 7.29.7 '@babel/types': 7.29.7 @@ -7112,33 +7126,33 @@ snapshots: '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 - '@babel/helper-compilation-targets@7.28.6': + '@babel/helper-compilation-targets@7.29.7': dependencies: - '@babel/compat-data': 7.29.3 - '@babel/helper-validator-option': 7.27.1 + '@babel/compat-data': 7.29.7 + '@babel/helper-validator-option': 7.29.7 browserslist: 4.28.2 lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-globals@7.28.0': {} + '@babel/helper-globals@7.29.7': {} - '@babel/helper-module-imports@7.28.6': + '@babel/helper-module-imports@7.29.7': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.7 '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-imports': 7.28.6 + '@babel/core': 7.29.7 + '@babel/helper-module-imports': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-plugin-utils@7.28.6': {} + '@babel/helper-plugin-utils@7.29.7': {} '@babel/helper-string-parser@7.27.1': {} @@ -7148,11 +7162,11 @@ snapshots: '@babel/helper-validator-identifier@7.29.7': {} - '@babel/helper-validator-option@7.27.1': {} + '@babel/helper-validator-option@7.29.7': {} - '@babel/helpers@7.29.2': + '@babel/helpers@7.29.7': dependencies: - '@babel/template': 7.28.6 + '@babel/template': 7.29.7 '@babel/types': 7.29.7 '@babel/parser@7.29.2': @@ -7163,15 +7177,15 @@ snapshots: dependencies: '@babel/types': 7.29.7 - '@babel/plugin-transform-react-jsx-self@7.27.1(@babel/core@7.29.0)': + '@babel/plugin-transform-react-jsx-self@7.29.7(@babel/core@7.29.7)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/core': 7.29.7 + '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-transform-react-jsx-source@7.27.1(@babel/core@7.29.0)': + '@babel/plugin-transform-react-jsx-source@7.29.7(@babel/core@7.29.7)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/core': 7.29.7 + '@babel/helper-plugin-utils': 7.29.7 '@babel/runtime-corejs3@7.29.2': dependencies: @@ -7179,19 +7193,19 @@ snapshots: '@babel/runtime@7.29.2': {} - '@babel/template@7.28.6': + '@babel/template@7.29.7': dependencies: - '@babel/code-frame': 7.29.0 + '@babel/code-frame': 7.29.7 '@babel/parser': 7.29.7 '@babel/types': 7.29.7 - '@babel/traverse@7.29.0': + '@babel/traverse@7.29.7': dependencies: - '@babel/code-frame': 7.29.0 - '@babel/generator': 7.29.1 - '@babel/helper-globals': 7.28.0 + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.7 + '@babel/helper-globals': 7.29.7 '@babel/parser': 7.29.7 - '@babel/template': 7.28.6 + '@babel/template': 7.29.7 '@babel/types': 7.29.7 debug: 4.4.3 transitivePeerDependencies: @@ -9547,9 +9561,9 @@ snapshots: '@vitejs/plugin-react@5.2.0(vite@7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': dependencies: - '@babel/core': 7.29.0 - '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.29.0) - '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.29.0) + '@babel/core': 7.29.7 + '@babel/plugin-transform-react-jsx-self': 7.29.7(@babel/core@7.29.7) + '@babel/plugin-transform-react-jsx-source': 7.29.7(@babel/core@7.29.7) '@rolldown/pluginutils': 1.0.0-rc.3 '@types/babel__core': 7.20.5 react-refresh: 0.18.0 @@ -9837,16 +9851,16 @@ snapshots: transitivePeerDependencies: - supports-color - astro-expressive-code@0.42.0(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): + astro-expressive-code@0.42.0(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) rehype-expressive-code: 0.42.0 - astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3): + astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3): dependencies: '@astrojs/compiler': 4.0.0 - '@astrojs/internal-helpers': 0.9.1 - '@astrojs/markdown-remark': 7.1.2 + '@astrojs/internal-helpers': 0.10.0 + '@astrojs/markdown-remark': 7.2.0 '@astrojs/telemetry': 3.3.2 '@capsizecss/unpack': 4.0.0 '@clack/prompts': 1.4.0 @@ -9886,7 +9900,7 @@ snapshots: shiki: 4.1.0 smol-toml: 1.6.1 svgo: 4.0.1 - tinyclip: 0.1.12 + tinyclip: 0.1.13 tinyexec: 1.2.2 tinyglobby: 0.2.16 ultrahtml: 1.6.0 @@ -9968,7 +9982,7 @@ snapshots: base64-js@1.5.1: {} - baseline-browser-mapping@2.10.29: {} + baseline-browser-mapping@2.10.32: {} bcp-47-match@2.0.3: {} @@ -10028,10 +10042,10 @@ snapshots: browserslist@4.28.2: dependencies: - baseline-browser-mapping: 2.10.29 - caniuse-lite: 1.0.30001792 - electron-to-chromium: 1.5.356 - node-releases: 2.0.44 + baseline-browser-mapping: 2.10.32 + caniuse-lite: 1.0.30001793 + electron-to-chromium: 1.5.363 + node-releases: 2.0.46 update-browserslist-db: 1.2.3(browserslist@4.28.2) buffer-from@1.1.2: {} @@ -10071,7 +10085,7 @@ snapshots: callsites@3.1.0: {} - caniuse-lite@1.0.30001792: {} + caniuse-lite@1.0.30001793: {} ccount@2.0.1: {} @@ -10461,7 +10475,7 @@ snapshots: ee-first@1.1.1: {} - electron-to-chromium@1.5.356: {} + electron-to-chromium@1.5.363: {} emmet@2.4.11: dependencies: @@ -11649,7 +11663,7 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.5.0: {} + lru-cache@11.5.1: {} lru-cache@5.1.1: dependencies: @@ -12253,7 +12267,7 @@ snapshots: node-mock-http@1.0.4: {} - node-releases@2.0.44: {} + node-releases@2.0.46: {} normalize-path@3.0.0: {} @@ -13130,11 +13144,11 @@ snapshots: stackback@0.0.2: {} - starlight-links-validator@0.24.0(@astrojs/starlight@0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): + starlight-links-validator@0.24.0(@astrojs/starlight@0.39.2(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3))(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - '@astrojs/starlight': 0.39.2(astro@6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) + '@astrojs/starlight': 0.39.2(astro@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3))(typescript@5.9.3) '@types/picomatch': 4.0.3 - astro: 6.3.8(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) + astro: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(rollup@4.60.4)(tsx@4.22.3)(yaml@2.8.3) github-slugger: 2.0.0 hast-util-from-html: 2.0.3 is-absolute-url: 5.0.0 @@ -13364,7 +13378,7 @@ snapshots: tinybench@2.9.0: {} - tinyclip@0.1.12: {} + tinyclip@0.1.13: {} tinyexec@0.3.2: {} @@ -13602,7 +13616,7 @@ snapshots: chokidar: 5.0.0 destr: 2.0.5 h3: 1.15.11 - lru-cache: 11.5.0 + lru-cache: 11.5.1 node-fetch-native: 1.6.7 ofetch: 1.5.1 ufo: 1.6.4 diff --git a/website/package.json b/website/package.json index 53c145f46..0fc003aaa 100644 --- a/website/package.json +++ b/website/package.json @@ -35,7 +35,7 @@ }, "dependencies": { "@apidevtools/json-schema-ref-parser": "^15.3.5", - "@astrojs/react": "^5.0.5", + "@astrojs/react": "^5.0.6", "@astrojs/starlight": "^0.39.2", "@jsonforms/core": "^3.7.0", "@jsonforms/react": "^3.7.0", @@ -43,7 +43,7 @@ "@types/react": "^18.3.28", "@types/react-dom": "^18.3.7", "ajv": "^8.20.0", - "astro": "^6.3.8", + "astro": "^6.4.2", "js-yaml": "^4.1.1", "json-schema-merge-allof": "^0.8.1", "openapi-sampler": "^1.7.4", From fd14d4a6c7011aec95f42f5f654ab0e305670861 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:35:51 -0700 Subject: [PATCH 11/51] Update code.json metadata (#874) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- code.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code.json b/code.json index 32542407c..459b869be 100644 --- a/code.json +++ b/code.json @@ -23,7 +23,7 @@ "disclaimerURL": "", "disclaimerText": "", "vcs": "git", - "laborHours": 14312, + "laborHours": 14340, "reuseFrequency": { "forks": 9, "clones": 0 @@ -51,8 +51,8 @@ "partners": [], "date": { "created": "2024-12-19T21:14:35Z", - "lastModified": "2026-04-30T14:32:16Z", - "metadataLastUpdated": "2026-05-01T00:16:14.494Z" + "lastModified": "2026-05-28T21:05:17Z", + "metadataLastUpdated": "2026-06-01T00:21:07.832Z" }, "tags": [ "api", From ab1499bdd3c767476d1a4e14df88fc93291f1a42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:40:29 -0700 Subject: [PATCH 12/51] chore(deps-dev): bump the all-deps group across 1 directory with 2 updates (#859) Bumps the all-deps group with 2 updates in the /lib/python-sdk directory: [ruff](https://github.com/astral-sh/ruff) and [datamodel-code-generator](https://github.com/koxudaxi/datamodel-code-generator). Updates `ruff` from 0.15.13 to 0.15.15 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.15.13...0.15.15) Updates `datamodel-code-generator` from 0.57.0 to 0.59.0 - [Release notes](https://github.com/koxudaxi/datamodel-code-generator/releases) - [Changelog](https://github.com/koxudaxi/datamodel-code-generator/blob/main/CHANGELOG.md) - [Commits](https://github.com/koxudaxi/datamodel-code-generator/compare/0.57.0...0.59.0) --- updated-dependencies: - dependency-name: datamodel-code-generator dependency-version: 0.58.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: all-deps - dependency-name: ruff dependency-version: 0.15.14 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: all-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lib/python-sdk/poetry.lock | 53 ++++++++++++++++++----------------- lib/python-sdk/pyproject.toml | 2 +- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/lib/python-sdk/poetry.lock b/lib/python-sdk/poetry.lock index cf7bfbc62..35b44ed36 100644 --- a/lib/python-sdk/poetry.lock +++ b/lib/python-sdk/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -244,14 +244,14 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "datamodel-code-generator" -version = "0.57.0" +version = "0.59.0" description = "Datamodel Code Generator" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "datamodel_code_generator-0.57.0-py3-none-any.whl", hash = "sha256:d26bf5defe5154493d0aa5a822b7725332b9e9dd2abccc2f8856052286aa83b5"}, - {file = "datamodel_code_generator-0.57.0.tar.gz", hash = "sha256:0eda778ea06eaa476e542a5f1fe1d14cc3bbf686edb33a0ad6151c7d19089906"}, + {file = "datamodel_code_generator-0.59.0-py3-none-any.whl", hash = "sha256:c8c119ab618d24a619d635fef7aa9c96b69e069a4d287c9adfc148ae28368a69"}, + {file = "datamodel_code_generator-0.59.0.tar.gz", hash = "sha256:054e4d5568c27db5a993f6b3e1d34af53bd1f6d1b6c18b7166908b0f3dc04bd4"}, ] [package.dependencies] @@ -262,16 +262,17 @@ inflect = ">=4.1,<8" isort = ">=4.3.21,<9" jinja2 = ">=2.10.1,<4" pydantic = [ - {version = ">=2.12,<3", markers = "python_version >= \"3.14\""}, {version = ">=2,<3", markers = "python_version < \"3.14\""}, + {version = ">=2.12,<3", markers = "python_version >= \"3.14\""}, ] pyyaml = ">=6.0.1" [package.extras] -all = ["graphql-core (>=3.2.3)", "httpx (>=0.24.1)", "openapi-spec-validator (>=0.2.8,<0.8)", "prance (>=0.18.2)", "pysnooper (>=0.4.1,<2)", "ruff (>=0.9.10)", "watchfiles (>=1.1)"] +all = ["graphql-core (>=3.2.3)", "grpcio-tools (>=1.62,<2)", "httpx (>=0.24.1)", "openapi-spec-validator (>=0.2.8,<0.8)", "prance (>=0.18.2)", "pysnooper (>=0.4.1,<2)", "ruff (>=0.9.10)", "watchfiles (>=1.1)"] debug = ["pysnooper (>=0.4.1,<2)"] graphql = ["graphql-core (>=3.2.3)"] http = ["httpx (>=0.24.1)"] +protobuf = ["grpcio-tools (>=1.62,<2)"] ruff = ["ruff (>=0.9.10)"] ryaml = ["ryaml (>=0.5.1)"] validation = ["openapi-spec-validator (>=0.2.8,<0.8)", "prance (>=0.18.2)"] @@ -1162,30 +1163,30 @@ files = [ [[package]] name = "ruff" -version = "0.15.13" +version = "0.15.15" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.15.13-py3-none-linux_armv6l.whl", hash = "sha256:444b580fc72fd6887e650acd3e575e18cdc79dbcf42fb4030b491057921f61f8"}, - {file = "ruff-0.15.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6590d009e7cb7ebf36f83dbdd44a3fa48a0994ff6f1cdc1b08006abe58f98dc7"}, - {file = "ruff-0.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1c26d2f66163deeb6e08d8b39fbbe983ce3c71cea06a6d7591cfd1421793c629"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbd6f94b434f896308e4d57fb7bfde0d02b99f7a64b3bdab0fdfa6a864203a5"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3259f3be4d181bda591da5db2571aed6853c6a048157756448020bc6c5cd22"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae9c17e5eb4430c154e76abc25d79a318190f5a997f38fb6b114416c5319ffc9"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e2e39bff6c341f4b577a21b801326fab0b11847f48fcaa83f00a113c9b3cb55"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8d9a8e08013542e94d3220bc5b62cc3e5ef87c5f74bff367d3fac14fab013e6"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc411dfebe5eebe55ce041c6ae080eb7668955e866daa2fbb16692a784f1c4ca"}, - {file = "ruff-0.15.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:768494eb08b9cee54e2fd27969966f74db5a57f6eaa7a90fcb3306af34dfc4bd"}, - {file = "ruff-0.15.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fb75f9a3a7e42ffe117d734494e6c5e5cb3565d66e12612cb63d0e572a41a5b6"}, - {file = "ruff-0.15.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8cb74dd33bb2f6613faf7fc03b660053b5ac4f80e706d5788c6335e2a8048d51"}, - {file = "ruff-0.15.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7ef823f817fcd191dc934e984be9cf4094f808effa16f2542ad8e821ba02bbf2"}, - {file = "ruff-0.15.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f345a13937bd7f09f6f5d19fa0721b0c103e00e7f62bc67089a8e5e037719e0b"}, - {file = "ruff-0.15.13-py3-none-win32.whl", hash = "sha256:4044f94208b3b05ba0fc4a4abd0558cf4d6459bd18325eead7fd8cc66f909b41"}, - {file = "ruff-0.15.13-py3-none-win_amd64.whl", hash = "sha256:7064884d442b7d477b4e7473d12da7f08851d2b1982763c5d3f388a19468a1a4"}, - {file = "ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21"}, - {file = "ruff-0.15.13.tar.gz", hash = "sha256:f9d89f17f7ba7fb2ed42921f0df75da797a9a5d71bc39049e2c687cf2baf44b7"}, + {file = "ruff-0.15.15-py3-none-linux_armv6l.whl", hash = "sha256:cf93e5388f412e1b108b1f8b34a6e036b70fe8aff89393befad96fe48670311b"}, + {file = "ruff-0.15.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ac5a646d1f6a7dadd5d50842dae2c1f9862ac887ef5d1b1375e02def791fde6e"}, + {file = "ruff-0.15.15-py3-none-macosx_11_0_arm64.whl", hash = "sha256:77d955a431430c66f72dd94e379ad38a16daea3d25094872ac4edf9e797be530"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7614ee79c69788cf6cedd568069ade9cecc22a1ad20494efe8d0c9ebb4b622d4"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cdb1679e06a1f6b47bc384714ae96f6e2fb65ca441eb78c43d2ca554176ce1f"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2728b93d7b23a603ea2c0ac6eb73d760bd38ec9de35f35fb41e18f7a3fee7622"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be582fcc0db438902c7792b08d6ddf6c9b9e21addaa10092c2c741cfb09e5a45"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7aa77465b8ecaf1a27bea098d696f7fed5e1eccbd10b321b682d6de586ae5627"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48decfa11d740de4889de623be1463308346312f2409a56e24aa280c86162dc4"}, + {file = "ruff-0.15.15-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a5015088452ca0081387063649ec67f06d3d1d6b8b936a1f836b5e9657ecd48c"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f5294aab6356c81600fcdea3a62bb1b924dfd5e91767c12318d3f68f86af57cd"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:db5bd4d802415cca656dc1616070b725952d6ae95eb5d4831e49fbd94a38f75f"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:587a6278ed42059191c1a466e490bd7930fb50bd2e255398bc29616c895a61cb"}, + {file = "ruff-0.15.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:df0c1c084f5f4be9812f61518a45c440d3c30d69ce4bf6c5270e66d38338f02a"}, + {file = "ruff-0.15.15-py3-none-win32.whl", hash = "sha256:29428ea79694afbe756d45fd59b36f22b6b020dc0443cf7de0173046236964b9"}, + {file = "ruff-0.15.15-py3-none-win_amd64.whl", hash = "sha256:8df0323902e15e24bc4bf246da830573d3cf3352bd0b9a164eab335d111ff4a4"}, + {file = "ruff-0.15.15-py3-none-win_arm64.whl", hash = "sha256:3c8ceca6792f38196b8f589bc92eccd03eef286602da92e5dc05cc42ef6441b7"}, + {file = "ruff-0.15.15.tar.gz", hash = "sha256:b8dff018130b46d8e5bf0f926ef6b60cf871d6d5ae45fc9334e09632daa741d6"}, ] [[package]] @@ -1308,4 +1309,4 @@ tests = ["build", "coverage", "mypy", "ruff", "wheel"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "0d7167406b20022c2c46d61c38b218751b70e832bd330bd86252a48fe1202558" +content-hash = "5e4f9dcec868b7a548eaa7be8796bb56611d4487d9a615246df992de70eb0c4e" diff --git a/lib/python-sdk/pyproject.toml b/lib/python-sdk/pyproject.toml index 3c9637174..8061106f3 100644 --- a/lib/python-sdk/pyproject.toml +++ b/lib/python-sdk/pyproject.toml @@ -22,7 +22,7 @@ ruff = ">=0.11,<0.16" tomlkit = ">=0.13.3,<0.16.0" tomlq = "^0.1.0" types-pyyaml = "^6.0.12.20250915" -datamodel-code-generator = ">=0.43.1,<0.58.0" +datamodel-code-generator = ">=0.43.1,<0.60.0" [build-system] requires = ["poetry-core"] From e76dd456777e7f3f9e24690ae24493167611f71e Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Thu, 4 Jun 2026 08:40:19 -0700 Subject: [PATCH 13/51] [Issue #798] Transforms PoC: TypeScript (#825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Issue #798] Transforms PoC: TypeScript Port the Python transforms PoC (PR #810, branch 799-transform-poc-fetch) to @common-grants/sdk so the ADR-0022 / ADR-0017 contract is validated in both SDKs before either is locked in for full implementation. Public additions under @common-grants/sdk/extensions: - buildTransforms() — compile a pair of ADR-0017 mapping objects into typed (toCommon, fromCommon) callables with call-time structural validation. Optional commonModel Zod schema turns parse failures into PluginError[] rather than thrown exceptions. - TransformResult — unconditional { result, errors } return shape (ADR-0022 Decision #7). - PluginError — structured error class with path / handler / sourceValue / cause (ADR-0022 Decision #9). sourceValue and cause are stored non-enumerable; toJSON() omits them so JSON.stringify(err) is PII-safe. console.log(err) / util.inspect(err) still render [cause] via Node's default Error inspection — README PII callout calls this out and recommends a redacted projection. Zod-validation message is also data-bearing because Zod's default error map embeds rejected values; full-message sanitization tracked under #744. - transformFromMapping(), getFromPath(), DEFAULT_HANDLERS — mapping runtime; six built-in handlers (const, field, match, switch alias, numberToString, stringToNumber). - definePlugin() accepts optional meta and transformSchemas. Existing callers passing only `extensions` are unaffected. Security hardening (mapping JSON may be reconstituted from untrusted sources via mergeExtensions(), so the runtime must fail loud on hostile shapes): - buildTransforms() rejects custom handler names that collide with the default registry or shadow Object.prototype keys (constructor, toString, __proto__, etc.) at call time. - validateMapping() rejects `__proto__` as an output field name at build time; transformFromMapping() rejects it again at walk time so the JSON attack vector (own-enumerable __proto__ key from JSON.parse) fails fast in both places. Shared DEFAULT_MAX_TRANSFORM_DEPTH = 500 across both walkers so adversarial mapping JSON can't pass build-time validation only to blow the stack at runtime. - transformFromMapping() scrubs top-level own `__proto__` from plain-object handler returns (defense-in-depth: const / field / match can return JSON.parse-loaded objects with own __proto__ keys, and a downstream for-in deep-merge of the result would otherwise pollute Object.prototype). - getFromPath() uses Object.prototype.hasOwnProperty.call rather than `in` so attacker-controlled field paths cannot resolve to prototype-chain properties. - stringToNumber's error message does not embed the source value (would flow into PluginError.message and bypass the sourceValue PII guard). - Handler JSDoc documents two contracts custom-handler authors must respect: don't return objects with a `__proto__` key (the walker treats handler return values as opaque beyond the top-level scrub), and don't throw Errors with PII in .message (it flows verbatim into PluginError.message, which is rendered by Node Error inspection). Cross-SDK parity: - switchOnValue mirrors Python's `lookup.get(val, default)` exactly: only string source values are candidate keys; numeric / boolean / object source values short-circuit to default. Pinned by parity tests. - ClientConfig parity export added (Python `__init__.py` exports it as `dict[str, Any]`). - Six placeholder type exports (ObjectSchemas, ObjectMappings, PluginExtensions, PluginExtensionsObjectConfig, PluginCapability, PluginMeta) define the ADR-0022 contract surface and parallel the Python PoC's extensions/__init__.py exports. Documented divergences from ADR-0022's TS code blocks (justified in findings.md): - DefinePluginOptions.transformSchemas (not `schemas`) avoids collision with the existing Plugin.schemas field. Resolution deferred to #756. - BuildTransformsOptions.commonModel uses `z.ZodType` to admit schemas with input/output asymmetry (e.g. .transform() producing Date from string). ZodType would reject the SDK's own OpportunityBaseSchema. Out of scope (matches Python PoC; deferred to full SDK): - Auto-generation of transforms from declarative extensions.schemas[obj].mappings inside definePlugin() (Decision #6 TODO). - Always-on commonModel validation inside definePlugin() — opt-in at buildTransforms() for now (Decision #7 TODO). - Sanitizing Zod's default error map output before it lands in PluginError.message — tracked in #744. - Defensive output-key rejection of __defineGetter__ / __defineSetter__ / constructor / toString — tracked in #744. - Recursive (nested) sanitization of __proto__ in handler return values — tracked in #744 alongside other Decision #8 hardening. Includes: - examples/transforms.ts round-trip (`pnpm example:transforms`) - README "Plugin transformations" section + API reference table - 14 define-plugin specs, 35 transformation-handler specs, 19 buildTransforms specs (439 tests across 24 suites, all passing) - Minor changeset bump for @common-grants/sdk Targets HOLD-transforms per the SDK Plugin Enhancements branching strategy. * review: stringToNumber rejects unsafe integers and empty strings Two foot-guns flagged in code review on PR #825: 1. Integer-shaped strings beyond Number.MAX_SAFE_INTEGER silently lost precision via Number(s) (e.g. "9999999999999999999" → 1e19). Plugin authors round-tripping 64-bit IDs through stringToNumber would have seen silent corruption. Guard the integer branch with Number.isSafeInteger and throw on overflow; callers needing arbitrary precision should declare the field as a string or write a custom handler returning BigInt. 2. Empty and whitespace-only strings coerced to 0 via Number("") / Number(" "), turning an implicit-absent CSV cell into a real zero on the transformed side. Explicit s === "" check after .trim() throws on both cases. Docstring rewritten to document both divergences from Python's int() alongside the existing "42.0" decimal-fallback note. Six new tests pin the boundaries (MAX/MIN_SAFE_INTEGER accepted, beyond-safe rejected on both signs, empty and whitespace-only rejected). pnpm run checks clean; pnpm test 443/443; example:transforms round-trip still verifies (legacy ID 12345 well within safe-integer range). * review: validateMapping rejects sibling keys at handler-dispatch nodes The runtime walker is first-key-wins, so `{ field: "x", const: "y" }` would silently drop `const`. Almost always an author bug. Match Python PoC `_validate_mapping` and fail loud at buildTransforms() call time. The low-level transformFromMapping walker keeps lenient behavior so programmatic users composing partial mappings aren't forced into the strict shape. * review: ADR-0022 alignment + parity-claim corrections + test gaps Two behavior changes: - PluginError: drop non-enumerable + toJSON() hardening to match ADR-0022 Decision #9 ("the SDK does not redact by default"). README + JSDoc document the adopter-supplied redacted projection as the supported PII-safety path. Existing PII test inverted. - switchOnValue: throw on non-object spec instead of silent undefined. Walker wraps as HandlerError → PluginError(handler: "match"). JSDoc / comment fixes: reframe "Mirrors Python PoC" claims that were factually wrong (Python's _validate_mapping doesn't reject siblings; Python has no __proto__ defenses; lookup.get accepts non-string val) as honest "TS-only hardening; cf. #810 for parallel Python proposal." Complete @throws lists on buildTransforms and transformFromMapping. Document stringToNumber null/undefined return. Correct misleading "re-exported" wording on DEFAULT_MAX_TRANSFORM_DEPTH. Correct "trimmed to match Python's __init__.py" comment on the barrel (TS exports a superset). Test gap coverage (ADR-0022 Decision #8): getFromPath prototype-chain safety; handler-shadow rejection for __proto__/toString/default-name collisions beyond \`field\`; build-time __proto__ rejection in validateMapping; fromCommonMapping depth-cap. README + CI: add missing ClientConfig and TransformFromMappingOptions rows to the API table; chain example:transforms into pnpm run ci as a smoke step. Simplify: extract deepMapping(levels) helper; PII test suite uses beforeEach so both tests assert on the same PluginError instance. Verification: pnpm run ci passes end-to-end (445 → 449 tests). Cross-SDK follow-ups (file against #810 / amend ADR-0022 separately): TS-only sibling-key rejection, __proto__ defenses, switchOnValue non-object throw, stringToNumber strictness, handler-shadow rejection; and the ADR-0022 Decision #9 default-redaction question itself. * review: revert sibling-key rejection (d52dd72) for cross-SDK parity Python's `_validate_mapping` accepts handler-dispatch nodes with sibling keys; the TS `validateMapping` was rejecting them at build time, creating the only genuine "TS adds mandatory validation Python doesn't" divergence in this PR. The first-key-wins walker behavior already matches Python, so both SDKs now share the same foot-gun: `{ field: "x", const: "fallback" }` silently drops `const` in both. Replaces the two rejection tests with one parity-pinning test (`accepts (does not reject) sibling keys ... cross-SDK parity`) that asserts `.not.toThrow()` — locks the decision so a future regression re-introducing build-time rejection (and re-creating the divergence) fails loudly. Updates `validateMapping` JSDoc to call out the parity decision and `transformFromMapping` JSDoc to drop the (now false) reference to a stricter build-time check. If we later want both SDKs to fail loud on this shape, the cleanest path is to add the check to Python first and then re-introduce TS — keeps divergence at zero. Verification: pnpm run ci passes end-to-end (449 → 448 tests; -2 removed rejection tests, +1 added parity-pinning test). * review: array-spec rejection, scrub clones source, README projection Three findings from a /review pass over the PoC. All three tighten existing contracts; none introduce new shapes. - switchOnValue: reject arrays as spec. Arrays pass `typeof === "object"` and non-null but lack the structural shape — a mapping like `{ match: ["posted", "archived"] }` would otherwise silently resolve to s.field/s.case/s.default all-undefined and return undefined. Same fail-loud direction as the existing non-object guards. - __proto__ scrub: shallow-clone on the pollution path rather than `delete`-in-place. `fieldValue` returns references plucked from caller input via `getFromPath`, so an in-place delete would silently mutate the caller's data — surprising for plugin authors caching parsed source records across `toCommon` calls (common in long-running adapter processes and multi-tenant deployments). Spread is the correct copy primitive here: CopyDataProperties -> CreateDataProperty bypasses the prototype setter. Object.assign would mutate the target's prototype chain instead — the source comment now warns against the regression. - README error-handling snippet: log a named redacted projection rather than building the message via template-string interpolation. Makes the "what gets logged" surface a single audit point and matches the projection in the prose PII callout and in the redacted-projection test verbatim. Verified: pnpm --filter @common-grants/sdk run ci passes (449 tests across 24 suites, +1 source-preservation test; example:transforms round-trip OK). * consolidate customFields onto ObjectSchemasInput (Python a156d31 parity) Mirror the Python PoC PR #838 commit a156d31 so plugin authors add a single per-object entry under transformSchemas[Object] when introducing a new object, rather than splitting customFields across a separate top-level extensions dict. - types.ts: add customFields?: Record to ObjectSchemasInput; drop customFields from PluginExtensionsObjectConfig (now mappings-only, matching Python's PluginExtensionsSchema). - define-plugin.ts: make extensions optional in DefinePluginOptions and Plugin; definePlugin() sources customFields from transformSchemas[name].customFields first, falling back to the legacy extensions[name] surface so existing customFields-only plugins keep working. - examples/transforms.ts: consolidate to a single transformSchemas entry carrying customFields + toCommon + fromCommon together. - tests: cover the consolidated path (customFields on transformSchemas[obj]), no-extensions-arg case, and transformSchemas-wins-over-extensions priority. Open question — ADR-0022 as written places customFields inside PluginExtensions.schemas[obj] specifically so declarations can be combined across packages via mergeExtensions() (Decision driver line 23, Decision #4). This commit follows Jeff's Python move out of that serializable surface, which drops customFields from the cross-package merge contract. Pending an ADR-0022 amendment formalizing the trade-off; flagged inline on ObjectSchemasInput and PluginExtensionsObjectConfig. Type inference for plugin.schemas[obj] still flows through the legacy extensions parameter — the runtime applies customFields from either source, but the compiled-schema type only narrows when customFields are declared via extensions. Wiring the generic through transformSchemas is a follow-up (parallel to the transformSchemas → schemas rename deferred to #756). * align transforms-layer handlers with ADR-0024 three-state contract Preserve explicit `null` source values ("publisher asserts doesn't apply") across `toCommon` / `fromCommon` instead of collapsing them to `undefined` ("not provided"). Without this, ADR-0024's third state would be unobservable through the transforms layer — the ADR audited the Zod/Pydantic validation surface and explicitly left SDKs untouched, but the handler layer is downstream of that. Handler changes (lib/ts-sdk/src/extensions/transformation.ts): - numberToString / stringToNumber: null source -> null output (was undefined). Return types widen to string | null | undefined / number | null | undefined. - switchOnValue: null source passes through as null by default; mapping authors opt in to target-side translation via a `case: { "null": ... }` key. `default` is NOT consulted for null source (default is for unrecognized values, not publisher assertions). - field / getFromPath: unchanged — already preserves terminal null. JSDoc updated to document intermediate-null behavior as scoped-out ("parent N/A propagates to child paths"). - The walker places handler-returned null onto the output object as a real null (distinct from an absent key), so the three-state distinction survives end-to-end. Docs: - New "Null handling" subsection in src/extensions/README.md documenting the three-state contract for custom-handler authors. - API reference table updated to reflect the new return semantics. - Module-level JSDoc on transformation.ts documents the contract and the cross-SDK divergence (TS leads, Python PoC #810 parity tracked there). Example (examples/transforms.ts): - Source record now includes `source_url: null` demonstrating the publisher's "doesn't apply" assertion. - Round-trip spot-check fails if a future regression coerces it back to undefined. Tests: - Existing "returns undefined on null or missing" tests split into two for both coercing handlers (null -> null; absent -> undefined). - New switchOnValue coverage for null pass-through, opt-in "null" case-key precedence, and the no-default-for-null rule. - New walker test pinning that handler-returned null lands on the output. - New buildTransforms round-trip + commonModel Zod assertion using `source` (the .nullish() field on OpportunityBaseSchema). - fieldValue regression pin for terminal-null preservation and intermediate- null collapse. 466 tests pass (was 452, +14). pnpm run checks / build / example:transforms all clean. * review: fix doc drift surfaced by three-lens self-review Code/runtime behavior unchanged — these are JSDoc, README, and example-comment fixes for code-vs-docs drift that accumulated across review iterations. - `PluginError` JSDoc: rename the "Recommended redacted projection" snippet to "partial-redaction pattern" with an inline CAUTION on the data-bearing `message` field. The previous label invited copy-paste that contradicts the warning paragraph immediately below (Zod's default error map embeds runtime values into `issue.message`, which flows verbatim into `PluginError.message`). - `Handler` JSDoc: remove stale reference to `PluginError.toJSON()`. The toJSON()/non-enumerable hardening was removed earlier in this PR for ADR-0022 Decision #9 alignment; the JSDoc still described a protection mechanism that no longer exists. Replace with a pointer to the README PII warning and to `stringToNumber`'s generic-message pattern. - `Handler` JSDoc: note the cosmetic ADR-0022 parameter-name divergence (`(data, arg)` vs ADR's `(value, context)`) so a custom-handler author reading the ADR interface block doesn't trip on the rename. - `switchOnValue` and `DEFAULT_HANDLERS` JSDoc + README handler table: drop the phantom "ADR-0022 Decision #3" citation. Decision #3 is about per-object schema grouping, not handler naming. The `switch` alias is a convenience — no prior SDK API defined `switch`, so there is no backward-compatibility obligation. Cross-SDK parity rationale (if any) tracked separately. - README API reference table: update `ObjectSchemasInput` row to include `customFields?` (added in d9d9bb2) and update `PluginExtensionsObjectConfig` row to drop `customFields?` (moved out in the same commit). Both rows had drifted from the implementing interfaces. - `examples/transforms.ts`: add PII-warning comments on the two error-path `e.message` formatting blocks. Safe in this example (fixed PII-free source data) but adopters copy this shape into their own pipelines. Checks: - `pnpm run check:lint` — clean. - `pnpm run check:format` — clean (prettier --write applied to README after the table-row edit shifted column widths). - `pnpm run check:types` — clean. - `pnpm run test` — 466 / 466 pass across 24 suites. - `pnpm run example:transforms` — round-trip + ADR-0024 three-state spot-check both pass. * review: omit undefined output keys so in-memory shape carries three-state (ADR-0024) Resolves the walker/doc/test mismatch flagged in self-review (Important #1). Behavior change (transformFromMapping output-shape branch): a child that transforms to `undefined` (absent source — ADR-0024 "not provided") is now omitted from the output object instead of written as `out[k] = undefined`. `null` ("doesn't apply") is still written as a present key. So the in-memory object now distinguishes all three states the same way the wire does: absent → key omitted, `null` → present `null`, value → present value. Why this shape: - The wire output is unchanged — `JSON.stringify` already dropped `undefined`-valued keys, so absent→missing on the wire was already correct. This only fixes the in-memory representation to match. - Makes the in-memory object a faithful three-state carrier: consumers can use key presence (`hasOwnProperty` / `in`) to tell "not provided" from "doesn't apply", which a present-`undefined` key made impossible. - Aligns with the dict shape the Python SDK lands on once it adopts the three-state contract: Python has no `undefined`, so an absent field is a missing dict key there. (Python's PoC currently collapses absent+null into `None` — known non-conformant, per the cross-SDK note in the README.) Doc/test corrections that depended on the old behavior: - README "Null handling": the "distinct from an absent key" claim is now literally true; reworded to state the absent→omitted / null→present rule and that key presence is the probe. - transformFromMapping JSDoc: output-shape bullet documents undefined-omission. - transforms.spec.ts "distinguishes absent from null": the test comment claimed an absent output key but only asserted `toBeUndefined()` (satisfied by a present-undefined key too). Added `hasOwnProperty` assertions that pin the real contract — absent key for absent source, present key for null. - transformation.spec.ts: new low-level test asserting an undefined-returning handler omits the key while a sibling present key is still written. Checks: lint / prettier / tsc clean; test 467/467 (was 466, +1 walker test); example:transforms round-trip + source_url:null three-state spot-check pass. * review: correct stale cross-SDK parity note in transforms README The "Null handling" cross-SDK note claimed the Python ADR-0024 parity follow-up was "tracked there" and linked #810. Reading #810 directly (body + diff) confirms it is the base Python Transform PoC (closes #799, merged 2026-05-13), scoped "to the transform layer only" with no null/three-state content — its coercing handlers collapse absent and null into `None` (`str(val) if val is not None else None`; `if val is None: return None`). ADR-0024 (PR #855) was created and merged 2026-05-26, 13 days after #810, so #810 neither did the parity work nor tracks it. No dedicated Python-parity issue exists. Reword: drop the false "tracked there" claim, link ADR-0024 directly as the contract Python must meet, state plainly that Python predates the ADR and that handler parity is a pending follow-up. No tracker is asserted. Docs-only; prettier clean. * review: attribute scrub-time throws to the handler (move __proto__ scrub inside try) The walker's __proto__ scrub on a handler's return value ran *outside* the handler try/catch. A custom handler returning a trap-throwing exotic value (e.g. a Proxy whose getPrototypeOf/hasOwnProperty trap throws, or one that throws on spread) made the scrub throw, which escaped as an unattributed PluginError via runMapping's generic catch instead of HandlerError. Not a security issue — handler code is trusted (author-supplied, registered per buildTransforms call); ADR-0022 Decision #8's threat model is untrusted mapping JSON, not handler code. The failure was already contained (PluginError in errors[], no crash/pollution). This only restores handler attribution on that exotic path. Fix: move the scrub inside the existing handler try/catch. The scrub operates on the handler's return value, so a throw there is correctly attributed to the handler via HandlerError -> PluginError(handler). Added a comment marking the placement as deliberate ("don't hoist it out") and a test: a handler returning a Proxy with a throwing getPrototypeOf trap now yields PluginError(handler). Checks: lint / prettier / tsc clean; test 468/468 (+1); example:transforms passes. * fix: reject sibling keys at handler-dispatch nodes (restore d52dd72) The 3e557b9 revert removed TS's build-time sibling-key rejection on the premise that Python's `_validate_mapping` accepts the shape. It does not: `build_transforms` -> `_validate_mapping` (python-sdk extensions/transforms.py) raises on a handler key with sibling keys, and has since the Python PoC (#810). The revert conflated Python's lenient runtime walker (transform_from_mapping, first-key-wins) with its strict build-time validator, so it created the cross-SDK divergence it claimed to remove. Re-instates build-time rejection in validateMapping (mirroring Python), restores the two rejection tests, and corrects the now-false JSDoc in transforms.ts and transformation.ts. The low-level transformFromMapping walker stays lenient, matching Python's runtime walker. Verified: check:types + prettier + build + vitest (469 passed). * docs(ts-sdk/extensions): trim cross-reference cruft from transforms PoC Pointers cost per occurrence and rot at different rates; keep the inline reasoning, thin the citations. - Remove all `#838` / commit `a156d31` references (unmerged-branch pointers that describe Python code not on the base branch). Replaced with "pending ADR-0022 amendment" framing that states the trade-off in its own terms. - Collapse the ADR-0024 three-state explanation to one canonical block in transformation.ts + the README; handlers now note only their own departures instead of re-deriving the contract. - Remove the 4 duplicated per-handler "Python #810 parity" notes; keep the statement once in the README (#810 is merged, durable). - Thin repeated `(ADR-0022 Decision #8)` prototype-hardening tags in the walker; the security reasoning stays at every guard, one ADR anchor kept. Comment/doc-only; no behavior change. tsc/eslint/prettier clean, extension tests pass, example:transforms round-trips. * docs(ts-sdk/extensions): drop ungrounded "pending ADR-0022 amendment" framing The "pending amendment / open question" language asserted a governance artifact that exists nowhere: ADR-0022's doc never mentions it, no issue tracks it, and the only files in the repo referencing an "ADR-0022 amendment" were this PR's own. Same failure mode as the #838 pointer — a reference to something that isn't real. Replace with the verifiable behavior + trade-off: customFields can live on transformSchemas[Object] (preferred) or the legacy `extensions` surface (kept for mergeExtensions cross-package composition); transformSchemas wins when both are set. State the trade-off, drop the speculative future-ADR claim. Comment/doc-only. tsc/prettier clean, 159 extension tests pass. * docs(ts-sdk/extensions): second cruft pass — uniform across the whole PR The first pass cherry-picked the loudest offenders and left the broader "talks a lot about other issues/PRs/ADRs" problem (originally flagged on define-plugin.ts) largely intact. This applies one policy uniformly across all PR files: - ADR-0017: one canonical mention (README status line); elsewhere "mapping objects" stands alone. - ADR-0022 "Decision #N" tags: keep the reasoning; one anchor each for the two security decisions (#8 prototype guard in getFromPath, #9 PII in the README warning). Drop the rest, including in test names. - "full SDK will… (tracked under #756)": state PoC scope plainly; #756 kept once (define-plugin.ts). - Cross-SDK Python-parity pointers ("Mirrors the Python SDK's X", parity export, superset-of-__init__, .-join-matches-Python): consolidated to the one README cross-SDK note. Behavioral divergence docs (int(s) semantics, safe-integer rationale) kept — they describe this code. - #798/#810: one mention each (README/changeset opening). - Test names/comments: bare ADR/Decision numbers stripped, behavioral phrasing kept ("doesn't apply", "hardening", sibling-key "Python PoC parity"). De-referenced a rot-prone transforms.ts:218 line pointer. Comment/doc-only; no behavior change. tsc/eslint/prettier clean, 159 extension tests pass, example:transforms round-trips. * docs(ts-sdk/examples): explain the join/split custom handlers in transforms example Address review on PR #825: the Step 2 handler block had only a section banner. Expand it to cover the handler dispatch mechanism (name-as-key, the built-in set, the Step 4 handlers map, absolute data paths) and add a one-line behavior note above joinFields and splitField, framing them as an inverse pair for the derived compositeLabel round trip. * Flattening plugin contract to match ADR * Removing extraneous key checks * Updating documentation and fixing CI issue * Updating dependencies * Fixing CI * Fixing tests * Updating changeset * Code cleanup * Removing merge extensions artifacts --------- Co-authored-by: jcrichlake --- .changeset/transforms-poc-typescript.md | 32 + lib/changelog-emitter/package.json | 2 +- lib/changelog-emitter/vitest.config.ts | 1 + .../__tests__/commands/check/check.test.ts | 7 +- .../commands/compile/compile.test.ts | 6 +- .../src/__tests__/commands/init/init.test.ts | 6 +- .../commands/preview/preview.test.ts | 6 +- .../extensions/define-plugin.spec.ts | 240 +++--- lib/ts-sdk/__tests__/extensions/index.spec.ts | 96 ++- .../extensions/merge-extensions.spec.ts | 181 ----- .../extensions/transformation.spec.ts | 395 +++++++++ .../__tests__/extensions/transforms.spec.ts | 510 ++++++++++++ lib/ts-sdk/examples/plugins.ts | 105 +-- lib/ts-sdk/examples/transforms.ts | 321 ++++++++ lib/ts-sdk/package.json | 5 +- lib/ts-sdk/src/extensions/README.md | 419 +++++++--- lib/ts-sdk/src/extensions/define-plugin.ts | 180 +++-- lib/ts-sdk/src/extensions/index.ts | 27 +- lib/ts-sdk/src/extensions/merge-extensions.ts | 181 ----- lib/ts-sdk/src/extensions/transformation.ts | 347 ++++++++ lib/ts-sdk/src/extensions/transforms.ts | 232 ++++++ lib/ts-sdk/src/extensions/types.ts | 253 +++++- lib/ts-sdk/vitest.config.ts | 1 + pnpm-lock.yaml | 758 ++---------------- pnpm-workspace.yaml | 6 +- .../governance/adr/0022-plugin-framework.mdx | 62 +- 26 files changed, 2869 insertions(+), 1510 deletions(-) create mode 100644 .changeset/transforms-poc-typescript.md delete mode 100644 lib/ts-sdk/__tests__/extensions/merge-extensions.spec.ts create mode 100644 lib/ts-sdk/__tests__/extensions/transformation.spec.ts create mode 100644 lib/ts-sdk/__tests__/extensions/transforms.spec.ts create mode 100644 lib/ts-sdk/examples/transforms.ts delete mode 100644 lib/ts-sdk/src/extensions/merge-extensions.ts create mode 100644 lib/ts-sdk/src/extensions/transformation.ts create mode 100644 lib/ts-sdk/src/extensions/transforms.ts diff --git a/.changeset/transforms-poc-typescript.md b/.changeset/transforms-poc-typescript.md new file mode 100644 index 000000000..4729ae771 --- /dev/null +++ b/.changeset/transforms-poc-typescript.md @@ -0,0 +1,32 @@ +--- +"@common-grants/sdk": minor +--- + +Add a TypeScript proof-of-concept for the plugin transformation framework (issue #798), mirroring the Python PoC in PR #810. Plugin authors can now compile declarative mapping objects into typed `(toCommon, fromCommon)` callables, validate `toCommon` output against an extended Zod schema, and attach those callables to a plugin via `definePlugin({ schemas })`. + +**New public surface (under `@common-grants/sdk/extensions`):** + +- `buildTransforms(toCommonMapping, fromCommonMapping, handlers?, commonModel?)` — compiles a pair of mapping objects into typed `(toCommon, fromCommon)` callables with call-time structural validation. `handlers` is a `Map` for custom handler registration. Optional `commonModel` Zod schema turns parse failures into `PluginError[]` instead of thrown exceptions. +- `TransformResult` — unconditional `{ result, errors }` return shape. +- `PluginError` — structured error class carrying `path`, `handler`, `sourceValue`, `cause`. +- `transformFromMapping()`, `getFromPath()`, `DEFAULT_HANDLERS` — lower-level mapping runtime pieces; `DEFAULT_HANDLERS` is a `Map` of six built-in handlers (`const`, `field`, `match` / `switch` alias, `numberToString`, `stringToNumber`). +- `definePlugin()` accepts optional `meta: PluginMeta` and `schemas: SchemasInput`. All per-object declarations (custom fields, native schema, transforms) are co-located under `schemas[Object]` — `customFields` lives on `schemas[Object].customFields` rather than on the `extensions` key. The compiled `plugin.schemas[Object].common` holds the extended Zod schema. +- New supporting types: `Handler`, `SchemasInput`, `ObjectSchemasInput`, `ObjectSchemas`, `PluginMeta`, `PluginCapability`, `ObjectMappings`, `PluginExtensionsObjectConfig`, `PluginExtensions`. + +**Three-state null handling (ADR-0024)** for optional fields: + +- `numberToString` and `stringToNumber` now preserve `null` source values as `null` (the publisher's "doesn't apply" assertion) instead of collapsing to `undefined`. Return types widen from `string | undefined` / `number | undefined` to `string | null | undefined` / `number | null | undefined`. +- `match` / `switch` passes `null` source through by default; opt in to target-side translation via a `"null"` key in the `case` map. `default` is not consulted for `null` source values. +- `field` / `getFromPath` already preserve terminal `null`; intermediate-null short-circuits the path (documented as propagating "doesn't apply"). +- The walker places handler-returned `null` onto the output object as a real `null`, distinct from an absent key — so consumers can read the three states (absent / `null` / value) end-to-end through `toCommon` and `fromCommon`. + +**Removed:** + +- `mergeExtensions` has been removed from the public surface. Consumers who previously used `mergeExtensions` to combine extension objects should merge them manually (e.g. with object spread) before passing to `definePlugin`. + +**Out of scope** (deferred to full SDK): + +- Auto-generation of transforms from declarative `extensions.schemas[obj].mappings` inside `definePlugin()`. +- Always-on `commonModel` validation inside `definePlugin()` — opt-in at `buildTransforms()` for now. + +Runnable example: `pnpm --filter @common-grants/sdk example:transforms` (round-trips a synthetic grants.gov record through `toCommon` and `fromCommon` with custom `join` / `split` handlers, extended-schema validation, and three-state null preservation on `source_url: null`). diff --git a/lib/changelog-emitter/package.json b/lib/changelog-emitter/package.json index 8112ec068..a6167ddb0 100644 --- a/lib/changelog-emitter/package.json +++ b/lib/changelog-emitter/package.json @@ -29,7 +29,7 @@ "vitest": "catalog:" }, "scripts": { - "build": "tsc", + "build": "tsc && node --input-type=module -e \"import{copyFileSync}from'node:fs';copyFileSync('package.json','dist/package.json')\"", "prepare": "pnpm build", "watch": "tsc --watch", "test": "vitest run", diff --git a/lib/changelog-emitter/vitest.config.ts b/lib/changelog-emitter/vitest.config.ts index 0332b5e22..1c7b84b01 100644 --- a/lib/changelog-emitter/vitest.config.ts +++ b/lib/changelog-emitter/vitest.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", + exclude: ["dist/**", "node_modules/**"], // testTimeout: 10000, // Uncomment to increase the default timeout isolate: false, // Your test shouldn't have side effects doing this will improve performance. }, diff --git a/lib/cli/src/__tests__/commands/check/check.test.ts b/lib/cli/src/__tests__/commands/check/check.test.ts index 90d5b94ea..1f98f8285 100644 --- a/lib/cli/src/__tests__/commands/check/check.test.ts +++ b/lib/cli/src/__tests__/commands/check/check.test.ts @@ -8,10 +8,9 @@ const mockCheckSpec = vi.fn(); // Mock the service with consistent implementation vi.mock("../../../commands/check/check-service.ts", () => ({ - DefaultCheckService: vi.fn(() => ({ - checkApi: mockCheckApi, - checkSpec: mockCheckSpec, - })), + DefaultCheckService: vi.fn(function () { + return { checkApi: mockCheckApi, checkSpec: mockCheckSpec }; + }), })); describe("checkCommand", () => { diff --git a/lib/cli/src/__tests__/commands/compile/compile.test.ts b/lib/cli/src/__tests__/commands/compile/compile.test.ts index 912563618..a4ca93475 100644 --- a/lib/cli/src/__tests__/commands/compile/compile.test.ts +++ b/lib/cli/src/__tests__/commands/compile/compile.test.ts @@ -5,9 +5,9 @@ import { compileCommand } from "../../../commands/compile/compile"; const mockCompile = vi.fn(); vi.mock("../../../commands/compile/compile-service", () => ({ - DefaultCompileService: vi.fn(() => ({ - compile: mockCompile, - })), + DefaultCompileService: vi.fn(function () { + return { compile: mockCompile }; + }), })); describe("compileCommand", () => { diff --git a/lib/cli/src/__tests__/commands/init/init.test.ts b/lib/cli/src/__tests__/commands/init/init.test.ts index 9d014380a..ac1aa4741 100644 --- a/lib/cli/src/__tests__/commands/init/init.test.ts +++ b/lib/cli/src/__tests__/commands/init/init.test.ts @@ -27,9 +27,9 @@ describe("initCommand", () => { const mockListTemplates = vi .fn<() => Promise>() .mockResolvedValue(["template1", "template2"]); - (DefaultInitService as Mock).mockImplementation(() => ({ - listTemplates: mockListTemplates, - })); + (DefaultInitService as Mock).mockImplementation(function () { + return { listTemplates: mockListTemplates }; + }); initCommand(program); const cmd = program.commands.find(cmd => cmd.name() === "init"); diff --git a/lib/cli/src/__tests__/commands/preview/preview.test.ts b/lib/cli/src/__tests__/commands/preview/preview.test.ts index e4c576a3d..aff694fbe 100644 --- a/lib/cli/src/__tests__/commands/preview/preview.test.ts +++ b/lib/cli/src/__tests__/commands/preview/preview.test.ts @@ -7,9 +7,9 @@ const mockPreviewSpec = vi.fn(); // Mock the service with consistent implementation vi.mock("../../../commands/preview/preview-service", () => ({ - DefaultPreviewService: vi.fn(() => ({ - previewSpec: mockPreviewSpec, - })), + DefaultPreviewService: vi.fn(function () { + return { previewSpec: mockPreviewSpec }; + }), })); describe("previewCommand", () => { diff --git a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts index 48e952e40..0e6f0b729 100644 --- a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { z } from "zod"; -import { definePlugin, mergeExtensions, type SchemaExtensions } from "@/extensions"; +import { buildTransforms, definePlugin, type PluginMeta, type TransformResult } from "@/extensions"; import { OpportunityBaseSchema } from "@/schemas/zod/models"; import { CustomFieldType } from "@/constants"; @@ -32,25 +32,22 @@ describe("definePlugin", () => { // ############################################################################ describe("basic structure", () => { - it("should return a Plugin with .extensions and .schemas", () => { - const extensions = { - Opportunity: { - legacyId: { fieldType: CustomFieldType.string }, + it("should return a Plugin with .schemas", () => { + const plugin = definePlugin({ + schemas: { + Opportunity: { + customFields: { + legacyId: { fieldType: CustomFieldType.string }, + }, + }, }, - } as const; - - const plugin = definePlugin({ extensions }); + } as const); - expect(plugin).toHaveProperty("extensions"); expect(plugin).toHaveProperty("schemas"); }); - it("should preserve extensions input by reference on .extensions", () => { - const extensions = { - Opportunity: { - legacyId: { fieldType: CustomFieldType.string }, - }, - } as const; + it("should preserve meta input by reference", () => { + const extensions = { meta: { name: "test", sourceSystem: "test" } }; const plugin = definePlugin({ extensions }); @@ -63,23 +60,25 @@ describe("definePlugin", () => { // ############################################################################ describe("extensible schemas", () => { - it("should parse payloads with custom fields via plugin.schemas.Opportunity", () => { + it("should parse payloads with custom fields via plugin.schemas.Opportunity.common", () => { const plugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { - fieldType: CustomFieldType.object, - value: LegacyIdValueSchema, - }, - category: { - fieldType: CustomFieldType.string, - description: "Grant category", + customFields: { + legacyId: { + fieldType: CustomFieldType.object, + value: LegacyIdValueSchema, + }, + category: { + fieldType: CustomFieldType.string, + description: "Grant category", + }, }, }, }, } as const); - const result = plugin.schemas.Opportunity.parse({ + const result = plugin.schemas.Opportunity.common.parse({ ...validOpp, customFields: { legacyId: { @@ -102,17 +101,19 @@ describe("definePlugin", () => { it("should reject invalid custom field values via safeParse", () => { const plugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { - fieldType: CustomFieldType.object, - value: LegacyIdValueSchema, + customFields: { + legacyId: { + fieldType: CustomFieldType.object, + value: LegacyIdValueSchema, + }, }, }, }, } as const); - const result = plugin.schemas.Opportunity.safeParse({ + const result = plugin.schemas.Opportunity.common.safeParse({ ...validOpp, customFields: { legacyId: { @@ -128,105 +129,162 @@ describe("definePlugin", () => { }); // ############################################################################ - // Empty extensions + // Empty schemas // ############################################################################ - describe("empty extensions", () => { - it("should return base schemas when extensions is {}", () => { - const plugin = definePlugin({ extensions: {} }); + describe("empty schemas", () => { + it("should return base schema under .common when no customFields are provided", () => { + const plugin = definePlugin({}); - expect(plugin.schemas.Opportunity).toBe(OpportunityBaseSchema); + expect(plugin.schemas.Opportunity.common).toBe(OpportunityBaseSchema); }); - it("should return base Opportunity schema when specs are empty", () => { - const plugin = definePlugin({ extensions: { Opportunity: {} } }); + it("should return base schema under .common when customFields is empty", () => { + const plugin = definePlugin({ + schemas: { Opportunity: { customFields: {} } }, + }); - expect(plugin.schemas.Opportunity).toBe(OpportunityBaseSchema); + expect(plugin.schemas.Opportunity.common).toBe(OpportunityBaseSchema); }); }); // ############################################################################ - // Plugin composition + // Type compatibility // ############################################################################ - describe("plugin composition", () => { - it("should work with mergeExtensions for multi-plugin composition", () => { - const pluginA = definePlugin({ - extensions: { + describe("type compatibility", () => { + it("should have the expected Plugin shape at runtime", () => { + const plugin = definePlugin({ + schemas: { Opportunity: { - legacyId: { - fieldType: CustomFieldType.object, - value: LegacyIdValueSchema, + customFields: { + legacyId: { fieldType: CustomFieldType.string }, }, }, }, } as const); - const pluginB = definePlugin({ - extensions: { - Opportunity: { - category: { - fieldType: CustomFieldType.string, - description: "Grant category", - }, - }, + expect(plugin).toHaveProperty("schemas"); + expect(typeof plugin.schemas).toBe("object"); + }); + }); + + // ############################################################################ + // meta + // ############################################################################ + + describe("meta", () => { + it("preserves meta on the returned plugin", () => { + const meta: PluginMeta = { + name: "grants.gov", + version: "1.0.0", + sourceSystem: "grants.gov", + capabilities: ["customFields", "transforms"], + }; + + const plugin = definePlugin({ meta }); + + expect(plugin.meta).toBe(meta); + }); + + it("leaves meta undefined when not provided", () => { + const plugin = definePlugin({}); + + expect(plugin.meta).toBeUndefined(); + }); + }); + + // ############################################################################ + // schemas (transform callables) + // ############################################################################ + + describe("schemas (transforms)", () => { + it("preserves toCommon and fromCommon on the returned plugin schemas", () => { + const { toCommon, fromCommon } = buildTransforms( + { title: { field: "data.opportunity_title" } }, + { data: { opportunity_title: { field: "title" } } } + ); + + const plugin = definePlugin({ + schemas: { + Opportunity: { toCommon, fromCommon }, }, - } as const); + }); - const merged = mergeExtensions([pluginA.extensions, pluginB.extensions]); - const combined = definePlugin({ extensions: merged }); + expect(plugin.schemas.Opportunity.toCommon).toBe(toCommon); + expect(plugin.schemas.Opportunity.fromCommon).toBe(fromCommon); + }); - const result = combined.schemas.Opportunity.parse({ - ...validOpp, - customFields: { - legacyId: { - name: "legacyId", - fieldType: "object", - value: { system: "legacy", id: 99 }, - }, - category: { - name: "category", - fieldType: "string", - value: "Education", - }, + it("invokes the stored transform callable via plugin.schemas", () => { + const { toCommon, fromCommon } = buildTransforms( + { title: { field: "data.opportunity_title" } }, + { data: { opportunity_title: { field: "title" } } } + ); + + const plugin = definePlugin({ + schemas: { + Opportunity: { toCommon, fromCommon }, }, }); - expect(result.customFields?.legacyId).toBeDefined(); - expect(result.customFields?.category).toBeDefined(); + const out = plugin.schemas.Opportunity.toCommon?.({ + data: { opportunity_title: "Hello" }, + }) as TransformResult<{ title: string }>; + + expect(out.errors).toEqual([]); + expect(out.result.title).toBe("Hello"); + }); + + it("leaves toCommon and fromCommon undefined when not provided", () => { + const plugin = definePlugin({}); + + expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); + expect(plugin.schemas.Opportunity.fromCommon).toBeUndefined(); }); }); // ############################################################################ - // Type compatibility + // schemas.customFields — single surface // ############################################################################ - describe("type compatibility", () => { - it("should have the same shape as base Plugin", () => { + describe("schemas.customFields (single surface)", () => { + it("applies customFields from schemas[obj] to the compiled common schema", () => { const plugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { fieldType: CustomFieldType.string }, + customFields: { + legacyId: { + fieldType: CustomFieldType.object, + value: LegacyIdValueSchema, + }, + }, }, }, - } as const); + }); - // Verify Plugin has the expected Plugin shape at runtime - expect(plugin).toHaveProperty("extensions"); - expect(plugin).toHaveProperty("schemas"); - expect(typeof plugin.extensions).toBe("object"); - expect(typeof plugin.schemas).toBe("object"); + const result = plugin.schemas.Opportunity.common.parse({ + ...validOpp, + customFields: { + legacyId: { + name: "legacyId", + fieldType: "object", + value: { system: "legacy", id: 42 }, + }, + }, + }); + + const legacy = result.customFields?.legacyId?.value as { system: string; id: number }; + expect(legacy.system).toBe("legacy"); + expect(legacy.id).toBe(42); }); - it("should accept SchemaExtensions as input type", () => { - const extensions: SchemaExtensions = { - Opportunity: { - category: { fieldType: "string" }, - }, - }; + it("works without a schemas argument at all", () => { + const plugin = definePlugin({}); - const plugin = definePlugin({ extensions }); - expect(plugin.schemas.Opportunity).toBeDefined(); + expect(plugin.extensions).toBeUndefined(); + // Common schema defaults to base schema when no customFields provided. + const result = plugin.schemas.Opportunity.common.parse(validOpp); + expect(result.title).toBe("Test Opportunity"); }); }); }); diff --git a/lib/ts-sdk/__tests__/extensions/index.spec.ts b/lib/ts-sdk/__tests__/extensions/index.spec.ts index 22d87d947..3dac63802 100644 --- a/lib/ts-sdk/__tests__/extensions/index.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/index.spec.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { z } from "zod"; -import { withCustomFields, getCustomFieldValue, definePlugin, mergeExtensions } from "@/extensions"; +import { withCustomFields, getCustomFieldValue, definePlugin } from "@/extensions"; import { OpportunityBaseSchema } from "@/schemas"; import { CustomFieldType } from "@/constants"; @@ -289,10 +289,10 @@ describe("withCustomFields + getCustomFieldValue integration", () => { }); // ############################################################################ -// Plugin composition integration tests +// Plugin integration tests // ############################################################################ -describe("plugin composition", () => { +describe("plugin integration", () => { // Shared test data const validOpp = { id: "573525f2-8e15-4405-83fb-e6523511d893", @@ -305,12 +305,14 @@ describe("plugin composition", () => { // Plugin 1: Legacy System — adds legacyId (object with system + id) const legacyPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { - fieldType: CustomFieldType.object, - value: LegacyIdValueSchema, - description: "Maps to the opportunity_id in the legacy system", + customFields: { + legacyId: { + fieldType: CustomFieldType.object, + value: LegacyIdValueSchema, + description: "Maps to the opportunity_id in the legacy system", + }, }, }, }, @@ -318,15 +320,17 @@ describe("plugin composition", () => { // Plugin 2: Classification — adds category (string) and priority (integer) const classificationPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - category: { - fieldType: CustomFieldType.string, - description: "Grant category", - }, - priority: { - fieldType: CustomFieldType.integer, - description: "Processing priority (1 = highest)", + customFields: { + category: { + fieldType: CustomFieldType.string, + description: "Grant category", + }, + priority: { + fieldType: CustomFieldType.integer, + description: "Processing priority (1 = highest)", + }, }, }, }, @@ -344,7 +348,7 @@ describe("plugin composition", () => { }, }, }; - const legacyResult = legacyPlugin.schemas.Opportunity.parse(legacyData); + const legacyResult = legacyPlugin.schemas.Opportunity.common.parse(legacyData); const legacyId = getCustomFieldValue(legacyResult, "legacyId", LegacyIdValueSchema); expect(legacyId).toEqual({ system: "grants-v1", id: 42 }); @@ -364,27 +368,35 @@ describe("plugin composition", () => { }, }, }; - const classResult = classificationPlugin.schemas.Opportunity.parse(classificationData); + const classResult = classificationPlugin.schemas.Opportunity.common.parse(classificationData); const category = getCustomFieldValue(classResult, "category", z.string()); const priority = getCustomFieldValue(classResult, "priority", z.number().int()); expect(category).toBe("STEM Education"); expect(priority).toBe(1); }); - it("should merge extensions from multiple plugins without conflict", () => { - const merged = mergeExtensions([legacyPlugin.extensions, classificationPlugin.extensions]); - - // All three fields should be present under Opportunity - expect(merged.Opportunity).toBeDefined(); - expect(Object.keys(merged.Opportunity!)).toEqual( - expect.arrayContaining(["legacyId", "category", "priority"]) - ); - expect(Object.keys(merged.Opportunity!)).toHaveLength(3); - }); - - it("should parse a payload with all custom fields from the combined plugin", () => { - const merged = mergeExtensions([legacyPlugin.extensions, classificationPlugin.extensions]); - const combinedPlugin = definePlugin({ extensions: merged }); + it("should parse a payload with multiple custom fields defined in a single plugin", () => { + const combinedPlugin = definePlugin({ + schemas: { + Opportunity: { + customFields: { + legacyId: { + fieldType: CustomFieldType.object, + value: LegacyIdValueSchema, + description: "Maps to the opportunity_id in the legacy system", + }, + category: { + fieldType: CustomFieldType.string, + description: "Grant category", + }, + priority: { + fieldType: CustomFieldType.integer, + description: "Processing priority (1 = highest)", + }, + }, + }, + }, + } as const); const fullData = { ...validOpp, @@ -407,7 +419,7 @@ describe("plugin composition", () => { }, }; - const parsed = combinedPlugin.schemas.Opportunity.parse(fullData); + const parsed = combinedPlugin.schemas.Opportunity.common.parse(fullData); // Extract all custom field values const legacyId = parsed.customFields?.legacyId?.value; @@ -421,9 +433,19 @@ describe("plugin composition", () => { expect(priority).toBe(1); }); - it("should reject invalid data in the combined schema", () => { - const merged = mergeExtensions([legacyPlugin.extensions, classificationPlugin.extensions]); - const combinedPlugin = definePlugin({ extensions: merged }); + it("should reject invalid data for a custom field type", () => { + const combinedPlugin = definePlugin({ + schemas: { + Opportunity: { + customFields: { + priority: { + fieldType: CustomFieldType.integer, + description: "Processing priority (1 = highest)", + }, + }, + }, + }, + } as const); const invalidData = { ...validOpp, @@ -436,7 +458,7 @@ describe("plugin composition", () => { }, }; - const result = combinedPlugin.schemas.Opportunity.safeParse(invalidData); + const result = combinedPlugin.schemas.Opportunity.common.safeParse(invalidData); expect(result.success).toBe(false); if (!result.success) { expect(result.error.issues[0].path).toContain("customFields"); diff --git a/lib/ts-sdk/__tests__/extensions/merge-extensions.spec.ts b/lib/ts-sdk/__tests__/extensions/merge-extensions.spec.ts deleted file mode 100644 index 19d987996..000000000 --- a/lib/ts-sdk/__tests__/extensions/merge-extensions.spec.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { mergeExtensions } from "@/extensions"; -import type { SchemaExtensions } from "@/extensions"; - -describe("mergeExtensions", () => { - // ########################################################################## - // Base cases - // ########################################################################## - describe("base cases", () => { - it("returns an empty object for an empty array", () => { - const result = mergeExtensions([]); - expect(result).toEqual({}); - }); - - it("returns the same reference for a single source", () => { - const source: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "string" } }, - }; - const result = mergeExtensions([source]); - expect(result).toBe(source); - }); - }); - - // ########################################################################## - // Non-overlapping merges - // ########################################################################## - - describe("non-overlapping merges", () => { - it("merges disjoint fields on the same model", () => { - const a: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "string" } }, - }; - const b: SchemaExtensions = { - Opportunity: { category: { fieldType: "string" } }, - }; - - const result = mergeExtensions([a, b]); - expect(result).toEqual({ - Opportunity: { - legacyId: { fieldType: "string" }, - category: { fieldType: "string" }, - }, - }); - }); - - it("merges sources that extend different models (no overlap)", () => { - // TODO: Expand this test to include other schemas when they are supported - const a: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "string" } }, - }; - const b: SchemaExtensions = {}; - - const result = mergeExtensions([a, b]); - expect(result).toEqual({ - Opportunity: { legacyId: { fieldType: "string" } }, - }); - }); - }); - - // ########################################################################## - // Conflict handling - // ########################################################################## - - describe("conflict handling", () => { - // Default behavior - it('throws on duplicate field names with default onConflict ("error")', () => { - const a: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "string" } }, - }; - const b: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "number" } }, - }; - - expect(() => mergeExtensions([a, b])).toThrowError( - 'mergeExtensions: duplicate field "legacyId" on model "Opportunity"' - ); - }); - - // Explicit error behavior - it('throws on duplicate field names with explicit onConflict: "error"', () => { - const a: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "string" } }, - }; - const b: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "number" } }, - }; - - expect(() => mergeExtensions([a, b], { onConflict: "error" })).toThrowError( - 'mergeExtensions: duplicate field "legacyId" on model "Opportunity"' - ); - }); - - // Last wins behavior - it('keeps the last definition with onConflict: "lastWins"', () => { - const a: SchemaExtensions = { - Opportunity: { - legacyId: { fieldType: "string", description: "first" }, - }, - }; - const b: SchemaExtensions = { - Opportunity: { - legacyId: { fieldType: "number", description: "second" }, - }, - }; - - const result = mergeExtensions([a, b], { onConflict: "lastWins" }); - expect(result).toEqual({ - Opportunity: { - legacyId: { fieldType: "number", description: "second" }, - }, - }); - }); - - // First wins behavior - it('keeps the first definition with onConflict: "firstWins"', () => { - const a: SchemaExtensions = { - Opportunity: { - legacyId: { fieldType: "string", description: "first" }, - }, - }; - const b: SchemaExtensions = { - Opportunity: { - legacyId: { fieldType: "number", description: "second" }, - }, - }; - - const result = mergeExtensions([a, b], { onConflict: "firstWins" }); - expect(result).toEqual({ - Opportunity: { - legacyId: { fieldType: "string", description: "first" }, - }, - }); - }); - }); - - // ########################################################################## - // Three+ sources - // ########################################################################## - - describe("three+ sources", () => { - it("merges three or more sources correctly", () => { - const a: SchemaExtensions = { - Opportunity: { legacyId: { fieldType: "string" } }, - }; - const b: SchemaExtensions = { - Opportunity: { category: { fieldType: "string" } }, - }; - const c: SchemaExtensions = { - Opportunity: { priority: { fieldType: "integer" } }, - }; - - const result = mergeExtensions([a, b, c]); - expect(result).toEqual({ - Opportunity: { - legacyId: { fieldType: "string" }, - category: { fieldType: "string" }, - priority: { fieldType: "integer" }, - }, - }); - }); - - it("applies lastWins across three sources with overlapping fields", () => { - const a: SchemaExtensions = { - Opportunity: { tag: { fieldType: "string", description: "v1" } }, - }; - const b: SchemaExtensions = { - Opportunity: { tag: { fieldType: "string", description: "v2" } }, - }; - const c: SchemaExtensions = { - Opportunity: { tag: { fieldType: "string", description: "v3" } }, - }; - - const result = mergeExtensions([a, b, c], { onConflict: "lastWins" }); - expect(result).toEqual({ - Opportunity: { - tag: { fieldType: "string", description: "v3" }, - }, - }); - }); - }); -}); diff --git a/lib/ts-sdk/__tests__/extensions/transformation.spec.ts b/lib/ts-sdk/__tests__/extensions/transformation.spec.ts new file mode 100644 index 000000000..ae02eb0c3 --- /dev/null +++ b/lib/ts-sdk/__tests__/extensions/transformation.spec.ts @@ -0,0 +1,395 @@ +import { describe, it, expect } from "vitest"; + +// Public barrel only re-exports the high-level surface. Individual handler +// functions are imported directly from the source module for unit testing — +// they remain module-internal otherwise. +import { DEFAULT_HANDLERS, getFromPath, transformFromMapping } from "@/extensions"; +import { + constValue, + HandlerError, + numberToString, + fieldValue, + stringToNumber, + switchOnValue, +} from "@/extensions/transformation"; + +// ############################################################################ +// getFromPath +// ############################################################################ + +describe("getFromPath", () => { + it("returns the value at a dot-notation path", () => { + expect(getFromPath({ a: { b: { c: 42 } } }, "a.b.c")).toBe(42); + }); + + it("returns undefined when an intermediate step is missing", () => { + expect(getFromPath({ a: { b: 1 } }, "a.c.d")).toBeUndefined(); + }); + + it("returns undefined when a step traverses into a non-object", () => { + expect(getFromPath({ a: 1 }, "a.b")).toBeUndefined(); + }); + + it("returns the entire input on empty path", () => { + const input = { a: 1 }; + expect(getFromPath(input, "")).toBe(input); + }); + + it("returns the provided default when missing", () => { + expect(getFromPath({ a: 1 }, "z", "fallback")).toBe("fallback"); + }); + + it("preserves the original value type (no coercion)", () => { + expect(getFromPath({ a: false }, "a")).toBe(false); + expect(getFromPath({ a: null }, "a")).toBeNull(); + expect(getFromPath({ a: 0 }, "a")).toBe(0); + }); +}); + +// ############################################################################ +// Individual handlers +// ############################################################################ + +describe("fieldValue", () => { + it("wraps getFromPath with the field path arg", () => { + expect(fieldValue({ x: { y: "z" } }, "x.y")).toBe("z"); + }); + + it("returns undefined on missing path", () => { + expect(fieldValue({ x: 1 }, "x.y")).toBeUndefined(); + }); + + it("preserves terminal null ('doesn't apply')", () => { + // `fieldValue` defers to `getFromPath`, which returns the terminal value + // verbatim — so an explicit `null` survives unchanged. This pins the + // three-state contract for the bare `field` handler. + expect(fieldValue({ a: null }, "a")).toBeNull(); + }); + + it("returns undefined when an intermediate null short-circuits the path", () => { + // `null` means "doesn't apply for THIS field." A null + // intermediate ("the parent doesn't apply") is treated as a propagating + // assertion: child paths return undefined ("not provided") rather than + // null, because the publisher made the assertion at the parent level. + // Scoped-out for now; see the README "Null handling" section. + expect(fieldValue({ a: null }, "a.b")).toBeUndefined(); + }); +}); + +describe("constValue", () => { + it("returns the literal regardless of source data", () => { + expect(constValue({ ignored: true }, "USD")).toBe("USD"); + expect(constValue({}, 42)).toBe(42); + expect(constValue(null, null)).toBeNull(); + }); +}); + +describe("switchOnValue", () => { + const spec = { + field: "status", + case: { posted: "open", archived: "closed" }, + default: "custom", + }; + + it("returns the case lookup when source value matches", () => { + expect(switchOnValue({ status: "posted" }, spec)).toBe("open"); + expect(switchOnValue({ status: "archived" }, spec)).toBe("closed"); + }); + + it("returns the default when no case matches", () => { + expect(switchOnValue({ status: "forecasted" }, spec)).toBe("custom"); + }); + + it("returns undefined when no default is provided and no case matches", () => { + expect(switchOnValue({ status: "x" }, { field: "status", case: { y: 1 } })).toBeUndefined(); + }); + + it("throws when spec is not an object (D2: fail loud on malformed mapping)", () => { + // Prior contract returned `undefined` silently when `spec` was non-object, + // which let a typo'd mapping like `{ match: "literal-where-spec-belongs" }` + // produce missing fields with no error trail. New contract: throw, so the + // walker wraps as `HandlerError` and the boundary materializes a + // `PluginError` with `handler: "match"`. + expect(() => switchOnValue({ status: "posted" }, "garbage")).toThrow(/spec must be an object/); + expect(() => switchOnValue({ status: "posted" }, null)).toThrow(/spec must be an object/); + expect(() => switchOnValue({ status: "posted" }, 42)).toThrow(/spec must be an object/); + // Arrays pass `typeof === "object"` and non-null, but are not the + // structural-object shape `switchOnValue` expects. A mapping like + // `{ match: ["posted", "archived"] }` (plausible from a plugin author + // hand-writing JSON config) would otherwise silently resolve to + // `s.field`/`s.case`/`s.default` all-undefined and return undefined. + expect(() => switchOnValue({ status: "posted" }, ["posted", "archived"])).toThrow( + /spec must be an object/ + ); + }); + + it("does not coerce numeric source values to string-keyed cases", () => { + // TS uses an explicit `typeof val === "string"` guard. Python's PoC uses + // bare `dict.get(val, default)` which would natively support non-string + // keys; but for JSON-loaded mappings (where `case` keys are always + // strings) the practical behavior matches — string-keyed lookup misses + // a numeric `val`, falls through to `default`. TS pins this explicitly. + expect(switchOnValue({ n: 1 }, { field: "n", case: { "1": "yes" }, default: "no" })).toBe("no"); + // String "1" still matches. + expect(switchOnValue({ n: "1" }, { field: "n", case: { "1": "yes" }, default: "no" })).toBe( + "yes" + ); + }); + + it("does not coerce boolean source values to string-keyed cases", () => { + expect(switchOnValue({ b: true }, { field: "b", case: { true: "yes" }, default: "no" })).toBe( + "no" + ); + expect(switchOnValue({ b: false }, { field: "b", case: { false: "yes" }, default: "no" })).toBe( + "no" + ); + }); + + it("returns the default when the field is missing (no `undefined` string coercion)", () => { + // Prior implementation coerced missing-field `undefined` to the literal + // string "undefined" and matched a `case: { undefined: ... }` entry. The + // current contract returns `default` when the source value is non-string, + // even if a `case` entry happens to be keyed `"undefined"`. + expect( + switchOnValue({}, { field: "missing", case: { undefined: "matched" }, default: "no" }) + ).toBe("no"); + }); + + it("returns the default when `field` is omitted (object source not key-coercible)", () => { + // With no `field`, `getFromPath` returns the whole `data` object; + // an object value is non-string and short-circuits to `default`. + expect(switchOnValue({ a: 1 }, { case: { a: "b" }, default: "fallback" })).toBe("fallback"); + }); + + // Three-state preservation for null source values. The mapping + // author opts in to target-side translation via a `"null"` case key; + // otherwise the publisher's "doesn't apply" assertion passes through + // unchanged. `default` is NOT consulted for null — `default` belongs to + // "unrecognized value," not to "publisher asserts irrelevant." + + it("passes null source through as null when no `case.null` is provided", () => { + expect(switchOnValue({ status: null }, spec)).toBeNull(); + }); + + it("does NOT fall through to `default` for a null source (default is for unrecognized values)", () => { + // The current spec has `default: "custom"`. If null source incorrectly + // fell through to default, this would be "custom" instead of null. + expect(switchOnValue({ status: null }, spec)).not.toBe("custom"); + }); + + it("uses a `case.null` mapping when the author opts in (target-side translation)", () => { + // A mapping author who wants to translate the publisher's "doesn't + // apply" assertion into a target-side sentinel (e.g. an `n_a` status + // token) opts in by adding a `"null"` key to the case map. + expect( + switchOnValue( + { status: null }, + { field: "status", case: { posted: "open", null: "n_a" }, default: "custom" } + ) + ).toBe("n_a"); + }); + + it("opt-in null mapping wins over the pass-through default", () => { + // Pin the precedence: case.null takes priority over the pass-through + // null behavior. (Documents that the author's explicit decision + // overrides the SDK's default treatment.) + expect(switchOnValue({ status: null }, { field: "status", case: { null: "translated" } })).toBe( + "translated" + ); + }); +}); + +describe("numberToString", () => { + it("coerces a number to its string form", () => { + expect(numberToString({ amount: 1000 }, "amount")).toBe("1000"); + expect(numberToString({ x: 1.5 }, "x")).toBe("1.5"); + }); + + it("returns null on explicit null source ('doesn't apply')", () => { + // The publisher asserted the field is irrelevant for this record. The + // handler must preserve that assertion instead of collapsing it to + // `undefined` (which would be indistinguishable from "not provided"). + // `String(null)` is bypassed — it would otherwise emit the literal "null". + expect(numberToString({ a: null }, "a")).toBeNull(); + }); + + it("returns undefined on absent source ('not provided')", () => { + expect(numberToString({}, "a")).toBeUndefined(); + }); +}); + +describe("stringToNumber", () => { + it("parses an integer string as a number", () => { + expect(stringToNumber({ a: "42" }, "a")).toBe(42); + }); + + it("parses a decimal string via the float fallback", () => { + expect(stringToNumber({ a: "1.5" }, "a")).toBe(1.5); + }); + + it("returns null on explicit null source ('doesn't apply')", () => { + // Parallels numberToString — null is the publisher's assertion that the + // field doesn't apply. Preserve it as data; don't collapse to undefined. + expect(stringToNumber({ a: null }, "a")).toBeNull(); + }); + + it("returns undefined on absent source ('not provided')", () => { + expect(stringToNumber({}, "x")).toBeUndefined(); + }); + + it("throws on non-numeric input", () => { + expect(() => stringToNumber({ a: "abc" }, "a")).toThrow(/cannot convert/); + }); + + it("throws on the empty string (would otherwise coerce to 0)", () => { + // `Number("")` returns 0 in JavaScript, which would silently turn an + // implicit-absent CSV cell into a real zero on the transformed side. + expect(() => stringToNumber({ a: "" }, "a")).toThrow(/cannot convert/); + }); + + it("throws on a whitespace-only string (post-trim empty)", () => { + // `Number(" ")` also coerces to 0; the trim-then-empty-check covers it. + expect(() => stringToNumber({ a: " " }, "a")).toThrow(/cannot convert/); + }); + + it("accepts integer strings up to Number.MAX_SAFE_INTEGER", () => { + expect(stringToNumber({ a: String(Number.MAX_SAFE_INTEGER) }, "a")).toBe( + Number.MAX_SAFE_INTEGER + ); + expect(stringToNumber({ a: String(Number.MIN_SAFE_INTEGER) }, "a")).toBe( + Number.MIN_SAFE_INTEGER + ); + }); + + it("throws on integer strings beyond the safe-integer range (no silent precision loss)", () => { + // Without the safe-integer guard, `Number("9999999999999999999")` would + // return `1e19` — a different value than the input. Plugin authors + // round-tripping 64-bit IDs would see silent corruption. Reject instead. + expect(() => stringToNumber({ a: "9999999999999999999" }, "a")).toThrow(/cannot convert/); + expect(() => stringToNumber({ a: "-9999999999999999999" }, "a")).toThrow(/cannot convert/); + }); +}); + +// ############################################################################ +// DEFAULT_HANDLERS registry +// ############################################################################ + +describe("DEFAULT_HANDLERS", () => { + it("registers all built-in handler names", () => { + expect([...DEFAULT_HANDLERS.keys()].sort()).toEqual( + ["const", "field", "match", "numberToString", "stringToNumber", "switch"].sort() + ); + }); + + it("points `match` and `switch` at the same handler function (alias)", () => { + expect(DEFAULT_HANDLERS.get("match")).toBe(DEFAULT_HANDLERS.get("switch")); + }); +}); + +// ############################################################################ +// transformFromMapping +// ############################################################################ + +describe("transformFromMapping", () => { + it("returns primitives unchanged", () => { + expect(transformFromMapping({}, "literal")).toBe("literal"); + expect(transformFromMapping({}, 42)).toBe(42); + expect(transformFromMapping({}, null)).toBeNull(); + }); + + it("preserves output object shape and recurses on each value", () => { + const data = { foo: { bar: "baz" } }; + const mapping = { + a: { field: "foo.bar" }, + b: { value: { field: "foo.bar" }, currency: "USD" }, + }; + expect(transformFromMapping(data, mapping)).toEqual({ + a: "baz", + b: { value: "baz", currency: "USD" }, + }); + }); + + it("dispatches built-in handlers via the registry", () => { + expect( + transformFromMapping( + { s: "posted" }, + { + match: { field: "s", case: { posted: "open" }, default: "custom" }, + } + ) + ).toBe("open"); + }); + + it("supports the `switch` alias for `match`", () => { + expect( + transformFromMapping( + { s: "posted" }, + { + switch: { field: "s", case: { posted: "open" }, default: "custom" }, + } + ) + ).toBe("open"); + }); + + it("reads only the first key of a handler-dispatch node — sibling keys are silently ignored", () => { + // Locks in the documented walker contract: a node whose first key is a + // handler dispatches that handler with the corresponding arg, and the + // remaining keys are dropped. Mixing handlers in one node is unsupported + // by design; this test prevents a regression that would start treating + // the node as an output shape. + const mapping = { field: "x", const: "fallback-ignored" }; + expect(transformFromMapping({ x: "value-from-field" }, mapping)).toBe("value-from-field"); + }); + + it("wraps handler exceptions in HandlerError carrying the handler name", () => { + let caught: unknown; + try { + transformFromMapping({ x: "not-a-number" }, { stringToNumber: "x" }); + } catch (e) { + caught = e; + } + expect(caught).toBeInstanceOf(HandlerError); + expect((caught as HandlerError).handler).toBe("stringToNumber"); + expect((caught as Error).message).toMatch(/cannot convert/); + }); + + it("treats unregistered keys as output field names, not handler dispatches", () => { + const mapping: Record = {}; + mapping["toString"] = { field: "x" }; + const out = transformFromMapping({ x: 1 }, mapping); + expect(out).toEqual({ toString: 1 }); + }); + + it("preserves a handler-returned null on the output object (three-state)", () => { + // The walker must place `null` returned by a handler onto the output + // shape — not drop the key, not coerce to undefined. Combined with the + // null-aware handlers, this preserves the publisher's "doesn't apply" + // assertion end-to-end through `toCommon` / `fromCommon`. + expect(transformFromMapping({ a: null }, { x: { numberToString: "a" } })).toEqual({ + x: null, + }); + expect(transformFromMapping({ a: null }, { x: { stringToNumber: "a" } })).toEqual({ + x: null, + }); + expect( + transformFromMapping({ status: null }, { value: { match: { field: "status" } } }) + ).toEqual({ value: null }); + }); + + it("omits an output key whose handler returned undefined ('not provided')", () => { + // Counterpart to the null-preservation test above: an absent source field + // produces `undefined` from the handler, and the walker must OMIT the key + // rather than write `{ x: undefined }`. This is what makes the in-memory + // object distinguish "not provided" (key absent) from "doesn't apply" + // (key present, value null). + const out = transformFromMapping({}, { x: { numberToString: "missing" } }); + expect(out).not.toHaveProperty("x"); + // A sibling present key is still written — only the undefined child is dropped. + const mixed = transformFromMapping( + { a: 42 }, + { present: { numberToString: "a" }, absent: { field: "nope" } } + ); + expect(mixed).toEqual({ present: "42" }); + expect(mixed).not.toHaveProperty("absent"); + }); +}); diff --git a/lib/ts-sdk/__tests__/extensions/transforms.spec.ts b/lib/ts-sdk/__tests__/extensions/transforms.spec.ts new file mode 100644 index 000000000..e06b617bf --- /dev/null +++ b/lib/ts-sdk/__tests__/extensions/transforms.spec.ts @@ -0,0 +1,510 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { z } from "zod"; + +import { PluginError, buildTransforms, getFromPath, withCustomFields } from "@/extensions"; +import { OpportunityBaseSchema } from "@/schemas/zod/models"; +import { CustomFieldType } from "@/constants"; + +// ############################################################################ +// Test fixtures +// ############################################################################ + +const SOURCE_DATA = { + data: { + agency_name: "Department of Examples", + created_at: "2025-01-15T09:00:00Z", + last_modified_at: "2025-04-01T12:30:00Z", + opportunity_description: "Funding to advance research into conservation techniques.", + opportunity_id: 12345, + opportunity_number: "ABC-123-XYZ-001", + opportunity_status: "posted", + opportunity_title: "Research into conservation techniques", + opportunity_uuid: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + }, +}; + +const toCommonMapping = { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + createdAt: { field: "data.created_at" }, + lastModifiedAt: { field: "data.last_modified_at" }, + status: { + value: { + match: { + field: "data.opportunity_status", + case: { posted: "open", archived: "closed", forecasted: "forecasted" }, + default: "custom", + }, + }, + }, +}; + +const fromCommonMapping = { + data: { + opportunity_title: { field: "title" }, + opportunity_uuid: { field: "id" }, + }, +}; + +// ############################################################################ +// Call-time validation +// ############################################################################ + +describe("buildTransforms — call-time validation", () => { + it("rejects custom handler names that collide with defaults", () => { + expect(() => + buildTransforms({}, {}, new Map([["field", (_d: unknown, _a: unknown) => null]])) + ).toThrow(/collide with defaults/); + // Also `match` — confirms the collision check isn't hardcoded to `field`. + expect(() => + buildTransforms({}, {}, new Map([["match", (_d: unknown, _a: unknown) => null]])) + ).toThrow(/collide with defaults/); + }); + + it("rejects mappings whose nodes are structurally malformed (array where scalar expected)", () => { + expect(() => + buildTransforms({ a: ["unexpected", "array"] as unknown as Record }, {}) + ).toThrow(/Invalid mapping node/); + }); + + it("accepts a well-formed pair of mappings", () => { + const built = buildTransforms(toCommonMapping, fromCommonMapping); + expect(typeof built.toCommon).toBe("function"); + expect(typeof built.fromCommon).toBe("function"); + }); + + it("rejects sibling keys alongside a handler key — two handlers in one node (Python PoC parity)", () => { + // The runtime walker is first-key-wins, so `{ field, const }` would + // silently drop `const` — almost always an author bug — so reject at + // build time. + expect(() => buildTransforms({ value: { field: "data.x", const: "fallback" } }, {})).toThrow( + /cannot have sibling keys/ + ); + }); + + it("rejects a handler key alongside a non-handler sibling at the same node", () => { + // The non-handler sibling would also be silently dropped by the runtime + // walker once the handler dispatches. Catch the typo early. + expect(() => buildTransforms({ value: { field: "data.x", extra: "literal" } }, {})).toThrow( + /cannot have sibling keys/ + ); + }); +}); + +// ############################################################################ +// toCommon happy path +// ############################################################################ + +describe("buildTransforms — toCommon", () => { + it("returns the transformed result with an empty error list on success", () => { + const { toCommon } = buildTransforms(toCommonMapping, fromCommonMapping); + const out = toCommon(SOURCE_DATA); + + expect(out.errors).toEqual([]); + expect(out.result).toEqual({ + id: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + title: "Research into conservation techniques", + description: "Funding to advance research into conservation techniques.", + createdAt: "2025-01-15T09:00:00Z", + lastModifiedAt: "2025-04-01T12:30:00Z", + status: { value: "open" }, + }); + }); + + it("wraps a handler exception as a PluginError carrying handler + cause", () => { + const { toCommon } = buildTransforms({ amount: { stringToNumber: "data.bogus" } }, {}); + // Use a source value that fails coercion so stringToNumber throws. + const out = toCommon({ data: { bogus: "abc" } }); + + expect(out.errors).toHaveLength(1); + const [err] = out.errors; + expect(err).toBeInstanceOf(PluginError); + expect(err.handler).toBe("stringToNumber"); + expect(err.cause).toBeInstanceOf(Error); + // On handler exception, result is an empty object — no partial fields. + // `toStrictEqual` (not `toEqual`) so the contract rejects null/undefined- + // shaped "empty" sentinels — the documented shape is a literal `{}`. + expect(out.result).toStrictEqual({}); + }); + + it("short-circuits on the first HandlerError — two failing fields produce a single PluginError", () => { + // Locks in the documented asymmetry: handler-failure mode is first-error-stops, + // while the Zod-validation path aggregates every issue. Two `stringToNumber` + // calls both fail on `"abc"`, but only one PluginError surfaces. + const { toCommon } = buildTransforms( + { + a: { stringToNumber: "data.bogus_a" }, + b: { stringToNumber: "data.bogus_b" }, + }, + {} + ); + + const out = toCommon({ data: { bogus_a: "abc", bogus_b: "def" } }); + + expect(out.errors).toHaveLength(1); + expect(out.errors[0]).toBeInstanceOf(PluginError); + expect(out.errors[0].handler).toBe("stringToNumber"); + }); + + it("flattens Zod validation issues into PluginError[] when commonModel is provided", () => { + const { toCommon } = buildTransforms( + // Output is intentionally missing required CG fields so Zod fails. + { title: { field: "data.opportunity_title" } }, + {}, + undefined, + OpportunityBaseSchema + ); + + const out = toCommon(SOURCE_DATA); + + expect(out.errors.length).toBeGreaterThan(0); + expect(out.errors[0]).toBeInstanceOf(PluginError); + // Path is a non-empty dot-joined string for field-level issues, or + // `undefined` for root-level issues from schema-wide `.refine()` calls. + expect( + out.errors.every( + e => e.path === undefined || (typeof e.path === "string" && e.path.length > 0) + ) + ).toBe(true); + // result preserves the raw transformed object so callers can inspect malformed data. + expect(out.result).toEqual({ title: "Research into conservation techniques" }); + }); + + it("produces one PluginError per ZodIssue (aggregation contract — not first-issue-only)", () => { + // Pin the documented asymmetry from buildTransforms() JSDoc: handler + // failures short-circuit (one PluginError per call), Zod failures + // aggregate every issue. Build a small schema that produces a + // deterministic two-issue failure so a regression to + // `[parsed.error.issues[0]]` would fail this test. + const TwoFieldSchema = z.object({ + a: z.string().min(5), + b: z.number().int(), + }); + + const { toCommon } = buildTransforms( + { + a: { const: "x" }, + b: { const: "not-a-number" }, + }, + {}, + undefined, + TwoFieldSchema + ); + + const out = toCommon({}); + + expect(out.errors).toHaveLength(2); + expect(out.errors.every(e => e instanceof PluginError)).toBe(true); + const paths = out.errors.map(e => e.path).sort(); + expect(paths).toEqual(["a", "b"]); + }); + + it("validates against the fully extended schema produced by withCustomFields", () => { + const extendedOpp = withCustomFields(OpportunityBaseSchema, { + legacyId: { fieldType: CustomFieldType.integer, value: z.number().int() }, + }); + + const { toCommon } = buildTransforms( + { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + createdAt: { field: "data.created_at" }, + lastModifiedAt: { field: "data.last_modified_at" }, + status: { value: { const: "open" } }, + customFields: { + legacyId: { + value: { field: "data.opportunity_id" }, + name: "legacyId", + fieldType: "integer", + }, + }, + }, + {}, + undefined, + extendedOpp + ); + + const out = toCommon(SOURCE_DATA); + + expect(out.errors).toEqual([]); + const result = out.result as z.infer; + expect(result.customFields?.legacyId?.value).toBe(12345); + }); +}); + +// ############################################################################ +// fromCommon +// ############################################################################ + +describe("buildTransforms — fromCommon", () => { + it("transforms CG → native using the fromCommon mapping", () => { + const { fromCommon } = buildTransforms(toCommonMapping, fromCommonMapping); + const out = fromCommon({ + id: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + title: "Research into conservation techniques", + }); + + expect(out.errors).toEqual([]); + expect(out.result).toEqual({ + data: { + opportunity_title: "Research into conservation techniques", + opportunity_uuid: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + }, + }); + }); + + it("supports a round trip on fields covered by both directions", () => { + const { toCommon, fromCommon } = buildTransforms(toCommonMapping, fromCommonMapping); + + const cg = toCommon(SOURCE_DATA); + expect(cg.errors).toEqual([]); + + const back = fromCommon(cg.result); + expect(back.errors).toEqual([]); + expect((back.result as { data: { opportunity_title: string } }).data.opportunity_title).toBe( + "Research into conservation techniques" + ); + }); + + it("wraps a handler exception as a PluginError carrying handler + cause", () => { + const { fromCommon } = buildTransforms({}, { data: { amount: { stringToNumber: "bogus" } } }); + // Use a CG-shaped value where `bogus` fails coercion so stringToNumber throws. + const out = fromCommon({ bogus: "abc" } as never); + + expect(out.errors).toHaveLength(1); + const [err] = out.errors; + expect(err).toBeInstanceOf(PluginError); + expect(err.handler).toBe("stringToNumber"); + expect(err.cause).toBeInstanceOf(Error); + // Same contract as the toCommon side: literal `{}`, not null/undefined. + expect(out.result).toStrictEqual({}); + }); +}); + +// ############################################################################ +// Null preservation — three-state contract +// ############################################################################ + +describe("buildTransforms — null preservation (three-state)", () => { + // Optional fields carry three distinct states on the wire: absent ("not + // provided"), explicit `null` ("doesn't apply"), and a value. The transform + // handlers preserve all three so the publisher's assertion survives + // end-to-end through `toCommon` / `fromCommon`. + + it("preserves explicit null on the toCommon side (publisher 'doesn't apply')", () => { + // Source has `description: null` — publisher asserts the field doesn't + // apply for this record. The `field` handler returns the terminal null; + // the walker places it on the output object as a real `null`, distinct + // from an absent key. + const { toCommon } = buildTransforms( + { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + }, + {} + ); + + const out = toCommon({ + data: { + opportunity_uuid: "uuid-1", + opportunity_title: "T", + opportunity_description: null, + }, + }); + + expect(out.errors).toEqual([]); + expect(out.result).toEqual({ + id: "uuid-1", + title: "T", + description: null, + }); + // Pin the absent vs. null distinction explicitly. + expect((out.result as Record).description).toBeNull(); + expect(Object.prototype.hasOwnProperty.call(out.result, "description")).toBe(true); + }); + + it("validates a null-bearing toCommon result against a .nullish() Zod schema", () => { + // SDKs already accept null on `.nullish()` fields; this + // pins it inside the buildTransforms() Zod path so a future schema + // change that swapped `.nullish()` for plain `.optional()` would surface + // here as a PluginError. `source` is `.nullish()` on OpportunityBaseSchema; + // `description` is required (z.string()) so it can't carry the null state. + const { toCommon } = buildTransforms( + { + ...toCommonMapping, + source: { field: "data.source_url" }, + }, + {}, + undefined, + OpportunityBaseSchema + ); + + const out = toCommon({ + data: { + ...SOURCE_DATA.data, + source_url: null, + }, + }); + + expect(out.errors).toEqual([]); + expect((out.result as { source: string | null }).source).toBeNull(); + }); + + it("preserves null through a full toCommon → fromCommon round trip", () => { + // The reverse direction must also carry null verbatim — `null` on the + // CommonGrants side ("publisher said doesn't apply") survives back to + // the native shape. + const { toCommon, fromCommon } = buildTransforms( + { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + }, + { + data: { + opportunity_uuid: { field: "id" }, + opportunity_title: { field: "title" }, + opportunity_description: { field: "description" }, + }, + } + ); + + const cg = toCommon({ + data: { + opportunity_uuid: "uuid-1", + opportunity_title: "T", + opportunity_description: null, + }, + }); + expect(cg.errors).toEqual([]); + expect((cg.result as { description: string | null }).description).toBeNull(); + + const back = fromCommon(cg.result); + expect(back.errors).toEqual([]); + expect( + (back.result as { data: { opportunity_description: string | null } }).data + .opportunity_description + ).toBeNull(); + }); + + it("preserves null through the numberToString handler in a real mapping", () => { + // Cross-check that the coercing handlers carry null in a buildTransforms + // context, not just at the unit-test level. + const { toCommon } = buildTransforms( + { legacyId: { numberToString: "data.opportunity_id" } }, + {} + ); + + const out = toCommon({ data: { opportunity_id: null } }); + + expect(out.errors).toEqual([]); + expect(out.result).toEqual({ legacyId: null }); + }); + + it("distinguishes absent from null at the buildTransforms boundary", () => { + // Pin the asymmetry the three-state contract depends on: absent source + // produces an absent output KEY; null source produces a present, explicit + // null. The distinction is by key presence, not just by value — both an + // absent key and a present-`undefined` key read as `undefined` via + // property access, so this test asserts `hasOwnProperty` directly. + const { toCommon } = buildTransforms( + { + absentField: { field: "data.does_not_exist" }, + nullField: { field: "data.declared_null" }, + }, + {} + ); + + const out = toCommon({ data: { declared_null: null } }); + + expect(out.errors).toEqual([]); + const result = out.result as Record; + // null source ("doesn't apply") → present key with value null. + expect(result).toHaveProperty("nullField"); + expect(result.nullField).toBeNull(); + // absent source ("not provided") → key omitted entirely. The walker skips + // `undefined`-valued children so the in-memory object matches the wire + // shape `JSON.stringify` produces. + expect(result).not.toHaveProperty("absentField"); + expect(result.absentField).toBeUndefined(); + }); +}); + +// ############################################################################ +// Custom handlers +// ############################################################################ + +describe("PluginError — serialization", () => { + // The SDK does not redact by default. Both tests + // assert on the same PluginError instance — one without redaction (PII + // flows), one with the adopter-supplied projection (PII contained). + // Forcing one shared setup keeps "redacted vs. raw is the only delta" a + // structural property of the test code rather than just narration. + let err: PluginError; + + beforeEach(() => { + const { toCommon } = buildTransforms({ amount: { stringToNumber: "data.bogus" } }, {}); + [err] = toCommon({ data: { bogus: "abc", ssn: "PII_PAYLOAD_123" } }).errors; + }); + + it("includes sourceValue and cause in JSON.stringify by default — adopters redact", () => { + expect(err).toBeInstanceOf(PluginError); + expect(err.sourceValue).toEqual({ data: { bogus: "abc", ssn: "PII_PAYLOAD_123" } }); + expect(err.cause).toBeInstanceOf(Error); + + // No redaction by default: PII flows through JSON.stringify. + const serialized = JSON.stringify(err); + expect(serialized).toContain("PII_PAYLOAD_123"); + expect(serialized).toContain("sourceValue"); + }); + + it("supports an adopter-provided redacted projection for safe logging", () => { + // The projection adopters are documented to use (see README). + const safe = { + name: err.name, + message: err.message, + path: err.path, + handler: err.handler, + }; + const serialized = JSON.stringify(safe); + expect(serialized).not.toContain("PII_PAYLOAD_123"); + expect(JSON.parse(serialized)).toEqual({ + name: "PluginError", + message: expect.any(String), + handler: "stringToNumber", + }); + }); +}); + +describe("buildTransforms — custom handlers", () => { + it("invokes a custom handler registered for this call only", () => { + const join = (data: unknown, spec: unknown) => { + const s = spec as { fields?: string[]; sep?: string }; + const sep = s.sep ?? " "; + const parts = (s.fields ?? []) + .map(path => getFromPath(data, path)) + .filter(v => v !== undefined && v !== null) + .map(String); + return parts.length > 0 ? parts.join(sep) : undefined; + }; + + const { toCommon } = buildTransforms( + { + label: { + join: { fields: ["data.opportunity_number", "data.opportunity_title"], sep: " — " }, + }, + }, + {}, + new Map([["join", join]]) + ); + + const out = toCommon(SOURCE_DATA); + expect(out.errors).toEqual([]); + expect((out.result as { label: string }).label).toBe( + "ABC-123-XYZ-001 — Research into conservation techniques" + ); + }); +}); diff --git a/lib/ts-sdk/examples/plugins.ts b/lib/ts-sdk/examples/plugins.ts index 849c9c6d6..9d18a3370 100644 --- a/lib/ts-sdk/examples/plugins.ts +++ b/lib/ts-sdk/examples/plugins.ts @@ -2,17 +2,16 @@ * Example script demonstrating the plugin framework. * * This example shows how to: - * 1. Define independent plugins with definePlugin() - * 2. Use a standalone plugin's typed schemas - * 3. Compose plugins with mergeExtensions() - * 4. Access typed custom fields from the combined schema + * 1. Define a plugin with custom fields via `definePlugin({ schemas })` + * 2. Access the typed `common` schema for validation + * 3. Read typed custom field values directly * * Run with: pnpm example:plugins */ import { z } from "zod"; import { CustomFieldType } from "../src/constants"; -import { definePlugin, mergeExtensions } from "../src/extensions"; +import { definePlugin } from "../src/extensions"; // ############################################################################ // Step 1: Define independent plugins @@ -25,12 +24,14 @@ const LegacyIdValueSchema = z.object({ }); const legacyPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { - fieldType: CustomFieldType.object, - value: LegacyIdValueSchema, - description: "Maps to the opportunity_id in the legacy system", + customFields: { + legacyId: { + fieldType: CustomFieldType.object, + value: LegacyIdValueSchema, + description: "Maps to the opportunity_id in the legacy system", + }, }, }, }, @@ -38,15 +39,17 @@ const legacyPlugin = definePlugin({ // Plugin for grant categorization const classificationPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - category: { - fieldType: CustomFieldType.string, - description: "Grant category", - }, - priority: { - fieldType: CustomFieldType.integer, - description: "Processing priority (1 = highest)", + customFields: { + category: { + fieldType: CustomFieldType.string, + description: "Grant category", + }, + priority: { + fieldType: CustomFieldType.integer, + description: "Processing priority (1 = highest)", + }, }, }, }, @@ -70,7 +73,7 @@ function demonstrateStandalonePlugins() { console.log("--- Standalone plugins ---\n"); // Legacy plugin schema validates and types its own fields - const legacyOpp = legacyPlugin.schemas.Opportunity.parse({ + const legacyOpp = legacyPlugin.schemas.Opportunity.common.parse({ ...baseData, customFields: { legacyId: { @@ -87,7 +90,7 @@ function demonstrateStandalonePlugins() { console.log(` legacyId.id: ${legacyId?.id} (typed as number)\n`); // Classification plugin schema validates its fields - const classOpp = classificationPlugin.schemas.Opportunity.parse({ + const classOpp = classificationPlugin.schemas.Opportunity.common.parse({ ...baseData, customFields: { category: { @@ -108,57 +111,24 @@ function demonstrateStandalonePlugins() { } // ############################################################################ -// Step 3: Compose plugins -// ############################################################################ - -function demonstrateComposition() { - console.log("\n--- Composed plugins ---\n"); - - // Merge extensions from both plugins into one - const merged = mergeExtensions([legacyPlugin.extensions, classificationPlugin.extensions]); - - // Build a combined plugin with all custom fields - const combinedPlugin = definePlugin({ extensions: merged }); - - // Parse data with all custom fields - const opportunity = combinedPlugin.schemas.Opportunity.parse({ - ...baseData, - customFields: { - legacyId: { - name: "legacyId", - fieldType: CustomFieldType.object, - value: { system: "grants-v1", id: 42 }, - }, - category: { - name: "category", - fieldType: CustomFieldType.string, - value: "STEM Education", - }, - priority: { - name: "priority", - fieldType: CustomFieldType.integer, - value: 1, - }, - }, - }); - - // All fields are accessible with full type safety - console.log(` ${opportunity.title}\n`); - console.log(` legacyId.system: ${opportunity.customFields?.legacyId?.value.system}`); - console.log(` legacyId.id: ${opportunity.customFields?.legacyId?.value.id}`); - console.log(` category: ${opportunity.customFields?.category?.value}`); - console.log(` priority: ${opportunity.customFields?.priority?.value}`); -} - -// ############################################################################ -// Step 4: Demonstrate validation +// Step 3: Demonstrate validation // ############################################################################ function demonstrateValidation() { console.log("\n--- Validation ---\n"); - const merged = mergeExtensions([legacyPlugin.extensions, classificationPlugin.extensions]); - const combinedPlugin = definePlugin({ extensions: merged }); + const combinedPlugin = definePlugin({ + schemas: { + Opportunity: { + customFields: { + priority: { + fieldType: CustomFieldType.integer, + description: "Processing priority (1 = highest)", + }, + }, + }, + }, + } as const); const oppData = { ...baseData, @@ -175,7 +145,7 @@ function demonstrateValidation() { console.log(JSON.stringify(oppData, null, 2)); console.log(); - const result = combinedPlugin.schemas.Opportunity.safeParse(oppData); + const result = combinedPlugin.schemas.Opportunity.common.safeParse(oppData); if (!result.success) { const issue = result.error.issues[0]; @@ -192,7 +162,6 @@ function demonstrateValidation() { function main() { console.log("=== Plugins Example ===\n"); demonstrateStandalonePlugins(); - demonstrateComposition(); demonstrateValidation(); console.log("\n=== Example Complete ==="); } diff --git a/lib/ts-sdk/examples/transforms.ts b/lib/ts-sdk/examples/transforms.ts new file mode 100644 index 000000000..55db2fb25 --- /dev/null +++ b/lib/ts-sdk/examples/transforms.ts @@ -0,0 +1,321 @@ +/** + * Example script demonstrating bidirectional transforms. Shows: + * 1. Defining `toCommon` / `fromCommon` mappings. + * 2. Registering a custom mapping handler (`join`) for this call only. + * 3. Validating `toCommon` output against the fully extended Zod schema + * (`withCustomFields(OpportunityBaseSchema, ...)`) — passing the base + * schema would silently weaken validation of typed custom fields. + * 4. Exposing the compiled transform via `definePlugin({ schemas })`. + * 5. Round-tripping `native → common → native` and printing both directions. + * + * Run with: `pnpm example:transforms` + * + * @remarks + * Zod's default `.parse()` strips unknown keys, so source-system fields that + * have no home in the CommonGrants schema must round-trip through + * `customFields` (declared on the extended schema). The example treats + * `opportunity_number` this way. + */ + +import { z } from "zod"; + +import { CustomFieldType } from "../src/constants"; +import { + buildTransforms, + definePlugin, + getFromPath, + withCustomFields, + type Handler, +} from "../src/extensions"; +import { OpportunityBaseSchema } from "../src/schemas/zod/models"; + +// ############################################################################ +// Step 1 — Sample grants.gov source data +// ############################################################################ + +// Note the `source_url: null` below — this is the publisher actively asserting +// "doesn't apply" for the source URL (three-state null). The transforms +// preserve it as `null` end-to-end rather than collapsing to absent, so a +// downstream consumer can distinguish "publisher said N/A" from "publisher +// didn't supply this." +const SOURCE_DATA = { + data: { + agency_name: "Department of Examples", + created_at: "2025-01-15T09:00:00Z", + last_modified_at: "2025-04-01T12:30:00Z", + opportunity_description: + "Funding to advance research into conservation techniques for endangered ecosystems.", + opportunity_id: 12345, + opportunity_number: "ABC-123-XYZ-001", + opportunity_status: "posted", + opportunity_title: "Research into conservation techniques", + opportunity_uuid: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + source_url: null, + summary: { + applicant_types: ["state_governments"], + }, + }, +}; + +// ############################################################################ +// Step 2 — Custom handlers (joined-label round trip) +// ############################################################################ + +// A mapping leaf like `{ field: "data.x" }` dispatches to a *handler* — a +// `(data, spec) => value` function looked up by the leaf's key. `buildTransforms` +// ships built-ins (`field`, `const`, `match`/`switch`, `numberToString`, +// `stringToNumber`); anything beyond those is a custom handler you register on +// the `handlers` map in Step 4. A mapping invokes one by name — `{ join: {...} }` +// runs `joinFields` with `{...}` as its `spec`. `data` is always the whole object +// under transform, so handler paths are absolute from its root (hence the +// `data.`-prefixed paths below). A custom name that collides with a built-in is +// rejected when `buildTransforms` runs. +// +// `join` and `split` are an inverse pair, here to demo a *derived* custom field +// with no single source column: `join` composes `compositeLabel` +// (" — ") on the toCommon side, and `split` recovers +// `opportunity_number` back out of it on the fromCommon side. The round trip is +// lossless only while the separator never occurs inside a constituent value — +// see the NOTE on the `compositeLabel` mapping in Step 4. + +// join: concatenate the values at `spec.fields`, in order, joined by `spec.sep` +// (default " "). undefined/null parts are dropped; when nothing survives it +// returns undefined so the field is omitted rather than emitted as an empty "". +const joinFields: Handler = (data, spec) => { + const s = (spec ?? {}) as { fields?: string[]; sep?: string }; + const sep = s.sep ?? " "; + const parts = (s.fields ?? []) + .map(path => getFromPath(data, path)) + .filter(v => v !== undefined && v !== null) + .map(String); + return parts.length > 0 ? parts.join(sep) : undefined; +}; + +// split: the inverse of join. Read the string at `spec.field`, split it on +// `spec.sep` (default " "), and return the segment at `spec.index` (default 0). +// Returns undefined when the source value is absent or the index is out of range. +const splitField: Handler = (data, spec) => { + const s = (spec ?? {}) as { field?: string; sep?: string; index?: number }; + const value = getFromPath(data, s.field ?? ""); + if (value === undefined || value === null) return undefined; + const parts = String(value).split(s.sep ?? " "); + const idx = s.index ?? 0; + return idx < parts.length ? parts[idx] : undefined; +}; + +// ############################################################################ +// Step 3 — Custom field declarations + extended schema for validation +// ############################################################################ + +const customFieldSpecs = { + legacyId: { + name: "legacyId", + fieldType: CustomFieldType.integer, + value: z.number().int(), + description: "Numeric ID from the legacy database (round-trip preserved).", + }, + agencyName: { + name: "agencyName", + fieldType: CustomFieldType.string, + value: z.string(), + description: "Name of the agency hosting this opportunity.", + }, + applicantTypes: { + name: "applicantTypes", + fieldType: CustomFieldType.array, + value: z.array(z.string()), + description: "Types of applicants eligible for this opportunity.", + }, + // Derived field composed by the `join` handler; the `split` handler in + // fromCommon recovers opportunity_number from this value. Survives Zod + // validation because it's declared on the extended schema as a custom field. + compositeLabel: { + name: "compositeLabel", + fieldType: CustomFieldType.string, + value: z.string(), + description: "Composite label '<opportunity_number> — <opportunity_title>'.", + }, +} as const; + +const ExtendedOpportunitySchema = withCustomFields(OpportunityBaseSchema, customFieldSpecs); + +// ############################################################################ +// Step 4 — Compile bidirectional transforms +// ############################################################################ + +const { toCommon, fromCommon } = buildTransforms( + { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + createdAt: { field: "data.created_at" }, + lastModifiedAt: { field: "data.last_modified_at" }, + // Three-state demo: native `source_url: null` carries the + // publisher's "doesn't apply" assertion. The `field` handler preserves + // the terminal null; the walker places it on the output as a real null + // (distinct from an absent key). Zod's `.nullish()` accepts it. + source: { field: "data.source_url" }, + status: { + value: { + match: { + field: "data.opportunity_status", + case: { + posted: "open", + archived: "closed", + forecasted: "forecasted", + }, + default: "custom", + }, + }, + }, + customFields: { + legacyId: { + value: { field: "data.opportunity_id" }, + name: "legacyId", + fieldType: "integer", + }, + agencyName: { + value: { field: "data.agency_name" }, + name: "agencyName", + fieldType: "string", + }, + applicantTypes: { + value: { field: "data.summary.applicant_types" }, + name: "applicantTypes", + fieldType: "array", + }, + // Compose a derived label via the `join` custom handler; fromCommon + // recovers opportunity_number from it via `split`. + // NOTE: the separator must not appear inside any of the constituent + // field values, or `split` will produce a wrong result on the way back. + compositeLabel: { + value: { + join: { + fields: ["data.opportunity_number", "data.opportunity_title"], + sep: " — ", + }, + }, + name: "compositeLabel", + fieldType: "string", + }, + }, + }, + { + data: { + opportunity_uuid: { field: "id" }, + opportunity_title: { field: "title" }, + opportunity_description: { field: "description" }, + created_at: { field: "createdAt" }, + last_modified_at: { field: "lastModifiedAt" }, + // Recover opportunity_number from the joined label via `split`. The + // separator (` — `) must match what the `toCommon` side used to join, + // and it must not appear inside any constituent field value, or the + // split index will land on the wrong segment. See the join side above. + opportunity_number: { + split: { field: "customFields.compositeLabel.value", sep: " — ", index: 0 }, + }, + opportunity_id: { field: "customFields.legacyId.value" }, + agency_name: { field: "customFields.agencyName.value" }, + // Round-trip the "doesn't apply" assertion back to native: the null + // sourced from `source` on the CG side becomes `source_url: null` again. + source_url: { field: "source" }, + summary: { + applicant_types: { field: "customFields.applicantTypes.value" }, + }, + }, + }, + new Map([ + ["join", joinFields], + ["split", splitField], + ]), + ExtendedOpportunitySchema +); + +// ############################################################################ +// Step 5 — Plug the compiled transforms into a plugin definition +// ############################################################################ + +// All per-object input — customFields, toCommon, and fromCommon — lives on +// the same schemas[Opportunity] entry. See ObjectSchemasInput in +// extensions/types.ts for details. +const grantsGovPlugin = definePlugin({ + meta: { + name: "grants.gov", + version: "0.1.0", + sourceSystem: "grants.gov", + capabilities: ["customFields", "transforms"], + }, + schemas: { + Opportunity: { customFields: customFieldSpecs, toCommon, fromCommon }, + }, +} as const); + +// ############################################################################ +// Step 6 — Run the round trip and report +// ############################################################################ + +function fail(message: string): never { + console.error(message); + process.exit(1); +} + +const toCommonResult = grantsGovPlugin.schemas.Opportunity.toCommon?.(SOURCE_DATA); +if (!toCommonResult) fail("schemas.Opportunity.toCommon missing"); +if (toCommonResult.errors.length > 0) { + // The source data in this example is fixed and PII-free, so embedding + // `e.message` here is safe. Production adopters: `PluginError.message` can + // carry source values on the Zod-validation path (Zod's default error map + // embeds runtime values). See the README PII warning before copying this + // logging shape. + fail( + `toCommon failed: ${toCommonResult.errors + .map(e => `[${e.path ?? "?"}] ${e.message}`) + .join("; ")}` + ); +} + +// The console output below dumps the entire transform result for demonstration. +// Production callers should not log `result` without a PII review — applicant +// records, EINs, and free-text fields routinely flow through `customFields`. +console.log("=== toCommon (native → CommonGrants) ==="); +console.log(JSON.stringify(toCommonResult.result, null, 2)); + +const fromCommonResult = grantsGovPlugin.schemas.Opportunity.fromCommon?.(toCommonResult.result); +if (!fromCommonResult) fail("schemas.Opportunity.fromCommon missing"); +if (fromCommonResult.errors.length > 0) { + // Same PII caveat as the toCommon error block above — `e.message` may carry + // source values on the Zod path. Safe here because the example data is fixed. + fail( + `fromCommon failed: ${fromCommonResult.errors + .map(e => `[${e.path ?? "?"}] ${e.message}`) + .join("; ")}` + ); +} + +console.log("\n=== fromCommon (CommonGrants → native) ==="); +console.log(JSON.stringify(fromCommonResult.result, null, 2)); + +// Spot-check a covered field round-trips. +const native = fromCommonResult.result as { + data: { opportunity_number: string; source_url: string | null | undefined }; +}; +if (native.data.opportunity_number !== SOURCE_DATA.data.opportunity_number) { + fail( + `round-trip mismatch on opportunity_number: ${native.data.opportunity_number} ≠ ${SOURCE_DATA.data.opportunity_number}` + ); +} + +// Three-state pin: an explicit `null` ("doesn't apply") on the +// source side must survive both transforms as a real `null`, not collapse +// to undefined ("not provided"). A future regression that put `undefined` +// here instead of `null` would fail this check. +if (native.data.source_url !== null) { + fail( + `three-state mismatch on source_url: expected explicit null ("doesn't apply"), got ${JSON.stringify( + native.data.source_url + )}` + ); +} + +console.log("\n✓ round-trip verified for fields covered by both mappings"); +console.log("✓ three-state null preserved: source_url null ('doesn't apply') round-tripped"); diff --git a/lib/ts-sdk/package.json b/lib/ts-sdk/package.json index f9244bbed..488b922fd 100644 --- a/lib/ts-sdk/package.json +++ b/lib/ts-sdk/package.json @@ -72,7 +72,7 @@ "check:format": "prettier --check .", "check:types": "tsc --noEmit", "checks": "pnpm run check:lint && pnpm run check:format && pnpm run check:types", - "ci": "pnpm run checks && pnpm run build && pnpm run test", + "ci": "pnpm run checks && pnpm run build && pnpm run test && pnpm run example:transforms", "audit": "pnpm audit", "typespec": "tsp compile lib/main.tsp", "example:list": "tsx examples/list-opportunities.ts", @@ -81,7 +81,8 @@ "example:custom-fields": "tsx examples/custom-fields.ts", "example:get-custom-fields": "tsx examples/get-opportunity-with-custom-fields.ts", "example:plugins": "tsx examples/plugins.ts", - "example:server": "tsx examples/mock-api-server.ts" + "example:server": "tsx examples/mock-api-server.ts", + "example:transforms": "tsx examples/transforms.ts" }, "dependencies": { "zod": "catalog:" diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index db42ab84d..b03b8c5c7 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -22,29 +22,34 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor - [What is a plugin?](#what-is-a-plugin) - [Defining a plugin](#defining-a-plugin) - [Publishing a plugin](#publishing-a-plugin) - - [Combining plugins](#combining-plugins) - [Using plugins with the API client](#using-plugins-with-the-api-client) +- [Plugin transformations (PoC)](#plugin-transformations-poc) + - [Defining bidirectional transforms](#defining-bidirectional-transforms) + - [Built-in mapping handlers](#built-in-mapping-handlers) + - [Null handling](#null-handling) + - [Custom handlers](#custom-handlers) + - [Validating against the extended schema](#validating-against-the-extended-schema) + - [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) + - [Error handling](#error-handling) - [Best practices](#best-practices) - [Export value schemas alongside your plugin](#export-value-schemas-alongside-your-plugin) - [Use `peerDependencies` for `@common-grants/sdk`](#use-peerdependencies-for-common-grantssdk) - [Keep plugins focused](#keep-plugins-focused) - - [Avoid `"firstWins"` / `"lastWins"` in published plugins](#avoid-firstwins--lastwins-in-published-plugins) - [API reference](#api-reference) - [Plugin creation](#plugin-creation) - [Schema utilities](#schema-utilities) - - [Merging and composition](#merging-and-composition) + - [Transforms (PoC)](#transforms-poc) - [Shared types](#shared-types) ## Key concepts Here are some key concepts that are used to define custom fields and plugins that extend base schemas from the CommonGrants protocol. -| Concept | Description | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | -| **`CustomFieldSpec`** | A TypeScript object that _describes_ a custom field: its `fieldType`, optional `value` (a Zod schema for validating the custom field's value), and optional `name` and `description`. | -| **`SchemaExtensions`** | A mapping of extensible model names (e.g. `"Opportunity"`) to records of `CustomFieldSpec` objects. This is the shape that `definePlugin()` and `withCustomFields()` accept. | -| **`Plugin`** | An object with `.extensions` (the raw `SchemaExtensions`) and `.schemas` (Zod schemas with typed `customFields` applied). Created by `definePlugin()`. | +| Concept | Description | +| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | +| **`CustomFieldSpec`** | A TypeScript object that _describes_ a custom field: its `fieldType`, optional `value` (a Zod schema for validating the custom field's value), and optional `name` and `description`. | +| **`Plugin`** | An object returned by `definePlugin()` with `.schemas` (per-object compiled output: `.common` Zod schema, `.native`, `.toCommon`, `.fromCommon`) and optional `.extensions` / `.meta`. | ## Extending base models with custom fields @@ -99,19 +104,21 @@ Use `definePlugin()` when you want to create a **reusable, shareable** set of cu import { definePlugin } from "@common-grants/sdk/extensions"; const legacyPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { - fieldType: "object", - value: LegacyIdValueSchema, - description: "Maps to the opportunity_id in the legacy system", + customFields: { + legacyId: { + fieldType: "object", + value: LegacyIdValueSchema, + description: "Maps to the opportunity_id in the legacy system", + }, }, }, }, } as const); // The plugin exposes typed schemas for every extensible model -const opportunity = legacyPlugin.schemas.Opportunity.parse(data); +const opportunity = legacyPlugin.schemas.Opportunity.common.parse(data); opportunity.customFields?.legacyId?.value.id; // number ``` @@ -172,13 +179,14 @@ const missing = getCustomFieldValue(opp, "nonexistent", z.string()); A plugin is any object that satisfies the `Plugin` interface: ```typescript -interface Plugin<T extends SchemaExtensions = SchemaExtensions> { - extensions: T; +interface Plugin<T extends SchemasInput = SchemasInput> { schemas: PluginSchemas<T>; + extensions?: PluginExtensions; + meta?: PluginMeta; } ``` -The `Plugin` interface uses [structural typing](https://www.typescriptlang.org/docs/handbook/type-compatibility.html), so any object with the right shape qualifies as a plugin, whether it comes from a local file, a monorepo package, or an installed npm package. There is no base class to extend or registry to sign up for. In practice, you'll almost always create plugins with `definePlugin()`, which handles building the `.schemas` from your `.extensions` automatically. +The `Plugin` interface uses [structural typing](https://www.typescriptlang.org/docs/handbook/type-compatibility.html), so any object with the right shape qualifies as a plugin, whether it comes from a local file, a monorepo package, or an installed npm package. There is no base class to extend or registry to sign up for. In practice, you'll almost always create plugins with `definePlugin()`, which builds `.schemas` (including the `.common` Zod schema) from your `schemas` input automatically. For the full interface definition, see [define-plugin.ts](./define-plugin.ts). @@ -194,20 +202,22 @@ const LegacyIdValueSchema = z.object({ }); const myPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { - fieldType: "object", - value: LegacyIdValueSchema, - description: "Maps to the opportunity_id in the legacy system", - }, - category: { - fieldType: "string", - description: "Grant category", - }, - priority: { - fieldType: "integer", - description: "Processing priority (1 = highest)", + customFields: { + legacyId: { + fieldType: "object", + value: LegacyIdValueSchema, + description: "Maps to the opportunity_id in the legacy system", + }, + category: { + fieldType: "string", + description: "Grant category", + }, + priority: { + fieldType: "integer", + description: "Processing priority (1 = highest)", + }, }, }, }, @@ -217,10 +227,13 @@ const myPlugin = definePlugin({ > [!IMPORTANT] > Always pass `as const` to the options object for `definePlugin()` (and the specs object for `withCustomFields()`). Without it, TypeScript widens literal types like `"string"` to `string`, which prevents the type system from inferring the correct `value` type for each custom field. -The returned `Plugin` object has two main properties: +The returned `Plugin` object has three properties: -- **`myPlugin.extensions`**: the raw `SchemaExtensions` you passed in, preserved by reference. Useful for introspection or for passing to `mergeExtensions()`. -- **`myPlugin.schemas`**: a record of Zod schemas, one per extensible model. Each schema has typed `customFields` based on the specs you provided. Models without extensions pass through with their base schema. +- **`myPlugin.schemas`**: a record of per-object compiled output, one entry per extensible model. Each entry has: + - `.common` — the Zod schema with typed `customFields` applied (use this to parse data). + - `.native`, `.toCommon`, `.fromCommon` — populated when transforms are configured. +- **`myPlugin.extensions`**: optional serializable config (mappings, meta) — safe to store as JSON. +- **`myPlugin.meta`**: optional plugin identity (`name`, `version`, `sourceSystem`, `capabilities`). ### Publishing a plugin @@ -248,16 +261,18 @@ export const ProgramAreaValueSchema = z.object({ }); const plugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - programArea: { - fieldType: "object", - value: ProgramAreaValueSchema, - description: "The HHS program area for this opportunity", - }, - cfda: { - fieldType: "string", - description: "CFDA number", + customFields: { + programArea: { + fieldType: "object", + value: ProgramAreaValueSchema, + description: "The HHS program area for this opportunity", + }, + cfda: { + fieldType: "string", + description: "CFDA number", + }, }, }, }, @@ -319,8 +334,7 @@ export default plugin; ```typescript import plugin from "./"; - plugin.extensions.Opportunity.programArea; // CustomFieldSpec - plugin.schemas.Opportunity.parse({} as any); // fully typed result + plugin.schemas.Opportunity.common.parse({} as any); // fully typed result ``` 3. **Publish** with `npm publish` (or your preferred registry workflow). @@ -332,41 +346,11 @@ After installing the plugin (e.g. `npm install @commongrants/hhs-plugin`): ```typescript import hhs from "@commongrants/hhs-plugin"; -const opp = hhs.schemas.Opportunity.parse(data); +const opp = hhs.schemas.Opportunity.common.parse(data); opp.customFields?.programArea?.value.code; // string opp.customFields?.cfda?.value; // string ``` -### Combining plugins - -Use `mergeExtensions()` to combine extensions from multiple plugins into a single set, then pass the result to `definePlugin()`: - -```typescript -import { definePlugin, mergeExtensions } from "@common-grants/sdk/extensions"; - -const merged = mergeExtensions([legacyPlugin.extensions, classificationPlugin.extensions]); - -const combinedPlugin = definePlugin({ extensions: merged }); - -// All custom fields from both plugins are available with full type safety -const opp = combinedPlugin.schemas.Opportunity.parse(data); -opp.customFields?.legacyId?.value.id; // number (from legacyPlugin) -opp.customFields?.category?.value; // string (from classificationPlugin) -``` - -**Conflict resolution:** By default, `mergeExtensions()` throws an error if two sources define the same field name on the same model. You can change this behavior with the `onConflict` option: - -```typescript -// Keep the first definition encountered -mergeExtensions([a.extensions, b.extensions], { onConflict: "firstWins" }); - -// Keep the last definition encountered -mergeExtensions([a.extensions, b.extensions], { onConflict: "lastWins" }); -``` - -> [!WARNING] -> When using `"firstWins"` or `"lastWins"`, the return type falls back to the base `SchemaExtensions` type because conflict resolution makes static typing unreliable for overlapping field names. The default `"error"` strategy preserves full type inference via intersection types. - ## Using plugins with the API client Pass a plugin's extended schema to the API client via the `schema` option. The client uses it to parse API responses into fully typed objects: @@ -376,10 +360,12 @@ import { Client, Auth } from "@common-grants/sdk/client"; import { definePlugin } from "@common-grants/sdk/extensions"; const myPlugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - legacyId: { fieldType: "integer", description: "Legacy system ID" }, - category: { fieldType: "string", description: "Grant category" }, + customFields: { + legacyId: { fieldType: "integer", description: "Legacy system ID" }, + category: { fieldType: "string", description: "Grant category" }, + }, }, }, } as const); @@ -391,14 +377,14 @@ const client = new Client({ // Get a single opportunity with typed custom fields const opp = await client.opportunities.get(oppId, { - schema: myPlugin.schemas.Opportunity, + schema: myPlugin.schemas.Opportunity.common, }); opp.customFields?.legacyId?.value; // typed as number opp.customFields?.category?.value; // typed as string // List with the same schema const response = await client.opportunities.list({ - schema: myPlugin.schemas.Opportunity, + schema: myPlugin.schemas.Opportunity.common, }); for (const opp of response.items) { console.log(opp.customFields?.category?.value); @@ -408,12 +394,207 @@ for (const opp of response.items) { const results = await client.opportunities.search({ query: "health", statuses: ["open"], - schema: myPlugin.schemas.Opportunity, + schema: myPlugin.schemas.Opportunity.common, }); ``` The `schema` option is accepted by `get()`, `list()`, and `search()`. When omitted, the client falls back to `OpportunityBaseSchema` (with untyped `customFields`). +## Plugin transformations (PoC) + +> **Status:** Proof-of-concept (issue [#798](https://github.com/HHS/simpler-grants-protocol/issues/798)). Mirrors the Python PoC in [PR #810](https://github.com/HHS/simpler-grants-protocol/pull/810). Contract follows [ADR-0022](https://commongrants.org/governance/adr/0022-plugin-framework/) and [ADR-0017](https://commongrants.org/governance/adr/0017-mapping-format/). + +Plugins can declare bidirectional transforms that convert between a source system's native shape and the CommonGrants protocol. `toCommon` maps `native → CommonGrants`; `fromCommon` reverses it. Both directions are author-provided — the SDK does not invert one into the other, because many-to-one handlers (like `match`) are not reversible. + +### Defining bidirectional transforms + +Use `buildTransforms()` to compile a pair of mapping objects into typed callables: + +```typescript +import { buildTransforms } from "@common-grants/sdk/extensions"; + +const { toCommon, fromCommon } = buildTransforms( + // toCommonMapping: native → CommonGrants + { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + createdAt: { field: "data.created_at" }, + lastModifiedAt: { field: "data.last_modified_at" }, + status: { + value: { + match: { + field: "data.opportunity_status", + case: { posted: "open", archived: "closed", forecasted: "forecasted" }, + default: "custom", + }, + }, + }, + }, + // fromCommonMapping: CommonGrants → native + { + data: { + opportunity_uuid: { field: "id" }, + opportunity_title: { field: "title" }, + }, + } +); + +const result = toCommon(sourceData); +if (result.errors.length === 0) { + use(result.result); +} +``` + +Each callable returns a `TransformResult<T>` of `{ result, errors }` unconditionally. Partial failures surface as `PluginError[]` rather than thrown exceptions — consumers choose their own strict-vs-lenient rule. + +### Built-in mapping handlers + +Mapping objects are nested literals where keys are either output field names or registered handler names. The handler-keyed node dispatches the handler with `(data, handlerArg)`. Bare primitives are treated as literals. + +| Handler | Spec shape | Behavior | +| ---------------- | ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `field` | `{ field: "dot.notation.path" }` | Plucks a value from the source via dot-notation. Terminal `null` is preserved; absent / intermediate-null returns `undefined`. See [Null handling](#null-handling). | +| `const` | `{ const: <literal> }` | Returns the literal value, ignoring source data. | +| `match` | `{ match: { field, case: { ... }, default? } }` | Case-based lookup on a source field value. `null` source passes through unchanged; opt-in target-side translation via `case: { "null": ... }`. | +| `switch` | Same as `match` | Convenience alias for `match` — both point at the same handler function. | +| `numberToString` | `{ numberToString: "dot.notation.path" }` | Plucks a value and coerces to string via `String()`. Returns `null` on `null` source ("doesn't apply"); `undefined` on absent. | +| `stringToNumber` | `{ stringToNumber: "dot.notation.path" }` | Plucks a value, parses as int when possible, falls back to float. Returns `null` on `null` source; `undefined` on absent. Throws on non-numeric input. | + +### Null handling + +The transforms layer respects [ADR-0024](https://commongrants.org/governance/adr/0024-optional-field-nullability/)'s three-state contract for optional fields. Optional values carry three distinct states on the wire, each preserved through the transform: + +| State | Meaning | Handler output | +| ---------- | ---------------------------------------------------------------------------------------- | -------------- | +| **absent** | "Not provided" — the publisher did not supply this data | `undefined` | +| **`null`** | "Doesn't apply" — the publisher actively asserts the field is irrelevant for this record | `null` | +| **value** | "Has a value" | coerced value | + +The built-in coercing handlers (`numberToString`, `stringToNumber`) pass `null` through unchanged instead of collapsing it to `undefined`. `field` defers to `getFromPath`, which preserves terminal `null` and treats an intermediate `null` as a propagating absence (`{ a: null }` at path `"a.b"` → `undefined`). + +`match` / `switch` adds an opt-in for target-side translation. By default a `null` source passes through: + +```typescript +// status = null on input → status = null on output (publisher's "doesn't apply" survives) +{ match: { field: "status", case: { posted: "open", archived: "closed" }, default: "custom" } } +``` + +To translate "doesn't apply" into a target-side sentinel (e.g. an `n_a` status token), opt in via a `"null"` case key: + +```typescript +// status = null on input → status = "n_a" on output (author-chosen translation) +{ match: { field: "status", case: { posted: "open", "null": "n_a" }, default: "custom" } } +``` + +`default` is **not** consulted for `null` source values — `default` belongs to "unrecognized value," not to "publisher asserts irrelevant." The opt-in `"null"` case key is the only path from `null` source to a non-`null` target. + +**For custom-handler authors:** preserve the three-state contract when you write your own handlers. Return `undefined` for "not provided," return `null` for "doesn't apply," return a value otherwise. The walker omits keys whose handler returned `undefined` and writes `null` returns as a real, present `null` — so the output object distinguishes the three states the same way the wire does: absent → key omitted, `null` → present `null`, value → present value. Consumers can check for key presence to tell "not provided" from "doesn't apply." + +> **Cross-SDK note.** The TS PoC leads on [ADR-0024](https://commongrants.org/governance/adr/0024-optional-field-nullability/) alignment: it preserves the three-state distinction (absent / `null` / value) at the transform layer. The Python PoC ([#810](https://github.com/HHS/simpler-grants-protocol/pull/810)) predates ADR-0024 and still collapses `None` source into the "not provided" path for the coercing handlers — bringing the Python handlers to parity is a pending follow-up. + +### Custom handlers + +Register additional handlers per `buildTransforms()` call. Name collisions with built-ins raise at call time. Custom handlers should follow the three-state contract from [Null handling](#null-handling) above: + +```typescript +import { buildTransforms, getFromPath } from "@common-grants/sdk/extensions"; + +// `join` is a special case: string concatenation has no meaningful null +// behavior, so the filter below drops both `undefined` and `null` source +// values. This is appropriate for `join` specifically — most coercing +// handlers should follow `numberToString` / `stringToNumber` and preserve +// the three-state contract (return `null` on null source, `undefined` on +// absent) instead. +const join = (data: unknown, spec: unknown) => { + const s = spec as { fields?: string[]; sep?: string }; + const parts = (s.fields ?? []) + .map(p => getFromPath(data, p)) + .filter(v => v !== undefined && v !== null) + .map(String); + return parts.length > 0 ? parts.join(s.sep ?? " ") : undefined; +}; + +const { toCommon } = buildTransforms( + { label: { join: { fields: ["a.b", "c.d"], sep: " — " } } }, + {}, + new Map([["join", join]]) +); +``` + +### Validating against the extended schema + +Pass an optional `commonModel` to validate `toCommon` output. **Use the fully extended schema** (the result of `withCustomFields()`), not the base schema — passing the base silently weakens validation of typed custom fields: + +```typescript +import { buildTransforms, withCustomFields } from "@common-grants/sdk/extensions"; +import { OpportunityBaseSchema } from "@common-grants/sdk/schemas"; + +const ExtendedOpportunity = withCustomFields(OpportunityBaseSchema, { + legacyId: { fieldType: "integer", value: z.number().int() }, +}); + +const { toCommon } = buildTransforms( + { + /* toCommonMapping ... */ + }, + {}, + undefined, + ExtendedOpportunity +); + +const out = toCommon(sourceData); +// On validation failure, `out.result` holds the raw transformed object so +// callers can inspect malformed data alongside `out.errors`. +``` + +### Wiring transforms into a plugin + +Pass `toCommon` / `fromCommon` and `customFields` together under `schemas.<Object>`: + +```typescript +const plugin = definePlugin({ + meta: { + name: "grants.gov", + version: "0.1.0", + sourceSystem: "grants.gov", + capabilities: ["customFields", "transforms"], + }, + schemas: { + Opportunity: { + customFields: { + /* customFieldSpecs */ + }, + toCommon, + fromCommon, + }, + }, +}); + +// Invoke at runtime: +const cg = plugin.schemas.Opportunity?.toCommon?.(sourceData); +``` + +For a complete runnable round-trip with custom handlers and `commonModel` validation, see [`examples/transforms.ts`](../../examples/transforms.ts) (`pnpm example:transforms`). + +### Error handling + +`PluginError` carries structured context — `path`, `handler`, `sourceValue`, `cause` — so consumers can reason about failures programmatically without parsing error text: + +```typescript +const out = toCommon(sourceData); +for (const err of out.errors) { + // Build a redacted projection — err.sourceValue / err.cause carry input data + // by design; the projection enumerates only safe + // fields. On the Zod-validation path err.message is also data-bearing — + // see the PII warning below for the full picture and tracking issue. + const safe = { name: err.name, message: err.message, path: err.path, handler: err.handler }; + console.warn(safe); +} +``` + +> **PII warning (ADR-0022 Decision #9):** The SDK does **not** redact by default. `PluginError.sourceValue` and `cause` are plain enumerable fields and flow through `JSON.stringify(err)`, `util.inspect(err)`, `console.log(err)`, and any logger that enumerates own properties. `sourceValue` is populated with the entire input record passed to `toCommon` / `fromCommon` — not just the value at the failing field. Log a redacted projection instead — e.g. `{ name: err.name, message: err.message, path: err.path, handler: err.handler }`. On the Zod-validation path (when `commonModel` is passed to `buildTransforms()`), `PluginError.message` is also data-bearing — Zod's default error map embeds the rejected value into `issue.message`, which flows verbatim into `PluginError.message`. Redact `message` alongside `sourceValue` and `cause`. Full-message sanitization is tracked under [#744](https://github.com/HHS/simpler-grants-protocol/issues/744). + ## Best practices ### Export value schemas alongside your plugin @@ -432,12 +613,14 @@ export const ProgramAreaValueSchema = z.object({ }); const plugin = definePlugin({ - extensions: { + schemas: { Opportunity: { - programArea: { - fieldType: "object", - value: ProgramAreaValueSchema, - description: "The HHS program area for this opportunity", + customFields: { + programArea: { + fieldType: "object", + value: ProgramAreaValueSchema, + description: "The HHS program area for this opportunity", + }, }, }, }, @@ -452,7 +635,7 @@ This allows consumers to use `getCustomFieldValue()` with the same schema the pl import hhs, { ProgramAreaValueSchema } from "@commongrants/hhs-plugin"; import { getCustomFieldValue } from "@common-grants/sdk/extensions"; -const opp = hhs.schemas.Opportunity.parse(data); +const opp = hhs.schemas.Opportunity.common.parse(data); // Extract the value with full type safety using the exported schema const area = getCustomFieldValue(opp, "programArea", ProgramAreaValueSchema); @@ -465,13 +648,7 @@ Declare `@common-grants/sdk` as a `peerDependency` in your plugin's `package.jso ### Keep plugins focused -A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, use `mergeExtensions()` to combine separate plugins rather than bundling everything into one. - -### Avoid `"firstWins"` / `"lastWins"` in published plugins - -When calling `mergeExtensions()` with `"firstWins"` or `"lastWins"`, the return type widens to `SchemaExtensions`, losing specific field-level type inference. This is fine for local or ad hoc usage, but if you publish a package that uses one of these strategies internally, the widened type propagates to your consumers. They'll see `SchemaExtensions` instead of the precise field types, with no indication of why. - -Prefer the default `"error"` strategy in published plugins. If your extensions genuinely overlap with another plugin, resolve the conflicts explicitly before publishing rather than deferring the resolution to a lossy merge strategy. +A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, define a combined plugin with all fields declared under a single `definePlugin({ schemas: { ... } })` call rather than splitting them across separate plugins. ## API reference @@ -479,11 +656,11 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, ### Plugin creation -| Export | Kind | Description | Demonstrated in | -| ------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| [`definePlugin()`](./define-plugin.ts) | function | Creates a `Plugin` from a `SchemaExtensions` config. Returns an object with `.extensions` (the raw input) and `.schemas` (Zod schemas with typed `customFields`). | [Defining a plugin](#defining-a-plugin) | -| [`Plugin`](./define-plugin.ts) | interface | The object returned by `definePlugin()`. | [What is a plugin?](#what-is-a-plugin) | -| [`DefinePluginOptions`](./define-plugin.ts) | interface | Options accepted by `definePlugin()`. Contains the `extensions` property. | [Defining a plugin](#defining-a-plugin) | +| Export | Kind | Description | Demonstrated in | +| ------------------------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| [`definePlugin()`](./define-plugin.ts) | function | Creates a `Plugin` from `DefinePluginOptions`. Returns an object with `.schemas` (per-object output: `.common`, `.native`, `.toCommon`, `.fromCommon`) and optional `.extensions` / `.meta`. | [Defining a plugin](#defining-a-plugin) | +| [`Plugin`](./define-plugin.ts) | interface | The object returned by `definePlugin()`. | [What is a plugin?](#what-is-a-plugin) | +| [`DefinePluginOptions`](./define-plugin.ts) | interface | Options for `definePlugin()`. `schemas` carries per-object input (custom fields, native schema, transforms); `extensions` is serializable-only config; `meta` is plugin identity. | [Defining a plugin](#defining-a-plugin) | ### Schema utilities @@ -493,20 +670,34 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, | [`WithCustomFieldsResult`](./with-custom-fields.ts) | type | The return type of `withCustomFields()`. A Zod object schema where `customFields` is replaced with a typed version. | | | [`getCustomFieldValue()`](./get-custom-field-value.ts) | function | Safely extracts and parses a custom field value from an `ExtensibleObject`. Returns the parsed value, `undefined` if missing, or throws `ZodError` if invalid. | [Extracting custom field values](#extracting-custom-field-values) | -### Merging and composition - -| Export | Kind | Description | Demonstrated in | -| ------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| [`mergeExtensions()`](./merge-extensions.ts) | function | Combines multiple `SchemaExtensions` objects into one. Throws on field name conflicts by default; supports `"firstWins"` and `"lastWins"` strategies. | [Combining plugins](#combining-plugins) | -| [`MergeExtensionsOptions`](./merge-extensions.ts) | interface | Controls conflict resolution for `mergeExtensions()`. | [Combining plugins](#combining-plugins) | -| [`MergedSchemaExtensions`](./merge-extensions.ts) | type | The return type of `mergeExtensions()` with the default `"error"` strategy. Intersects field specs from each source. | | +### Transforms (PoC) + +| Export | Kind | Description | Demonstrated in | +| ---------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| [`buildTransforms()`](./transforms.ts) | function | Compiles a pair of mapping objects into typed `(toCommon, fromCommon)` callables. Positional params: `(toCommonMapping, fromCommonMapping, handlers?, commonModel?)`. Validates mapping structure at call time; collisions with built-ins throw. | [Defining bidirectional transforms](#defining-bidirectional-transforms) | +| [`BuiltTransforms`](./transforms.ts) | interface | Return shape of `buildTransforms()` — `{ toCommon, fromCommon }`. | | +| [`transformFromMapping()`](./transformation.ts) | function | Low-level mapping walker used by `buildTransforms()`. Useful if you want to drive a single mapping pass without the call-time validation or error-wrapping layer. | | +| [`TransformFromMappingOptions`](./transformation.ts) | interface | Options for `transformFromMapping()`: optional `handlers` registry (`Map<string, Handler>`). | | +| [`DEFAULT_HANDLERS`](./transformation.ts) | const | `Map<string, Handler>` of built-in handlers: `const`, `field`, `match`, `numberToString`, `stringToNumber`, `switch`. | [Built-in mapping handlers](#built-in-mapping-handlers) | +| [`getFromPath()`](./transformation.ts) | function | Walks an object via dot-notation; returns `undefined` (or a provided default) when the path is missing or traverses a non-object. | | +| [`TransformResult`](./types.ts) | interface | Unconditional return shape `{ result, errors }` for `toCommon` / `fromCommon`. | [Defining bidirectional transforms](#defining-bidirectional-transforms) | +| [`PluginError`](./types.ts) | class | Structured transformation error carrying `path`, `handler`, `sourceValue`, `cause`. Extends `Error`. | [Error handling](#error-handling) | +| [`Handler`](./types.ts) | type | Signature for mapping handler functions: `(data, arg) => unknown`. | [Custom handlers](#custom-handlers) | +| [`PluginMeta`](./types.ts) | interface | Plugin identity: `name` (required), `sourceSystem` (required), optional `version` and `capabilities`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | +| [`PluginCapability`](./types.ts) | type | Literal union of capability names: `"customFields" \| "customFilters" \| "transforms" \| "client"`. | | +| [`SchemasInput`](./define-plugin.ts) | type | Map from extensible model name to `ObjectSchemasInput`. The shape of `DefinePluginOptions.schemas`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | +| [`ObjectSchemasInput`](./types.ts) | interface | Author-provided input per object: `{ native?, customFields?, toCommon?, fromCommon? }`. Passed inside `definePlugin({ schemas })`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | +| [`ObjectSchemas`](./types.ts) | interface | Compiled runtime shape: `{ native, common, toCommon, fromCommon }`. Accessed via `plugin.schemas.<Name>`. | | +| [`ObjectMappings`](./types.ts) | interface | Serializable `{ toCommon?, fromCommon? }` mapping dicts. Stored inside `PluginExtensionsObjectConfig.mappings`. | | +| [`PluginExtensionsObjectConfig`](./types.ts) | interface | Per-object slot inside `PluginExtensions.schemas`: `{ mappings? }`. | | +| [`PluginExtensions`](./types.ts) | interface | Serializable plugin config carrying `meta?: Partial<PluginMeta>` and per-object `schemas`. | | +| [`ClientConfig`](./types.ts) | type | The per-plugin client configuration shape. Concrete shape is deferred to the full SDK. | | ### Shared types -| Export | Kind | Description | Demonstrated in | -| ------------------------------------ | --------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------- | -| [`CustomFieldSpec`](./types.ts) | interface | Describes a single custom field: its `fieldType`, optional `value`, and optional `name`/`description`. | [Key concepts](#key-concepts) | -| [`SchemaExtensions`](./types.ts) | type | Maps extensible model names to records of `CustomFieldSpec` objects. Plugins only need to declare models they actually extend. | [Key concepts](#key-concepts) | -| [`ExtensibleSchemaName`](./types.ts) | type | Union of model names that support `customFields` extensions. Currently: `"Opportunity"`. | | -| [`HasCustomFields`](./types.ts) | type | A Zod object schema whose shape includes a `customFields` property. Constrains `withCustomFields()` inputs at compile time. | | -| [`ExtensibleObject`](./types.ts) | interface | An object with an optional `customFields` property. Constrains `getCustomFieldValue()` inputs at compile time. | | +| Export | Kind | Description | Demonstrated in | +| ------------------------------------ | --------- | --------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | +| [`CustomFieldSpec`](./types.ts) | interface | Describes a single custom field: its `fieldType`, optional `value`, and optional `name`/`description`. | [Key concepts](#key-concepts) | +| [`ExtensibleSchemaName`](./types.ts) | type | Union of model names that support `customFields` extensions. Currently: `"Opportunity"`. | | +| [`HasCustomFields`](./types.ts) | type | A Zod object schema whose shape includes a `customFields` property. Constrains `withCustomFields()` inputs at compile time. | | +| [`ExtensibleObject`](./types.ts) | interface | An object with an optional `customFields` property. Constrains `getCustomFieldValue()` inputs at compile time. | | diff --git a/lib/ts-sdk/src/extensions/define-plugin.ts b/lib/ts-sdk/src/extensions/define-plugin.ts index f97b44c4e..dcefae23a 100644 --- a/lib/ts-sdk/src/extensions/define-plugin.ts +++ b/lib/ts-sdk/src/extensions/define-plugin.ts @@ -4,41 +4,80 @@ * @module @common-grants/sdk/extensions */ -import { z } from "zod"; import type { ExtensibleSchemaName, HasCustomFields, - SchemaExtensions, CustomFieldSpec, + ObjectSchemasInput, + PluginMeta, + PluginExtensions, } from "./types"; import { EXTENSIBLE_SCHEMA_MAP } from "./types"; import { withCustomFields, type WithCustomFieldsResult } from "./with-custom-fields"; // ############################################################################ -// Public types - DefinePluginOptions, Plugin +// Public types - SchemasInput, DefinePluginOptions, Plugin // ############################################################################ +/** + * Per-object schemas input keyed by extensible model name. + * + * Plugin authors populate this with hand-written or `buildTransforms()`-generated + * `toCommon` / `fromCommon` callables, an optional `native` schema, and optional + * `customFields` specs. Passed as `DefinePluginOptions.schemas`. + */ +// Per-entry (TNative, TCommon) pairs only meet at the `buildTransforms()` +// boundary. `unknown` would reject legitimate caller schemas at contravariant +// positions; the widening lives only at this dictionary storage layer. +export type SchemasInput = Partial< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Record<ExtensibleSchemaName, ObjectSchemasInput<any, any>> +>; + /** * Options for `definePlugin()`. * - * Accepts an `extensions` property with custom field specifications. + * `schemas` carries the consolidated per-object input (custom fields, + * native schema, transforms). `extensions` is the serializable-only surface + * for JSON-safe config (mappings, meta). Custom field declarations belong + * exclusively on `schemas[Object].customFields`. + * * Structured as an options object for forward-compatibility with future * properties like `namespace`. */ -export interface DefinePluginOptions<T extends SchemaExtensions = SchemaExtensions> { - /** Custom field specifications for extensible models */ - extensions: T; +export interface DefinePluginOptions<T extends SchemasInput = SchemasInput> { + /** + * Serializable plugin config — mappings and meta, safe to store as JSON. + * + * Does not carry `customFields`; those belong on `schemas[Object].customFields`. + */ + extensions?: PluginExtensions; + /** Optional plugin identity and capability declaration. */ + meta?: PluginMeta; + /** + * Per-object transform input — `native` schema, `customFields` specs, + * and `toCommon` / `fromCommon` callables — for each extensible model. + * + * This is the single surface for custom field declarations. + * + * Stored as-is in the PoC (no compilation, no Zod-wrap); the full SDK compiles + * `ObjectSchemasInput` → `ObjectSchemas` and injects the generated `common` model. + */ + schemas?: T; } /** * Configuration object returned by `definePlugin()`. * - * - `extensions` — the original `SchemaExtensions` input (preserved by reference) - * - `schemas` — extensible schemas with custom fields applied where applicable + * - `extensions` — serializable plugin config (mappings, meta), preserved by reference + * - `schemas` — per-object compiled output: `common` (extended Zod schema), `native`, + * `toCommon`, and `fromCommon` for each extensible model + * - `meta` — plugin identity passed through from options */ -export interface Plugin<T extends SchemaExtensions = SchemaExtensions> { - extensions: T; +export interface Plugin<T extends SchemasInput = SchemasInput> { + extensions?: PluginExtensions; schemas: PluginSchemas<T>; + meta?: PluginMeta; } // ############################################################################ @@ -48,90 +87,105 @@ export interface Plugin<T extends SchemaExtensions = SchemaExtensions> { /** * Creates a `Plugin` from the given options. * - * Iterates over extensible schemas. For those with specs in `extensions`, - * applies `withCustomFields()` to produce a typed schema. Others pass - * through unchanged. + * Iterates over extensible schemas. For each model, looks up `customFields` + * specs from `schemas[name].customFields`. When specs are present, applies + * `withCustomFields()` to produce a typed `common` schema; otherwise the base + * schema passes through unchanged. The per-object result is wrapped under + * `.common` alongside any `native`, `toCommon`, and `fromCommon` provided. * - * @param options - Options containing custom field specifications - * @returns A `Plugin` with `.extensions` and `.schemas` + * @param options - Options containing schemas and/or serializable extensions + * @returns A `Plugin` with `.extensions`, `.schemas`, and `.meta` * * @example * ```typescript * const plugin = definePlugin({ - * extensions: { + * schemas: { * Opportunity: { - * legacyId: { fieldType: "string" }, - * category: { fieldType: "string", description: "Grant category" }, + * customFields: { + * legacyId: { fieldType: "string" }, + * category: { fieldType: "string", description: "Grant category" }, + * }, + * toCommon, + * fromCommon, * }, * }, * } as const); * - * // plugin.schemas.Opportunity has typed customFields + * // Access the extended Zod schema: + * const opp = plugin.schemas.Opportunity.common.parse(rawData); + * // Access the transform callables: + * const result = plugin.schemas.Opportunity.toCommon?.(nativeData); * ``` */ -export function definePlugin<const T extends SchemaExtensions>( +export function definePlugin<const T extends SchemasInput>( options: DefinePluginOptions<T> ): Plugin<T> { - const { extensions } = options; - const schemas: Record<string, z.ZodTypeAny> = {}; + const { extensions, meta, schemas: schemasInput } = options; + const schemas: Record<string, object> = {}; - // Walk every extensible model. If the caller supplied specs for it, - // produce a schema with typed customFields; otherwise keep the base schema. for (const [name, extensibleSchema] of Object.entries(EXTENSIBLE_SCHEMA_MAP) as [ ExtensibleSchemaName, HasCustomFields, ][]) { - const specs = extensions[name as ExtensibleSchemaName]; - if (specs && Object.keys(specs).length > 0) { - schemas[name] = withCustomFields(extensibleSchema, specs); - } else { - schemas[name] = extensibleSchema; - } + const specs = schemasInput?.[name]?.customFields; + const common = + specs && Object.keys(specs).length > 0 + ? withCustomFields(extensibleSchema, specs) + : extensibleSchema; + schemas[name] = { + common, + native: schemasInput?.[name]?.native, + toCommon: schemasInput?.[name]?.toCommon, + fromCommon: schemasInput?.[name]?.fromCommon, + }; } // Cast is safe — the runtime loop mirrors the PluginSchemas<T> mapped type, // but TypeScript can't verify that from the dynamic Object.entries() iteration. - return { extensions, schemas } as Plugin<T>; + return { extensions, schemas, meta } as Plugin<T>; } // ############################################################################ // Internal - type inference utilities // ############################################################################ +/** Looks up the base Zod schema for an extensible model name. */ +type BaseZodSchema<K extends ExtensibleSchemaName> = (typeof EXTENSIBLE_SCHEMA_MAP)[K]; + /** - * Computes the schema type for each extensible schema given extensions `T`. - * - * This mapped type iterates over every `ExtensibleSchemaName` (currently just - * `"Opportunity"`) and decides what Zod schema type to assign: - * - * 1. `K extends keyof T` — does the plugin declare extensions for this model? - * 2. `T[K] extends Record<string, CustomFieldSpec>` — are those extensions - * a non-empty specs record? + * Extracts the `customFields` record from `T[K]`, or `never` if absent. * - * If both are true, the schema is the result of `withCustomFields()` applied - * to the base schema for that model. Otherwise the base schema passes through - * unchanged. - * - * @example - * ```typescript - * // Given extensions that customize Opportunity: - * type T = { Opportunity: { legacyId: { fieldType: "string" } } }; - * - * // PluginSchemas<T> resolves to: - * // { Opportunity: WithCustomFieldsResult<typeof OpportunityBaseSchema, T["Opportunity"]> } - * ``` + * Used to feed the custom-fields spec into `WithCustomFieldsResult` while + * keeping the base schema as the fallback when no specs are declared. */ -/** Looks up the base Zod schema for an extensible model name. */ -type BaseZodSchema<K extends ExtensibleSchemaName> = (typeof EXTENSIBLE_SCHEMA_MAP)[K]; +type ExtractCustomFields<K extends ExtensibleSchemaName, T extends SchemasInput> = K extends keyof T + ? NonNullable<T[K]> extends { customFields?: infer CF } + ? CF extends Record<string, CustomFieldSpec> + ? CF + : never + : never + : never; -/** Resolves the schema for a single model: applies extensions if present, else returns base. */ -type ResolveSchema<K extends ExtensibleSchemaName, T extends SchemaExtensions> = K extends keyof T - ? T[K] extends Record<string, CustomFieldSpec> - ? WithCustomFieldsResult<BaseZodSchema<K>, T[K]> - : BaseZodSchema<K> - : BaseZodSchema<K>; +/** Resolves the `common` Zod schema for a single model. */ +type ResolveCommonSchema<K extends ExtensibleSchemaName, T extends SchemasInput> = [ + ExtractCustomFields<K, T>, +] extends [never] + ? BaseZodSchema<K> + : WithCustomFieldsResult<BaseZodSchema<K>, ExtractCustomFields<K, T>>; -/** Maps each extensible model to its resolved schema. */ -type PluginSchemas<T extends SchemaExtensions> = { - [K in ExtensibleSchemaName]: ResolveSchema<K, T>; +/** + * Maps each extensible model to its compiled per-object output. + * + * Each entry contains: + * - `common` — the fully extended Zod schema (base + custom fields) + * - `native` — the optional native-format Zod schema + * - `toCommon` / `fromCommon` — typed transform callables derived from `T[K]` + */ +type PluginSchemas<T extends SchemasInput> = { + [K in ExtensibleSchemaName]: { + common: ResolveCommonSchema<K, T>; + native: K extends keyof T ? NonNullable<T[K]>["native"] : undefined; + toCommon: K extends keyof T ? NonNullable<T[K]>["toCommon"] : undefined; + fromCommon: K extends keyof T ? NonNullable<T[K]>["fromCommon"] : undefined; + }; }; diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index 316c10a2f..d016a461d 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -9,14 +9,33 @@ */ // Plugin creation — define and compose plugins -export type { Plugin, DefinePluginOptions } from "./define-plugin"; -export type { SchemaExtensions, CustomFieldSpec, HasCustomFields, ExtensibleObject } from "./types"; +export type { Plugin, DefinePluginOptions, SchemasInput } from "./define-plugin"; +export type { CustomFieldSpec, HasCustomFields, ExtensibleObject } from "./types"; export { definePlugin } from "./define-plugin"; -export { mergeExtensions } from "./merge-extensions"; // Schema utilities — lower-level tools for working with custom fields -export type { MergeExtensionsOptions, MergedSchemaExtensions } from "./merge-extensions"; export type { WithCustomFieldsResult } from "./with-custom-fields"; export type { ExtensibleSchemaName } from "./types"; export { withCustomFields } from "./with-custom-fields"; export { getCustomFieldValue } from "./get-custom-field-value"; + +// Transforms — bidirectional plugin transformation contract +export type { + ClientConfig, + Handler, + ObjectMappings, + ObjectSchemas, + ObjectSchemasInput, + PluginCapability, + PluginExtensions, + PluginExtensionsObjectConfig, + PluginMeta, + TransformResult, +} from "./types"; +export { PluginError } from "./types"; +export type { BuiltTransforms } from "./transforms"; +export { buildTransforms } from "./transforms"; +export type { TransformFromMappingOptions } from "./transformation"; +// Individual handler functions (fieldValue, constValue, switchOnValue, etc.) +// stay internal; reach them through DEFAULT_HANDLERS when needed. +export { DEFAULT_HANDLERS, getFromPath, transformFromMapping } from "./transformation"; diff --git a/lib/ts-sdk/src/extensions/merge-extensions.ts b/lib/ts-sdk/src/extensions/merge-extensions.ts deleted file mode 100644 index 972431d15..000000000 --- a/lib/ts-sdk/src/extensions/merge-extensions.ts +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Provides the `mergeExtensions()` utility function. - * - * @module @common-grants/sdk/extensions - */ - -import type { SchemaExtensions, ExtensibleSchemaName, CustomFieldSpec } from "./types"; - -// ############################################################################ -// Public type - MergeExtensionsOptions -// ############################################################################ - -/** - * Options for controlling how field-name conflicts are resolved - * when merging multiple `SchemaExtensions` sources. - */ -export interface MergeExtensionsOptions { - /** - * Strategy for handling duplicate field names within the same model. - * - * - `"error"` (default) — throw an error on conflict - * - `"firstWins"` — keep the first definition encountered - * - `"lastWins"` — use the last definition encountered - * - * @remarks - * WARNING: Using `"firstWins"` or `"lastWins"` causes the return type to fall back to - * `SchemaExtensions`, losing specific field-level type inference. If you publish - * a package that calls `mergeExtensions()` with one of these strategies, the - * widened type will propagate to your consumers. Prefer the default `"error"` - * strategy in published plugins to preserve full type safety for downstream users. - */ - onConflict?: "error" | "firstWins" | "lastWins"; -} - -// ############################################################################ -// Public function - mergeExtensions() -// ############################################################################ - -/** - * Merges multiple `SchemaExtensions` objects into a single combined result. - * - * When using the default `"error"` conflict strategy (or no options), the - * return type preserves the specific field names and specs from each source - * via TypeScript intersection types. This enables fully typed `customFields` - * access after passing the result to `definePlugin()`. - * - * When using `"firstWins"` or `"lastWins"`, the return type falls back to - * `SchemaExtensions` because conflict resolution makes static typing - * unreliable for overlapping field names. - * - * @param sources - Array of `SchemaExtensions` to merge - * @param options - Conflict resolution options - * @returns A single merged `SchemaExtensions` object - * @throws Error if duplicate field names are found and `onConflict` is `"error"` - * - * @example - * ```typescript - * const merged = mergeExtensions([ - * legacyPlugin.extensions, - * classificationPlugin.extensions, - * ]); - * // merged preserves both plugins' field types - * const combined = definePlugin({ extensions: merged }); - * // combined.schemas.Opportunity has typed customFields for all fields - * ``` - */ -// Overload: when onConflict is "error" (the default) - preserve specific field types -export function mergeExtensions<const T extends readonly SchemaExtensions[]>( - sources: [...T], - options?: { onConflict?: "error" } -): MergedSchemaExtensions<T>; - -/** - * Overload for `"firstWins"` or `"lastWins"` conflict strategies. - * - * @remarks - * WARNING: Using `"firstWins"` or `"lastWins"` causes the return type to fall back to - * `SchemaExtensions`, losing specific field-level type inference. If you publish - * a package that calls `mergeExtensions()` with one of these strategies, the - * widened type will propagate to your consumers. Prefer the default `"error"` - * strategy in published plugins to preserve full type safety for downstream users. - */ -export function mergeExtensions( - sources: SchemaExtensions[], - options: MergeExtensionsOptions -): SchemaExtensions; - -// Implementation signature — not visible to callers. -export function mergeExtensions( - sources: SchemaExtensions[], - options: MergeExtensionsOptions = {} -): SchemaExtensions { - if (sources.length === 0) return {}; - if (sources.length === 1) return sources[0]; - - const { onConflict = "error" } = options; - const result: Record<string, Record<string, CustomFieldSpec>> = {}; - - for (const source of sources) { - mergeSource(result, source, onConflict); - } - - return result as SchemaExtensions; -} - -// ############################################################################ -// Internal functions - mergeSource(), mergeFields() -// ############################################################################ - -/** Merges all models from a single source into the accumulated result. */ -function mergeSource( - result: Record<string, Record<string, CustomFieldSpec>>, - source: SchemaExtensions, - onConflict: NonNullable<MergeExtensionsOptions["onConflict"]> -): void { - for (const [model, fields] of Object.entries(source) as [ - ExtensibleSchemaName, - Record<string, CustomFieldSpec>, - ][]) { - result[model] ??= {}; - mergeFields(result[model], fields, model, onConflict); - } -} - -/** Merges fields from a single model into the target, applying conflict resolution. */ -function mergeFields( - target: Record<string, CustomFieldSpec>, - source: Record<string, CustomFieldSpec>, - model: string, - onConflict: NonNullable<MergeExtensionsOptions["onConflict"]> -): void { - for (const [fieldName, spec] of Object.entries(source)) { - if (fieldName in target) { - switch (onConflict) { - case "error": - throw new Error(`mergeExtensions: duplicate field "${fieldName}" on model "${model}"`); - case "firstWins": - break; - case "lastWins": - target[fieldName] = spec; - break; - } - } else { - target[fieldName] = spec; - } - } -} - -// ############################################################################ -// Internal type-level merge utilities -// ############################################################################ - -/** Converts a union type to an intersection type. */ -type UnionToIntersection<U> = (U extends unknown ? (k: U) => void : never) extends ( - k: infer I -) => void - ? I - : never; - -/** - * Extracts the field specs for a given model from a single `SchemaExtensions` - * source. Uses `keyof` + `NonNullable` so that `Partial` keys are handled - * correctly (the optional `undefined` is stripped). - * - * Distributes over union types when `S` = `T[number]`, so each tuple element - * contributes its own fields independently. - */ -type ExtractModelFields<S, K extends string> = K extends keyof S ? NonNullable<S[K]> : never; - -/** - * Computes the merged `SchemaExtensions` type from a tuple of sources. - * - * For each extensible model, intersects the field-spec records from every - * source. When sources declare disjoint fields the intersection simply - * combines them; when they declare the same field name (a conflict) the - * runtime `onConflict: "error"` strategy throws, so the intersection is - * still sound — both definitions would need to agree at the type level. - */ -export type MergedSchemaExtensions<T extends readonly SchemaExtensions[]> = { - [K in ExtensibleSchemaName]: UnionToIntersection<ExtractModelFields<T[number], K>>; -}; diff --git a/lib/ts-sdk/src/extensions/transformation.ts b/lib/ts-sdk/src/extensions/transformation.ts new file mode 100644 index 000000000..2a8d551bd --- /dev/null +++ b/lib/ts-sdk/src/extensions/transformation.ts @@ -0,0 +1,347 @@ +/** + * Mapping-runtime utilities for declarative bidirectional transforms. + * + * Used by `buildTransforms()` in `./transforms`. Re-exported by `./index`. + * + * ## Null handling (three-state contract, ADR-0024) + * + * Optional fields carry three distinct states, each preserved through every + * built-in handler rather than collapsing `null` into `undefined`: + * + * - **absent** — "not provided" (publisher did not supply this data) + * - **`null`** — "doesn't apply" (publisher actively asserts irrelevant for this record) + * - **value** — "has a value" + * + * `fieldValue` / `getFromPath` pass terminal `null` through verbatim; the + * coercing handlers (`numberToString`, `stringToNumber`, `switchOnValue`) + * return `null` on `null` source. Custom-handler authors should do the same. + * This is the canonical description — handlers below note only their own + * departures from it. + * + * @module @common-grants/sdk/extensions + */ + +import type { Handler } from "./types"; + +// ############################################################################ +// Public utilities - getFromPath, handlers, transformFromMapping +// ############################################################################ + +/** + * Extract a value from an object using dot-notation. + * + * Walks the path through nested objects. Returns the default when any + * intermediate step is missing or non-object. + * + * Null handling: a terminal `null` is returned verbatim (preserves "doesn't + * apply" at the leaf); a `null` intermediate step short-circuits to + * `defaultValue` (typically `undefined`) — a null parent makes its children + * "not provided" by extension. Authors needing richer per-leaf null semantics + * inside a null parent should write a custom handler. + * + * @example + * ```ts + * getFromPath({ a: { b: 1 } }, "a.b"); // 1 + * getFromPath({ a: { b: 1 } }, "a.c"); // undefined (absent) + * getFromPath({ a: null }, "a"); // null (terminal null preserved) + * getFromPath({ a: null }, "a.b"); // undefined (intermediate null → absent) + * getFromPath({ a: 1 }, ""); // { a: 1 } (empty path returns input) + * ``` + */ +export function getFromPath<T = unknown>( + data: unknown, + path: string, + defaultValue: T | undefined = undefined +): T | undefined { + if (path === "") return data as T; + const parts = path.split("."); + let cursor: unknown = data; + for (const part of parts) { + if (typeof cursor !== "object" || cursor === null) return defaultValue; + cursor = (cursor as Record<string, unknown>)[part]; + } + return cursor === undefined ? defaultValue : (cursor as T); +} + +/** + * `field` handler — pluck a value by dot-notation path. + */ +export function fieldValue(data: unknown, fieldPath: unknown): unknown { + return getFromPath(data, String(fieldPath ?? "")); +} + +/** + * `const` handler — return a fixed literal value, ignoring source data. + */ +export function constValue(_data: unknown, value: unknown): unknown { + return value; +} + +/** + * `match` / `switch` handler — case-based lookup on a field's value. + * + * Spec shape: `{ field: "path", case: { sourceValue: targetValue, ... }, default?: any }`. + * Returns the mapped value if `data[field]` is a `case` key, otherwise `default` + * (or undefined when no default is provided). + * + * Only string source values are candidate lookup keys: a non-string `val` + * short-circuits to `default`. `case: { "1": "yes" }` does not match a source + * value of `1`. Authors who need to map non-string source values should coerce + * upstream (e.g. via a custom handler that `String()`-coerces before + * dispatching `match`). + * + * Null handling. When the source field resolves to an explicit `null`, the + * handler returns `case["null"]` if the author opted in to target-side + * translation, otherwise passes `null` through unchanged. `default` is NOT + * consulted for `null` source — `default` is for unrecognized values, not for + * the publisher's "doesn't apply" assertion; use `case: { "null": ... }` for a + * target-side sentinel. (See the module-level null-handling note.) + * + * A missing field resolves to `undefined` via `getFromPath`, which fails the + * string-only guard and falls through to `default`. To distinguish "field + * absent" from "field present and equal to undefined," guard upstream. + * + * If `field` is omitted or `""`, `getFromPath` returns the entire `data` + * object — non-string, so the handler falls back to `default`. A `match` spec + * with no `field` is functionally equivalent to `const: <default>`; prefer + * `const` for clarity when the constant case is what you want. + * + * Cross-SDK divergence: Python's PoC uses bare `dict.get(val, default)`, which + * accepts non-string `val` natively and collapses `null` into `default`. This + * handler instead preserves `null` (three-state) and fails loud with a + * descriptive error on a malformed spec. + * + * `match` is the canonical handler name; `switch` is provided as a + * convenience alias — both point at the same handler function. (No prior SDK + * API defined `switch`; the alias is for ergonomic preference, not a + * backward-compatibility obligation.) + * + * @throws Error when `spec` is not a non-null object. The walker wraps this + * as a `HandlerError`; `buildTransforms` surfaces it as a `PluginError` + * with `handler: "match"`. + */ +export function switchOnValue(data: unknown, spec: unknown): unknown { + if (typeof spec !== "object" || spec === null || Array.isArray(spec)) { + throw new Error("match/switch handler: spec must be an object"); + } + const s = spec as { field?: string; case?: Record<string, unknown>; default?: unknown }; + const val = getFromPath(data, s.field ?? ""); + const lookup = new Map(Object.entries(s.case ?? {})); + // null source = publisher asserts "doesn't apply." + // Author opts in to target-side translation via a `"null"` case key. + // Otherwise pass null through unchanged — do NOT fall through to default, + // which is for unrecognized values, not for publisher assertions. + if (val === null) { + return lookup.has("null") ? lookup.get("null") : null; + } + if (typeof val === "string" && lookup.has(val)) { + return lookup.get(val); + } + return s.default; +} + +/** + * `numberToString` handler — pluck a value and coerce it to string via `String()`. + * + * Null handling: absent → `undefined`; `null` source → `null` (`String(null)` + * is bypassed so the literal "null" never lands in output); value → + * `String(val)`. (See the module-level null-handling note.) + * + * The handler is named for its primary use case (numeric source values) but the + * coercion is `String()`, so booleans (`"true"` / `"false"`), arrays + * (`"a,b,c"`), and other non-null values pass through unchanged. Plugin authors + * who need strict number-only behavior should validate the source value before + * the handler runs (e.g. via a custom handler) or use `field` plus a downstream + * Zod check. + */ +export function numberToString(data: unknown, fieldPath: unknown): string | null | undefined { + const val = getFromPath(data, String(fieldPath ?? "")); + if (val === undefined) return undefined; + if (val === null) return null; + return String(val); +} + +/** + * `stringToNumber` handler — pluck a value, coerce to an integer when the + * string is a pure integer, otherwise fall back to a general `Number()` + * coercion. Non-numeric inputs throw. + * + * Null handling: absent → `undefined`; `null` source → `null` (no coercion + * attempted); value → coerced via the integer / float / safe-integer rules + * below. (See the module-level null-handling note.) + * + * Divergences from Python's `int(s)` semantics, both intentional: + * + * - `int("42.0")` raises `ValueError` in Python; this handler falls through + * to `Number(s)` and returns `42`. Plugin authors porting a handler that + * relies on the Python behavior should add their own decimal-rejecting regex. + * - Python's `int()` is arbitrary precision; JavaScript numbers are IEEE 754 + * doubles with a safe-integer ceiling of `Number.MAX_SAFE_INTEGER` + * (2^53 − 1). An integer-shaped string outside that range cannot be + * represented without precision loss, so this handler throws rather than + * silently returning a corrupted value. Plugin authors round-tripping + * 64-bit IDs should declare the field as a string (and skip this handler) + * or write a custom handler that returns a `BigInt`. + * + * Empty and whitespace-only strings throw — `Number("")` and `Number(" ")` + * both coerce to `0` in JavaScript, which would silently turn an + * implicit-absent CSV cell into a real zero on the transformed side. + * Callers who want absent input to surface as `undefined` should null the + * field upstream. + */ +export function stringToNumber(data: unknown, fieldPath: unknown): number | null | undefined { + const val = getFromPath(data, String(fieldPath ?? "")); + if (val === undefined) return undefined; + if (val === null) return null; + const s = String(val).trim(); + // Empty / whitespace-only strings would otherwise coerce to 0 via `Number()`. + if (s === "") { + throw new Error("stringToNumber: cannot convert source value to a number"); + } + // Integer-shaped strings parse as integers; anything else falls through to + // `Number(s)`. The integer branch is + // intentional even though `Number(s)` would handle integer inputs too — + // it pins the int-vs-float distinction at the call site. + if (/^-?\d+$/.test(s)) { + const n = Number(s); + // Outside the safe-integer range, `Number(s)` silently loses precision + // (e.g. `"9999999999999999999"` → `1e19`). Reject rather than corrupt. + if (Number.isFinite(n) && Number.isSafeInteger(n)) return n; + throw new Error("stringToNumber: cannot convert source value to a number"); + } + const f = Number(s); + if (Number.isFinite(f)) return f; + // Don't embed the source value in the error message — it could be PII when + // transforming applicant data. The handler name and cause flow into + // `PluginError` separately for programmatic reasoning. Adopters who need + // the offending value can still read it from `PluginError.sourceValue` + // (which carries its own PII warning). + throw new Error("stringToNumber: cannot convert source value to a number"); +} + +/** + * Raised when a handler function throws. Carries the handler name for attribution. + * + * `buildTransforms()` catches this and wraps it as a `PluginError`, so callers + * of the public `toCommon` / `fromCommon` pair will not see `HandlerError` + * directly. The class is **internal**: not re-exported from the package barrel, + * not part of the published `package.json` `exports` map, and not a supported + * import path for adopters. Tests that drive `transformFromMapping` itself + * import it from the source file for `instanceof` checks; consumers of + * `transformFromMapping()` through the public surface should treat thrown + * values as plain `Error`s. + */ +export class HandlerError extends Error { + readonly handler: string; + readonly cause: unknown; + constructor(handler: string, cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause)); + this.name = "HandlerError"; + this.handler = handler; + this.cause = cause; + } +} + +/** + * Registry of built-in mapping handlers. + * + * `match` is the canonical name; `switch` is a convenience alias + * — both point at the same handler function. + */ +export const DEFAULT_HANDLERS: Map<string, Handler> = new Map([ + ["const", constValue], + ["field", fieldValue], + ["match", switchOnValue], + ["numberToString", numberToString], + ["stringToNumber", stringToNumber], + ["switch", switchOnValue], +]); + +/** + * Options for {@link transformFromMapping}. + */ +export interface TransformFromMappingOptions { + /** Handler registry. Defaults to {@link DEFAULT_HANDLERS}. */ + handlers?: Map<string, Handler>; +} + +/** + * Transform a data object according to a declarative mapping spec. + * + * The mapping is a nested object where: + * - Primitive leaves (string, number, boolean, null) pass through as literals. + * - Object nodes whose first key is a registered handler dispatch to that + * handler with `(data, handlerArg)`. The handler's return value is the + * transformed node. Sibling keys on a handler-dispatch node are silently + * ignored here — only the first key is read, so `{ field: "x", const: "fallback" }` + * drops `const`. This low-level walker stays lenient; callers entering + * through `buildTransforms()` get a stricter `validateMapping` pass that + * rejects the shape at build time. + * - Object nodes whose first key is not a handler are treated as output + * shapes — each child is transformed recursively. A child that transforms to + * `undefined` (absent source) is omitted from the output object; `null` + * ("doesn't apply") is written as a present key. So absent → missing key, + * `null` → present `null`, value → present value. + * + * @example + * ```ts + * transformFromMapping( + * { opportunity_status: "posted", opportunity_amount: 1000 }, + * { + * status: { field: "opportunity_status" }, + * amount: { value: { field: "opportunity_amount" }, currency: "USD" }, + * } + * ); + * // => { status: "posted", amount: { value: 1000, currency: "USD" } } + * ``` + * + * @throws {@link HandlerError} when a registered handler throws. + */ +export function transformFromMapping( + data: unknown, + mapping: unknown, + options: TransformFromMappingOptions = {} +): unknown { + const handlers = options.handlers ?? DEFAULT_HANDLERS; + + const transformNode = (node: unknown): unknown => { + // Primitives, null, and arrays pass through as literals. Arrays in handler + // args are opaque to the walker by design — handlers own their own shape. + if (typeof node !== "object" || node === null || Array.isArray(node)) { + return node; + } + + const entries = Object.entries(node as Record<string, unknown>); + if (entries.length === 0) return {}; + + // Check the first key. If it names a handler, dispatch with the handler + // argument. Otherwise treat the node as an output shape and recurse over + // every child. + const [firstKey] = entries[0]; + if (handlers.has(firstKey)) { + const handlerFn = handlers.get(firstKey)!; + const handlerArg = (node as Record<string, unknown>)[firstKey]; + try { + return handlerFn(data, handlerArg); + } catch (exc) { + throw new HandlerError(firstKey, exc); + } + } + + const out: Record<string, unknown> = {}; + for (const [k, v] of entries) { + // Three-state: a child that transforms to `undefined` (absent source) is + // omitted entirely rather than written as `out[k] = undefined`, so the + // object carries all three states — absent → key omitted, `null` → + // present `null`, value → present. This matches what `JSON.stringify` + // produces on the wire (it drops `undefined` keys). + const child = transformNode(v); + if (child !== undefined) { + out[k] = child; + } + } + return out; + }; + + return transformNode(mapping); +} diff --git a/lib/ts-sdk/src/extensions/transforms.ts b/lib/ts-sdk/src/extensions/transforms.ts new file mode 100644 index 000000000..3b4699117 --- /dev/null +++ b/lib/ts-sdk/src/extensions/transforms.ts @@ -0,0 +1,232 @@ +/** + * `buildTransforms()` — compile a pair of declarative mapping objects into typed + * `(toCommon, fromCommon)` callables. + * + * Each direction is author-provided — this utility never inverts one into the + * other, because many-to-one handlers like `switch` are not reversible. + * + * @module @common-grants/sdk/extensions + */ + +import { z } from "zod"; + +import { PluginError, type Handler, type TransformResult } from "./types"; +import { DEFAULT_HANDLERS, HandlerError, transformFromMapping } from "./transformation"; + +// ############################################################################ +// Internal - mapping structure validation +// ############################################################################ + +/** + * Walk the mapping tree and throw on structural malformation. + * + * Each node must be a primitive (`string` / `number` / `boolean` / `null` / + * `undefined`) or a plain object. Arrays and class instances are rejected. + * + * Handler arguments are runtime-only and skipped — they may legitimately be + * arrays, deeply nested specs, or anything else the handler accepts. The + * walker can't detect *unknown* handler names at static analysis time (an + * unknown key is indistinguishable from an output field name); that + * detection is deferred to the full SDK. + * + * Sibling keys at a handler-dispatch node are rejected here. The runtime + * walker is first-key-wins, so `{ field: "x", const: "fallback" }` would + * silently drop `const` — almost always an author typo — so fail loud at + * build time instead (parity with the Python PoC's build-time validation). + * The low-level `transformFromMapping` walker stays lenient so programmatic + * callers composing partial mappings aren't forced into the strict shape. + * + * @internal + */ +function validateMapping(mapping: unknown, knownHandlers: Set<string>, path = ""): void { + if (mapping === null || mapping === undefined) return; + const t = typeof mapping; + if (t === "string" || t === "number" || t === "boolean") return; + + if (t !== "object" || Array.isArray(mapping)) { + throw new Error( + `Invalid mapping node at '${path}': expected object, string, number, boolean, or null, got ${ + Array.isArray(mapping) ? "array" : t + }` + ); + } + + // A handler invocation must be the sole key in its node — see the function + // docstring above for the first-key-wins rationale and cross-SDK parity. + const nodeKeys = Object.keys(mapping as Record<string, unknown>); + const handlerKeys = nodeKeys.filter(k => knownHandlers.has(k)); + if (handlerKeys.length > 0 && nodeKeys.length > 1) { + const siblings = nodeKeys.filter(k => !knownHandlers.has(k)).sort(); + throw new Error( + `Invalid mapping node at '${path === "" ? "<root>" : path}': handler key '${ + handlerKeys[0] + }' cannot have sibling keys ${JSON.stringify(siblings)}. ` + + `A handler invocation must be the only key in its dict.` + ); + } + + for (const [key, value] of Object.entries(mapping as Record<string, unknown>)) { + const childPath = path === "" ? key : `${path}.${key}`; + if (knownHandlers.has(key)) { + // Handler invocation — argument is runtime-only, do not recurse. + continue; + } + validateMapping(value, knownHandlers, childPath); + } +} + +// ############################################################################ +// Public - buildTransforms +// ############################################################################ + +/** + * Return shape of {@link buildTransforms}. + */ +export interface BuiltTransforms<TNative, TCommon> { + toCommon: (native: TNative) => TransformResult<TCommon>; + fromCommon: (common: TCommon) => TransformResult<TNative>; +} + +/** + * Compile a pair of declarative mapping objects into typed + * `(toCommon, fromCommon)` callables. + * + * @example + * ```ts + * const { toCommon, fromCommon } = buildTransforms( + * { + * id: { field: "data.opportunity_uuid" }, + * title: { field: "data.opportunity_title" }, + * status: { + * value: { + * match: { + * field: "data.opportunity_status", + * case: { posted: "open", archived: "closed" }, + * default: "custom", + * }, + * }, + * }, + * }, + * { data: { opportunity_title: { field: "title" } } }, + * ); + * + * const result = toCommon(sourceData); + * if (result.errors.length === 0) use(result.result); + * ``` + * + * @remarks + * Error aggregation is asymmetric across the two failure modes: + * + * - Handler failures (a registered handler throws): the mapping walk + * short-circuits on the first failure, so `errors` carries exactly one + * `PluginError` even when several fields would have failed. + * - Zod-validation failures (`commonModel` provided): every `ZodIssue` is + * flattened into a separate `PluginError`, so `errors` carries the full + * set. + * + * Callers writing strict-mode handling should treat any non-empty `errors` + * as failure regardless of length. + * + * @param toCommonMapping - ADR-0017 mapping from native format → CommonGrants. + * @param fromCommonMapping - ADR-0017 mapping from CommonGrants → native format. + * @param handlers - Optional custom handlers registered for this call only. + * Name collisions with {@link DEFAULT_HANDLERS} raise a `TypeError` at call + * time rather than silently shadowing the default. + * @param commonModel - Optional Zod schema to validate `toCommon` output against. + * Must be the fully extended schema (e.g. result of `withCustomFields(...)`) — + * not the base schema. Passing the base schema silently weakens validation of + * typed custom fields. When provided, `safeParse()` runs on the transform + * result and Zod issues are flattened into `TransformResult.errors`. + * `PluginError.path` for Zod-flattened issues uses dot notation including + * numeric indices (e.g. `"customFields.items.0.value"`). + * + * @throws TypeError when custom handler names collide with built-in defaults. + * @throws Error when either mapping is structurally malformed (sibling keys + * on a handler-dispatch node). + */ +export function buildTransforms<TNative = unknown, TCommon = unknown>( + toCommonMapping: Record<string, unknown>, + fromCommonMapping: Record<string, unknown>, + handlers?: Map<string, Handler>, + // Bivariant `any` accepts schemas with input/output asymmetry; `unknown` + // would reject them at the contravariant input position. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + commonModel?: z.ZodType<TCommon, z.ZodTypeDef, any> +): BuiltTransforms<TNative, TCommon> { + if (handlers) { + const collisions = [...handlers.keys()].filter(k => DEFAULT_HANDLERS.has(k)); + if (collisions.length > 0) { + throw new TypeError( + `buildTransforms: handler names collide with defaults: ${JSON.stringify(collisions.sort())}` + ); + } + } + + const merged = new Map([...DEFAULT_HANDLERS, ...(handlers ?? [])]); + const known = new Set(merged.keys()); + + // Validate mapping structure up front so structurally malformed mappings + // fail at build time, not on first invocation. + validateMapping(toCommonMapping, known); + validateMapping(fromCommonMapping, known); + + const runMapping = ( + data: unknown, + mapping: Record<string, unknown> + ): { ok: true; value: unknown } | { ok: false; error: PluginError } => { + try { + return { ok: true, value: transformFromMapping(data, mapping, { handlers: merged }) }; + } catch (exc) { + if (exc instanceof HandlerError) { + const cause = exc.cause; + return { + ok: false, + error: new PluginError(cause instanceof Error ? cause.message : String(cause), { + handler: exc.handler, + sourceValue: data, + cause, + }), + }; + } + return { + ok: false, + error: new PluginError(exc instanceof Error ? exc.message : String(exc), { + sourceValue: data, + cause: exc, + }), + }; + } + }; + + const toCommon = (native: TNative): TransformResult<TCommon> => { + const ran = runMapping(native, toCommonMapping); + if (!ran.ok) return { result: {} as TCommon, errors: [ran.error] }; + + if (commonModel === undefined) { + return { result: ran.value as TCommon, errors: [] }; + } + + const parsed = commonModel.safeParse(ran.value); + if (parsed.success) { + return { result: parsed.data, errors: [] }; + } + const errors = parsed.error.issues.map(issue => { + // Root-level issues (e.g. from `.refine()` on the schema itself) have an + // empty `path` — leave PluginError.path undefined so the "if known" contract + // in the docstring holds. + const joined = issue.path.length > 0 ? issue.path.map(p => String(p)).join(".") : undefined; + return new PluginError(issue.message, { path: joined }); + }); + // Return the raw transformed object alongside errors so callers can + // inspect malformed data. + return { result: ran.value as TCommon, errors }; + }; + + const fromCommon = (common: TCommon): TransformResult<TNative> => { + const ran = runMapping(common, fromCommonMapping); + if (!ran.ok) return { result: {} as TNative, errors: [ran.error] }; + return { result: ran.value as TNative, errors: [] }; + }; + + return { toCommon, fromCommon }; +} diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index a1d4daa8d..0ac44be9c 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -91,7 +91,7 @@ export interface ExtensibleObject { } // ############################################################################ -// Public types - ExtensibleSchemaName, SchemaExtensions +// Public types - ExtensibleSchemaName // ############################################################################ /** @@ -117,46 +117,225 @@ export const EXTENSIBLE_SCHEMA_MAP = { Opportunity: OpportunityBaseSchema, } as const satisfies Record<ExtensibleSchemaName, HasCustomFields>; +// ############################################################################ +// Public types - Transform contract +// ############################################################################ + /** - * Maps extensible model names to their custom field specifications. + * Features a plugin can declare in `PluginMeta.capabilities`. + */ +export type PluginCapability = "customFields" | "customFilters" | "transforms" | "client"; + +/** + * Loose configuration object for plugin-provided HTTP clients. * - * Each key is the name of a model that supports `customFields`, and the value - * is a record mapping custom field keys to their `CustomFieldSpec` definitions. - * The `Partial` type is used so plugins only need to declare models they actually extend. + * Loose per-plugin client configuration. The PoC does not constrain the shape; + * the full SDK's `client` capability work decides the precise type. + */ +export type ClientConfig = Record<string, unknown>; + +/** + * Handler signature for transform mapping handlers. * - * @example - * The following `SchemaExtensions` object: + * - First arg: the source data being transformed (where field paths resolve from). + * - Second arg: the handler argument from the mapping spec. + * - Return: the transformed value. * - * ```typescript - * const extensions: SchemaExtensions = { - * Opportunity: { - * legacyId: { name: "Legacy ID", fieldType: "integer" }, - * category: { name: "Category", fieldType: "string", description: "Grant category" }, - * }, + * @remarks + * One contract custom-handler authors should respect: + * + * **Do not throw `Error`s whose `.message` embeds source data when that + * data may contain PII.** `buildTransforms()` wraps a handler exception's + * message verbatim into the resulting `PluginError.message`, which is + * enumerable on `Error.prototype` and rendered by `util.inspect` / + * `console.log(err)`. The SDK does not redact by default — + * `PluginError.sourceValue` and `.cause` are enumerable, and + * `.message` flows through verbatim. The built-in `stringToNumber` handler + * follows this rule by throwing a generic "cannot convert source value to a + * number" message; see the README's `PluginError` PII warning for the + * adopter-side redaction pattern. + */ +export type Handler = (data: unknown, arg: unknown) => unknown; + +/** + * Unconditional return shape for `toCommon` / `fromCommon`. + * + * `result` is the transformed value (may be partial on handler error or validation + * failure). `errors` is the aggregated `PluginError` list, empty on full success. + * + * Consumers apply their own strict-vs-lenient rule — strict adopters treat any + * non-empty `errors` as failure; lenient adopters use `result` despite warnings + * and inspect `errors` for context. + */ +export interface TransformResult<T> { + result: T; + errors: PluginError[]; +} + +/** + * Structured transformation error. + * + * Carries field path, handler name, source value, and underlying cause so + * consumers can reason about failures programmatically without parsing error text. + * + * @remarks + * **The SDK does not redact by default.** + * `sourceValue` and `cause` are plain enumerable fields and flow through + * `JSON.stringify(err)`, `util.inspect(err)`, and any logger that enumerates + * own properties. When populated by `buildTransforms()`, `sourceValue` is the + * entire input record passed to `toCommon` / `fromCommon` — not just the + * value at the failing field — so adopters whose source data may contain PII + * must redact before logging. + * + * Partial-redaction pattern (strips `sourceValue` and `cause` only — see + * caveat on `message` below): + * ```ts + * const partiallySafe = { + * name: err.name, + * // CAUTION: `message` is data-bearing on the Zod-validation path. + * // Strip or transform it before logging if your source data may contain PII. + * message: err.message, + * path: err.path, + * handler: err.handler, * }; * ``` - * Corresponds to the following customFields object on the Opportunity schema: - * ```json - * { - * "id": "573525f2-8e15-4405-83fb-e6523511d893", - * "title": "Test Opportunity", - * "status": { "value": "open" }, - * "customFields": { - * "legacyId": { - * "name": "Legacy ID", - * "fieldType": "integer", - * "value": 12345, - * }, - * "category": { - * "name": "Category", - * "fieldType": "string", - * "value": "Education", - * "description": "Grant category", - * } - * } - * } - * ``` + * + * `PluginError.message` is data-bearing on the Zod-validation path + * (`buildTransforms({ commonModel })`): Zod's default error map embeds the + * received runtime value into `issue.message`, which flows verbatim into + * `PluginError.message`. Adopters whose source data may contain PII must redact + * `message` alongside `sourceValue` and `cause`. Full-message sanitization is + * tracked under #744. */ -export type SchemaExtensions = Partial< - Record<ExtensibleSchemaName, Record<string, CustomFieldSpec>> ->; +export class PluginError extends Error { + /** Dot-notation field path where the error occurred, if known. */ + path?: string; + /** Name of the handler that raised, if applicable. */ + handler?: string; + /** The source value that triggered the error (may contain PII — redact before logging). */ + sourceValue?: unknown; + /** Underlying cause of the error, if any (may contain PII — redact before logging). */ + cause?: unknown; + + constructor( + message: string, + options?: { + path?: string; + handler?: string; + sourceValue?: unknown; + cause?: unknown; + } + ) { + super(message); + this.name = "PluginError"; + this.path = options?.path; + this.handler = options?.handler; + this.sourceValue = options?.sourceValue; + this.cause = options?.cause; + } +} + +/** + * Author-provided input for a single extensible object, passed inside + * `DefinePluginOptions.schemas`. + * + * Plugin authors supply `toCommon` and `fromCommon` as plain callables — either + * hand-written or generated via `buildTransforms()`. `native` is the optional Zod + * schema for the source format. `customFields` declares any extra fields this + * object exposes beyond the base CommonGrants schema; `definePlugin()` applies + * them to the compiled schema via `withCustomFields()`. + * + * @remarks + * `common` is intentionally absent here. It is injected by `definePlugin()` + * during compilation from `ObjectSchemasInput` → `ObjectSchemas`, resolved from + * the generated model classes produced by the code generator. Plugin config + * files cannot import from `generated/` (which is the input to generation). + * + * `customFields` is co-located with the transform callables so authors add a + * single per-object entry under `DefinePluginOptions.schemas` rather than + * splitting declarations across two top-level keys. This matches the Python SDK's + * `ObjectSchemasInput` shape, which also carries `custom_fields` alongside + * `to_common` / `from_common`. + */ +export interface ObjectSchemasInput<TNative = unknown, TCommon = unknown> { + native?: z.ZodType<TNative>; + customFields?: Record<string, CustomFieldSpec>; + toCommon?: (native: TNative) => TransformResult<TCommon>; + fromCommon?: (common: TCommon) => TransformResult<TNative>; +} + +/** + * Runtime compiled type produced by `definePlugin()` — not provided directly by authors. + * + * In the PoC, `definePlugin()` stores `ObjectSchemasInput` as-is on the returned + * plugin's `transformSchemas` field. Full compilation (adding `common` from the + * base CG model, wrapping with Zod validation) is deferred to the full SDK. + */ +export interface ObjectSchemas<TNative, TCommon> { + native: z.ZodType<TNative>; + common: z.ZodType<TCommon>; + toCommon: (native: TNative) => TransformResult<TCommon>; + fromCommon: (common: TCommon) => TransformResult<TNative>; +} + +/** + * Plugin identity and capability declaration. + * + * `name` and `sourceSystem` are required so that plugin registries and + * dependency-injection surfaces always have a reliable display label and + * provenance string. `version` and `capabilities` remain optional because + * they can be inferred or omitted during early development. + */ +export interface PluginMeta { + /** Plugin display name (e.g. `"grants.gov"`). */ + name: string; + /** Plugin version (semver, e.g. `"1.0.0"`). */ + version?: string; + /** Name of the native source system (e.g. `"grants.gov"`). */ + sourceSystem: string; + /** Features the plugin provides. */ + capabilities?: PluginCapability[]; +} + +/** + * Declarative mapping dicts for a single object, stored in the serializable + * extensions config. + * + * Each direction is author-provided — `buildTransforms()` does not invert one + * direction into the other, because many-to-one handlers like `switch` are not + * reversible. + */ +export interface ObjectMappings { + toCommon?: Record<string, unknown>; + fromCommon?: Record<string, unknown>; +} + +/** + * Per-object config inside the serializable `PluginExtensions.schemas` dict. + * + * `mappings` carries optional declarative mappings; when present and no + * explicit `toCommon` / `fromCommon` is supplied in + * `DefinePluginOptions.schemas`, `definePlugin()` will auto-invoke + * `buildTransforms()` on these. Deferred to the full SDK. + * + * @remarks + * `customFields` lives on {@link ObjectSchemasInput} (inside + * `DefinePluginOptions.schemas[obj]`) rather than here, so authors keep + * all per-object declarations — custom fields, native schema, and transforms + * — in one entry. This matches the Python SDK's `ObjectSchemasInput` shape. + * Cross-package composition of custom field declarations is done by defining + * a combined plugin with all fields under `schemas[Object].customFields`. + */ +export interface PluginExtensionsObjectConfig { + mappings?: ObjectMappings; +} + +/** + * Serializable portion of plugin config — safe to store as JSON. + * + * Used to store JSON-safe serializable config alongside a plugin. + */ +export interface PluginExtensions { + meta?: Partial<PluginMeta>; + schemas?: Partial<Record<ExtensibleSchemaName, PluginExtensionsObjectConfig>>; +} diff --git a/lib/ts-sdk/vitest.config.ts b/lib/ts-sdk/vitest.config.ts index 6aa22d9f0..0534b241d 100644 --- a/lib/ts-sdk/vitest.config.ts +++ b/lib/ts-sdk/vitest.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ }, test: { environment: "node", + exclude: ["dist/**", "node_modules/**"], coverage: { provider: "v8", reporter: ["text", "json", "html"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1930ee25c..8df8b08b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,8 +34,8 @@ catalogs: specifier: ^0.81.0 version: 0.81.0 '@vitest/coverage-v8': - specifier: ^3.2.4 - version: 3.2.4 + specifier: ^4.1.8 + version: 4.1.8 eslint: specifier: ^9.39.4 version: 9.39.4 @@ -64,15 +64,15 @@ catalogs: specifier: ^8.59.3 version: 8.59.3 vitest: - specifier: ^3.2.4 - version: 3.2.4 + specifier: ^4.1.8 + version: 4.1.8 zod: specifier: ^3.25.76 version: 3.25.76 website: vitest: - specifier: ^4.1.6 - version: 4.1.6 + specifier: ^4.1.8 + version: 4.1.8 overrides: axios@<1.15.0: '>=1.15.0' @@ -116,7 +116,7 @@ importers: version: 1.10.0 '@vitest/coverage-v8': specifier: 'catalog:' - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) + version: 4.1.8(vitest@4.1.8) eslint: specifier: 'catalog:' version: 9.39.4 @@ -131,7 +131,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: 'catalog:' - version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) + version: 4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) lib/cli: dependencies: @@ -207,7 +207,7 @@ importers: version: 0.81.0(@typespec/compiler@1.11.0(@types/node@20.19.41)) '@vitest/coverage-v8': specifier: 'catalog:' - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) + version: 4.1.8(vitest@4.1.8) eslint: specifier: 'catalog:' version: 9.39.4 @@ -219,7 +219,7 @@ importers: version: 5.5.5(eslint-config-prettier@9.1.2(eslint@9.39.4))(eslint@9.39.4)(prettier@3.8.3) eslint-plugin-vitest: specifier: 'catalog:' - version: 0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) + version: 0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@4.1.8) prettier: specifier: 'catalog:' version: 3.8.3 @@ -240,7 +240,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: 'catalog:' - version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) + version: 4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) lib/core: dependencies: @@ -337,7 +337,7 @@ importers: version: 0.81.0(@typespec/compiler@1.11.0(@types/node@20.19.41)) '@vitest/coverage-v8': specifier: 'catalog:' - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) + version: 4.1.8(vitest@4.1.8) ajv: specifier: ^8.20.0 version: 8.20.0 @@ -376,7 +376,7 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: 'catalog:' - version: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) + version: 4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) website: dependencies: @@ -515,17 +515,13 @@ importers: version: 8.59.3(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: catalog:website - version: 4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) + version: 4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) wrangler: specifier: ^4.95.0 version: 4.95.0 packages: - '@ampproject/remapping@2.3.0': - resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} - engines: {node: '>=6.0.0'} - '@apidevtools/json-schema-ref-parser@11.7.2': resolution: {integrity: sha512-4gY54eEGEstClvEkGnwVkTkrx0sqwemEFG5OSRRn3tD91XH0+Q8XIkYIfo7IwEWPpJZwILb9GUXeShtplRc/eA==} engines: {node: '>= 16'} @@ -667,10 +663,6 @@ packages: resolution: {integrity: sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==} engines: {node: '>=6.9.0'} - '@babel/helper-string-parser@7.27.1': - resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} - engines: {node: '>=6.9.0'} - '@babel/helper-string-parser@7.29.7': resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} engines: {node: '>=6.9.0'} @@ -691,11 +683,6 @@ packages: resolution: {integrity: sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==} engines: {node: '>=6.9.0'} - '@babel/parser@7.29.2': - resolution: {integrity: sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==} - engines: {node: '>=6.0.0'} - hasBin: true - '@babel/parser@7.29.7': resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} engines: {node: '>=6.0.0'} @@ -729,10 +716,6 @@ packages: resolution: {integrity: sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==} engines: {node: '>=6.9.0'} - '@babel/types@7.29.0': - resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} - engines: {node: '>=6.9.0'} - '@babel/types@7.29.7': resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} engines: {node: '>=6.9.0'} @@ -2106,10 +2089,6 @@ packages: resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} - '@istanbuljs/schema@0.1.3': - resolution: {integrity: sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==} - engines: {node: '>=8'} - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -2249,277 +2228,139 @@ packages: rollup: optional: true - '@rollup/rollup-android-arm-eabi@4.60.0': - resolution: {integrity: sha512-WOhNW9K8bR3kf4zLxbfg6Pxu2ybOUbB2AjMDHSQx86LIF4rH4Ft7vmMwNt0loO0eonglSNy4cpD3MKXXKQu0/A==} - cpu: [arm] - os: [android] - '@rollup/rollup-android-arm-eabi@4.60.4': resolution: {integrity: sha512-F5QXMSiFebS9hKZj02XhWLLnRpJ3B3AROP0tWbFBSj+6kCbg5m9j5JoHKd4mmSVy5mS/IMQloYgYxCuJC0fxEQ==} cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.60.0': - resolution: {integrity: sha512-u6JHLll5QKRvjciE78bQXDmqRqNs5M/3GVqZeMwvmjaNODJih/WIrJlFVEihvV0MiYFmd+ZyPr9wxOVbPAG2Iw==} - cpu: [arm64] - os: [android] - '@rollup/rollup-android-arm64@4.60.4': resolution: {integrity: sha512-GxxTKApUpzRhof7poWvCJHRF51C67u1R7D6DiluBE8wKU1u5GWE8t+v81JvJYtbawoBFX1hLv5Ei4eVjkWokaw==} cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.60.0': - resolution: {integrity: sha512-qEF7CsKKzSRc20Ciu2Zw1wRrBz4g56F7r/vRwY430UPp/nt1x21Q/fpJ9N5l47WWvJlkNCPJz3QRVw008fi7yA==} - cpu: [arm64] - os: [darwin] - '@rollup/rollup-darwin-arm64@4.60.4': resolution: {integrity: sha512-tua0TaJxMOB1R0V0RS1jFZ/RpURFDJIOR2A6jWwQeawuFyS4gBW+rntLRaQd0EQ4bd6Vp44Z2rXW+YYDBsj6IA==} cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.60.0': - resolution: {integrity: sha512-WADYozJ4QCnXCH4wPB+3FuGmDPoFseVCUrANmA5LWwGmC6FL14BWC7pcq+FstOZv3baGX65tZ378uT6WG8ynTw==} - cpu: [x64] - os: [darwin] - '@rollup/rollup-darwin-x64@4.60.4': resolution: {integrity: sha512-CSKq7MsP+5PFIcydhAiR1K0UhEI1A2jWXVKHPCBZ151yOutENwvnPocgVHkivu2kviURtCEB6zUQw0vs8RrhMg==} cpu: [x64] os: [darwin] - '@rollup/rollup-freebsd-arm64@4.60.0': - resolution: {integrity: sha512-6b8wGHJlDrGeSE3aH5mGNHBjA0TTkxdoNHik5EkvPHCt351XnigA4pS7Wsj/Eo9Y8RBU6f35cjN9SYmCFBtzxw==} - cpu: [arm64] - os: [freebsd] - '@rollup/rollup-freebsd-arm64@4.60.4': resolution: {integrity: sha512-+O8OkVdyvXMtJEciu2wS/pzm1IxntEEQx3z5TAVy4l32G0etZn+RsA48ARRrFm6Ri8fvqPQfgrvNxSjKAbnd3g==} cpu: [arm64] os: [freebsd] - '@rollup/rollup-freebsd-x64@4.60.0': - resolution: {integrity: sha512-h25Ga0t4jaylMB8M/JKAyrvvfxGRjnPQIR8lnCayyzEjEOx2EJIlIiMbhpWxDRKGKF8jbNH01NnN663dH638mA==} - cpu: [x64] - os: [freebsd] - '@rollup/rollup-freebsd-x64@4.60.4': resolution: {integrity: sha512-Iw3oMskH3AfNuhU0MSN7vNbdi4me/NiYo2azqPz/Le16zHSa+3RRmliCMWWQmh4lcndccU40xcJuTYJZxNo/lw==} cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.60.0': - resolution: {integrity: sha512-RzeBwv0B3qtVBWtcuABtSuCzToo2IEAIQrcyB/b2zMvBWVbjo8bZDjACUpnaafaxhTw2W+imQbP2BD1usasK4g==} - cpu: [arm] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-arm-gnueabihf@4.60.4': resolution: {integrity: sha512-EIPRXTVQpHyF8WOo219AD2yEltPehLTcTMz2fn6JsatLYSzQf00hj3rulF+yauOlF9/FtM2WpkT/hJh/KJFGhA==} cpu: [arm] os: [linux] libc: [glibc] - '@rollup/rollup-linux-arm-musleabihf@4.60.0': - resolution: {integrity: sha512-Sf7zusNI2CIU1HLzuu9Tc5YGAHEZs5Lu7N1ssJG4Tkw6e0MEsN7NdjUDDfGNHy2IU+ENyWT+L2obgWiguWibWQ==} - cpu: [arm] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-arm-musleabihf@4.60.4': resolution: {integrity: sha512-J3Yh9PzzF1Ovah2At+lHiGQdsYgArxBbXv/zHfSyaiFQEqvNv7DcW98pCrmdjCZBrqBiKrKKe2V+aaSGWuBe/w==} cpu: [arm] os: [linux] libc: [musl] - '@rollup/rollup-linux-arm64-gnu@4.60.0': - resolution: {integrity: sha512-DX2x7CMcrJzsE91q7/O02IJQ5/aLkVtYFryqCjduJhUfGKG6yJV8hxaw8pZa93lLEpPTP/ohdN4wFz7yp/ry9A==} - cpu: [arm64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-arm64-gnu@4.60.4': resolution: {integrity: sha512-BFDEZMYfUvLn37ONE1yMBojPxnMlTFsdyNoqncT0qFq1mAfllL+ATMMJd8TeuVMiX84s1KbcxcZbXInmcO2mRg==} cpu: [arm64] os: [linux] libc: [glibc] - '@rollup/rollup-linux-arm64-musl@4.60.0': - resolution: {integrity: sha512-09EL+yFVbJZlhcQfShpswwRZ0Rg+z/CsSELFCnPt3iK+iqwGsI4zht3secj5vLEs957QvFFXnzAT0FFPIxSrkQ==} - cpu: [arm64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-arm64-musl@4.60.4': resolution: {integrity: sha512-pc9EYOSlOgdQ2uPl1o9PF6/kLSgaUosia7gOuS8mB69IxJvlclko1MECXysjs5ryez1/5zjYqx3+xYU0TU6R1A==} cpu: [arm64] os: [linux] libc: [musl] - '@rollup/rollup-linux-loong64-gnu@4.60.0': - resolution: {integrity: sha512-i9IcCMPr3EXm8EQg5jnja0Zyc1iFxJjZWlb4wr7U2Wx/GrddOuEafxRdMPRYVaXjgbhvqalp6np07hN1w9kAKw==} - cpu: [loong64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-loong64-gnu@4.60.4': resolution: {integrity: sha512-NxnomyxYerDh5n4iLrNa+sH+Z+U4BMEE46V2PgQ/hoB909i8gV1M5wPojWg9fk1jWpO3IQnOs20K4wyZuFLEFQ==} cpu: [loong64] os: [linux] libc: [glibc] - '@rollup/rollup-linux-loong64-musl@4.60.0': - resolution: {integrity: sha512-DGzdJK9kyJ+B78MCkWeGnpXJ91tK/iKA6HwHxF4TAlPIY7GXEvMe8hBFRgdrR9Ly4qebR/7gfUs9y2IoaVEyog==} - cpu: [loong64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-loong64-musl@4.60.4': resolution: {integrity: sha512-nbJnQ8a3z1mtmrwImCYhc6BGpThAyYVRQxw9uKSKG4wR6aAYno9sVjJ0zaZcW9BPJX1GbrDPf+SvdWjgTuDmnw==} cpu: [loong64] os: [linux] libc: [musl] - '@rollup/rollup-linux-ppc64-gnu@4.60.0': - resolution: {integrity: sha512-RwpnLsqC8qbS8z1H1AxBA1H6qknR4YpPR9w2XX0vo2Sz10miu57PkNcnHVaZkbqyw/kUWfKMI73jhmfi9BRMUQ==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-ppc64-gnu@4.60.4': resolution: {integrity: sha512-2EU6acNrQLd8tYvo/LXW535wupT3m6fo7HKo6lr7ktQoItxTyOL1ZCR/GfGCuXl2vR+zmfI6eRXkSemafv+iVg==} cpu: [ppc64] os: [linux] libc: [glibc] - '@rollup/rollup-linux-ppc64-musl@4.60.0': - resolution: {integrity: sha512-Z8pPf54Ly3aqtdWC3G4rFigZgNvd+qJlOE52fmko3KST9SoGfAdSRCwyoyG05q1HrrAblLbk1/PSIV+80/pxLg==} - cpu: [ppc64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-ppc64-musl@4.60.4': resolution: {integrity: sha512-WeBtoMuaMxiiIrO2IYP3xs6GMWkJP2C0EoT8beTLkUPmzV1i/UcOSVw1d5r9KBODtHKilG5yFxsGRnBbK3wJ4A==} cpu: [ppc64] os: [linux] libc: [musl] - '@rollup/rollup-linux-riscv64-gnu@4.60.0': - resolution: {integrity: sha512-3a3qQustp3COCGvnP4SvrMHnPQ9d1vzCakQVRTliaz8cIp/wULGjiGpbcqrkv0WrHTEp8bQD/B3HBjzujVWLOA==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-riscv64-gnu@4.60.4': resolution: {integrity: sha512-FJHFfqpKUI3A10WrWKiFbBZ7yVbGT4q4B5o1qKFFojqpaYoh9LrQgqWCmmcxQzVSXYtyB5bzkXrYzlHTs21MYA==} cpu: [riscv64] os: [linux] libc: [glibc] - '@rollup/rollup-linux-riscv64-musl@4.60.0': - resolution: {integrity: sha512-pjZDsVH/1VsghMJ2/kAaxt6dL0psT6ZexQVrijczOf+PeP2BUqTHYejk3l6TlPRydggINOeNRhvpLa0AYpCWSQ==} - cpu: [riscv64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-riscv64-musl@4.60.4': resolution: {integrity: sha512-mcEl6CUT5IAUmQf1m9FYSmVqCJlpQ8r8eyftFUHG8i9OhY7BkBXSUdnLH5DOf0wCOjcP9v/QO93zpmF1SptCCw==} cpu: [riscv64] os: [linux] libc: [musl] - '@rollup/rollup-linux-s390x-gnu@4.60.0': - resolution: {integrity: sha512-3ObQs0BhvPgiUVZrN7gqCSvmFuMWvWvsjG5ayJ3Lraqv+2KhOsp+pUbigqbeWqueGIsnn+09HBw27rJ+gYK4VQ==} - cpu: [s390x] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-s390x-gnu@4.60.4': resolution: {integrity: sha512-ynt3JxVd2w2buzoKDWIyiV1pJW93xlQic1THVLXilz429oijRpSHivZAgp65KBu+cMcgf1eVVjdnTLvPxgCuoQ==} cpu: [s390x] os: [linux] libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.60.0': - resolution: {integrity: sha512-EtylprDtQPdS5rXvAayrNDYoJhIz1/vzN2fEubo3yLE7tfAw+948dO0g4M0vkTVFhKojnF+n6C8bDNe+gDRdTg==} - cpu: [x64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.60.4': resolution: {integrity: sha512-Boiz5+MsaROEWDf+GGEwF8VMHGhlUoQMtIPjOgA5fv4osupqTVnJteQNKJwUcnUog2G55jYXH7KZFFiJe0TEzQ==} cpu: [x64] os: [linux] libc: [glibc] - '@rollup/rollup-linux-x64-musl@4.60.0': - resolution: {integrity: sha512-k09oiRCi/bHU9UVFqD17r3eJR9bn03TyKraCrlz5ULFJGdJGi7VOmm9jl44vOJvRJ6P7WuBi/s2A97LxxHGIdw==} - cpu: [x64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-x64-musl@4.60.4': resolution: {integrity: sha512-+qfSY27qIrFfI/Hom04KYFw3GKZSGU4lXus51wsb5EuySfFlWRwjkKWoE9emgRw/ukoT4Udsj4W/+xxG8VbPKg==} cpu: [x64] os: [linux] libc: [musl] - '@rollup/rollup-openbsd-x64@4.60.0': - resolution: {integrity: sha512-1o/0/pIhozoSaDJoDcec+IVLbnRtQmHwPV730+AOD29lHEEo4F5BEUB24H0OBdhbBBDwIOSuf7vgg0Ywxdfiiw==} - cpu: [x64] - os: [openbsd] - '@rollup/rollup-openbsd-x64@4.60.4': resolution: {integrity: sha512-VpTfOPHgVXEBeeR8hZ2O0F3aSso+JDWqTWmTmzcQKted54IAdUVbxE+j/MVxUsKa8L20HJhv3vUezVPoquqWjA==} cpu: [x64] os: [openbsd] - '@rollup/rollup-openharmony-arm64@4.60.0': - resolution: {integrity: sha512-pESDkos/PDzYwtyzB5p/UoNU/8fJo68vcXM9ZW2V0kjYayj1KaaUfi1NmTUTUpMn4UhU4gTuK8gIaFO4UGuMbA==} - cpu: [arm64] - os: [openharmony] - '@rollup/rollup-openharmony-arm64@4.60.4': resolution: {integrity: sha512-IPOsh5aRYuLv/nkU51X10Bf75Bsf6+gZdx1X+QP5QM6lIJFHHqbHLG0uJn/hWthzo13UAc2umiUorqZy3axoZg==} cpu: [arm64] os: [openharmony] - '@rollup/rollup-win32-arm64-msvc@4.60.0': - resolution: {integrity: sha512-hj1wFStD7B1YBeYmvY+lWXZ7ey73YGPcViMShYikqKT1GtstIKQAtfUI6yrzPjAy/O7pO0VLXGmUVWXQMaYgTQ==} - cpu: [arm64] - os: [win32] - '@rollup/rollup-win32-arm64-msvc@4.60.4': resolution: {integrity: sha512-4QzE9E81OohJ/HKzHhsqU+zcYYojVOXlFMs1DdyMT6qXl/niOH7AVElmmEdUNHHS/oRkc++d5k6Vy85zFs0DEw==} cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.60.0': - resolution: {integrity: sha512-SyaIPFoxmUPlNDq5EHkTbiKzmSEmq/gOYFI/3HHJ8iS/v1mbugVa7dXUzcJGQfoytp9DJFLhHH4U3/eTy2Bq4w==} - cpu: [ia32] - os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.60.4': resolution: {integrity: sha512-zTPgT1YuHHcd+Tmx7h8aml0FWFVelV5N54oHow9SLj+GfoDy/huQ+UV396N/C7KpMDMiPspRktzM1/0r1usYEA==} cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-gnu@4.60.0': - resolution: {integrity: sha512-RdcryEfzZr+lAr5kRm2ucN9aVlCCa2QNq4hXelZxb8GG0NJSazq44Z3PCCc8wISRuCVnGs0lQJVX5Vp6fKA+IA==} - cpu: [x64] - os: [win32] - '@rollup/rollup-win32-x64-gnu@4.60.4': resolution: {integrity: sha512-DRS4G7mi9lJxqEDezIkKCaUIKCrLUUDCUaCsTPCi/rtqaC6D/jjwslMQyiDU50Ka0JKpeXeRBFBAXwArY52vBw==} cpu: [x64] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.60.0': - resolution: {integrity: sha512-PrsWNQ8BuE00O3Xsx3ALh2Df8fAj9+cvvX9AIA6o4KpATR98c9mud4XtDWVvsEuyia5U4tVSTKygawyJkjm60w==} - cpu: [x64] - os: [win32] - '@rollup/rollup-win32-x64-msvc@4.60.4': resolution: {integrity: sha512-QVTUovf40zgTqlFVrKA1uXMVvU2QWEFWfAH8Wdc48IxLvrJMQVMBRjuQyUpzZCDkakImib9eVazbWlC6ksWtJw==} cpu: [x64] @@ -3086,23 +2927,20 @@ packages: peerDependencies: vite: 6.4.2 - '@vitest/coverage-v8@3.2.4': - resolution: {integrity: sha512-EyF9SXU6kS5Ku/U82E259WSnvg6c8KTjppUncuNdm5QHpe17mwREHnjDzozC8x9MZ0xfBUFSaLkRv4TMA75ALQ==} + '@vitest/coverage-v8@4.1.8': + resolution: {integrity: sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==} peerDependencies: - '@vitest/browser': 3.2.4 - vitest: 3.2.4 + '@vitest/browser': 4.1.8 + vitest: 4.1.8 peerDependenciesMeta: '@vitest/browser': optional: true - '@vitest/expect@3.2.4': - resolution: {integrity: sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==} + '@vitest/expect@4.1.8': + resolution: {integrity: sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==} - '@vitest/expect@4.1.6': - resolution: {integrity: sha512-7EHDquPthALSV0jhhjgEW8FXaviMx7rSqu8W6oqCoAuOhKov814P99QDV1pxMA3QPv21YudvJngIhjrNI4opLg==} - - '@vitest/mocker@3.2.4': - resolution: {integrity: sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==} + '@vitest/mocker@4.1.8': + resolution: {integrity: sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==} peerDependencies: msw: ^2.4.9 vite: 6.4.2 @@ -3112,46 +2950,20 @@ packages: vite: optional: true - '@vitest/mocker@4.1.6': - resolution: {integrity: sha512-MCFc63czMjEInOlcY2cpQCvCN+KgbAn+60xu9cMgP4sKaLC5JNAKw7JH8QdAnoAC88hW1IiSNZ+GgVXlN1UcMQ==} - peerDependencies: - msw: ^2.4.9 - vite: 6.4.2 - peerDependenciesMeta: - msw: - optional: true - vite: - optional: true - - '@vitest/pretty-format@3.2.4': - resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} - - '@vitest/pretty-format@4.1.6': - resolution: {integrity: sha512-h5SxD/IzNhZYnrSZRsUZQIC+vD0GY8cUvq0iwsmkFKixRCKLLWqCXa/FIQ4S1R+sI+PGoojkHsdNrbZiM9Qpgw==} - - '@vitest/runner@3.2.4': - resolution: {integrity: sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==} - - '@vitest/runner@4.1.6': - resolution: {integrity: sha512-nOPCmn2+yD0ZNmKdsXGv/UxMMWbMuKeD6GyYncNwdkYDxpQvrPSKYj2rWuDjC2Y4b6w6hjip5dBKFzEUuZe3vA==} - - '@vitest/snapshot@3.2.4': - resolution: {integrity: sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==} + '@vitest/pretty-format@4.1.8': + resolution: {integrity: sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==} - '@vitest/snapshot@4.1.6': - resolution: {integrity: sha512-YhsdE6xAVfTDmzjxL2ZDUvjj+ZsgyOKe+TdQzqkD72wIOmHka8NuGQ6NpTNZv9D2Z63fbwWKJPeVpEw4EQgYxw==} + '@vitest/runner@4.1.8': + resolution: {integrity: sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==} - '@vitest/spy@3.2.4': - resolution: {integrity: sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==} + '@vitest/snapshot@4.1.8': + resolution: {integrity: sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==} - '@vitest/spy@4.1.6': - resolution: {integrity: sha512-JFKxMx6udhwKh/Ldo270e17QX710vgunMkuPAvXjHSvC6oqLWAHhVhjg/I71q0u0CBSErIODV1Kjv0FQNSWjdg==} + '@vitest/spy@4.1.8': + resolution: {integrity: sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==} - '@vitest/utils@3.2.4': - resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} - - '@vitest/utils@4.1.6': - resolution: {integrity: sha512-FxIY+U81R3LGKCxaHHFRQ5+g6/iRgGLmeHWdp2Amj4ljQRrEIWHmZyDfDYBRZlpyqA7qKxtS9DD1dhk8RnRIVQ==} + '@vitest/utils@4.1.8': + resolution: {integrity: sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==} '@volar/kit@2.4.28': resolution: {integrity: sha512-cKX4vK9dtZvDRaAzeoUdaAJEew6IdxHNCRrdp5Kvcl6zZOqb6jTOfk3kXkIkG3T7oTFXguEMt5+9ptyqYR84Pg==} @@ -3301,8 +3113,8 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} - ast-v8-to-istanbul@0.3.12: - resolution: {integrity: sha512-BRRC8VRZY2R4Z4lFIL35MwNXmwVqBityvOIwETtsCSwvjl0IdgFsy9NhdaA6j74nUdtJJlIypeRhpDam19Wq3g==} + ast-v8-to-istanbul@1.0.3: + resolution: {integrity: sha512-jCMQ6ZylLPudp0CDfBmQBZUsrh1/8psbmu9ibeVWKuHWD0YrH9YABwlKu5kVEFoT0GCQQW9Z/SxfuEbbkGQCRg==} astring@1.9.0: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} @@ -3418,10 +3230,6 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} - cac@6.7.14: - resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} - engines: {node: '>=8'} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -3447,10 +3255,6 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} - chai@5.3.3: - resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} - engines: {node: '>=18'} - chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -3485,10 +3289,6 @@ packages: chardet@2.1.1: resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} - check-error@2.1.3: - resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} - engines: {node: '>= 16'} - chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -3732,10 +3532,6 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} - deep-eql@5.0.2: - resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} - engines: {node: '>=6'} - deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -3905,9 +3701,6 @@ packages: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} - es-module-lexer@1.7.0: - resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} - es-module-lexer@2.1.0: resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} @@ -4627,10 +4420,6 @@ packages: resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} engines: {node: '>=10'} - istanbul-lib-source-maps@5.0.6: - resolution: {integrity: sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==} - engines: {node: '>=10'} - istanbul-reports@3.2.0: resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} engines: {node: '>=8'} @@ -4647,9 +4436,6 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-tokens@9.0.1: - resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} - js-yaml@3.14.2: resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==} hasBin: true @@ -4831,9 +4617,6 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true - loupe@3.2.1: - resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} - lowlight@1.20.0: resolution: {integrity: sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==} @@ -4850,9 +4633,6 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - magicast@0.3.5: - resolution: {integrity: sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==} - magicast@0.5.3: resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} @@ -5092,10 +4872,6 @@ packages: resolution: {integrity: sha512-bjdr2xW1dBCMsMGGsUeqM4eFI60m94+szhxWys+B1ztIt6gWSfeGBdSVCIawezeHYLYn0j6zrsXdQS/JllBzww==} engines: {node: '>=6'} - minimatch@10.2.4: - resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} - engines: {node: 18 || 20 || >=22} - minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -5370,10 +5146,6 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pathval@2.0.1: - resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} - engines: {node: '>= 14.16'} - piccolore@0.1.3: resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} @@ -5418,10 +5190,6 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} - postcss@8.5.8: - resolution: {integrity: sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==} - engines: {node: ^10 || ^12 || >=14} - prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -5728,11 +5496,6 @@ packages: resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} hasBin: true - rollup@4.60.0: - resolution: {integrity: sha512-yqjxruMGBQJ2gG4HtjZtAfXArHomazDHoFwFFmZZl0r7Pdo7qCIXKqKHZc8yeoMgzJJ+pO6pEEHa+V7uzWlrAQ==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - rollup@4.60.4: resolution: {integrity: sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -5934,9 +5697,6 @@ packages: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} - std-env@3.10.0: - resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} - std-env@4.0.0: resolution: {integrity: sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==} @@ -5977,9 +5737,6 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} - strip-literal@3.1.0: - resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} - strnum@2.3.0: resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==} @@ -6058,10 +5815,6 @@ packages: resolution: {integrity: sha512-qFAy10MTMwjzjU8U16YS4YoZD+NQLHzLssFMNqgravjbvIPNiqkGFR4yjhJfmY9R5OFU7+yHxc6y+uGHkKwLRA==} engines: {node: '>=20'} - test-exclude@7.0.2: - resolution: {integrity: sha512-u9E6A+ZDYdp7a4WnarkXPZOx8Ilz46+kby6p1yZ8zsGTz9gYa6FIS7lj2oezzNKmtdyyJNNmmXDppga5GB7kSw==} - engines: {node: '>=18'} - tiny-inflate@1.0.3: resolution: {integrity: sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==} @@ -6072,13 +5825,6 @@ packages: resolution: {integrity: sha512-8OqlXQ35euK9+e7L68u8UwcODxkHoIkjbGsgXuARKNyQ5G6xt8nw1YPeMbxMLgCPFkToU+UEK5j05t2t8edKpQ==} engines: {node: ^16.14.0 || >= 17.3.0} - tinyexec@0.3.2: - resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} - - tinyexec@1.1.2: - resolution: {integrity: sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA==} - engines: {node: '>=18'} - tinyexec@1.2.2: resolution: {integrity: sha512-M/Q0B2cp4K7kynaT/vnED1j8TlLY+Pp7C6Wl2bl/7u/F0mUVwdyOpwomQb8JpYLitHUssAJRmLZdMCGsrx7i+g==} engines: {node: '>=18'} @@ -6091,22 +5837,10 @@ packages: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} - tinypool@1.1.1: - resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} - engines: {node: ^18.0.0 || >=20.0.0} - - tinyrainbow@2.0.0: - resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} - engines: {node: '>=14.0.0'} - tinyrainbow@3.1.0: resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} - tinyspy@4.0.4: - resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} - engines: {node: '>=14.0.0'} - to-buffer@1.2.2: resolution: {integrity: sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==} engines: {node: '>= 0.4'} @@ -6416,11 +6150,6 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} - vite-node@3.2.4: - resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} - hasBin: true - vite@6.4.2: resolution: {integrity: sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -6509,48 +6238,20 @@ packages: vite: optional: true - vitest@3.2.4: - resolution: {integrity: sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} - hasBin: true - peerDependencies: - '@edge-runtime/vm': '*' - '@types/debug': ^4.1.12 - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 - '@vitest/browser': 3.2.4 - '@vitest/ui': 3.2.4 - happy-dom: '*' - jsdom: '*' - peerDependenciesMeta: - '@edge-runtime/vm': - optional: true - '@types/debug': - optional: true - '@types/node': - optional: true - '@vitest/browser': - optional: true - '@vitest/ui': - optional: true - happy-dom: - optional: true - jsdom: - optional: true - - vitest@4.1.6: - resolution: {integrity: sha512-6lvjbS3p9b4CrdCmguzbh2/4uoXhGE2q71R4OX5sqF9R1bo9Xd6fGrMAfvp5wnCzlBnFVdCOp6onuTQVbo8iUQ==} + vitest@4.1.8: + resolution: {integrity: sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@opentelemetry/api': ^1.9.0 '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.6 - '@vitest/browser-preview': 4.1.6 - '@vitest/browser-webdriverio': 4.1.6 - '@vitest/coverage-istanbul': 4.1.6 - '@vitest/coverage-v8': 4.1.6 - '@vitest/ui': 4.1.6 + '@vitest/browser-playwright': 4.1.8 + '@vitest/browser-preview': 4.1.8 + '@vitest/browser-webdriverio': 4.1.8 + '@vitest/coverage-istanbul': 4.1.8 + '@vitest/coverage-v8': 4.1.8 + '@vitest/ui': 4.1.8 happy-dom: '*' jsdom: '*' vite: 6.4.2 @@ -6842,11 +6543,6 @@ packages: snapshots: - '@ampproject/remapping@2.3.0': - dependencies: - '@jridgewell/gen-mapping': 0.3.13 - '@jridgewell/trace-mapping': 0.3.31 - '@apidevtools/json-schema-ref-parser@11.7.2': dependencies: '@jsdevtools/ono': 7.1.3 @@ -7154,8 +6850,6 @@ snapshots: '@babel/helper-plugin-utils@7.29.7': {} - '@babel/helper-string-parser@7.27.1': {} - '@babel/helper-string-parser@7.29.7': {} '@babel/helper-validator-identifier@7.28.5': {} @@ -7169,10 +6863,6 @@ snapshots: '@babel/template': 7.29.7 '@babel/types': 7.29.7 - '@babel/parser@7.29.2': - dependencies: - '@babel/types': 7.29.7 - '@babel/parser@7.29.7': dependencies: '@babel/types': 7.29.7 @@ -7211,11 +6901,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/types@7.29.0': - dependencies: - '@babel/helper-string-parser': 7.27.1 - '@babel/helper-validator-identifier': 7.28.5 - '@babel/types@7.29.7': dependencies: '@babel/helper-string-parser': 7.29.7 @@ -8303,8 +7988,6 @@ snapshots: dependencies: minipass: 7.1.3 - '@istanbuljs/schema@0.1.3': {} - '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -8469,153 +8152,78 @@ snapshots: optionalDependencies: rollup: 4.60.4 - '@rollup/rollup-android-arm-eabi@4.60.0': - optional: true - '@rollup/rollup-android-arm-eabi@4.60.4': optional: true - '@rollup/rollup-android-arm64@4.60.0': - optional: true - '@rollup/rollup-android-arm64@4.60.4': optional: true - '@rollup/rollup-darwin-arm64@4.60.0': - optional: true - '@rollup/rollup-darwin-arm64@4.60.4': optional: true - '@rollup/rollup-darwin-x64@4.60.0': - optional: true - '@rollup/rollup-darwin-x64@4.60.4': optional: true - '@rollup/rollup-freebsd-arm64@4.60.0': - optional: true - '@rollup/rollup-freebsd-arm64@4.60.4': optional: true - '@rollup/rollup-freebsd-x64@4.60.0': - optional: true - '@rollup/rollup-freebsd-x64@4.60.4': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.60.0': - optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.60.4': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.60.0': - optional: true - '@rollup/rollup-linux-arm-musleabihf@4.60.4': optional: true - '@rollup/rollup-linux-arm64-gnu@4.60.0': - optional: true - '@rollup/rollup-linux-arm64-gnu@4.60.4': optional: true - '@rollup/rollup-linux-arm64-musl@4.60.0': - optional: true - '@rollup/rollup-linux-arm64-musl@4.60.4': optional: true - '@rollup/rollup-linux-loong64-gnu@4.60.0': - optional: true - '@rollup/rollup-linux-loong64-gnu@4.60.4': optional: true - '@rollup/rollup-linux-loong64-musl@4.60.0': - optional: true - '@rollup/rollup-linux-loong64-musl@4.60.4': optional: true - '@rollup/rollup-linux-ppc64-gnu@4.60.0': - optional: true - '@rollup/rollup-linux-ppc64-gnu@4.60.4': optional: true - '@rollup/rollup-linux-ppc64-musl@4.60.0': - optional: true - '@rollup/rollup-linux-ppc64-musl@4.60.4': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.60.0': - optional: true - '@rollup/rollup-linux-riscv64-gnu@4.60.4': optional: true - '@rollup/rollup-linux-riscv64-musl@4.60.0': - optional: true - '@rollup/rollup-linux-riscv64-musl@4.60.4': optional: true - '@rollup/rollup-linux-s390x-gnu@4.60.0': - optional: true - '@rollup/rollup-linux-s390x-gnu@4.60.4': optional: true - '@rollup/rollup-linux-x64-gnu@4.60.0': - optional: true - '@rollup/rollup-linux-x64-gnu@4.60.4': optional: true - '@rollup/rollup-linux-x64-musl@4.60.0': - optional: true - '@rollup/rollup-linux-x64-musl@4.60.4': optional: true - '@rollup/rollup-openbsd-x64@4.60.0': - optional: true - '@rollup/rollup-openbsd-x64@4.60.4': optional: true - '@rollup/rollup-openharmony-arm64@4.60.0': - optional: true - '@rollup/rollup-openharmony-arm64@4.60.4': optional: true - '@rollup/rollup-win32-arm64-msvc@4.60.0': - optional: true - '@rollup/rollup-win32-arm64-msvc@4.60.4': optional: true - '@rollup/rollup-win32-ia32-msvc@4.60.0': - optional: true - '@rollup/rollup-win32-ia32-msvc@4.60.4': optional: true - '@rollup/rollup-win32-x64-gnu@4.60.0': - optional: true - '@rollup/rollup-win32-x64-gnu@4.60.4': optional: true - '@rollup/rollup-win32-x64-msvc@4.60.0': - optional: true - '@rollup/rollup-win32-x64-msvc@4.60.4': optional: true @@ -9571,105 +9179,58 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@3.2.4(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': + '@vitest/coverage-v8@4.1.8(vitest@4.1.8)': dependencies: - '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 - ast-v8-to-istanbul: 0.3.12 - debug: 4.4.3 + '@vitest/utils': 4.1.8 + ast-v8-to-istanbul: 1.0.3 istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 5.0.6 istanbul-reports: 3.2.0 - magic-string: 0.30.21 - magicast: 0.3.5 - std-env: 3.10.0 - test-exclude: 7.0.2 - tinyrainbow: 2.0.0 - vitest: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - transitivePeerDependencies: - - supports-color - - '@vitest/expect@3.2.4': - dependencies: - '@types/chai': 5.2.3 - '@vitest/spy': 3.2.4 - '@vitest/utils': 3.2.4 - chai: 5.3.3 - tinyrainbow: 2.0.0 + magicast: 0.5.3 + obug: 2.1.1 + std-env: 4.0.0 + tinyrainbow: 3.1.0 + vitest: 4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) - '@vitest/expect@4.1.6': + '@vitest/expect@4.1.8': dependencies: '@standard-schema/spec': 1.1.0 '@types/chai': 5.2.3 - '@vitest/spy': 4.1.6 - '@vitest/utils': 4.1.6 + '@vitest/spy': 4.1.8 + '@vitest/utils': 4.1.8 chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@3.2.4(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': + '@vitest/mocker@4.1.8(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': dependencies: - '@vitest/spy': 3.2.4 + '@vitest/spy': 4.1.8 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - '@vitest/mocker@4.1.6(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3))': - dependencies: - '@vitest/spy': 4.1.6 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - - '@vitest/pretty-format@3.2.4': - dependencies: - tinyrainbow: 2.0.0 - - '@vitest/pretty-format@4.1.6': + '@vitest/pretty-format@4.1.8': dependencies: tinyrainbow: 3.1.0 - '@vitest/runner@3.2.4': - dependencies: - '@vitest/utils': 3.2.4 - pathe: 2.0.3 - strip-literal: 3.1.0 - - '@vitest/runner@4.1.6': - dependencies: - '@vitest/utils': 4.1.6 - pathe: 2.0.3 - - '@vitest/snapshot@3.2.4': + '@vitest/runner@4.1.8': dependencies: - '@vitest/pretty-format': 3.2.4 - magic-string: 0.30.21 + '@vitest/utils': 4.1.8 pathe: 2.0.3 - '@vitest/snapshot@4.1.6': + '@vitest/snapshot@4.1.8': dependencies: - '@vitest/pretty-format': 4.1.6 - '@vitest/utils': 4.1.6 + '@vitest/pretty-format': 4.1.8 + '@vitest/utils': 4.1.8 magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/spy@3.2.4': - dependencies: - tinyspy: 4.0.4 - - '@vitest/spy@4.1.6': {} + '@vitest/spy@4.1.8': {} - '@vitest/utils@3.2.4': + '@vitest/utils@4.1.8': dependencies: - '@vitest/pretty-format': 3.2.4 - loupe: 3.2.1 - tinyrainbow: 2.0.0 - - '@vitest/utils@4.1.6': - dependencies: - '@vitest/pretty-format': 4.1.6 + '@vitest/pretty-format': 4.1.8 convert-source-map: 2.0.0 tinyrainbow: 3.1.0 @@ -9826,7 +9387,7 @@ snapshots: assertion-error@2.0.1: {} - ast-v8-to-istanbul@0.3.12: + ast-v8-to-istanbul@1.0.3: dependencies: '@jridgewell/trace-mapping': 0.3.31 estree-walker: 3.0.3 @@ -10062,8 +9623,6 @@ snapshots: bytes@3.1.2: {} - cac@6.7.14: {} - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -10089,14 +9648,6 @@ snapshots: ccount@2.0.1: {} - chai@5.3.3: - dependencies: - assertion-error: 2.0.1 - check-error: 2.1.3 - deep-eql: 5.0.2 - loupe: 3.2.1 - pathval: 2.0.1 - chai@6.2.2: {} chalk-template@1.1.2: @@ -10122,8 +9673,6 @@ snapshots: chardet@2.1.1: {} - check-error@2.1.3: {} - chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -10382,8 +9931,6 @@ snapshots: dependencies: character-entities: 2.0.2 - deep-eql@5.0.2: {} - deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -10515,8 +10062,6 @@ snapshots: es-errors@1.3.0: {} - es-module-lexer@1.7.0: {} - es-module-lexer@2.1.0: {} es-object-atoms@1.1.1: @@ -10700,12 +10245,12 @@ snapshots: optionalDependencies: eslint-config-prettier: 9.1.2(eslint@9.39.4) - eslint-plugin-vitest@0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)): + eslint-plugin-vitest@0.5.4(eslint@9.39.4)(typescript@5.9.3)(vitest@4.1.8): dependencies: '@typescript-eslint/utils': 7.18.0(eslint@9.39.4)(typescript@5.9.3) eslint: 9.39.4 optionalDependencies: - vitest: 3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) + vitest: 4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) transitivePeerDependencies: - supports-color - typescript @@ -11488,14 +11033,6 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 - istanbul-lib-source-maps@5.0.6: - dependencies: - '@jridgewell/trace-mapping': 0.3.31 - debug: 4.4.3 - istanbul-lib-coverage: 3.2.2 - transitivePeerDependencies: - - supports-color - istanbul-reports@3.2.0: dependencies: html-escaper: 2.0.2 @@ -11513,8 +11050,6 @@ snapshots: js-tokens@4.0.0: {} - js-tokens@9.0.1: {} - js-yaml@3.14.2: dependencies: argparse: 1.0.10 @@ -11654,8 +11189,6 @@ snapshots: dependencies: js-tokens: 4.0.0 - loupe@3.2.1: {} - lowlight@1.20.0: dependencies: fault: 1.0.4 @@ -11673,12 +11206,6 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - magicast@0.3.5: - dependencies: - '@babel/parser': 7.29.2 - '@babel/types': 7.29.0 - source-map-js: 1.2.1 - magicast@0.5.3: dependencies: '@babel/parser': 7.29.7 @@ -12199,10 +11726,6 @@ snapshots: dependencies: lodash: 4.18.1 - minimatch@10.2.4: - dependencies: - brace-expansion: 5.0.6 - minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -12455,8 +11978,6 @@ snapshots: pathe@2.0.3: {} - pathval@2.0.1: {} - piccolore@0.1.3: {} picocolors@1.1.1: {} @@ -12496,12 +12017,6 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postcss@8.5.8: - dependencies: - nanoid: 3.3.11 - picocolors: 1.1.1 - source-map-js: 1.2.1 - prelude-ls@1.2.1: {} prettier-linter-helpers@1.0.1: @@ -12864,37 +12379,6 @@ snapshots: dependencies: glob: 10.5.0 - rollup@4.60.0: - dependencies: - '@types/estree': 1.0.8 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.60.0 - '@rollup/rollup-android-arm64': 4.60.0 - '@rollup/rollup-darwin-arm64': 4.60.0 - '@rollup/rollup-darwin-x64': 4.60.0 - '@rollup/rollup-freebsd-arm64': 4.60.0 - '@rollup/rollup-freebsd-x64': 4.60.0 - '@rollup/rollup-linux-arm-gnueabihf': 4.60.0 - '@rollup/rollup-linux-arm-musleabihf': 4.60.0 - '@rollup/rollup-linux-arm64-gnu': 4.60.0 - '@rollup/rollup-linux-arm64-musl': 4.60.0 - '@rollup/rollup-linux-loong64-gnu': 4.60.0 - '@rollup/rollup-linux-loong64-musl': 4.60.0 - '@rollup/rollup-linux-ppc64-gnu': 4.60.0 - '@rollup/rollup-linux-ppc64-musl': 4.60.0 - '@rollup/rollup-linux-riscv64-gnu': 4.60.0 - '@rollup/rollup-linux-riscv64-musl': 4.60.0 - '@rollup/rollup-linux-s390x-gnu': 4.60.0 - '@rollup/rollup-linux-x64-gnu': 4.60.0 - '@rollup/rollup-linux-x64-musl': 4.60.0 - '@rollup/rollup-openbsd-x64': 4.60.0 - '@rollup/rollup-openharmony-arm64': 4.60.0 - '@rollup/rollup-win32-arm64-msvc': 4.60.0 - '@rollup/rollup-win32-ia32-msvc': 4.60.0 - '@rollup/rollup-win32-x64-gnu': 4.60.0 - '@rollup/rollup-win32-x64-msvc': 4.60.0 - fsevents: 2.3.3 - rollup@4.60.4: dependencies: '@types/estree': 1.0.8 @@ -13163,8 +12647,6 @@ snapshots: statuses@2.0.2: {} - std-env@3.10.0: {} - std-env@4.0.0: {} stream-replace-string@2.0.0: {} @@ -13208,10 +12690,6 @@ snapshots: strip-json-comments@3.1.1: {} - strip-literal@3.1.0: - dependencies: - js-tokens: 9.0.1 - strnum@2.3.0: {} style-to-js@1.1.21: @@ -13368,22 +12846,12 @@ snapshots: ansi-escapes: 7.3.0 supports-hyperlinks: 4.4.0 - test-exclude@7.0.2: - dependencies: - '@istanbuljs/schema': 0.1.3 - glob: 10.5.0 - minimatch: 10.2.4 - tiny-inflate@1.0.3: {} tinybench@2.9.0: {} tinyclip@0.1.13: {} - tinyexec@0.3.2: {} - - tinyexec@1.1.2: {} - tinyexec@1.2.2: {} tinyglobby@0.2.15: @@ -13396,14 +12864,8 @@ snapshots: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - tinypool@1.1.1: {} - - tinyrainbow@2.0.0: {} - tinyrainbow@3.1.0: {} - tinyspy@4.0.4: {} - to-buffer@1.2.2: dependencies: isarray: 2.0.5 @@ -13678,34 +13140,13 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite-node@3.2.4(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): - dependencies: - cac: 6.7.14 - debug: 4.4.3 - es-module-lexer: 1.7.0 - pathe: 2.0.3 - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - transitivePeerDependencies: - - '@types/node' - - jiti - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml - vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): dependencies: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.8 - rollup: 4.60.0 + postcss: 8.5.15 + rollup: 4.60.4 tinyglobby: 0.2.16 optionalDependencies: '@types/node': 20.19.41 @@ -13733,57 +13174,15 @@ snapshots: optionalDependencies: vite: 7.3.3(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - vitest@3.2.4(@types/debug@4.1.13)(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3): - dependencies: - '@types/chai': 5.2.3 - '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) - '@vitest/pretty-format': 3.2.4 - '@vitest/runner': 3.2.4 - '@vitest/snapshot': 3.2.4 - '@vitest/spy': 3.2.4 - '@vitest/utils': 3.2.4 - chai: 5.3.3 - debug: 4.4.3 - expect-type: 1.3.0 - magic-string: 0.30.21 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 3.10.0 - tinybench: 2.9.0 - tinyexec: 0.3.2 - tinyglobby: 0.2.15 - tinypool: 1.1.1 - tinyrainbow: 2.0.0 - vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - vite-node: 3.2.4(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/debug': 4.1.13 - '@types/node': 20.19.41 - transitivePeerDependencies: - - jiti - - less - - lightningcss - - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml - - vitest@4.1.6(@types/node@20.19.41)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)): + vitest@4.1.8(@types/node@20.19.41)(@vitest/coverage-v8@4.1.8)(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)): dependencies: - '@vitest/expect': 4.1.6 - '@vitest/mocker': 4.1.6(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) - '@vitest/pretty-format': 4.1.6 - '@vitest/runner': 4.1.6 - '@vitest/snapshot': 4.1.6 - '@vitest/spy': 4.1.6 - '@vitest/utils': 4.1.6 + '@vitest/expect': 4.1.8 + '@vitest/mocker': 4.1.8(vite@6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3)) + '@vitest/pretty-format': 4.1.8 + '@vitest/runner': 4.1.8 + '@vitest/snapshot': 4.1.8 + '@vitest/spy': 4.1.8 + '@vitest/utils': 4.1.8 es-module-lexer: 2.1.0 expect-type: 1.3.0 magic-string: 0.30.21 @@ -13792,13 +13191,14 @@ snapshots: picomatch: 4.0.4 std-env: 4.0.0 tinybench: 2.9.0 - tinyexec: 1.1.2 + tinyexec: 1.2.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 vite: 6.4.2(@types/node@20.19.41)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.41 + '@vitest/coverage-v8': 4.1.8(vitest@4.1.8) transitivePeerDependencies: - msw diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5d5634354..c70fa3aa3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -16,7 +16,7 @@ catalog: '@typespec/openapi3': ^1.11.0 '@typespec/rest': ^0.81.0 '@typespec/versioning': ^0.81.0 - '@vitest/coverage-v8': ^3.2.4 + '@vitest/coverage-v8': ^4.1.8 eslint: ^9.39.4 eslint-config-prettier: ^9.1.2 eslint-plugin-prettier: ^5.5.5 @@ -26,12 +26,12 @@ catalog: ts-node: ^10.9.2 typescript: ^5.9.3 typescript-eslint: ^8.59.3 - vitest: ^3.2.4 + vitest: ^4.1.8 zod: ^3.25.76 catalogs: website: - vitest: ^4.1.6 + vitest: ^4.1.8 overrides: axios@<1.15.0: '>=1.15.0' diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 151f20255..5320e1b4e 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -20,7 +20,7 @@ _How should the Plugin object be structured to support both custom field declara ### Decision drivers - The framework must be implementable in both the Python and TypeScript SDKs with as consistent an interface as possible. -- The `extensions` config must be serializable and able to pass validation (JSON-safe), and must be combinable across multiple extension packages via `mergeExtensions()` (TypeScript) / `merge_extensions()` (Python). +- The `extensions` config must be serializable and able to pass validation (JSON-safe). - Existing plugin packages that declare only custom fields should remain valid with minimal changes. - The SDK interface should support clean dependency injection — it must be possible to pass `client` or `schemas` as a coherent unit without reassembling them from per-object branches. - Auth, transport, and rate-limiting are system-level concerns that belong to a single client, not distributed across per-object branches. @@ -36,7 +36,7 @@ We decided to: 3. **Use per-object grouping inside `schemas`** where it reflects real coupling: each object's native schema, CommonGrants schema, and bidirectional transforms are tightly coupled and change together. -4. **Expand `definePlugin()` to accept all top-level fields** — `meta`, `client`, `schemas`, and `extensions` — rather than only `extensions`. `extensions` will contain any parts of the plugin that can be merged with other plugins by using `mergeExtensions()`, which has a flag for handling key conflicts. +4. **Expand `definePlugin()` to accept all top-level fields** — `meta`, `client`, `schemas`, and `extensions` — rather than only `extensions`. `extensions` holds serializable-only config (mappings, meta); all per-object declarations (custom fields, native schema, transforms) live under `schemas.<Object>`. 5. **Make all top-level Plugin fields optional** so adopters can publish a plugin that provides only the features they need — for example, custom fields only — and expand to include transforms, client config, or additional schemas incrementally over time. @@ -44,7 +44,7 @@ We decided to: 7. **`toCommon` / `fromCommon` return a `TransformResult<T>` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. In the current PoC, this validation is opt-in at the `buildTransforms()` call site via the `commonModel` / `common_model` parameter — when supplied, validation runs inside `toCommon` against the fully extended generated schema. In the full SDK, `definePlugin()` will additionally inject validation when auto-generating transforms from `extensions.schemas.<Object>.mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. -8. **Custom handlers are registered per utility call, not globally.** `buildTransforms()` accepts an optional `handlers` argument for registering additional handler names. Per-call scoping keeps behavior explicit and testable; name collisions with the default set raise at `buildTransforms()` call time rather than silently shadowing them. Handler-name lookup must not resolve inherited attributes, because mapping JSON can be reconstituted from untrusted sources via `mergeExtensions()`. +8. **Custom handlers are registered per utility call, not globally.** `buildTransforms()` accepts an optional `handlers` argument (`Map<string, Handler>`) for registering additional handler names. Per-call scoping keeps behavior explicit and testable; name collisions with the default set raise at `buildTransforms()` call time rather than silently shadowing them. The registry is a `Map` rather than a plain object so that handler-name lookup uses `Map.has()` — which does not walk the prototype chain — rather than `in` or own-property checks on a plain object. 9. **Transformation errors carry structured context.** SDK-emitted transformation errors extend a single `PluginError` base carrying field path, handler name, source value, and underlying cause, enabling programmatic reasoning without parsing error text. The source value may contain PII when transforming applicant data; adopters are responsible for redacting it before logging or re-raising, and the SDK does not redact by default. @@ -61,7 +61,7 @@ plugin.schemas.<Name>.to_common // callable: native → TransformResult[com plugin.schemas.<Name>.from_common // callable: common → TransformResult[native] (None if not configured) ``` -**Python note:** In the Python SDK, `define_plugin()` returns a `PluginConfig` (build-time input) rather than a fully compiled `Plugin`. The code generator (`generate.py`) compiles `PluginConfig → Plugin` by injecting the generated model classes as the `common` schema, wrapping `get_client` with `functools.lru_cache`, and auto-generating `build_transforms()` calls for any objects that have `extensions.schemas[obj].mappings` but no explicit `to_common`/`from_common` in `schemas[obj]`. This split is necessary because `cg_config.py` cannot import from `generated/` — it is the input to code generation. In the Python SDK, `custom_fields` is declared on `ObjectSchemasInput` (inside `schemas`) rather than on `PluginExtensionsSchema` (inside `extensions`). +**Python note:** In the Python SDK, `define_plugin()` returns a `PluginConfig` (build-time input) rather than a fully compiled `Plugin`. The code generator (`generate.py`) compiles `PluginConfig → Plugin` by injecting the generated model classes as the `common` schema, wrapping `get_client` with `functools.lru_cache`, and auto-generating `build_transforms()` calls for any objects that have `extensions.schemas[obj].mappings` but no explicit `to_common`/`from_common` in `schemas[obj]`. This split is necessary because `cg_config.py` cannot import from `generated/` — it is the input to code generation. Both SDKs declare `custom_fields` / `customFields` on `ObjectSchemasInput` (inside `schemas`), not on `PluginExtensionsSchema` / `PluginExtensionsObjectConfig` (inside `extensions`). ### Example interface @@ -111,8 +111,11 @@ interface ObjectSchemas<TNative, TCommon> { // common is intentionally absent: the plugin config file cannot import from generated/ // since it is the input to generation. definePlugin() injects common during compilation // from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. +// customFields declares extra fields beyond the base CG schema; definePlugin() extends +// the base schema with them to produce the typed common schema. interface ObjectSchemasInput<TNative = unknown, TCommon = unknown> { native?: ZodType<TNative>; // defaults to Record<string, unknown> if omitted + customFields?: Record<string, CustomFieldSpec>; toCommon?: (native: TNative) => TransformResult<TCommon>; fromCommon?: (common: TCommon) => TransformResult<TNative>; } @@ -125,9 +128,11 @@ interface CustomFilterSpec { description?: string; } -// Per-object config shape inside extensions.schemas — mirrors Python's PluginExtensionsSchema +// Per-object config shape inside extensions.schemas — mirrors Python's PluginExtensionsSchema. +// customFields is intentionally absent here; it lives on ObjectSchemasInput so that +// all per-object declarations (custom fields, native schema, transforms) are co-located +// under schemas.<Object> rather than split across two top-level keys. interface PluginExtensionsObjectConfig { - customFields?: Record<string, CustomFieldSpec>; // Optional declarative mappings in ADR-0017 format. When present and no explicit // toCommon / fromCommon is supplied in schemas.<Object>, definePlugin() auto-invokes // buildTransforms() on these. Each direction is author-provided; see Decision #6. @@ -139,7 +144,7 @@ interface PluginExtensionsObjectConfig { // Serializable portion of the plugin config — safe to store as JSON. // schemas keys are restricted to ExtensibleSchemaName (the known set of CommonGrants -// objects that support custom fields), following the existing SchemaExtensions pattern. +// objects that support custom fields). interface PluginExtensions { meta?: Partial<PluginMeta>; schemas?: Partial<Record<ExtensibleSchemaName, PluginExtensionsObjectConfig>>; @@ -197,13 +202,6 @@ interface DefinePluginOptions { // to the errors array rather than throwing (see Decision #7). function definePlugin(options: DefinePluginOptions): Plugin; -// Combine multiple extension objects (e.g. from separate packages). -// Exact signature shape (overloads, param structure) is provisional pending SDK pin in #744. -function mergeExtensions( - extensions: PluginExtensions[], - options?: { onConflict?: "error" | "firstWins" | "lastWins" }, // defaults to "error" -): PluginExtensions; - // Handler signature matches ADR-0017 runtime conventions. type Handler = (value: unknown, context: unknown) => unknown; @@ -223,7 +221,7 @@ type Handler = (value: unknown, context: unknown) => unknown; function buildTransforms<TNative, TCommon>( toCommonMapping: Record<string, unknown>, // ADR-0017 mapping from native → CommonGrants fromCommonMapping: Record<string, unknown>, // ADR-0017 mapping from CommonGrants → native - handlers?: Record<string, Handler>, + handlers?: Map<string, Handler>, // Map (not plain object) — Map.has() is prototype-safe commonModel?: ZodType<TCommon>, // must be the generated extended schema, not the base ): { toCommon: (native: TNative) => TransformResult<TCommon>; @@ -485,35 +483,27 @@ const plugin = definePlugin({ maxItems: config.maxItems, auth: config.auth, }), - extensions: { - schemas: { - Opportunity: { - customFields: { - programArea: { - fieldType: CustomFieldType.String, - description: "HHS program area code", - }, - legacyGrantId: { - fieldType: CustomFieldType.Integer, - description: "Numeric ID from the legacy grants system", - }, - }, - }, - }, - }, + // All per-object declarations — customFields, native schema, and transforms — live + // under schemas.<Object> so authors have one entry per object rather than two. schemas: { Opportunity: { native: GrantsGovOpportunitySchema, + customFields: { + programArea: { + fieldType: CustomFieldType.String, + description: "HHS program area code", + }, + legacyGrantId: { + fieldType: CustomFieldType.Integer, + description: "Numeric ID from the legacy grants system", + }, + }, toCommon, fromCommon, }, }, }); -// Combine extensions from multiple packages before constructing the plugin -const merged = mergeExtensions([baseExtensions, grantsGovExtensions]); -const mergedPlugin = definePlugin({ extensions: merged }); - // Calling getClient() with a config object — memoized, so repeated calls return the same instance const client = plugin.getClient({ auth: Auth.bearer("token"), pageSize: 50 }); ``` @@ -668,7 +658,7 @@ print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms", " - Client stays singular — `getClient()` / `get_client()` is memoized (in TypeScript by `definePlugin()`; in Python by the code generator wrapping it with `functools.lru_cache`), so one source system always produces one `Client` instance regardless of how many times it is called - Top-level surface (`meta`, `client`, `schemas`, `extensions`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only - Dependency injection works along functional lines: pass `getClient`, pass `Schemas`, pass `Extensions` as coherent units without needing to reassemble from per-object branches - - `mergeExtensions()` / `merge_extensions()` operates on flat, serializable data at the root, not on deeply nested per-object branches + - `extensions` (serializable config) is flat and JSON-safe — it can be stored, versioned, or passed across package boundaries without carrying runtime Zod/Pydantic schemas - Per-object grouping inside `schemas` preserves the real coupling between native schema, CommonGrants schema, and bidirectional transforms — they share type signatures and change together - Mirrors the SDK module structure (`client`, `schemas`, `extensions`), so Plugin reads as a system-specific version of the existing SDK rather than a different mental model - `toCommon`/`fromCommon` can be plain hand-written functions, generated via `buildTransforms()` and passed in `schemas`, or auto-generated by `definePlugin()` from mappings declared in `extensions` — plugin authors are not required to use a declarative mapping format From c654e3e0713c90800612a70e7db6b654b796cca1 Mon Sep 17 00:00:00 2001 From: jcrichlake <145698165+jcrichlake@users.noreply.github.com> Date: Mon, 8 Jun 2026 16:40:24 -0400 Subject: [PATCH 14/51] Expanding transform logic (#880) * Expanding transform logic * Update transforms doctoring Co-authored-by: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> * Updating test file * Updating tests --------- Co-authored-by: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> --- .changeset/transforms-poc-typescript.md | 6 +- .../extensions/define-plugin.spec.ts | 203 +++++++++++++++++- .../__tests__/extensions/transforms.spec.ts | 85 ++++++++ lib/ts-sdk/src/extensions/define-plugin.ts | 55 ++++- lib/ts-sdk/src/extensions/transforms.ts | 47 +++- lib/ts-sdk/src/extensions/types.ts | 12 +- 6 files changed, 389 insertions(+), 19 deletions(-) diff --git a/.changeset/transforms-poc-typescript.md b/.changeset/transforms-poc-typescript.md index 4729ae771..a13cc10dd 100644 --- a/.changeset/transforms-poc-typescript.md +++ b/.changeset/transforms-poc-typescript.md @@ -11,6 +11,7 @@ Add a TypeScript proof-of-concept for the plugin transformation framework (issue - `PluginError` — structured error class carrying `path`, `handler`, `sourceValue`, `cause`. - `transformFromMapping()`, `getFromPath()`, `DEFAULT_HANDLERS` — lower-level mapping runtime pieces; `DEFAULT_HANDLERS` is a `Map<string, Handler>` of six built-in handlers (`const`, `field`, `match` / `switch` alias, `numberToString`, `stringToNumber`). - `definePlugin()` accepts optional `meta: PluginMeta` and `schemas: SchemasInput`. All per-object declarations (custom fields, native schema, transforms) are co-located under `schemas[Object]` — `customFields` lives on `schemas[Object].customFields` rather than on the `extensions` key. The compiled `plugin.schemas[Object].common` holds the extended Zod schema. +- `definePlugin()` **auto-wires transforms** from declarative `extensions.schemas[Name].mappings` at call time when no explicit `toCommon`/`fromCommon` callables are provided in `schemas[Name]`. The auto-wired path also runs `validateOutputPaths()` against the resolved schema (base or extended) so key-name mismatches are caught at `definePlugin()` call time rather than at runtime. Auto-wiring is all-or-nothing per object: any explicit callable disables it for that object. - New supporting types: `Handler`, `SchemasInput`, `ObjectSchemasInput`, `ObjectSchemas`, `PluginMeta`, `PluginCapability`, `ObjectMappings`, `PluginExtensionsObjectConfig`, `PluginExtensions`. **Three-state null handling (ADR-0024)** for optional fields: @@ -24,9 +25,8 @@ Add a TypeScript proof-of-concept for the plugin transformation framework (issue - `mergeExtensions` has been removed from the public surface. Consumers who previously used `mergeExtensions` to combine extension objects should merge them manually (e.g. with object spread) before passing to `definePlugin`. -**Out of scope** (deferred to full SDK): +**Deferred to full SDK:** -- Auto-generation of transforms from declarative `extensions.schemas[obj].mappings` inside `definePlugin()`. -- Always-on `commonModel` validation inside `definePlugin()` — opt-in at `buildTransforms()` for now. +- Always-on `commonModel` validation inside `definePlugin()` — opt-in at `buildTransforms()` call site for now (pass the fully extended schema as `commonModel` to enable Zod validation on `toCommon` output). Runnable example: `pnpm --filter @common-grants/sdk example:transforms` (round-trips a synthetic grants.gov record through `toCommon` and `fromCommon` with custom `join` / `split` handlers, extended-schema validation, and three-state null preservation on `source_url: null`). diff --git a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts index 0e6f0b729..d462cf42c 100644 --- a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { z } from "zod"; import { buildTransforms, definePlugin, type PluginMeta, type TransformResult } from "@/extensions"; import { OpportunityBaseSchema } from "@/schemas/zod/models"; @@ -288,3 +288,204 @@ describe("definePlugin", () => { }); }); }); + +// ############################################################################ +// Auto-wiring from mappings +// ############################################################################ + +// Mappings used across auto-wiring tests. toCommon covers all required fields of +// OpportunityBaseSchema so that Zod validation (run because `common` is passed as +// commonModel in the auto-wiring call) does not produce errors. +// `status: { value: { const: "open" } }` produces `{ value: "open" }` because +// `value` is an output key and `{ const: "open" }` dispatches the const handler. +const autoWireToCommonMapping = { + id: { field: "native_id" }, + title: { field: "native_title" }, + description: { const: "Test opportunity" }, + createdAt: { const: "2025-01-01T00:00:00Z" }, + lastModifiedAt: { const: "2025-01-01T00:00:00Z" }, + status: { value: { const: "open" } }, +}; +const autoWireFromCommonMapping = { + native_title: { field: "title" }, + native_id: { field: "id" }, +}; + +describe("definePlugin — auto-wiring from mappings", () => { + it("auto-generates working toCommon/fromCommon from extensions.schemas.Opportunity.mappings", () => { + const plugin = definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + fromCommon: autoWireFromCommonMapping, + }, + }, + }, + }, + }); + + const nativeData = { + native_id: "573525f2-8e15-4405-83fb-e6523511d893", + native_title: "Test Opp", + }; + const result = plugin.schemas.Opportunity.toCommon?.(nativeData); + expect(result?.errors).toHaveLength(0); + expect(result?.result).toMatchObject({ + id: "573525f2-8e15-4405-83fb-e6523511d893", + title: "Test Opp", + }); + }); + + it("explicit callables take priority — any explicit callable disables auto-wiring for that object", () => { + const explicitToCommon = vi.fn( + (_: unknown): TransformResult<unknown> => ({ result: { id: "explicit" }, errors: [] }) + ); + const explicitFromCommon = vi.fn( + (_: unknown): TransformResult<unknown> => ({ result: {}, errors: [] }) + ); + + const plugin = definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + fromCommon: autoWireFromCommonMapping, + }, + }, + }, + }, + schemas: { + Opportunity: { + toCommon: explicitToCommon, + fromCommon: explicitFromCommon, + }, + }, + }); + + plugin.schemas.Opportunity.toCommon?.({}); + expect(explicitToCommon).toHaveBeenCalledOnce(); + }); + + it("disables auto-wiring when only one explicit callable is provided", () => { + const explicitToCommon = vi.fn( + (_: unknown): TransformResult<unknown> => ({ result: {}, errors: [] }) + ); + + const plugin = definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + fromCommon: autoWireFromCommonMapping, + }, + }, + }, + }, + schemas: { + Opportunity: { toCommon: explicitToCommon }, + }, + }); + + // fromCommon is not auto-wired because explicitToCommon disables auto-wiring + expect(plugin.schemas.Opportunity.fromCommon).toBeUndefined(); + plugin.schemas.Opportunity.toCommon?.({}); + expect(explicitToCommon).toHaveBeenCalledOnce(); + }); + + it("throws at definition time when mappings.fromCommon is absent", () => { + expect(() => + definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + // fromCommon intentionally absent + }, + }, + }, + }, + }) + ).toThrow(/Opportunity\.mappings\.fromCommon is required/); + }); + + it("throws at definition time when mappings.toCommon is absent", () => { + expect(() => + definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + // toCommon intentionally absent + fromCommon: autoWireFromCommonMapping, + }, + }, + }, + }, + }) + ).toThrow(/Opportunity\.mappings\.toCommon is required/); + }); + + it("auto-wired toCommon rejects unknown output fields via validateOutputPaths", () => { + expect(() => + definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + toCommon: { unknownFieldXyz: { field: "data.x" } }, + fromCommon: autoWireFromCommonMapping, + }, + }, + }, + }, + }) + ).toThrow(/unknown output fields.*"unknownFieldXyz"/); + }); + + it("preserves native schema on schemas[Name] in the auto-wired branch", () => { + const nativeSchema = z.object({ native_id: z.string(), native_title: z.string() }); + + const plugin = definePlugin({ + extensions: { + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + fromCommon: autoWireFromCommonMapping, + }, + }, + }, + }, + schemas: { + Opportunity: { + native: nativeSchema, + }, + }, + }); + + expect(plugin.schemas.Opportunity.native).toBe(nativeSchema); + }); + + it("no-ops cleanly when extensions is absent entirely", () => { + const plugin = definePlugin({}); + expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); + expect(plugin.schemas.Opportunity.fromCommon).toBeUndefined(); + }); + + it("no-ops cleanly when extensions.schemas has no key for this model name", () => { + const plugin = definePlugin({ extensions: { schemas: {} } }); + expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); + }); + + it("no-ops cleanly when extensions.schemas[Name].mappings is undefined", () => { + const plugin = definePlugin({ + extensions: { schemas: { Opportunity: { mappings: undefined } } }, + }); + expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); + }); +}); diff --git a/lib/ts-sdk/__tests__/extensions/transforms.spec.ts b/lib/ts-sdk/__tests__/extensions/transforms.spec.ts index e06b617bf..145a6ff25 100644 --- a/lib/ts-sdk/__tests__/extensions/transforms.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/transforms.spec.ts @@ -508,3 +508,88 @@ describe("buildTransforms — custom handlers", () => { ); }); }); + +// ############################################################################ +// Output key validation (commonModel provided) +// ############################################################################ + +describe("buildTransforms — output key validation (commonModel provided)", () => { + it("throws when a top-level mapping key is not a field on the schema", () => { + expect(() => + buildTransforms({ titlee: { field: "data.title" } }, {}, undefined, OpportunityBaseSchema) + ).toThrow(/unknown output fields.*"titlee"/); + }); + + it("accepts all valid top-level schema field names", () => { + expect(() => + buildTransforms({ title: { field: "data.title" } }, {}, undefined, OpportunityBaseSchema) + ).not.toThrow(); + }); + + it("accepts valid fields on a schema extended with withCustomFields()", () => { + const extended = withCustomFields(OpportunityBaseSchema, { + legacyId: { fieldType: CustomFieldType.string }, + }); + // Base fields remain valid on the extended schema + expect(() => + buildTransforms({ title: { field: "data.title" } }, {}, undefined, extended) + ).not.toThrow(); + }); + + it("does not throw when commonModel is absent", () => { + // No commonModel → output key validation is skipped entirely + expect(() => buildTransforms({ notAField: { field: "data.x" } }, {})).not.toThrow(); + }); + + it("silently passes when commonModel is not a ZodObject", () => { + // ZodRecord is not instanceof ZodObject — permissive fallback, no throw + const nonObjectModel = z.record(z.unknown()); + expect(() => + buildTransforms({ anyKey: { field: "data.x" } }, {}, undefined, nonObjectModel) + ).not.toThrow(); + }); + + it("does not validate fromCommonMapping output keys", () => { + // fromCommon maps to native — no commonModel shape to validate against + expect(() => + buildTransforms( + { title: { field: "data.title" } }, + { notAField: { field: "title" } }, + undefined, + OpportunityBaseSchema + ) + ).not.toThrow(); + }); + + it("reports multiple unknown fields in the same error", () => { + expect( + () => + buildTransforms( + { titlee: { field: "data.title" }, descripton: { field: "data.desc" } }, + {}, + undefined, + OpportunityBaseSchema + ) + // Keys are sorted before joining, so "descripton" comes before "titlee" + ).toThrow(/unknown output fields.*"descripton".*"titlee"/); + }); + + it("rejects a custom-field name placed at the top level, even on the extended schema", () => { + const extended = withCustomFields(OpportunityBaseSchema, { + legacyId: { fieldType: CustomFieldType.string }, + }); + // legacyId lives under `customFields`, never as a top-level key + expect(() => + buildTransforms({ legacyId: { field: "data.x" } }, {}, undefined, extended) + ).toThrow(/unknown output fields.*"legacyId"/); + }); + + it("accepts a custom field mapped under the top-level customFields key", () => { + const extended = withCustomFields(OpportunityBaseSchema, { + legacyId: { fieldType: CustomFieldType.string }, + }); + expect(() => + buildTransforms({ customFields: { legacyId: { field: "data.x" } } }, {}, undefined, extended) + ).not.toThrow(); + }); +}); diff --git a/lib/ts-sdk/src/extensions/define-plugin.ts b/lib/ts-sdk/src/extensions/define-plugin.ts index dcefae23a..cb0240b5d 100644 --- a/lib/ts-sdk/src/extensions/define-plugin.ts +++ b/lib/ts-sdk/src/extensions/define-plugin.ts @@ -14,6 +14,7 @@ import type { } from "./types"; import { EXTENSIBLE_SCHEMA_MAP } from "./types"; import { withCustomFields, type WithCustomFieldsResult } from "./with-custom-fields"; +import { buildTransforms } from "./transforms"; // ############################################################################ // Public types - SchemasInput, DefinePluginOptions, Plugin @@ -60,8 +61,10 @@ export interface DefinePluginOptions<T extends SchemasInput = SchemasInput> { * * This is the single surface for custom field declarations. * - * Stored as-is in the PoC (no compilation, no Zod-wrap); the full SDK compiles - * `ObjectSchemasInput` → `ObjectSchemas` and injects the generated `common` model. + * `definePlugin()` compiles this into runtime schemas: `common` is built via + * `withCustomFields()` when `customFields` are declared; `toCommon` / `fromCommon` + * are auto-wired from `extensions.schemas[Name].mappings` when no explicit + * callables are provided. Native input Zod-wrapping remains deferred. */ schemas?: T; } @@ -132,12 +135,48 @@ export function definePlugin<const T extends SchemasInput>( specs && Object.keys(specs).length > 0 ? withCustomFields(extensibleSchema, specs) : extensibleSchema; - schemas[name] = { - common, - native: schemasInput?.[name]?.native, - toCommon: schemasInput?.[name]?.toCommon, - fromCommon: schemasInput?.[name]?.fromCommon, - }; + + const explicitToCommon = schemasInput?.[name]?.toCommon; + const explicitFromCommon = schemasInput?.[name]?.fromCommon; + const mappings = extensions?.schemas?.[name]?.mappings; + const nativeSchema = schemasInput?.[name]?.native; + + // Auto-wire from declarative mappings when no explicit callables are supplied. + // All-or-nothing: any explicit callable disables auto-wiring for this object + // (mirrors Python's explicit_objs vs mappings_objs logic in generate.py). + let toCommon = explicitToCommon; + let fromCommon = explicitFromCommon; + + if ( + mappings !== undefined && + explicitToCommon === undefined && + explicitFromCommon === undefined + ) { + if (mappings.toCommon === undefined) { + throw new Error( + `definePlugin: ${name}.mappings.toCommon is required when auto-generating transforms. ` + + `Either provide both mapping directions or pass explicit toCommon/fromCommon callables.` + ); + } + if (mappings.fromCommon === undefined) { + throw new Error( + `definePlugin: ${name}.mappings.fromCommon is required when auto-generating transforms. ` + + `Either provide both mapping directions or pass explicit toCommon/fromCommon callables.` + ); + } + // Pass `common` as commonModel so validateOutputPaths runs against the + // fully-resolved schema (base or extended). Key-existence checking is correct + // regardless of whether customFields were declared — this is not the same as + // the "base schema weakens custom-field type validation" warning in buildTransforms' + // JSDoc, which applies to Zod-type checking of custom field values at runtime. + const built = buildTransforms(mappings.toCommon, mappings.fromCommon, undefined, common); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + toCommon = built.toCommon as any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fromCommon = built.fromCommon as any; + } + + schemas[name] = { common, native: nativeSchema, toCommon, fromCommon }; } // Cast is safe — the runtime loop mirrors the PluginSchemas<T> mapped type, diff --git a/lib/ts-sdk/src/extensions/transforms.ts b/lib/ts-sdk/src/extensions/transforms.ts index 3b4699117..73a012b9e 100644 --- a/lib/ts-sdk/src/extensions/transforms.ts +++ b/lib/ts-sdk/src/extensions/transforms.ts @@ -26,8 +26,9 @@ import { DEFAULT_HANDLERS, HandlerError, transformFromMapping } from "./transfor * Handler arguments are runtime-only and skipped — they may legitimately be * arrays, deeply nested specs, or anything else the handler accepts. The * walker can't detect *unknown* handler names at static analysis time (an - * unknown key is indistinguishable from an output field name); that - * detection is deferred to the full SDK. + * unknown key is indistinguishable from an output field name); detection of + * unknown top-level output keys is now handled by `validateOutputPaths` when + * `commonModel` is provided; detection at arbitrary nesting depth remains deferred. * * Sibling keys at a handler-dispatch node are rejected here. The runtime * walker is first-key-wins, so `{ field: "x", const: "fallback" }` would @@ -75,6 +76,45 @@ function validateMapping(mapping: unknown, knownHandlers: Set<string>, path = "" } } +/** + * Validate that every top-level output key in `mapping` that is not a known + * handler name is a real field on `commonModel`. + * + * Only runs when `commonModel` is an instance of `z.ZodObject` — when it is + * not (e.g. `ZodRecord`, `ZodUnion`), returns without error. In practice all + * schemas produced by `withCustomFields()` and `OpportunityBaseSchema` are + * `ZodObject`s (`.extend()` / `.merge()` preserve the concrete type), so the + * fallback is a safety net, not an expected code path. + * + * Called only for `toCommonMapping` — `fromCommonMapping` maps to native + * format whose shape is unknown to this layer. + * + * Cross-SDK note: Python raises `ValueError`; TypeScript uses `Error` for + * all structural build-time failures (consistent with `validateMapping`). + * + * @internal + */ +function validateOutputPaths( + mapping: Record<string, unknown>, + knownHandlers: Set<string>, + // Bivariant `any` at the input position matches buildTransforms's own + // commonModel signature and avoids a contravariant `unknown` type error. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + commonModel: z.ZodType<unknown, z.ZodTypeDef, any> +): void { + if (!(commonModel instanceof z.ZodObject)) return; + const validNames = new Set(Object.keys(commonModel.shape)); + const outputKeys = Object.keys(mapping).filter(k => !knownHandlers.has(k)); + const invalid = outputKeys.filter(k => !validNames.has(k)); + if (invalid.length === 0) return; + throw new Error( + `buildTransforms (toCommonMapping): unknown output fields ${JSON.stringify(invalid.sort())} for schema. ` + + `If these are custom fields, map them under the top-level "customFields" key ` + + `(e.g. { customFields: { yourField: ... } }), not as top-level keys, and declare them in schemas[Object].customFields. ` + + `Otherwise check the field name.` + ); +} + // ############################################################################ // Public - buildTransforms // ############################################################################ @@ -169,6 +209,9 @@ export function buildTransforms<TNative = unknown, TCommon = unknown>( // fail at build time, not on first invocation. validateMapping(toCommonMapping, known); validateMapping(fromCommonMapping, known); + if (commonModel !== undefined) { + validateOutputPaths(toCommonMapping, known, commonModel); + } const runMapping = ( data: unknown, diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index 0ac44be9c..a3b02ff0c 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -267,9 +267,11 @@ export interface ObjectSchemasInput<TNative = unknown, TCommon = unknown> { /** * Runtime compiled type produced by `definePlugin()` — not provided directly by authors. * - * In the PoC, `definePlugin()` stores `ObjectSchemasInput` as-is on the returned - * plugin's `transformSchemas` field. Full compilation (adding `common` from the - * base CG model, wrapping with Zod validation) is deferred to the full SDK. + * `definePlugin()` now compiles `ObjectSchemasInput` into this shape: `common` is + * injected from the base CG model (extended via `withCustomFields()` when custom + * fields are declared), and `toCommon` / `fromCommon` are auto-wired from declarative + * mappings when no explicit callables are provided. Native input Zod-wrapping remains + * deferred. */ export interface ObjectSchemas<TNative, TCommon> { native: z.ZodType<TNative>; @@ -315,8 +317,8 @@ export interface ObjectMappings { * * `mappings` carries optional declarative mappings; when present and no * explicit `toCommon` / `fromCommon` is supplied in - * `DefinePluginOptions.schemas`, `definePlugin()` will auto-invoke - * `buildTransforms()` on these. Deferred to the full SDK. + * `DefinePluginOptions.schemas`, `definePlugin()` auto-invokes + * `buildTransforms()` on these at call time. * * @remarks * `customFields` lives on {@link ObjectSchemasInput} (inside From debd55b678d57b18cfd0303435a3eaef1188f91b Mon Sep 17 00:00:00 2001 From: jcrichlake <jeff.crichlake@agile6.com> Date: Thu, 11 Jun 2026 15:24:36 -0400 Subject: [PATCH 15/51] Update examples file --- .../examples/plugins/grants_gov/cg_config.py | 25 +++++++++++++++++++ lib/python-sdk/examples/transforms.py | 18 ++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py index 4000d845d..d78a3f9d9 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -9,6 +9,8 @@ poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov """ +from typing import Any + from common_grants_sdk.extensions import ( CustomFieldSpec, ObjectSchemasInput, @@ -17,6 +19,15 @@ define_plugin, ) from common_grants_sdk.schemas.pydantic.fields import CustomFieldType +from common_grants_sdk.utils.transformation import get_from_path + + +def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + sep = spec.get("sep", " ") + parts = [get_from_path(data, path) for path in spec.get("fields", [])] + values = [str(p) for p in parts if p is not None] + return sep.join(values) if values else None + # --------------------------------------------------------------------------- # Bidirectional transforms @@ -32,6 +43,7 @@ # --------------------------------------------------------------------------- to_common, from_common = build_transforms( + handlers={"join": _join_fields}, # to_common: grants.gov native → CommonGrants Opportunity to_common_mapping={ "title": {"field": "data.opportunity_title"}, @@ -82,6 +94,14 @@ "priorityScore": { "value": {"stringToNumber": "data.priority_score_str"}, }, + "compositeLabel": { + "value": { + "join": { + "fields": ["data.opportunity_number", "data.opportunity_title"], + "sep": " — ", + } + }, + }, }, }, # from_common: CommonGrants Opportunity → grants.gov native @@ -151,6 +171,11 @@ name="Priority score", description="Numeric priority score coerced from a string via stringToNumber", ), + "compositeLabel": CustomFieldSpec( + field_type=CustomFieldType.STRING, + name="Composite label", + description="Composite label '<opportunity_number> — <opportunity_title>' for round-trip recovery", + ), }, to_common=to_common, from_common=from_common, diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index fc1dac7a3..26002e5d9 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -112,13 +112,15 @@ def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: } }, }, - "label": { - "join": { - "fields": ["data.opportunity_number", "data.opportunity_title"], - "sep": " — ", - } - }, "customFields": { + "compositeLabel": { + "value": { + "join": { + "fields": ["data.opportunity_number", "data.opportunity_title"], + "sep": " — ", + } + }, + }, "legacyId": { "value": {"field": "data.opportunity_id"}, }, @@ -264,6 +266,10 @@ def main() -> None: print( f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})" ) + if cf.composite_label: + print( + f" compositeLabel.value: {cf.composite_label.value!r} ({type(cf.composite_label.value).__name__})" + ) custom_native = from_common_with_custom( custom_cg.result if not custom_cg.errors else {} From 3f255c4aa97153e240d054ef6f4e00612a6e207d Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:21:34 -0700 Subject: [PATCH 16/51] Custom filters TS PoC (#868) (#884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(01-01): add failing tests for BooleanComparisonFilterSchema - Import BooleanComparisonFilterSchema from @/schemas - Add BooleanComparisonFilter Schema describe block with 4 cases: eq+true succeeds, neq+false succeeds, gt throws, string value throws * feat(01-01): add BooleanComparisonFilterSchema to filters Zod schemas (D-04) - Add BooleanComparisonFilterSchema using EquivalenceOperatorsEnum (eq|neq) and z.boolean() - Place after Money section with section-header comment style - No lib/core changes per D-04 (SDK/Zod level only) * feat(01-01): add CustomFilterType enum and filter declaration types (D-11) - Add 11-value CustomFilterType string-literal union (authoritative *Comparison naming) - Add CustomFilterSpec interface (filterType + optional description; no value field) - Add RouteDeclarations interface (optional filters record) - Add PluginRoutes type (resource → method → RouteDeclarations) - Placement: after CustomFieldSpec block, before HasCustomFields section * test(01-01): add failing routes pass-through tests for definePlugin - Import PluginRoutes type from @/extensions - Add describe("routes") block with 3 it() cases: routes pass through unchanged, routes undefined when omitted, routes accessible for later classification * feat(01-01): add routes key to definePlugin and barrel-export filter types (D-10) - Add PluginRoutes to import block in define-plugin.ts - Add routes?: PluginRoutes to DefinePluginOptions and Plugin interfaces - Destructure routes in definePlugin() and pass through in returned object - Add CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations to extensions/index.ts barrel (types only; runtime exports in Plan 02) * feat(01-02): implement custom-filters classifier, validators, and F helpers - classifyFilters: three-bucket consumer→OppFilters wire body (default→top-level, registered/ad-hoc→customFilters) - validateRoutes: registration-time validation (unknown filterType, duplicate names, default-name collisions) → PluginError - validateFilterCall: call-time validation (operator/schema for registered; shape-only for ad-hoc) → PluginError - F namespace: 12 helpers (eq, neq, gt, gte, lt, lte, in, notIn, like, notLike, between, outside) → {operator, value} - FILTER_TYPE_SCHEMAS map covers all 11 CustomFilterType values; integerComparison reuses NumberComparisonFilterSchema (finding candidate) - gov.<system>@<filterName> namespaced keys pass through to customFilters verbatim (D-16) * test(01-02): add Vitest coverage for classifyFilters, validators, and F helpers - describe("classifyFilters") with sub-blocks: three-bucket classification (7 tests including exact wire-body deep-equal), registration-time validation (5 tests for unknown filterType, default-name collisions, valid routes), call-time validation (8 tests for op/type mismatch, shape mismatch, ad-hoc passthrough) - describe("F helpers") with 14 tests covering all 12 helpers + DefaultFilterSchema compatibility - Uses @/ path alias; file-scope fixtures for grantsGovRoutes and mixedConsumerFilters - Runtime coverage only; compile-time as const narrowing tests deferred to Plan 03 * feat(01-02): add runtime barrel exports for custom-filters classifier and F helpers - Export classifyFilters, validateRoutes, validateFilterCall, F from ./custom-filters - Update comment from placeholder "Runtime exports added in Plan 02" to active exports - Type exports for CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations retained from Plan 01 * feat(01-03): add compile-time as const narrowing type-tests (D-13) - Add custom-filters-types.spec.ts: four @ts-expect-error assertions proving unknown key, operator/filterType mismatch, value-shape, and array-shape mismatches are compile errors WITH as const (check:types exit 0 = proof) - Add WITHOUT-as-const widening trap demonstration via Plugin type annotation - Extend Plugin<T> with second generic TRoutes to preserve literal routes type; backward-compatible (default = PluginRoutes); required for TypedConsumerFilters narrowing to propagate from definePlugin() return type (D-13) - Add FilterValueShape, FilterOperator, TypedFilter, TypedConsumerFilters, RouteFilterSpecs utility types as the compile-time narrowing layer * feat(01-03): add canonical grants.gov custom-filters example + wire ci script - Add examples/custom-filters.ts: runnable grants.gov demo with agency (stringArray), fundingProgram (stringComparison), ad-hoc legacyTag, and default status/closeDateRange; prints classified OppFilters wire body; assertions verify three-bucket classification; no network I/O - Wire example:custom-filters script in package.json and append to ci chain - Non-null assertion on plugin.routes for classifyFilters call (routes? is optional on Plugin interface; example declares it so the assertion is safe) * docs(01-04): add Plugin custom filters (PoC) section to extensions README - Add "Plugin custom filters (PoC)" section mirroring the transforms-PoC precedent - Document routes-vs-schemas critical distinction (filters on methods, fields on schemas) - Include filter-type catalog (11 values), F.* helpers table, classifyFilters usage - Add validation and as-const trap subsections with code examples - Add "Custom filters (PoC)" API reference table for all four new exports - Update table of contents with new section and API ref entry - Prettier-clean (check:format passes on README) * style(01): apply Prettier to phase 01 custom-filters files Plan 01-03 executors passed check:types and tests but never ran prettier --write; the whole-package check:format gate (part of pnpm ci) was failing on 6 files. Formatting only — no behavioral change. * fix(01-03): place @ts-expect-error inside literals so the build catches them Prettier reflow (commit e6a21c9) pushed the directives above the `const bad = {` line, but tsc reports object-literal excess-property/assignability errors on the PROPERTY line — so the directives went unused (TS2578) and the real errors surfaced, failing `pnpm build` (and any fresh consumer install). Moving each directive directly above its offending property is prettier-stable and matches where tsc reports, restoring the as-const narrowing proof under the real build. * docs(01): drop internal planning references from custom-filters comments Comments carried planning-process artifacts a reader of the published SDK cannot resolve: decision/task IDs, internal plan numbers, point-in-time "PoC" framing, a tech-spec reference, and one stray tooling-file reference. Rewrite to durable, self-contained comments — every ADR reference and the technical substance (e.g. integerComparison reusing NumberComparisonFilterSchema) is preserved. Comment-only; no behavior change. * fix(01): satisfy eslint — reference type-only fixture, drop stale disable CI check:lint (the one check not run before pushing) failed on: - custom-filters-types.spec.ts: `grantsGovPlugin` is consumed only via `typeof`; the eslint config sets no varsIgnorePattern, so reference it with `void` to satisfy no-unused-vars. - custom-filters.ts: the `in` helper takes `unknown[]`, not `any` — its no-explicit-any disable directive was unused; removed. * docs(01): fix validateFilterCall signature and range filter shape in extensions README The call-time validation example documented a five-argument validateFilterCall(routes, resource, method, name, value) that does not exist; the real signature is validateFilterCall(spec, filterName, filterValue). Correct the example to look the spec up from the route-method and pass the three real arguments. Also fix the F.between examples: range filters serialize their value as { min, max } (matching NumberRangeFilterSchema / DateRangeFilterSchema / MoneyRangeFilterSchema and the catalog table), not as a [min, max] array. * docs(01): correct classifyFilters default-bucket validation comment The inline comment claimed default fields are validated against their own schemas from OppDefaultFiltersSchema. They are not: default keys get shape-only validation via DefaultFilterSchema, the same treatment as ad-hoc keys. Update the comment to describe the actual behavior. * fix(ts-sdk): enforce int values for integerComparison; drop unused operator metadata and unreachable duplicate check - Add IntegerComparisonFilterSchema (.int()) so integerComparison no longer reuses the looser NumberComparisonFilterSchema - Flatten FILTER_TYPE_SCHEMAS to a plain CustomFilterType -> Zod schema map: the operatorEnum entries were never read (each filter schema already constrains its operator enum, so one parse covers operator + value) - Remove the duplicate-name check in validateRoutes: filter names are object keys, and JS object literals cannot represent duplicate keys * fix(ts-sdk): correct compile-time filter map for money types; lock with new compile-error tests The conditional-type chains diverged from the runtime schemas where no @ts-expect-error test exercised them: - moneyComparison operators were between|outside; MoneyComparisonFilterSchema accepts gt|gte|lt|lte (ComparisonOperatorsEnum) - moneyComparison/moneyRange values used amount: number; MoneySchema.amount is a decimal string Replace the two parallel operator/value conditional chains with a single FilterTypeMap lookup interface (one row per filterType, directly comparable to its Zod schema; the TypedFilter indexed access doubles as a completeness guard) and add two moneyComparison @ts-expect-error assertions so the row can't drift silently again. * docs(ts-sdk): correct operator catalog, clarify classification entry points, use request-body wording - Catalog table: dateComparison and moneyComparison support only gt|gte|lt|lte (eq|neq rows were wrong); integerComparison value is int - Explain that definePlugin registers declarations while classifyFilters is the search-time step the full SDK client will call internally (standalone here because client wiring is out of scope, tracked under #645), and that the PoC's definePlugin passes routes through unvalidated - Merge the routes-vs-schemas bullets into one combined definePlugin example - Cross-reference the transforms PII warning for PluginError.sourceValue - Describe the classifier output as the request body sent to the API rather than 'wire body' (README, example, test names) * ci(ts-sdk): drop custom-filters example from the ci script The example is slated to be reworked against a real endpoint, so the ci script should not gate on it. It remains runnable manually via `pnpm example:custom-filters`. * refactor(ts-sdk): reduce custom-filters type tests to pure compile-time assertions The vitest wrapper around these assertions could never fail at runtime — the real gate is `tsc --noEmit`, where each @ts-expect-error directive guards a line that only errors while `as const` narrowing is in effect (removing the narrowing makes the directive unused, ts2578, failing the type check). Drop the runtime test scaffolding and the widening-demo block (which compiled clean by design and asserted nothing), and rename .spec.ts -> .ts so vitest no longer collects the file. The as-const trap remains documented in the extensions README and the example's comment block. * fix(ts-sdk): revert integerComparison to NumberComparisonFilterSchema The spec defines no integer filter model (lib/core numeric filters are NumberComparisonFilter / NumberRangeFilter / NumberArrayFilter only), so the SDK-side IntegerComparisonFilterSchema added in 21127a9 had no spec counterpart. The spec is the source of truth and changes flow from the spec to the SDK; an int-enforcing schema can follow a spec change that introduces one. integerComparison validates as a plain number again. * docs(ts-sdk): drop PoC status banners from transforms and custom-filters sections The extensions README sections should read as ready-to-use once published; remove the two status blockquotes and the (PoC) heading markers along with their TOC entries. --- .../extensions/custom-filters-types.ts | 166 ++++++++ .../extensions/custom-filters.spec.ts | 355 ++++++++++++++++++ .../extensions/define-plugin.spec.ts | 54 ++- .../__tests__/schemas/zod/filters.spec.ts | 29 ++ lib/ts-sdk/examples/custom-filters.ts | 192 ++++++++++ lib/ts-sdk/package.json | 3 +- lib/ts-sdk/src/extensions/README.md | 193 +++++++++- lib/ts-sdk/src/extensions/custom-filters.ts | 302 +++++++++++++++ lib/ts-sdk/src/extensions/define-plugin.ts | 53 ++- lib/ts-sdk/src/extensions/index.ts | 4 + lib/ts-sdk/src/extensions/types.ts | 81 ++++ lib/ts-sdk/src/schemas/zod/filters.ts | 9 + 12 files changed, 1429 insertions(+), 12 deletions(-) create mode 100644 lib/ts-sdk/__tests__/extensions/custom-filters-types.ts create mode 100644 lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts create mode 100644 lib/ts-sdk/examples/custom-filters.ts create mode 100644 lib/ts-sdk/src/extensions/custom-filters.ts diff --git a/lib/ts-sdk/__tests__/extensions/custom-filters-types.ts b/lib/ts-sdk/__tests__/extensions/custom-filters-types.ts new file mode 100644 index 000000000..cdcba93c8 --- /dev/null +++ b/lib/ts-sdk/__tests__/extensions/custom-filters-types.ts @@ -0,0 +1,166 @@ +/** + * Compile-time type-narrowing assertions for the custom-filters `as const` surface. + * + * Checked by `tsc --noEmit` only (`pnpm --filter @common-grants/sdk run check:types`) + * — there is no runtime test. Each `@ts-expect-error` directive IS the assertion: + * it guards a line that actually fails to compile when `as const` narrowing is in + * effect. If the narrowing ever regresses, the guarded line compiles, the directive + * becomes unused (ts2578), and the type-check gate fails. + */ + +import { definePlugin, type CustomFilterSpec, type PluginRoutes } from "@/extensions"; + +// ############################################################################ +// Typed narrowing utilities +// ############################################################################ + +/** Compile-time mirror of `MoneySchema` — `amount` is a decimal STRING, not a number. */ +type Money = { amount: string; currency: string }; + +/** + * Maps each `CustomFilterType` literal to its typed `{operator, value}` filter shape. + * + * Compile-time mirror of the Zod schemas in `src/schemas/zod/filters.ts` (via + * FILTER_TYPE_SCHEMAS in `custom-filters.ts`) — each row is directly comparable + * to its runtime schema. + */ +interface FilterTypeMap { + stringComparison: { operator: "eq" | "neq" | "like" | "notLike"; value: string }; + stringArray: { operator: "in" | "notIn"; value: string[] }; + numberComparison: { operator: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"; value: number }; + numberArray: { operator: "in" | "notIn"; value: number[] }; + numberRange: { operator: "between" | "outside"; value: { min: number; max: number } }; + // TS has no integer type — the spec defines no integer filter model, so the + // value is a plain number at runtime too (NumberComparisonFilterSchema) + integerComparison: { operator: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"; value: number }; + booleanComparison: { operator: "eq" | "neq"; value: boolean }; + dateComparison: { operator: "gt" | "gte" | "lt" | "lte"; value: string }; + dateRange: { operator: "between" | "outside"; value: { min: string; max: string } }; + moneyComparison: { operator: "gt" | "gte" | "lt" | "lte"; value: Money }; + moneyRange: { operator: "between" | "outside"; value: { min: Money; max: Money } }; +} + +/** + * Typed `{operator, value}` pair for a single `CustomFilterSpec`. + * + * The indexed access doubles as a completeness guard: it fails to compile + * (ts2536) if `FilterTypeMap` is missing or misnames any `CustomFilterType` key. + */ +type TypedFilter<TSpec extends CustomFilterSpec> = FilterTypeMap[TSpec["filterType"]]; + +/** + * Narrows a `PluginRoutes` constant to the `filters` record for a specific + * route + method. Returns `never` when the `filterType` values widened to + * `string` (i.e. the plugin was NOT defined `as const`). + */ +type RouteFilterSpecs< + TRoutes extends PluginRoutes, + R extends keyof TRoutes & string, + M extends keyof TRoutes[R] & string, +> = TRoutes[R][M] extends { filters: infer F } + ? F extends Record<string, CustomFilterSpec> + ? F + : never + : never; + +/** + * Maps each registered filter key to its `TypedFilter` shape. Keys NOT in + * `TSpecs` are compile errors — the mapped type has no index signature. + */ +type TypedConsumerFilters<TSpecs extends Record<string, CustomFilterSpec>> = { + [K in keyof TSpecs]?: TypedFilter<TSpecs[K]>; +}; + +// ############################################################################ +// Plugin under test — defined WITH `as const`, the load-bearing form that +// preserves literal `filterType` values for per-key narrowing. +// ############################################################################ + +const grantsGovPlugin = definePlugin({ + meta: { name: "grants.gov", version: "0.1.0", sourceSystem: "grants.gov" }, + routes: { + opportunities: { + search: { + filters: { + agency: { filterType: "stringArray", description: "Filter by funding agency code" }, + fundingProgram: { + filterType: "stringComparison", + description: "Filter by funding program name", + }, + awardCount: { + filterType: "numberComparison", + description: "Filter by number of awards", + }, + awardFloor: { + filterType: "moneyComparison", + description: "Filter by minimum award amount", + }, + }, + }, + }, + }, +} as const); + +// `grantsGovPlugin` is consumed only via `typeof` in GrantsGovSpecs below. +void grantsGovPlugin; + +// The narrowed filter-specs record that TypedConsumerFilters maps over. +type GrantsGovSpecs = RouteFilterSpecs< + NonNullable<typeof grantsGovPlugin.routes>, + "opportunities", + "search" +>; + +// ############################################################################ +// The six compile-error assertions +// ############################################################################ + +// (1) Unknown filter key — TypedConsumerFilters<GrantsGovSpecs> has only the +// registered keys; there is no index signature for anything else. +const unknownKey: TypedConsumerFilters<GrantsGovSpecs> = { + // @ts-expect-error -- unknown key 'notARealFilter' is not a registered filter (ts2353) + notARealFilter: { operator: "eq", value: "x" }, +}; + +// (2) Operator/filterType mismatch — awardCount is numberComparison +// ("eq" | "neq" | "gt" | "gte" | "lt" | "lte"); "like" is a string operator. +const wrongOperator: TypedConsumerFilters<GrantsGovSpecs> = { + // @ts-expect-error -- "like" is a string operator, not valid for numberComparison (ts2322) + awardCount: { operator: "like", value: 5 }, +}; + +// (3) Value-shape mismatch — numberComparison value must be a number. +const wrongValueShape: TypedConsumerFilters<GrantsGovSpecs> = { + // @ts-expect-error -- awardCount is numberComparison; value must be number, not string (ts2322) + awardCount: { operator: "eq", value: "notANumber" }, +}; + +// (4) Array-value-shape mismatch — stringArray value must be string[]. +const scalarForArray: TypedConsumerFilters<GrantsGovSpecs> = { + // @ts-expect-error -- agency is stringArray; value must be string[], not a scalar string (ts2322) + agency: { operator: "in", value: "HHS" }, +}; + +// (5) Range operator on a comparison filter — moneyComparison allows only +// "gt" | "gte" | "lt" | "lte" (MoneyComparisonFilterSchema uses ComparisonOperatorsEnum). +const rangeOnComparison: TypedConsumerFilters<GrantsGovSpecs> = { + // @ts-expect-error -- "between" is a range operator, not valid for moneyComparison (ts2322) + awardFloor: { operator: "between", value: { amount: "100000.00", currency: "USD" } }, +}; + +// (6) Money.amount as a number — MoneySchema.amount is a decimal STRING +// (DecimalStringSchema). +const numericMoneyAmount: TypedConsumerFilters<GrantsGovSpecs> = { + // @ts-expect-error -- Money.amount is a decimal string, not a number (ts2322) + awardFloor: { operator: "gt", value: { amount: 100000, currency: "USD" } }, +}; + +// The consts exist only to host the assertions above. +void [ + unknownKey, + wrongOperator, + wrongValueShape, + scalarForArray, + rangeOnComparison, + numericMoneyAmount, +]; diff --git a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts new file mode 100644 index 000000000..07c88d2fa --- /dev/null +++ b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts @@ -0,0 +1,355 @@ +import { describe, expect, it } from "vitest"; +import { classifyFilters, F, validateFilterCall, validateRoutes } from "@/extensions"; +import { PluginError } from "@/extensions"; +import type { PluginRoutes } from "@/extensions"; +import { OppFiltersSchema } from "@/schemas/zod/models"; +import { DefaultFilterSchema } from "@/schemas/zod/filters"; + +// ############################################################################ +// File-scope fixtures +// ############################################################################ + +/** + * Grants.gov plugin routes — the canonical demo from the spec (CONTEXT.md §Specific Ideas): + * - `agency` → stringArray (custom, registered) + * - `fundingProgram` → stringComparison (custom, registered) + */ +const grantsGovRoutes: PluginRoutes = { + opportunities: { + search: { + filters: { + agency: { filterType: "stringArray", description: "Filter by funding agency" }, + fundingProgram: { + filterType: "stringComparison", + description: "Filter by funding program", + }, + }, + }, + }, +}; + +/** Mixed consumer filters: default + registered custom + ad-hoc */ +const mixedConsumerFilters: Record<string, unknown> = { + // Bucket 1 — default filter fields (top-level request-body fields) + status: { operator: "in", value: ["open", "closed"] }, + closeDateRange: { operator: "between", value: { min: "2025-01-01", max: "2025-12-31" } }, + // Bucket 2 — registered custom filters → customFilters record + agency: { operator: "in", value: ["HHS", "DOE"] }, + fundingProgram: { operator: "like", value: "SBIR%" }, + // Bucket 3 — ad-hoc filter (not registered, not a default field) + legacyTag: { operator: "eq", value: "legacy-2024" }, +}; + +// ############################################################################ +// classifyFilters tests +// ############################################################################ + +describe("classifyFilters", () => { + // ############################################################################ + // Three-bucket classification + // ############################################################################ + + describe("three-bucket classification", () => { + it("routes default filters to top-level named request-body fields", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + status: { operator: "in", value: ["open"] }, + }); + + expect(result.status).toEqual({ operator: "in", value: ["open"] }); + expect(result.customFilters).toBeUndefined(); + }); + + it("routes pre-registered custom filters to customFilters record", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + agency: { operator: "in", value: ["HHS"] }, + }); + + expect(result.customFilters?.agency).toEqual({ operator: "in", value: ["HHS"] }); + expect(result.status).toBeUndefined(); + }); + + it("routes ad-hoc filters to customFilters passthrough (no registration required)", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + legacyTag: { operator: "eq", value: "legacy-2024" }, + }); + + expect(result.customFilters?.legacyTag).toEqual({ operator: "eq", value: "legacy-2024" }); + }); + + it("builds exact ADR-0012 OppFilters request body for mixed default + custom + ad-hoc input", () => { + const result = classifyFilters( + grantsGovRoutes, + "opportunities", + "search", + mixedConsumerFilters + ); + + // Assert the exact request body shape + const expected: ReturnType<typeof OppFiltersSchema.parse> = { + status: { operator: "in", value: ["open", "closed"] }, + closeDateRange: { operator: "between", value: { min: "2025-01-01", max: "2025-12-31" } }, + customFilters: { + agency: { operator: "in", value: ["HHS", "DOE"] }, + fundingProgram: { operator: "like", value: "SBIR%" }, + legacyTag: { operator: "eq", value: "legacy-2024" }, + }, + }; + + expect(result).toEqual(expected); + }); + + it("passes gov.<system>@<filterName> namespaced keys through to customFilters verbatim", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + "gov.grants@announcementType": { operator: "eq", value: "NOFO" }, + }); + + expect(result.customFilters?.["gov.grants@announcementType"]).toEqual({ + operator: "eq", + value: "NOFO", + }); + }); + + it("returns only top-level fields when no custom or ad-hoc filters are provided", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + status: { operator: "in", value: ["open"] }, + }); + + expect(result).not.toHaveProperty("customFilters"); + }); + + it("handles an empty filters object gracefully", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", {}); + + expect(result).toEqual({}); + }); + }); + + // ############################################################################ + // Registration-time validation (validateRoutes) + // ############################################################################ + + describe("registration-time validation", () => { + it("throws PluginError on unknown filterType", () => { + const badRoutes: PluginRoutes = { + opportunities: { + search: { + filters: { + // @ts-expect-error — intentionally passing an invalid filterType to test runtime validation + myFilter: { filterType: "invalidType" }, + }, + }, + }, + }; + + expect(() => validateRoutes(badRoutes)).toThrow(PluginError); + }); + + it("PluginError for unknown filterType includes the path and sourceValue", () => { + const badRoutes: PluginRoutes = { + opportunities: { + search: { + filters: { + // @ts-expect-error — intentionally passing an invalid filterType + myFilter: { filterType: "unknownType" }, + }, + }, + }, + }; + + try { + validateRoutes(badRoutes); + expect.fail("Expected PluginError to be thrown"); + } catch (err) { + expect(err).toBeInstanceOf(PluginError); + const pluginErr = err as PluginError; + expect(pluginErr.path).toBe("routes.opportunities.search.filters.myFilter"); + expect(pluginErr.sourceValue).toMatchObject({ filterType: "unknownType" }); + } + }); + + it("throws PluginError when custom filter name collides with a default filter name (status)", () => { + const collidingRoutes: PluginRoutes = { + opportunities: { + search: { + filters: { + // `status` is a default-filter field name — collision + status: { filterType: "stringComparison" }, + }, + }, + }, + }; + + expect(() => validateRoutes(collidingRoutes)).toThrow(PluginError); + }); + + it("throws PluginError when custom filter name collides with closeDateRange", () => { + const collidingRoutes: PluginRoutes = { + opportunities: { + search: { + filters: { + closeDateRange: { filterType: "dateRange" }, + }, + }, + }, + }; + + expect(() => validateRoutes(collidingRoutes)).toThrow(PluginError); + }); + + it("does not throw for valid routes", () => { + expect(() => validateRoutes(grantsGovRoutes)).not.toThrow(); + }); + }); + + // ############################################################################ + // Call-time validation (validateFilterCall) + // ############################################################################ + + describe("call-time validation", () => { + it("throws PluginError on operator/filterType mismatch for a registered filter", () => { + // `like` operator is not valid for numberComparison (only gt/gte/lt/lte/eq/neq) + const spec = { filterType: "numberComparison" } as const; + + expect(() => validateFilterCall(spec, "amount", { operator: "like", value: "100" })).toThrow( + PluginError + ); + }); + + it("throws PluginError on value-shape mismatch for a registered stringArray filter", () => { + // stringArray requires value to be string[]; passing a plain string fails + const spec = { filterType: "stringArray" } as const; + + expect(() => + validateFilterCall(spec, "agency", { operator: "in", value: "not-an-array" }) + ).toThrow(PluginError); + }); + + it("throws PluginError on value-shape mismatch for a registered numberComparison filter", () => { + // numberComparison requires value to be a number; passing a string fails + const spec = { filterType: "numberComparison" } as const; + + expect(() => + validateFilterCall(spec, "amount", { operator: "eq", value: "not-a-number" }) + ).toThrow(PluginError); + }); + + it("passes a valid integerComparison filter without throwing", () => { + const spec = { filterType: "integerComparison" } as const; + + expect(() => + validateFilterCall(spec, "awardCount", { operator: "eq", value: 2 }) + ).not.toThrow(); + }); + + it("PluginError path includes the filter name", () => { + const spec = { filterType: "stringArray" } as const; + + try { + validateFilterCall(spec, "agency", { operator: "in", value: "wrong" }); + expect.fail("Expected PluginError to be thrown"); + } catch (err) { + expect(err).toBeInstanceOf(PluginError); + expect((err as PluginError).path).toBe("filters.agency"); + } + }); + + it("passes a valid registered filter without throwing", () => { + const spec = { filterType: "stringArray" } as const; + + expect(() => + validateFilterCall(spec, "agency", { operator: "in", value: ["HHS", "DOE"] }) + ).not.toThrow(); + }); + + it("passes an ad-hoc filter through with only a shape check (no operator enforcement)", () => { + // Ad-hoc (spec=undefined) — any valid DefaultFilter shape passes + expect(() => + validateFilterCall(undefined, "legacyTag", { operator: "eq", value: "legacy-2024" }) + ).not.toThrow(); + }); + + it("throws PluginError for an ad-hoc filter with an invalid shape", () => { + // Missing `operator` key — fails DefaultFilterSchema shape check (operator is required/enum) + expect(() => validateFilterCall(undefined, "badFilter", { value: "something" })).toThrow( + PluginError + ); + }); + + it("throws PluginError for an ad-hoc filter with an unknown operator", () => { + // `superCustomOp` is not in AllOperatorsEnum — fails DefaultFilterSchema + expect(() => + validateFilterCall(undefined, "badFilter", { + operator: "superCustomOp", + value: "x", + }) + ).toThrow(PluginError); + }); + }); +}); + +// ############################################################################ +// F helpers tests +// ############################################################################ + +describe("F helpers", () => { + it("F.eq compiles to { operator: 'eq', value }", () => { + expect(F.eq("open")).toEqual({ operator: "eq", value: "open" }); + }); + + it("F.neq compiles to { operator: 'neq', value }", () => { + expect(F.neq("closed")).toEqual({ operator: "neq", value: "closed" }); + }); + + it("F.gt compiles to { operator: 'gt', value }", () => { + expect(F.gt(100)).toEqual({ operator: "gt", value: 100 }); + }); + + it("F.gte compiles to { operator: 'gte', value }", () => { + expect(F.gte(100)).toEqual({ operator: "gte", value: 100 }); + }); + + it("F.lt compiles to { operator: 'lt', value }", () => { + expect(F.lt(500)).toEqual({ operator: "lt", value: 500 }); + }); + + it("F.lte compiles to { operator: 'lte', value }", () => { + expect(F.lte(500)).toEqual({ operator: "lte", value: 500 }); + }); + + it("F.in compiles to { operator: 'in', value: [...] }", () => { + expect(F.in(["HHS", "DOE"])).toEqual({ operator: "in", value: ["HHS", "DOE"] }); + }); + + it("F.notIn compiles to { operator: 'notIn', value: [...] }", () => { + expect(F.notIn(["archived"])).toEqual({ operator: "notIn", value: ["archived"] }); + }); + + it("F.like compiles to { operator: 'like', value }", () => { + expect(F.like("SBIR%")).toEqual({ operator: "like", value: "SBIR%" }); + }); + + it("F.notLike compiles to { operator: 'notLike', value }", () => { + expect(F.notLike("TEST%")).toEqual({ operator: "notLike", value: "TEST%" }); + }); + + it("F.between compiles to { operator: 'between', value: { min, max } }", () => { + expect(F.between(100, 500)).toEqual({ + operator: "between", + value: { min: 100, max: 500 }, + }); + }); + + it("F.outside compiles to { operator: 'outside', value: { min, max } }", () => { + expect(F.outside(0, 50)).toEqual({ + operator: "outside", + value: { min: 0, max: 50 }, + }); + }); + + it("F.* helpers produce values compatible with DefaultFilterSchema", () => { + // Spot-check that F helper outputs satisfy the request-body schema + expect(() => DefaultFilterSchema.parse(F.eq("test"))).not.toThrow(); + expect(() => DefaultFilterSchema.parse(F.in(["a", "b"]))).not.toThrow(); + expect(() => DefaultFilterSchema.parse(F.between(100, 500))).not.toThrow(); + }); +}); diff --git a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts index d462cf42c..316224f6a 100644 --- a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts @@ -1,6 +1,12 @@ import { describe, it, expect, vi } from "vitest"; import { z } from "zod"; -import { buildTransforms, definePlugin, type PluginMeta, type TransformResult } from "@/extensions"; +import { + buildTransforms, + definePlugin, + type PluginMeta, + type TransformResult, + type PluginRoutes, +} from "@/extensions"; import { OpportunityBaseSchema } from "@/schemas/zod/models"; import { CustomFieldType } from "@/constants"; @@ -194,6 +200,52 @@ describe("definePlugin", () => { }); }); + // ############################################################################ + // routes + // ############################################################################ + + describe("routes", () => { + it("passes routes through to plugin.routes unchanged", () => { + const routes: PluginRoutes = { + opportunities: { + search: { + filters: { + agency: { filterType: "stringArray" }, + fundingProgram: { filterType: "stringComparison", description: "Filter by program" }, + }, + }, + }, + }; + + const plugin = definePlugin({ routes } as const); + + expect(plugin.routes).toBe(routes); + }); + + it("plugin.routes is undefined when routes not provided", () => { + const plugin = definePlugin({}); + + expect(plugin.routes).toBeUndefined(); + }); + + it("routes are accessible for later filter classification (structure round-trip)", () => { + const plugin = definePlugin({ + routes: { + opportunities: { + search: { + filters: { + agency: { filterType: "stringArray" }, + }, + }, + }, + }, + } as const); + + const filterSpec = plugin.routes?.opportunities?.search?.filters?.agency; + expect(filterSpec?.filterType).toBe("stringArray"); + }); + }); + // ############################################################################ // schemas (transform callables) // ############################################################################ diff --git a/lib/ts-sdk/__tests__/schemas/zod/filters.spec.ts b/lib/ts-sdk/__tests__/schemas/zod/filters.spec.ts index 211523818..2e2605bf2 100644 --- a/lib/ts-sdk/__tests__/schemas/zod/filters.spec.ts +++ b/lib/ts-sdk/__tests__/schemas/zod/filters.spec.ts @@ -17,6 +17,7 @@ import { DateRangeFilterSchema, MoneyComparisonFilterSchema, MoneyRangeFilterSchema, + BooleanComparisonFilterSchema, } from "@/schemas"; // ############################################################################ @@ -486,3 +487,31 @@ describe("MoneyRangeFilter Schema", () => { ).toThrow(); }); }); + +// ############################################################################ +// Boolean comparison filter +// ############################################################################ + +describe("BooleanComparisonFilter Schema", () => { + it("should validate eq operator with true value", () => { + expect(BooleanComparisonFilterSchema.parse({ operator: "eq", value: true })).toEqual({ + operator: "eq", + value: true, + }); + }); + + it("should validate neq operator with false value", () => { + expect(BooleanComparisonFilterSchema.parse({ operator: "neq", value: false })).toEqual({ + operator: "neq", + value: false, + }); + }); + + it("should raise an error for gt operator (not in eq|neq)", () => { + expect(() => BooleanComparisonFilterSchema.parse({ operator: "gt", value: true })).toThrow(); + }); + + it("should raise an error when value is a string instead of boolean", () => { + expect(() => BooleanComparisonFilterSchema.parse({ operator: "eq", value: "true" })).toThrow(); + }); +}); diff --git a/lib/ts-sdk/examples/custom-filters.ts b/lib/ts-sdk/examples/custom-filters.ts new file mode 100644 index 000000000..edb9e9018 --- /dev/null +++ b/lib/ts-sdk/examples/custom-filters.ts @@ -0,0 +1,192 @@ +/** + * Example script demonstrating custom-filters registration and classification. Shows: + * 1. Defining a grants.gov plugin with route-keyed custom filter specs via + * `definePlugin({ routes: { ... } } as const)`. + * 2. Building a unified consumer `filters` object mixing default fields + * (status, closeDateRange), pre-registered custom filters (agency, + * fundingProgram), and an ad-hoc filter (legacyTag) — using `F.*` helpers. + * Control fields (query, maxResults, signal, schema) stay OUTSIDE `filters`. + * 3. Calling `classifyFilters` to produce the ADR-0012 request body: + * default fields at top-level, custom + ad-hoc under `customFilters`. + * 4. A COMMENT block (not executed) demonstrating the `as const` widening + * trap — see custom-filters-types.ts for the compile-time narrowing assertions. + * + * Run with: `pnpm example:custom-filters` + * + * @remarks + * The three-bucket classification rule (ADR-0012): + * - Default filters (status, closeDateRange, ...) → named top-level fields on the request body. + * - Pre-registered custom filters (agency, fundingProgram) → `customFilters` record. + * - Ad-hoc filters (legacyTag) → `customFilters` passthrough (shape-only validated). + * + * No network I/O: this example BUILDS the request body and prints it; it does NOT + * send it over the wire — transport is not handled here. + */ + +import { classifyFilters, definePlugin, F } from "../src/extensions"; + +// ############################################################################ +// Step 1 — Define the grants.gov plugin with route-keyed custom filters +// ############################################################################ + +// `as const` is load-bearing: it preserves literal `filterType` values so +// that TypeScript can narrow call-site filter keys, operators, and value shapes. +// Without `as const`, `filterType` widens to `string` and the typed guard is lost. +// See custom-filters-types.ts for the compile-time proof. +const grantsGovPlugin = definePlugin({ + meta: { + name: "grants.gov", + version: "0.1.0", + sourceSystem: "grants.gov", + capabilities: ["customFilters"], + }, + routes: { + opportunities: { + search: { + filters: { + agency: { + filterType: "stringArray", + description: "Filter by funding agency code (e.g. 'HHS', 'DOE')", + }, + fundingProgram: { + filterType: "stringComparison", + description: "Filter by funding program name", + }, + // NOTE: `as const` is what makes these `filterType` values + // literal strings ("stringArray", "stringComparison") instead of + // widened `string`. Without it, the typed narrowing layer collapses. + }, + }, + }, + }, +} as const); + +console.log("=== Step 1: Plugin registered ==="); +console.log(`Plugin name: ${grantsGovPlugin.meta?.name}`); +console.log( + `Registered filters: ${Object.keys(grantsGovPlugin.routes?.opportunities?.search?.filters ?? {}).join(", ")}` +); + +// ############################################################################ +// Step 2 — Build a unified consumer filters object +// ############################################################################ + +// The consumer-facing `filters` object is flat — it mixes all three filter types +// without the caller needing to know which bucket each belongs to. +// +// IMPORTANT: control fields (query, maxResults, signal, schema) stay OUTSIDE +// the `filters` object. The classifier handles only the filter predicates. +const searchParams = { + // Control fields — outside `filters` + query: "conservation research", + maxResults: 25, + + // The flat consumer filters object + filters: { + // Default filters — will land as named top-level fields on the request body + status: F.in(["open", "forecasted"]), + closeDateRange: F.between("2025-01-01", "2025-12-31"), + + // Pre-registered custom filters — will land in customFilters record + agency: F.in(["HHS", "DOE", "NSF"]), + fundingProgram: F.like("*Conservation*"), + + // Ad-hoc filter (not registered in the plugin) — flows to customFilters verbatim + // (shape-only validated; operator/filterType not enforced for ad-hoc keys) + legacyTag: F.eq("conservation-2024"), + }, +}; + +console.log("\n=== Step 2: Consumer filters built ==="); +console.log("Consumer-facing filters (flat):"); +console.log(JSON.stringify(searchParams.filters, null, 2)); + +// ############################################################################ +// Step 3 — Classify into the ADR-0012 OppFilters request body +// ############################################################################ + +// `classifyFilters` runs the three-bucket classification: +// 1. status, closeDateRange → top-level named request-body fields +// 2. agency, fundingProgram (registered) → customFilters record +// 3. legacyTag (ad-hoc) → customFilters passthrough +// `grantsGovPlugin.routes` is non-null here — we declared it above as a +// non-optional literal object. The `!` assertion removes the `undefined` from +// the union type that `Plugin.routes?:` introduces (routes is optional in the +// interface to support plugins that don't declare filters). +const requestBody = classifyFilters( + grantsGovPlugin.routes!, + "opportunities", + "search", + searchParams.filters +); + +console.log("\n=== Step 3: Classified request body (OppFilters) ==="); +console.log(JSON.stringify(requestBody, null, 2)); + +// ############################################################################ +// Assertions — verify the three-bucket classification +// ############################################################################ + +function fail(message: string): never { + console.error(`ASSERTION FAILED: ${message}`); + process.exit(1); +} + +// Default filters must appear as named top-level fields +if (!requestBody.status) fail("status should be a top-level field (default filter bucket)"); +if (!requestBody.closeDateRange) + fail("closeDateRange should be a top-level field (default filter bucket)"); + +// Custom + ad-hoc filters must land in customFilters record +if (!requestBody.customFilters) fail("customFilters should exist for registered + ad-hoc filters"); +if (!requestBody.customFilters.agency) + fail("agency should be in customFilters (registered custom filter)"); +if (!requestBody.customFilters.fundingProgram) + fail("fundingProgram should be in customFilters (registered custom filter)"); +if (!requestBody.customFilters.legacyTag) + fail("legacyTag should be in customFilters (ad-hoc passthrough)"); + +// Default filter keys must NOT appear under customFilters +if ((requestBody.customFilters as Record<string, unknown>).status) + fail("status must NOT be in customFilters — it is a default filter"); + +console.log("\n=== Assertions passed ==="); +console.log(" status → top-level (default filter bucket)"); +console.log(" closeDateRange → top-level (default filter bucket)"); +console.log(" agency → customFilters (registered custom filter)"); +console.log(" fundingProgram → customFilters (registered custom filter)"); +console.log(" legacyTag → customFilters (ad-hoc passthrough)"); + +// ############################################################################ +// Step 4 — The `as const` widening trap (comment block — not executed) +// ############################################################################ + +// If you forget `as const` on the definePlugin call, TypeScript widens the +// `filterType` values from literal strings to the broad `string` type. The +// TypedConsumerFilters narrowing layer then cannot distinguish filter keys or +// validate operator/value shapes at the call site — unknown keys and wrong +// value types silently pass the type checker. +// +// The compile-time narrowing assertions (the errors that fire WITH `as const`) +// live in: lib/ts-sdk/__tests__/extensions/custom-filters-types.ts +// +// Example of what NOT to do: +// +// const badPlugin = definePlugin({ +// routes: { +// opportunities: { +// search: { filters: { agency: { filterType: "stringArray" } } }, +// }, +// }, +// }); // ← MISSING `as const` +// +// // When `badPlugin` is stored as `Plugin` (no TRoutes generic), +// // or when `as const` is omitted and the function's const-generic cannot +// // infer the literal, the typed guard collapses and the following would NOT +// // be caught at compile time: +// // badPlugin.routes?.opportunities?.search?.filters // type: Record<string, CustomFilterSpec> +// // // → literal filterTypes are lost, so per-key narrowing is impossible +// // // → unknown keys, wrong operators, and wrong value shapes silently accepted + +console.log("\n✓ custom-filters example complete"); +console.log(" See __tests__/extensions/custom-filters-types.ts for compile-time narrowing proof"); diff --git a/lib/ts-sdk/package.json b/lib/ts-sdk/package.json index 488b922fd..c434f7002 100644 --- a/lib/ts-sdk/package.json +++ b/lib/ts-sdk/package.json @@ -82,7 +82,8 @@ "example:get-custom-fields": "tsx examples/get-opportunity-with-custom-fields.ts", "example:plugins": "tsx examples/plugins.ts", "example:server": "tsx examples/mock-api-server.ts", - "example:transforms": "tsx examples/transforms.ts" + "example:transforms": "tsx examples/transforms.ts", + "example:custom-filters": "tsx examples/custom-filters.ts" }, "dependencies": { "zod": "catalog:" diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index b03b8c5c7..e9b3a1812 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -23,7 +23,7 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor - [Defining a plugin](#defining-a-plugin) - [Publishing a plugin](#publishing-a-plugin) - [Using plugins with the API client](#using-plugins-with-the-api-client) -- [Plugin transformations (PoC)](#plugin-transformations-poc) +- [Plugin transformations](#plugin-transformations) - [Defining bidirectional transforms](#defining-bidirectional-transforms) - [Built-in mapping handlers](#built-in-mapping-handlers) - [Null handling](#null-handling) @@ -31,6 +31,13 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor - [Validating against the extended schema](#validating-against-the-extended-schema) - [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) - [Error handling](#error-handling) +- [Plugin custom filters](#plugin-custom-filters) + - [Routes vs. schemas — a critical distinction](#routes-vs-schemas--a-critical-distinction) + - [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) + - [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) + - [Classifying consumer filters into the request body](#classifying-consumer-filters-into-the-request-body) + - [Validation — registration-time and call-time](#validation--registration-time-and-call-time) + - [The `as const` trap](#the-as-const-trap) - [Best practices](#best-practices) - [Export value schemas alongside your plugin](#export-value-schemas-alongside-your-plugin) - [Use `peerDependencies` for `@common-grants/sdk`](#use-peerdependencies-for-common-grantssdk) @@ -39,6 +46,7 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor - [Plugin creation](#plugin-creation) - [Schema utilities](#schema-utilities) - [Transforms (PoC)](#transforms-poc) + - [Custom filters (PoC)](#custom-filters-poc) - [Shared types](#shared-types) ## Key concepts @@ -400,9 +408,7 @@ const results = await client.opportunities.search({ The `schema` option is accepted by `get()`, `list()`, and `search()`. When omitted, the client falls back to `OpportunityBaseSchema` (with untyped `customFields`). -## Plugin transformations (PoC) - -> **Status:** Proof-of-concept (issue [#798](https://github.com/HHS/simpler-grants-protocol/issues/798)). Mirrors the Python PoC in [PR #810](https://github.com/HHS/simpler-grants-protocol/pull/810). Contract follows [ADR-0022](https://commongrants.org/governance/adr/0022-plugin-framework/) and [ADR-0017](https://commongrants.org/governance/adr/0017-mapping-format/). +## Plugin transformations Plugins can declare bidirectional transforms that convert between a source system's native shape and the CommonGrants protocol. `toCommon` maps `native → CommonGrants`; `fromCommon` reverses it. Both directions are author-provided — the SDK does not invert one into the other, because many-to-one handlers (like `match`) are not reversible. @@ -595,6 +601,172 @@ for (const err of out.errors) { > **PII warning (ADR-0022 Decision #9):** The SDK does **not** redact by default. `PluginError.sourceValue` and `cause` are plain enumerable fields and flow through `JSON.stringify(err)`, `util.inspect(err)`, `console.log(err)`, and any logger that enumerates own properties. `sourceValue` is populated with the entire input record passed to `toCommon` / `fromCommon` — not just the value at the failing field. Log a redacted projection instead — e.g. `{ name: err.name, message: err.message, path: err.path, handler: err.handler }`. On the Zod-validation path (when `commonModel` is passed to `buildTransforms()`), `PluginError.message` is also data-bearing — Zod's default error map embeds the rejected value into `issue.message`, which flows verbatim into `PluginError.message`. Redact `message` alongside `sourceValue` and `cause`. Full-message sanitization is tracked under [#744](https://github.com/HHS/simpler-grants-protocol/issues/744). +## Plugin custom filters + +Plugins can declare custom filter specs per route-method. At search time, a single flat consumer `filters` object is classified into the body of the search request sent to the API (the `OppFilters` shape from ADR-0012): default fields land as named top-level fields; custom and ad-hoc keys land under `customFilters`. + +### Routes vs. schemas — a critical distinction + +> [!IMPORTANT] +> Custom **filters** attach to resource **methods** (routes), not to schemas. Custom **fields** attach to schemas. These two extension points use different keys on the same `definePlugin()` call: +> +> ```typescript +> definePlugin({ +> routes: { opportunities: { search: { filters: { ... } } } }, // custom filters (this section) +> schemas: { Opportunity: { customFields: { ... } } }, // custom fields (see "Build-time with plugins") +> } as const); +> ``` +> +> The reason: filter declarations vary per route-method (e.g. `opportunities.search` may support different filters than `applications.list`), whereas custom fields are schema-level concerns shared across all operations that return the model. + +### Declaring custom filters on a route + +Pass `routes` alongside (or instead of) `schemas` in `definePlugin()`. Always add `as const` — it is load-bearing for compile-time narrowing (see [The `as const` trap](#the-as-const-trap)): + +```typescript +import { definePlugin } from "@common-grants/sdk/extensions"; + +const grantsGovPlugin = definePlugin({ + meta: { + name: "grants.gov", + version: "0.1.0", + sourceSystem: "grants.gov", + capabilities: ["customFilters"], + }, + routes: { + opportunities: { + search: { + filters: { + agency: { + filterType: "stringArray", + description: "Filter by funding agency code (e.g. 'HHS', 'DOE')", + }, + fundingProgram: { + filterType: "stringComparison", + description: "Filter by funding program name", + }, + }, + }, + }, + }, +} as const); // ← `as const` is load-bearing — see "The as const trap" below +``` + +Each filter entry is a `CustomFilterSpec`: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType` — you do not author them. + +### Filter-type catalog and the `F.*` helpers + +`CustomFilterType` is an 11-value enum. Each value maps to a base filter schema with auto-derived operators: + +| `filterType` | Operators | Value shape | +| ------------------- | ------------------------------------- | -------------- | +| `stringComparison` | `eq`, `neq`, `like`, `notLike` | `string` | +| `stringArray` | `in`, `notIn` | `string[]` | +| `numberComparison` | `eq`, `neq`, `gt`, `gte`, `lt`, `lte` | `number` | +| `numberArray` | `in`, `notIn` | `number[]` | +| `numberRange` | `between`, `outside` | `{ min, max }` | +| `integerComparison` | `eq`, `neq`, `gt`, `gte`, `lt`, `lte` | `number` | +| `booleanComparison` | `eq`, `neq` | `boolean` | +| `dateComparison` | `gt`, `gte`, `lt`, `lte` | `string` (ISO) | +| `dateRange` | `between`, `outside` | `{ min, max }` | +| `moneyComparison` | `gt`, `gte`, `lt`, `lte` | `Money` | +| `moneyRange` | `between`, `outside` | `{ min, max }` | + +The `F` namespace provides helpers that compile `{ operator, value }` filter objects without manual construction: + +```typescript +import { F } from "@common-grants/sdk/extensions"; + +F.eq("open"); // { operator: "eq", value: "open" } +F.in(["HHS", "DOE"]); // { operator: "in", value: ["HHS", "DOE"] } +F.like("*Conservation*"); // { operator: "like", value: "*Conservation*" } +F.between("2025-01-01", "2025-12-31"); // { operator: "between", value: { min: "2025-01-01", max: "2025-12-31" } } +// Full set: eq, neq, gt, gte, lt, lte, in, notIn, like, notLike, between, outside +``` + +> **Cross-SDK note.** TypeScript uses `F.in` (`"in"` as an object key — valid JS). The Python sibling SDK ([#869](https://github.com/HHS/simpler-grants-protocol/issues/869)) uses `f.in_` (trailing underscore, Python convention for reserved words). This is a documented naming divergence across SDKs. + +### Classifying consumer filters into the request body + +`definePlugin()` registers filter **declarations** — which filter names exist and what type each one is. `classifyFilters()` is the separate **search-time** step: it takes the consumer's actual filter values and produces the body of the request sent to the API. In the full SDK the client will call it internally when you invoke a search method — consumers won't call it directly. The PoC exposes it standalone because client wiring is out of scope (tracked under [#645](https://github.com/HHS/simpler-grants-protocol/issues/645)). + +`classifyFilters()` accepts the plugin's `routes`, a resource name, a method name, and the consumer's flat `filters` object. It returns an `OppFilters` request body conforming to ADR-0012: + +```typescript +import { classifyFilters } from "@common-grants/sdk/extensions"; + +const consumerFilters = { + // Default filters → top-level named fields on the request body + status: F.in(["open", "forecasted"]), + closeDateRange: F.between("2025-01-01", "2025-12-31"), + + // Pre-registered custom filters → customFilters record + agency: F.in(["HHS", "DOE"]), + fundingProgram: F.like("*Conservation*"), + + // Ad-hoc filter (not declared in the plugin) → customFilters passthrough + legacyTag: F.eq("conservation-2024"), +}; + +const requestBody = classifyFilters( + grantsGovPlugin.routes!, // routes is optional on Plugin; assert non-null when known + "opportunities", + "search", + consumerFilters +); +// requestBody shape (ADR-0012): +// { +// status: { operator: "in", value: ["open", "forecasted"] }, +// closeDateRange: { operator: "between", value: { min: "2025-01-01", max: "2025-12-31" } }, +// customFilters: { +// agency: { operator: "in", value: ["HHS", "DOE"] }, +// fundingProgram: { operator: "like", value: "*Conservation*" }, +// legacyTag: { operator: "eq", value: "conservation-2024" }, +// } +// } +``` + +The three-bucket classification rule (ADR-0012): + +1. **Default filters** (`status`, `closeDateRange`, etc.) → named top-level fields on the request body. +2. **Pre-registered custom filters** (declared in `routes.*.*.filters`) → `customFilters` record, with operator/value-shape validation against the declared `filterType`. +3. **Ad-hoc filters** (not declared, including `gov.<system>@<filterName>` namespaced keys) → `customFilters` passthrough, shape-only validated. + +For a complete runnable example with assertions, see [`examples/custom-filters.ts`](../../examples/custom-filters.ts) (`pnpm example:custom-filters`). + +### Validation — registration-time and call-time + +`validateRoutes()` is called at registration time. It throws `PluginError` if: + +- A filter spec uses an unknown `filterType` value. +- A custom filter name collides with a default filter field name (e.g. registering `"status"` would shadow the protocol's standard `status` filter). + +`validateFilterCall()` validates individual filters at call time. For registered filters it validates the operator and value shape against the declared `filterType`. For ad-hoc filters it applies a shape-only check (`DefaultFilterSchema`). `classifyFilters()` already runs it for every key — you only call it directly when validating a single filter outside classification. + +In the PoC, `definePlugin()` passes `routes` through **unvalidated** — the full SDK client will run `validateRoutes()` when a plugin is attached and call-time validation inside each search. Until then, call `validateRoutes()` yourself after defining a plugin if you want registration-time errors surfaced early. + +> **PII note:** as with transforms, `PluginError.sourceValue` carries the raw input — here, the consumer's filter value. The [PII warning](#error-handling) above applies equally; log a redacted projection. + +```typescript +import { validateRoutes, validateFilterCall } from "@common-grants/sdk/extensions"; + +// Registration-time — throws PluginError on unknown filterType or collision +validateRoutes(grantsGovPlugin.routes!); + +// Call-time — throws PluginError on operator/value mismatch for registered filters. +// Pass the filter's CustomFilterSpec (looked up from the route-method's filters), +// its name, and the value. +const agencySpec = grantsGovPlugin.routes!.opportunities.search.filters!.agency; +validateFilterCall(agencySpec, "agency", F.in(["HHS"])); // valid: in operator + string array value match the stringArray filterType +``` + +### The `as const` trap + +> [!IMPORTANT] +> Always pass `as const` to `definePlugin()` when declaring `routes`. Without it, TypeScript widens literal `filterType` values from specific strings (e.g. `"stringArray"`) to the broad `string` type, and the `TypedConsumerFilters` narrowing layer is lost — unknown filter keys, wrong operators, and wrong value shapes then silently pass the type checker. + +The compile-time proof is in [`__tests__/extensions/custom-filters-types.ts`](../../__tests__/extensions/custom-filters-types.ts) — six `@ts-expect-error` directives guard actual compile errors that fire only while `as const` narrowing is in effect, checked by `tsc --noEmit`. + ## Best practices ### Export value schemas alongside your plugin @@ -693,6 +865,19 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, | [`PluginExtensions`](./types.ts) | interface | Serializable plugin config carrying `meta?: Partial<PluginMeta>` and per-object `schemas`. | | | [`ClientConfig`](./types.ts) | type | The per-plugin client configuration shape. Concrete shape is deferred to the full SDK. | | +### Custom filters (PoC) + +| Export | Kind | Description | Demonstrated in | +| --------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| [`classifyFilters()`](./custom-filters.ts) | function | Three-bucket classifier. Maps a flat consumer `filters` object to the ADR-0012 `OppFilters` request body: default fields → top-level named fields; registered custom + ad-hoc → `customFilters` record. | [Classifying consumer filters](#classifying-consumer-filters-into-the-request-body) | +| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `PluginError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | +| [`validateFilterCall()`](./custom-filters.ts) | function | Call-time validator. Validates a single filter against its declared `filterType` schema (registered) or shape-only (ad-hoc). Throws `PluginError` on mismatch. | [Validation](#validation--registration-time-and-call-time) | +| [`F`](./custom-filters.ts) | namespace | Helper namespace. `F.eq`, `F.neq`, `F.gt`, `F.gte`, `F.lt`, `F.lte`, `F.in`, `F.notIn`, `F.like`, `F.notLike`, `F.between`, `F.outside` — each compiles to `{ operator, value }`. Note: `F.in` is `"in"` as an object property key. | [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) | +| [`CustomFilterSpec`](./types.ts) | interface | Per-filter declaration: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType`; no `value` field. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | +| [`CustomFilterType`](./types.ts) | type | 11-value literal union: `stringComparison \| stringArray \| numberComparison \| numberArray \| numberRange \| integerComparison \| booleanComparison \| dateComparison \| dateRange \| moneyComparison \| moneyRange`. | [Filter-type catalog](#filter-type-catalog-and-the-f-helpers) | +| [`PluginRoutes`](./types.ts) | type | `Record<string, Record<string, RouteDeclarations>>` — the `routes` value on `DefinePluginOptions`. Keys are resource name → method name → `RouteDeclarations`. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | +| [`RouteDeclarations`](./types.ts) | interface | Per-method filter map: `{ filters?: Record<string, CustomFilterSpec> }`. | | + ### Shared types | Export | Kind | Description | Demonstrated in | diff --git a/lib/ts-sdk/src/extensions/custom-filters.ts b/lib/ts-sdk/src/extensions/custom-filters.ts new file mode 100644 index 000000000..b63073580 --- /dev/null +++ b/lib/ts-sdk/src/extensions/custom-filters.ts @@ -0,0 +1,302 @@ +/** + * Custom Filters Extension + * + * Provides the classifier, validators, and F helper namespace for the custom-filters surface. + * + * - `classifyFilters` — transforms a flat consumer `filters` object into the + * ADR-0012 `OppFilters` request body (three-bucket: default → named top-level fields; + * registered custom → `customFilters`; ad-hoc → `customFilters` passthrough). + * - `validateRoutes` — registration-time validation; rejects unknown `filterType` + * and custom names that collide with default-filter names. + * - `validateFilterCall` — call-time validation; rejects operator/filterType mismatch + * and value-shape mismatches for registered filters; shape-only check for ad-hoc. + * - `F` — helper namespace that compiles `{operator, value}` raw filter objects. + * + * Request-body contract: ADR-0012 / OppFiltersSchema. + * Core-field escape hatch: `gov.<system>@<filterName>` keys pass through as custom-filter + * keys verbatim. + * + * @module @common-grants/sdk/extensions + */ + +import { z } from "zod"; +import { DefaultFilterSchema } from "../schemas/zod/filters"; +import { + BooleanComparisonFilterSchema, + DateComparisonFilterSchema, + DateRangeFilterSchema, + MoneyComparisonFilterSchema, + MoneyRangeFilterSchema, + NumberArrayFilterSchema, + NumberComparisonFilterSchema, + NumberRangeFilterSchema, + StringArrayFilterSchema, + StringComparisonFilterSchema, +} from "../schemas/zod/filters"; +import { OppDefaultFiltersSchema, OppFiltersSchema } from "../schemas/zod/models"; +import type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; +import { PluginError } from "./types"; + +// ############################################################################ +// Internal — filter-type schema map +// ############################################################################ + +/** + * Maps each CustomFilterType to the Zod schema that validates its + * `{operator, value}` pair. Each schema constrains both the allowed operator + * enum and the value shape, so a single parse covers both checks. + */ +const FILTER_TYPE_SCHEMAS: Record<CustomFilterType, z.ZodTypeAny> = { + stringComparison: StringComparisonFilterSchema, + stringArray: StringArrayFilterSchema, + numberComparison: NumberComparisonFilterSchema, + numberArray: NumberArrayFilterSchema, + numberRange: NumberRangeFilterSchema, + // integerComparison reuses NumberComparisonFilterSchema — the spec defines no + // integer filter model, so the int constraint is not schema-enforced + integerComparison: NumberComparisonFilterSchema, + booleanComparison: BooleanComparisonFilterSchema, + dateComparison: DateComparisonFilterSchema, + dateRange: DateRangeFilterSchema, + moneyComparison: MoneyComparisonFilterSchema, + moneyRange: MoneyRangeFilterSchema, +}; + +/** + * All 11 valid CustomFilterType values — used for unknown-filterType detection. + */ +const VALID_FILTER_TYPES = new Set<string>(Object.keys(FILTER_TYPE_SCHEMAS)); + +/** + * The default-filter field names from `OppDefaultFiltersSchema`. + * Custom filter names must not collide with these. + */ +const DEFAULT_FILTER_NAMES = new Set<string>(Object.keys(OppDefaultFiltersSchema.shape)); + +// ############################################################################ +// Public — F helpers +// ############################################################################ + +/** + * Helper namespace for building `{operator, value}` raw filter objects. + * + * Each helper compiles to the `DefaultFilter` wire shape accepted by ADR-0012. + * Raw `{operator, value}` objects are also accepted by `classifyFilters` — F.* + * is a convenience layer, not a requirement. + * + * NOTE: `F.in` uses the TS reserved word as an object key — valid as a property + * key. The Python sibling uses `f.in_` to avoid the reserved-word restriction; + * this cross-SDK naming difference is a documented divergence across SDKs. + * + * @example + * ```typescript + * const filter = F.eq("open"); + * // → { operator: "eq", value: "open" } + * + * const range = F.between(100, 500); + * // → { operator: "between", value: { min: 100, max: 500 } } + * ``` + */ +export const F = { + /** Equals — `{ operator: "eq", value }` */ + eq: (value: unknown) => ({ operator: "eq" as const, value }), + /** Not equals — `{ operator: "neq", value }` */ + neq: (value: unknown) => ({ operator: "neq" as const, value }), + /** Greater than — `{ operator: "gt", value }` */ + gt: (value: unknown) => ({ operator: "gt" as const, value }), + /** Greater than or equal — `{ operator: "gte", value }` */ + gte: (value: unknown) => ({ operator: "gte" as const, value }), + /** Less than — `{ operator: "lt", value }` */ + lt: (value: unknown) => ({ operator: "lt" as const, value }), + /** Less than or equal — `{ operator: "lte", value }` */ + lte: (value: unknown) => ({ operator: "lte" as const, value }), + /** Array inclusion — `{ operator: "in", value: [...] }` */ + in: (value: unknown[]) => ({ operator: "in" as const, value }), + /** Array exclusion — `{ operator: "notIn", value: [...] }` */ + notIn: (value: unknown[]) => ({ operator: "notIn" as const, value }), + /** String pattern match — `{ operator: "like", value }` */ + like: (value: string) => ({ operator: "like" as const, value }), + /** String pattern non-match — `{ operator: "notLike", value }` */ + notLike: (value: string) => ({ operator: "notLike" as const, value }), + /** Range (inclusive) — `{ operator: "between", value: { min, max } }` */ + between: (min: unknown, max: unknown) => ({ operator: "between" as const, value: { min, max } }), + /** Range (exclusive) — `{ operator: "outside", value: { min, max } }` */ + outside: (min: unknown, max: unknown) => ({ operator: "outside" as const, value: { min, max } }), +}; + +// ############################################################################ +// Public — validateRoutes (registration-time validation) +// ############################################################################ + +/** + * Registration-time validation for a `PluginRoutes` declaration. + * + * Throws `PluginError` on: + * 1. Unknown `filterType` (not one of the 11 `CustomFilterType` values) + * 2. A custom filter name that collides with a default-filter field name + * (`status`, `closeDateRange`, `totalFundingAvailableRange`, + * `minAwardAmountRange`, `maxAwardAmountRange`) + * + * Duplicate filter names within a route-method need no check: filter names are + * object keys, and JS object literals cannot represent duplicate keys. + * + * Implements ASVS L1 input validation at the plugin-author trust boundary. + * + * @param routes - The PluginRoutes declaration to validate + * @throws {PluginError} on any constraint violation + */ +export function validateRoutes(routes: PluginRoutes): void { + for (const [resourceKey, methods] of Object.entries(routes)) { + for (const [methodKey, declarations] of Object.entries(methods)) { + const filters = (declarations as RouteDeclarations).filters; + if (!filters) continue; + + for (const [filterName, spec] of Object.entries(filters)) { + const path = `routes.${resourceKey}.${methodKey}.filters.${filterName}`; + + // Check for unknown filterType + if (!VALID_FILTER_TYPES.has(spec.filterType)) { + throw new PluginError( + `Unknown filterType "${spec.filterType}" for filter "${filterName}". ` + + `Must be one of: ${[...VALID_FILTER_TYPES].join(", ")}`, + { path, sourceValue: spec } + ); + } + + // Check for collision with default-filter field names + if (DEFAULT_FILTER_NAMES.has(filterName)) { + throw new PluginError( + `Custom filter name "${filterName}" collides with a default filter field. ` + + `Default filter names are reserved: ${[...DEFAULT_FILTER_NAMES].join(", ")}`, + { path, sourceValue: spec } + ); + } + } + } + } +} + +// ############################################################################ +// Public — validateFilterCall (call-time validation) +// ############################################################################ + +/** + * Call-time validation for a single filter value against its registered spec. + * + * - For REGISTERED filters (spec provided): validates the `{operator, value}` + * pair against the filterType's Zod schema — each schema constrains both the + * allowed operator enum and the value shape, so one parse covers both checks. + * - For AD-HOC filters (spec is undefined): shape-only check against + * `DefaultFilterSchema` (no operator/filterType enforcement — accepted trade-off). + * + * @param spec - The registered `CustomFilterSpec` for this filter, or `undefined` for ad-hoc + * @param filterName - The filter key (used in error `path`) + * @param filterValue - The raw filter value from the consumer `filters` object + * @throws {PluginError} on operator/filterType mismatch or value-shape mismatch + */ +export function validateFilterCall( + spec: CustomFilterSpec | undefined, + filterName: string, + filterValue: unknown +): void { + const path = `filters.${filterName}`; + + if (spec === undefined) { + // Ad-hoc filter — shape-only check against DefaultFilterSchema + const result = DefaultFilterSchema.safeParse(filterValue); + if (!result.success) { + throw new PluginError( + `Ad-hoc filter "${filterName}" has an invalid shape: ${result.error.message}`, + { path, sourceValue: filterValue } + ); + } + return; + } + + // Registered filter — validate against the filterType's schema + const schema = FILTER_TYPE_SCHEMAS[spec.filterType]; + if (!schema) { + // Should not reach here if validateRoutes was called first, but guard anyway + throw new PluginError( + `Unknown filterType "${spec.filterType}" for registered filter "${filterName}"`, + { path, sourceValue: filterValue } + ); + } + + // One parse validates both the operator enum and the value shape + const result = schema.safeParse(filterValue); + if (!result.success) { + throw new PluginError( + `Filter "${filterName}" (filterType: "${spec.filterType}") failed validation: ${result.error.message}`, + { path, sourceValue: filterValue } + ); + } +} + +// ############################################################################ +// Public — classifyFilters (three-bucket classifier) +// ############################################################################ + +/** + * Classifies a flat consumer `filters` object into the ADR-0012 `OppFilters` request body. + * + * Three-bucket classification: + * 1. **Default filters** — keys present in `OppDefaultFiltersSchema` (e.g. `status`, + * `closeDateRange`) → top-level named fields on the request body. + * 2. **Registered custom filters** — keys declared in the route-method's `filters` + * spec → `customFilters[name]`. + * 3. **Ad-hoc filters** — unregistered keys (not in defaults, not in spec) → + * `customFilters[name]` passthrough. + * + * `gov.<system>@<filterName>` namespaced keys are treated as ad-hoc custom + * filter keys and flow into `customFilters` verbatim — no auto-migration. + * + * Call-time validation (`validateFilterCall`) is run for each key during classification. + * + * @param routes - The `PluginRoutes` from the plugin definition + * @param resourceKey - The resource name (e.g. `"opportunities"`) + * @param methodKey - The method name (e.g. `"search"`) + * @param consumerFilters - The flat consumer-facing filters object + * @returns The classified `OppFilters` request body + */ +export function classifyFilters( + routes: PluginRoutes, + resourceKey: string, + methodKey: string, + consumerFilters: Record<string, unknown> +): z.infer<typeof OppFiltersSchema> { + // Resolve registered filter specs for this route-method + const registeredFilters: Record<string, CustomFilterSpec> = + routes[resourceKey]?.[methodKey]?.filters ?? {}; + + const defaultFields: Partial<z.infer<typeof OppDefaultFiltersSchema>> = {}; + const customFilters: Record<string, z.infer<typeof DefaultFilterSchema>> = {}; + + for (const [key, value] of Object.entries(consumerFilters)) { + // Look up registered spec (undefined for ad-hoc and gov.* namespaced keys) + const spec = registeredFilters[key] as CustomFilterSpec | undefined; + + if (DEFAULT_FILTER_NAMES.has(key)) { + // Bucket 1: default filter → top-level named field. + // Default keys get shape-only validation via DefaultFilterSchema (the same + // treatment as ad-hoc keys); per-field type enforcement against + // OppDefaultFiltersSchema is intentionally not applied here. The server + // validates default fields and reports any it cannot apply. + validateFilterCall(undefined, key, value); + (defaultFields as Record<string, unknown>)[key] = value; + } else { + // Bucket 2 (registered custom) or Bucket 3 (ad-hoc / gov.* namespaced) + // Run call-time validation — passes spec if registered, undefined if ad-hoc + validateFilterCall(spec, key, value); + customFilters[key] = value as z.infer<typeof DefaultFilterSchema>; + } + } + + // Build request body — omit customFilters key entirely when empty (match nullish shape) + const requestBody: z.infer<typeof OppFiltersSchema> = { + ...defaultFields, + ...(Object.keys(customFilters).length > 0 ? { customFilters } : {}), + }; + + return requestBody; +} diff --git a/lib/ts-sdk/src/extensions/define-plugin.ts b/lib/ts-sdk/src/extensions/define-plugin.ts index cb0240b5d..2ac6f7e36 100644 --- a/lib/ts-sdk/src/extensions/define-plugin.ts +++ b/lib/ts-sdk/src/extensions/define-plugin.ts @@ -11,6 +11,7 @@ import type { ObjectSchemasInput, PluginMeta, PluginExtensions, + PluginRoutes, } from "./types"; import { EXTENSIBLE_SCHEMA_MAP } from "./types"; import { withCustomFields, type WithCustomFieldsResult } from "./with-custom-fields"; @@ -67,6 +68,32 @@ export interface DefinePluginOptions<T extends SchemasInput = SchemasInput> { * callables are provided. Native input Zod-wrapping remains deferred. */ schemas?: T; + /** + * Route-keyed custom filter declarations. + * + * Passed through unchanged to `Plugin.routes`. Filters attach to resource + * methods (e.g. `opportunities.search.filters`), not to a schema key — because + * filters vary asymmetrically across methods. + * + * Registration-time validation (`validateRoutes`) and call-time classification + * (`classifyFilters`) consume these declarations. + * + * @example + * ```typescript + * definePlugin({ + * routes: { + * opportunities: { + * search: { + * filters: { + * agency: { filterType: "stringArray" }, + * }, + * }, + * }, + * }, + * } as const) + * ``` + */ + routes?: PluginRoutes; } /** @@ -76,11 +103,23 @@ export interface DefinePluginOptions<T extends SchemasInput = SchemasInput> { * - `schemas` — per-object compiled output: `common` (extended Zod schema), `native`, * `toCommon`, and `fromCommon` for each extensible model * - `meta` — plugin identity passed through from options + * - `routes` — route-keyed custom filter declarations; when defined `as const`, the + * literal `filterType` values are preserved so that `TypedConsumerFilters` + * can narrow call-site filter keys, operators, and value shapes. + * + * The second generic parameter `TRoutes` captures the literal routes type when the + * caller uses `as const`. Callers that do not care about typed narrowing can ignore it + * (the default is `PluginRoutes`). */ -export interface Plugin<T extends SchemasInput = SchemasInput> { +export interface Plugin< + T extends SchemasInput = SchemasInput, + TRoutes extends PluginRoutes = PluginRoutes, +> { extensions?: PluginExtensions; schemas: PluginSchemas<T>; meta?: PluginMeta; + /** Route-keyed custom filter declarations, passed through unchanged from `DefinePluginOptions.routes`. */ + routes?: TRoutes; } // ############################################################################ @@ -120,10 +159,11 @@ export interface Plugin<T extends SchemasInput = SchemasInput> { * const result = plugin.schemas.Opportunity.toCommon?.(nativeData); * ``` */ -export function definePlugin<const T extends SchemasInput>( - options: DefinePluginOptions<T> -): Plugin<T> { - const { extensions, meta, schemas: schemasInput } = options; +export function definePlugin< + const T extends SchemasInput, + const TRoutes extends PluginRoutes = PluginRoutes, +>(options: DefinePluginOptions<T> & { routes?: TRoutes }): Plugin<T, TRoutes> { + const { extensions, meta, schemas: schemasInput, routes } = options; const schemas: Record<string, object> = {}; for (const [name, extensibleSchema] of Object.entries(EXTENSIBLE_SCHEMA_MAP) as [ @@ -181,7 +221,8 @@ export function definePlugin<const T extends SchemasInput>( // Cast is safe — the runtime loop mirrors the PluginSchemas<T> mapped type, // but TypeScript can't verify that from the dynamic Object.entries() iteration. - return { extensions, schemas, meta } as Plugin<T>; + // The second generic TRoutes preserves the literal routes type from `as const` calls. + return { extensions, schemas, meta, routes } as Plugin<T, TRoutes>; } // ############################################################################ diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index d016a461d..546dc2ac5 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -13,6 +13,10 @@ export type { Plugin, DefinePluginOptions, SchemasInput } from "./define-plugin" export type { CustomFieldSpec, HasCustomFields, ExtensibleObject } from "./types"; export { definePlugin } from "./define-plugin"; +// Custom filters — route-keyed filter registration + classification +export type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; +export { classifyFilters, validateRoutes, validateFilterCall, F } from "./custom-filters"; + // Schema utilities — lower-level tools for working with custom fields export type { WithCustomFieldsResult } from "./with-custom-fields"; export type { ExtensibleSchemaName } from "./types"; diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index a3b02ff0c..7cf105c88 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -51,6 +51,87 @@ export interface CustomFieldSpec { description?: string; } +// ############################################################################ +// Public types - CustomFilterType, CustomFilterSpec, RouteDeclarations, PluginRoutes +// ############################################################################ + +/** + * The 11-value enum of supported custom filter types. + * + * Each value maps 1:1 to a base filter type with auto-derived operators. + * Operators are NOT authored — they are derived from the filterType at + * classification time (e.g. `booleanComparison` → `eq|neq`). + * + * NOTE: Build against these authoritative `*Comparison` names. + * Terse names (`"string"`, `"number"`) are NOT valid — always use the full + * `*Comparison` / `*Array` / `*Range` form. + */ +export type CustomFilterType = + | "stringComparison" + | "stringArray" + | "numberComparison" + | "numberArray" + | "numberRange" + | "integerComparison" + | "booleanComparison" + | "dateComparison" + | "dateRange" + | "moneyComparison" + | "moneyRange"; + +/** + * Specification for a custom filter to be registered on a route method. + * + * The record key is the filter name; no separate `name` field is needed. + * Operators are auto-derived from `filterType`; no Zod value schema is + * authored here (contrast with `CustomFieldSpec.value`). + * + * @example + * ```typescript + * const agencyFilter: CustomFilterSpec = { + * filterType: "stringArray", + * description: "Filter by funding agency", + * }; + * ``` + */ +export interface CustomFilterSpec { + /** The filter type — determines the allowed operators and value shape */ + filterType: CustomFilterType; + /** Optional human-readable description of the filter */ + description?: string; +} + +/** + * Filter declarations for a single route method (e.g. `search`). + * + * Filters are keyed by name; the record value is a `CustomFilterSpec`. + */ +export interface RouteDeclarations { + filters?: Record<string, CustomFilterSpec>; +} + +/** + * Route map for custom filter declarations: resource → method → declarations. + * + * Passed as `routes` in `definePlugin()`. Filters attach to resource methods + * (which vary asymmetrically across schemas), not to a schema. + * + * @example + * ```typescript + * const routes: PluginRoutes = { + * opportunities: { + * search: { + * filters: { + * agency: { filterType: "stringArray" }, + * fundingProgram: { filterType: "stringComparison" }, + * }, + * }, + * }, + * }; + * ``` + */ +export type PluginRoutes = Record<string, Record<string, RouteDeclarations>>; + // ############################################################################ // Public types - HasCustomFields, ExtensibleObject // ############################################################################ diff --git a/lib/ts-sdk/src/schemas/zod/filters.ts b/lib/ts-sdk/src/schemas/zod/filters.ts index 9522c9f25..a68d1c29b 100644 --- a/lib/ts-sdk/src/schemas/zod/filters.ts +++ b/lib/ts-sdk/src/schemas/zod/filters.ts @@ -106,3 +106,12 @@ export const MoneyRangeFilterSchema = z.object({ max: MoneySchema, }), }); + +// ############################################################################ +// Boolean filters +// ############################################################################ + +export const BooleanComparisonFilterSchema = z.object({ + operator: EquivalenceOperatorsEnum, + value: z.boolean(), +}); From 0f85b7962566a85590a70c9d450dc38e89dac6fb Mon Sep 17 00:00:00 2001 From: jcrichlake <jeff.crichlake@agile6.com> Date: Fri, 12 Jun 2026 14:41:54 -0400 Subject: [PATCH 17/51] Addressing PR comments --- lib/python-sdk/common_grants_sdk/__init__.py | 3 +- .../common_grants_sdk/extensions/__init__.py | 26 +- .../common_grants_sdk/extensions/generate.py | 53 ++- .../common_grants_sdk/extensions/plugin.py | 66 ++-- .../common_grants_sdk/extensions/specs.py | 113 +----- .../extensions/transforms.py | 106 ++++-- .../common_grants_sdk/extensions/types.py | 99 +++--- .../examples/plugin_custom_fields.py | 4 +- .../examples/plugins/grants_gov/__init__.py | 1 - .../examples/plugins/grants_gov/cg_config.py | 4 +- .../opportunity_extensions/__init__.py | 1 - .../opportunity_extensions/cg_config.py | 4 +- lib/python-sdk/examples/transforms.py | 4 +- .../tests/extensions/test_plugin.py | 111 ++++-- .../tests/extensions/test_transforms.py | 155 ++++++-- lib/python-sdk/tests/extensions/test_types.py | 99 +++--- .../tests/schemas/test_plugin_registry.py | 18 +- .../tests/utils/test_merge_extensions.py | 140 -------- .../tests/utils/test_plugin_generator.py | 86 +++-- .../extensions/define-plugin.spec.ts | 197 +++++------ lib/ts-sdk/__tests__/extensions/index.spec.ts | 9 +- .../extensions/transformation.spec.ts | 2 +- .../__tests__/extensions/transforms.spec.ts | 141 ++++++-- lib/ts-sdk/examples/plugins.ts | 6 +- lib/ts-sdk/examples/transforms.ts | 11 +- lib/ts-sdk/src/extensions/README.md | 98 +++--- lib/ts-sdk/src/extensions/define-plugin.ts | 257 ++++++++++---- lib/ts-sdk/src/extensions/index.ts | 20 +- lib/ts-sdk/src/extensions/transforms.ts | 118 ++++--- lib/ts-sdk/src/extensions/types.ts | 198 ++++++----- .../{extensions => utils}/transformation.ts | 58 +-- .../governance/adr/0022-plugin-framework.mdx | 330 ++++++------------ 32 files changed, 1299 insertions(+), 1239 deletions(-) delete mode 100644 lib/python-sdk/tests/utils/test_merge_extensions.py rename lib/ts-sdk/src/{extensions => utils}/transformation.ts (83%) diff --git a/lib/python-sdk/common_grants_sdk/__init__.py b/lib/python-sdk/common_grants_sdk/__init__.py index 0c57028b2..f70067807 100644 --- a/lib/python-sdk/common_grants_sdk/__init__.py +++ b/lib/python-sdk/common_grants_sdk/__init__.py @@ -8,7 +8,7 @@ from . import schemas from .client import Auth, Client, Config -from .extensions import Plugin, PluginConfig, define_plugin, merge_extensions +from .extensions import Plugin, PluginConfig, define_plugin __all__ = [ "schemas", @@ -17,6 +17,5 @@ "Config", "Plugin", "PluginConfig", - "merge_extensions", "define_plugin", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 8589e1fa8..0fb2c8148 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,23 +1,20 @@ """Public extension APIs for the CommonGrants Python SDK.""" from .plugin import Plugin, PluginConfig, define_plugin, inject_transforms -from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions, merge_extensions +from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions from .transforms import build_transforms from .types import ( Handler, - ObjectMappings, - ObjectSchemas, - ObjectSchemasInput, PluginCapability, - PluginError, - PluginExtensions, PluginExtensionsMeta, - PluginExtensionsSchema, + SchemaConfig, + SchemaInput, + SchemaMappings, + TransformError, TransformResult, ) __all__ = [ - # Existing exports (unchanged) "ConflictStrategy", "CustomFieldSpec", "Plugin", @@ -25,18 +22,13 @@ "SchemaExtensions", "define_plugin", "inject_transforms", - "merge_extensions", - # New: build_transforms "build_transforms", - # New: ADR-0022 types "Handler", - "ObjectMappings", - "ObjectSchemas", - "ObjectSchemasInput", "PluginCapability", - "PluginError", - "PluginExtensions", "PluginExtensionsMeta", - "PluginExtensionsSchema", + "SchemaConfig", + "SchemaInput", + "SchemaMappings", + "TransformError", "TransformResult", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py index a259f6788..5b6b222ba 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ b/lib/python-sdk/common_grants_sdk/extensions/generate.py @@ -275,7 +275,7 @@ def _model_blocks( spec=spec, resolved_type=resolved_type ) # Use spec.name as the runtime display name if provided, otherwise fall back - # to the field key (the dict key in ObjectSchemasInput.custom_fields). + # to the field key (the dict key in SchemaInput.custom_fields). field_name_default = spec.name or field_key # repr() produces a quoted string literal safe to embed directly in source code. description_default = repr(spec.description) if spec.description else "None" @@ -365,13 +365,13 @@ def _render_schemas_py( blocks = "\n\n\n".join(_model_blocks(custom_fields)) mappings_only: set[str] = mappings_only_objs or set() - # Build schema assignments: each attribute is an ObjectSchemas instance so - # callers get a unified interface (plugin.schemas.Opportunity.common for the + # Build schema assignments: each attribute is a SchemaConfig instance so + # callers get a unified interface (plugin.schemas.Opportunity.common_schema for the # model class, .to_common/.from_common for transforms). # to_common/from_common default to None here; root __init__.py injects the # real callables for any object that has transforms configured. assignments: list[str] = [ - f" self.{name} = ObjectSchemas(native=dict, common={name}, to_common=None, from_common=None)" + f" self.{name} = SchemaConfig(source_schema=dict, common_schema={name}, to_common=None, from_common=None)" for name in model_names ] for obj in sorted(mappings_only): @@ -382,7 +382,7 @@ def _render_schemas_py( ) base_class = MODEL_BASE_CLASS[obj] assignments.append( - f" self.{obj} = ObjectSchemas(native=dict, common={base_class}, to_common=None, from_common=None)" + f" self.{obj} = SchemaConfig(source_schema=dict, common_schema={base_class}, to_common=None, from_common=None)" ) schema_assignments = "\n".join(assignments or [" pass"]) @@ -401,7 +401,7 @@ def _render_schemas_py( "", "from pydantic import ConfigDict, Field", "", - "from common_grants_sdk.extensions.types import ObjectSchemas", + "from common_grants_sdk.extensions.types import SchemaConfig", "from common_grants_sdk.schemas.pydantic.base import CommonGrantsBaseModel", "from common_grants_sdk.schemas.pydantic.fields import CustomField, CustomFieldType", "from common_grants_sdk.schemas.pydantic.models import OpportunityBase", @@ -459,15 +459,15 @@ def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) else set() ) mappings_objs: set[str] = ( - {obj for obj, s in config.extensions.schemas.items() if s.mappings is not None} - if config.extensions and config.extensions.schemas + {obj for obj, s in config.schemas.items() if s.mappings is not None} + if config.schemas else set() ) needs_build_transforms = bool(mappings_objs - explicit_objs) # Build pre-plugin lines: inject transforms into the _Schemas object before - # constructing Plugin. schemas.py initialises each ObjectSchemas with + # constructing Plugin. schemas.py initialises each SchemaConfig with # to_common=None/from_common=None; we mutate those attrs here. inject_lines: list[str] = [] @@ -475,9 +475,9 @@ def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) for obj in sorted(mappings_objs - explicit_objs): inject_lines += [ f"_{obj}_to_common, _{obj}_from_common = build_transforms(", - f' to_common_mapping=config.extensions.schemas["{obj}"].mappings.to_common,', - f' from_common_mapping=config.extensions.schemas["{obj}"].mappings.from_common,', - f" common_model=schemas.{obj}.common,", + f' to_common_mapping=config.schemas["{obj}"].mappings.to_common,', + f' from_common_mapping=config.schemas["{obj}"].mappings.from_common,', + f" common_schema=schemas.{obj}.common_schema,", ")", f"schemas.{obj}.to_common = _{obj}_to_common", f"schemas.{obj}.from_common = _{obj}_from_common", @@ -514,7 +514,6 @@ def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) plugin_lines = [ f"{plugin_variable_name} = Plugin(", " schemas=schemas,", - " extensions=config.extensions,", " meta=config.meta,", ")", ] @@ -565,14 +564,14 @@ def generate_plugin(plugin_dir: Path) -> Path: mappings_only_objs: set[str] = ( { obj - for obj, s in config.extensions.schemas.items() + for obj, s in config.schemas.items() if s.mappings is not None and obj not in explicit_cf_objs } - if config.extensions and config.extensions.schemas + if config.schemas else set() ) # Third bucket: objects in config.schemas with explicit transforms but no - # custom_fields and no extensions.schemas entry. They also need a pass-through + # custom_fields and no mappings entry. They also need a pass-through # entry in _Schemas or schemas.<Obj> won't exist at import time. transforms_only_objs: set[str] = ( set(config.schemas.keys()) - explicit_cf_objs - mappings_only_objs @@ -581,20 +580,14 @@ def generate_plugin(plugin_dir: Path) -> Path: ) # Validate that auto-generated transform objects have both mapping directions. - # Auto-generated objects: have mappings in extensions.schemas but no explicit - # to_common/from_common in config.schemas. - if config.extensions and config.extensions.schemas: - explicit_schema_objs: set[str] = ( - { - obj - for obj, s in config.schemas.items() - if s.to_common is not None or s.from_common is not None - } - if config.schemas - else set() - ) - ext_schemas = config.extensions.schemas - for obj, schema in ext_schemas.items(): + # Auto-generated objects: have mappings in schemas[obj] but no explicit callables. + if config.schemas: + explicit_schema_objs: set[str] = { + obj + for obj, s in config.schemas.items() + if s.to_common is not None or s.from_common is not None + } + for obj, schema in config.schemas.items(): if schema.mappings is None or obj in explicit_schema_objs: continue if schema.mappings.to_common is None: diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 7f8c037fa..49cd62d26 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -5,10 +5,7 @@ from dataclasses import dataclass from typing import Any, Generic, TypeVar, overload -from .types import ( - PluginExtensions, - PluginExtensionsMeta, -) +from .types import PluginExtensionsMeta T = TypeVar("T") TSchemas = TypeVar("TSchemas") @@ -20,20 +17,18 @@ class PluginConfig(Generic[TSchemas]): """Build-time plugin config produced by define_plugin() and consumed by generate.py. Generic on TSchemas so the precise type of the schemas dict is preserved — e.g. - PluginConfig[dict[str, ObjectSchemasInput[MyNative, MyCg]]] — rather than being - widened to ObjectSchemasInput[Any, Any] at the storage boundary. + PluginConfig[dict[str, SchemaInput[MyNative, MyCg]]] — rather than being + widened to SchemaInput[Any, Any] at the storage boundary. Stores inputs as-is — no compilation occurs at define_plugin() call time. - generate.py compiles this into a fully resolved Plugin by: - - Injecting the generated Pydantic model class as the common schema for each - ObjectSchemasInput entry (schemas[obj].native + common → ObjectSchemas). - - Auto-generating build_transforms() calls for any object that has - extensions.schemas[obj].mappings but no explicit schemas[obj] entry. + generate.py compiles this into a fully resolved Plugin by injecting the generated + Pydantic model class as the common schema for each SchemaInput entry, and + auto-generating build_transforms() calls for any object that has + schemas[obj].mappings but no explicit schemas[obj].to_common / from_common. All fields are optional so adopters can start with only what they need. """ - extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None schemas: TSchemas | None = None @@ -42,25 +37,23 @@ class PluginConfig(Generic[TSchemas]): class Plugin(Generic[T]): """Runtime plugin container assembled by generate.py after code generation. - schemas: the _Schemas object from generated/schemas.py. Each attribute is an - ObjectSchemas instance providing unified access to the model class and + schemas: the _Schemas object from generated/schemas.py. Each attribute is a + SchemaConfig instance providing unified access to the model class and transforms for that object: - plugin.schemas.Opportunity.common → the Pydantic model class (includes - any custom fields declared by the plugin) - plugin.schemas.Opportunity.to_common → transform callable (or None) - plugin.schemas.Opportunity.from_common → transform callable (or None) - plugin.schemas.Opportunity.native → the source system's type (or dict) + plugin.schemas.Opportunity.common_schema → the Pydantic model class (includes + any custom fields declared by the plugin) + plugin.schemas.Opportunity.to_common → transform callable (or None) + plugin.schemas.Opportunity.from_common → transform callable (or None) + plugin.schemas.Opportunity.source_schema → the source system's type (or dict) """ schemas: T - extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None @overload def define_plugin( meta: PluginExtensionsMeta | None = ..., - extensions: PluginExtensions | None = ..., schemas: None = ..., ) -> PluginConfig[None]: ... @@ -68,29 +61,44 @@ def define_plugin( @overload def define_plugin( meta: PluginExtensionsMeta | None = ..., - extensions: PluginExtensions | None = ..., schemas: TSchemas = ..., ) -> PluginConfig[TSchemas]: ... def define_plugin( meta: PluginExtensionsMeta | None = None, - extensions: PluginExtensions | None = None, schemas: Any = None, ) -> PluginConfig[Any]: """Create a PluginConfig consumed by the code generator. No compilation occurs here — inputs are stored as-is. The code generator - (generate.py) compiles ObjectSchemasInput → ObjectSchemas by injecting - the common model from the generated schemas. + (generate.py) compiles SchemaInput → SchemaConfig by injecting + the common model from the generated schemas, and auto-wires build_transforms() + for any object that has schemas[obj].mappings but no explicit callables. The return type is generic on the schemas argument: passing a typed dict - (e.g. {"Opportunity": ObjectSchemasInput[MyNative, MyCg](...) }) preserves + (e.g. {"Opportunity": SchemaInput[MyNative, MyCg](...) }) preserves those per-object generics on the returned PluginConfig rather than widening them to Any. + + Raises: + ValueError: If any schema entry specifies both mappings and explicit + to_common/from_common callables (XOR constraint). """ + if schemas: + for obj_name, schema_input in schemas.items(): + has_mappings = schema_input.mappings is not None + has_callables = ( + schema_input.to_common is not None + or schema_input.from_common is not None + ) + if has_mappings and has_callables: + raise ValueError( + f"define_plugin: {obj_name} cannot specify both mappings and explicit " + f"to_common/from_common. " + f"Use mappings for declarative transforms or provide explicit callables, not both." + ) return PluginConfig( - extensions=extensions, meta=meta, schemas=schemas, ) @@ -102,7 +110,7 @@ def inject_transforms( """Wire transform callables from plugin config into the generated schemas container. Called by the generated plugin __init__.py to inject to_common/from_common - callables (and the native type) from cg_config into the ObjectSchemas instances + callables (and the native type) from cg_config into the SchemaConfig instances produced by the code generator. Iterates over all entries in config.schemas that have at least one callable, @@ -142,7 +150,7 @@ def inject_transforms( raise ValueError( f"Plugin object {obj_name!r}: from_common callable is required" ) - obj_schemas.native = schema_input.native or dict + obj_schemas.source_schema = schema_input.source_schema or dict obj_schemas.to_common = schema_input.to_common obj_schemas.from_common = schema_input.from_common return schemas diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index d6ef97852..4b4124b57 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -3,16 +3,10 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict +from typing import Any, Literal, Optional, TypedDict from ..schemas.pydantic.fields.custom import CustomFieldType -if TYPE_CHECKING: - from .types import ( - PluginExtensions, - PluginExtensionsMeta, - ) - ConflictStrategy = Literal["error", "first_wins", "last_wins"] @@ -27,109 +21,6 @@ class CustomFieldSpec: class SchemaExtensions(TypedDict, total=False): - """Maps extensible model names to custom field specifications. - - Retained for callers that still use the flat TypedDict form directly. - merge_extensions now accepts PluginExtensions instead. - """ + """Maps extensible model names to custom field specifications.""" Opportunity: dict[str, CustomFieldSpec] - - -def _merge_meta( - current: PluginExtensionsMeta | None, - incoming: PluginExtensionsMeta, - on_conflict: ConflictStrategy, -) -> PluginExtensionsMeta: - """Merge incoming meta into current, respecting on_conflict for non-None field collisions.""" - from .types import PluginExtensionsMeta as _PluginExtensionsMeta - - if current is None: - return incoming - - merged: dict[str, Any] = {} - for field_name in ("name", "version", "source_system", "capabilities"): - current_val = getattr(current, field_name) - incoming_val = getattr(incoming, field_name) - - if incoming_val is None: - merged[field_name] = current_val - elif current_val is None: - merged[field_name] = incoming_val - else: - # Both have non-None values — apply conflict strategy - if on_conflict == "error": - raise ValueError( - f'merge_extensions: duplicate meta field "{field_name}" ' - f"(existing: {current_val!r}, incoming: {incoming_val!r})" - ) - merged[field_name] = ( - current_val if on_conflict == "first_wins" else incoming_val - ) - - return _PluginExtensionsMeta( - name=merged["name"], - version=merged["version"], - sourceSystem=merged["source_system"], - capabilities=merged["capabilities"], - ) - - -def merge_extensions( - sources: list[PluginExtensions], on_conflict: ConflictStrategy = "error" -) -> PluginExtensions: - """Merge multiple PluginExtensions into one. - - Args: - sources: Ordered list of PluginExtensions to merge. - on_conflict: Strategy for duplicate field names per object. - - "error": raise on first duplicate (default). - - "first_wins": keep first seen value. - - "last_wins": overwrite with latest value. - """ - from .types import ObjectMappings as _ObjectMappings - from .types import PluginExtensions as _PluginExtensions - from .types import PluginExtensionsSchema as _PluginExtensionsSchema - - if on_conflict not in {"error", "first_wins", "last_wins"}: - raise ValueError( - 'merge_extensions: on_conflict must be "error", "first_wins", or "last_wins"' - ) - - if len(sources) == 0: - return _PluginExtensions() - if len(sources) == 1: - return sources[0] - - # Accumulate into plain dicts; construct PluginExtensions once at the end. - merged_mappings: dict[str, _ObjectMappings] = {} - merged_meta: PluginExtensionsMeta | None = None - - for source in sources: - if source.schemas: - for obj, src_schema in source.schemas.items(): - if src_schema.mappings: - if obj in merged_mappings: - if on_conflict == "error": - raise ValueError( - f'merge_extensions: duplicate mappings for object "{obj}"' - ) - if on_conflict == "first_wins": - continue - merged_mappings[obj] = src_schema.mappings - if source.meta: - merged_meta = _merge_meta(merged_meta, source.meta, on_conflict) - - return _PluginExtensions( - meta=merged_meta, - schemas=( - { - obj: _PluginExtensionsSchema( - mappings=merged_mappings.get(obj), - ) - for obj in merged_mappings - } - if merged_mappings - else None - ), - ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index 1ff5570c8..b4099a1ad 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -20,9 +20,10 @@ transform_from_mapping, ) -from .types import Handler, PluginError, TransformResult +from .types import Handler, TransformError, TransformResult TCommon = TypeVar("TCommon", bound=BaseModel) +TSource = TypeVar("TSource", bound=BaseModel) def _validate_output_paths( @@ -33,7 +34,7 @@ def _validate_output_paths( ) -> None: """Validate that top-level output keys in mapping are valid fields on model. - Called when common_model is supplied to build_transforms(). Custom fields + Called when common_schema is supplied to build_transforms(). Custom fields declared by the plugin appear as regular model fields on the generated common model and are therefore treated as valid output paths automatically. @@ -52,7 +53,7 @@ def _validate_output_paths( raise ValueError( f"build_transforms ({direction}_mapping): unknown output {noun} " f"{sorted(invalid)!r} for model {model.__name__}. " - f"Declare them as custom_fields in ObjectSchemasInput or check the field name." + f"Declare them as custom_fields in SchemaInput or check the field name." ) @@ -106,7 +107,8 @@ def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], handlers: dict[str, Handler] | None = ..., - common_model: None = ..., + common_schema: None = ..., + source_schema: None = ..., ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], @@ -118,10 +120,37 @@ def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], handlers: dict[str, Handler] | None = ..., - common_model: type[TCommon] = ..., + common_schema: type[TCommon] = ..., + source_schema: None = ..., ) -> tuple[ Callable[[Any], TransformResult[TCommon | dict[str, Any]]], - Callable[[Any], TransformResult[dict[str, Any]]], + Callable[[Any], TransformResult[Any]], +]: ... + + +@overload +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = ..., + common_schema: None = ..., + source_schema: type[TSource] = ..., +) -> tuple[ + Callable[[Any], TransformResult[Any]], + Callable[[Any], TransformResult[TSource | dict[str, Any]]], +]: ... + + +@overload +def build_transforms( + to_common_mapping: dict[str, Any], + from_common_mapping: dict[str, Any], + handlers: dict[str, Handler] | None = ..., + common_schema: type[TCommon] = ..., + source_schema: type[TSource] = ..., +) -> tuple[ + Callable[[Any], TransformResult[TCommon | dict[str, Any]]], + Callable[[Any], TransformResult[TSource | dict[str, Any]]], ]: ... @@ -129,7 +158,8 @@ def build_transforms( to_common_mapping: dict[str, Any], from_common_mapping: dict[str, Any], handlers: dict[str, Handler] | None = None, - common_model: type[BaseModel] | None = None, + common_schema: type[BaseModel] | None = None, + source_schema: type[BaseModel] | None = None, ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], @@ -137,8 +167,8 @@ def build_transforms( """Generate to_common and from_common callables from mapping dicts. Args: - to_common_mapping: mapping from native source → CommonGrants. - from_common_mapping: mapping from CommonGrants → native source. + to_common_mapping: mapping from source system → CommonGrants. + from_common_mapping: mapping from CommonGrants → source system. handlers: Optional additional handlers registered for this call only. Keys must not collide with DEFAULT_HANDLERS (raises ValueError if they do). Each ``build_transforms()`` call gets its own isolated handler registry — @@ -158,7 +188,7 @@ def handle_upper(data, path): {}, handlers={"upper": handle_upper}, ) - common_model: Optional Pydantic model class to validate the to_common output + common_schema: Optional Pydantic model class to validate the to_common output against. Must be the fully extended generated model class (e.g. the generated Opportunity from generated/schemas.py), NOT the base class (e.g. OpportunityBase). Passing a base class will silently weaken @@ -168,15 +198,22 @@ def handle_upper(data, path): called on the transform result and any ValidationErrors are appended to TransformResult.errors rather than raised. - Note on result shape: when common_model is set, TransformResult.result + Note on result shape: when common_schema is set, TransformResult.result holds the validated Pydantic instance on success, or the raw transformed dict on ValidationError (so callers can inspect the malformed data alongside the errors). This is intentional — check TransformResult.errors before consuming TransformResult.result. + source_schema: Optional Pydantic model class to validate the from_common output + against. Without this, from_common casts its result to a plain dict with + no runtime check, so the return type provides no real safety guarantee. + When provided, model_validate is called on the transform result and any + ValidationErrors are appended to TransformResult.errors rather than raised. + The result shape follows the same convention as common_schema: a validated + model instance on success, or the raw dict alongside errors on failure. Returns: A (to_common, from_common) tuple. Each callable accepts a dict and returns - TransformResult[Any]. Failures surface as PluginError entries in + TransformResult[Any]. Failures surface as TransformError entries in TransformResult.errors rather than being raised. Raises: @@ -202,15 +239,18 @@ def handle_upper(data, path): _validate_mapping(to_common_mapping, known) _validate_mapping(from_common_mapping, known) - # When common_model is provided, validate that to_common output keys are real fields - if common_model is not None: - _validate_output_paths(to_common_mapping, common_model, known, "to_common") + # When common_schema is provided, validate that to_common output keys are real fields + if common_schema is not None: + _validate_output_paths(to_common_mapping, common_schema, known, "to_common") + # When source_schema is provided, validate that from_common output keys are real fields + if source_schema is not None: + _validate_output_paths(from_common_mapping, source_schema, known, "from_common") def to_common(native: Any) -> TransformResult[Any]: try: result = transform_from_mapping(native, to_common_mapping, handlers=merged) except HandlerError as exc: - error = PluginError( + error = TransformError( str(exc.cause), path=None, handler=exc.handler, @@ -219,18 +259,18 @@ def to_common(native: Any) -> TransformResult[Any]: ) return TransformResult(result={}, errors=[error]) except Exception as exc: - error = PluginError(str(exc), path=None, source_value=native, cause=exc) + error = TransformError(str(exc), path=None, source_value=native, cause=exc) return TransformResult(result={}, errors=[error]) - if common_model is None: + if common_schema is None: return TransformResult(result=result, errors=[]) try: - validated = common_model.model_validate(result) + validated = common_schema.model_validate(result) return TransformResult(result=validated, errors=[]) except ValidationError as exc: errors = [ - PluginError( + TransformError( e["msg"], path=".".join(str(loc) for loc in e["loc"]), ) @@ -238,7 +278,7 @@ def to_common(native: Any) -> TransformResult[Any]: ] return TransformResult(result=result, errors=errors) except Exception as exc: - error = PluginError(str(exc), path=None, source_value=result, cause=exc) + error = TransformError(str(exc), path=None, source_value=result, cause=exc) return TransformResult(result=result, errors=[error]) def from_common(common: Any) -> TransformResult[Any]: @@ -246,9 +286,8 @@ def from_common(common: Any) -> TransformResult[Any]: result = transform_from_mapping( common, from_common_mapping, handlers=merged ) - return TransformResult(result=result, errors=[]) except HandlerError as exc: - error = PluginError( + error = TransformError( str(exc.cause), path=None, handler=exc.handler, @@ -257,7 +296,26 @@ def from_common(common: Any) -> TransformResult[Any]: ) return TransformResult(result={}, errors=[error]) except Exception as exc: - error = PluginError(str(exc), path=None, source_value=common, cause=exc) + error = TransformError(str(exc), path=None, source_value=common, cause=exc) return TransformResult(result={}, errors=[error]) + if source_schema is None: + return TransformResult(result=result, errors=[]) + + try: + validated = source_schema.model_validate(result) + return TransformResult(result=validated, errors=[]) + except ValidationError as exc: + errors = [ + TransformError( + e["msg"], + path=".".join(str(loc) for loc in e["loc"]), + ) + for e in exc.errors() + ] + return TransformResult(result=result, errors=errors) + except Exception as exc: + error = TransformError(str(exc), path=None, source_value=result, cause=exc) + return TransformResult(result=result, errors=[error]) + return to_common, from_common diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index eccacc519..493bba1eb 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -9,18 +9,18 @@ from .specs import CustomFieldSpec -TNative = TypeVar("TNative") +TSource = TypeVar("TSource") TCommon = TypeVar("TCommon") T = TypeVar("T") # Capability enum — Literal rather than StrEnum to stay JSON-safe. -PluginCapability = Literal["customFields", "customFilters", "transforms", "client"] +PluginCapability = Literal["customFields", "customFilters", "transforms"] # Type aliases Handler = Callable[[Any, Any], Any] -class PluginError(Exception): +class TransformError(Exception): """Structured transformation error per ADR-0022 Decision #9. Carries field path, handler name, source value, and underlying cause so @@ -52,7 +52,7 @@ class TransformResult(Generic[T]): """Unconditional return shape for to_common / from_common (ADR-0022 Decision #7). result: the transformed value (may be partial on error). - errors: aggregated PluginErrors; empty on full success. + errors: aggregated TransformErrors; empty on full success. Consumers apply their own strict-vs-lenient rule for what counts as success: - Strict: treat any non-empty errors as failure. @@ -60,11 +60,11 @@ class TransformResult(Generic[T]): """ result: T - errors: list[PluginError] + errors: list[TransformError] -class ObjectMappings(BaseModel): - """ADR-0017 mapping dicts for a single object, stored in the serializable extensions config. +class SchemaMappings(BaseModel): + """ADR-0017 declarative mapping dicts for a single object. Each direction is author-provided — build_transforms() does not invert one into the other because many-to-one handlers like switch are not reversible (Decision #6). @@ -77,82 +77,69 @@ class ObjectMappings(BaseModel): class PluginExtensionsMeta(BaseModel): - """Plugin identity and capability declaration. All fields are optional.""" + """Plugin identity and capability declaration. - model_config = ConfigDict(populate_by_name=True) - - name: str | None = None - version: str | None = None - source_system: str | None = Field(default=None, alias="sourceSystem") - capabilities: list[PluginCapability] | None = None - - -class PluginExtensionsSchema(BaseModel): - """Per-object config inside extensions.schemas. Holds declarative mappings only. - - mappings: optional ADR-0017 declarative mappings. When present and no explicit - to_common / from_common is supplied in schemas[obj], define_plugin() will - auto-invoke build_transforms() on these (TODO — ADR-0022 Decision #6). + name and source_system are required so that plugin registries and + dependency-injection surfaces always have a reliable display label and + provenance string. version and capabilities remain optional because + they can be inferred or omitted during early development. """ model_config = ConfigDict(populate_by_name=True) - mappings: ObjectMappings | None = None - - -class PluginExtensions(BaseModel): - """Serializable portion of plugin config — safe to store as JSON. - - Used by merge_extensions() to combine declarations from multiple plugin packages. - """ - - model_config = ConfigDict(populate_by_name=True) - - meta: PluginExtensionsMeta | None = None - schemas: dict[str, PluginExtensionsSchema] | None = None + name: str + source_system: str = Field(alias="sourceSystem") + version: str | None = None + capabilities: list[PluginCapability] | None = None @dataclass -class ObjectSchemasInput(Generic[TNative, TCommon]): +class SchemaInput(Generic[TSource, TCommon]): """Input type provided by plugin authors inside define_plugin(schemas=...). - Plugin authors supply to_common and from_common as plain callables — either - hand-written or generated via build_transforms(). native defaults to - dict[str, Any] if omitted. + This is the single surface for all per-object declarations. Plugin authors supply + to_common and from_common as plain callables — either hand-written or generated + via build_transforms(). source_schema defaults to dict[str, Any] if omitted. custom_fields declares any extra fields this object exposes beyond the base CommonGrants schema. The code generator reads these and emits typed subclasses. - common is intentionally absent here. It is injected by define_plugin() during - compilation from ObjectSchemasInput → ObjectSchemas, resolved from the generated + mappings holds optional declarative mappings. When present and no + explicit to_common / from_common is supplied, the code generator auto-invokes + build_transforms() on these. Explicit callables take priority and disable + auto-wiring for that object. + + common_schema is intentionally absent here. It is injected by define_plugin() during + compilation from SchemaInput → SchemaConfig, resolved from the generated model classes produced by the code generator. Plugin authors never set it directly — cg_config.py cannot import from generated/ (it is the input to generation). """ - native: type[TNative] | None = None + source_schema: type[TSource] | None = None custom_fields: dict[str, CustomFieldSpec] | None = None - to_common: Callable[[TNative], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TNative]] | None = None + mappings: SchemaMappings | None = None + to_common: Callable[[TSource], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TSource]] | None = None @dataclass -class ObjectSchemas(Generic[TNative, TCommon]): +class SchemaConfig(Generic[TSource, TCommon]): """Runtime compiled schema container for a single object (ADR-0022). Bundles the type information and transform callables for one schema object (e.g. Opportunity). Accessed via attribute lookup on the plugin's schemas container: plugin.schemas.Opportunity. - native: The source system's Python type (defaults to dict when not specified). - common: The CommonGrants-format Pydantic model class produced by the generator. - If the plugin declares custom_fields, this is a generated subclass of - the base CG model (e.g. OpportunityBase) with those fields already - baked in as typed attributes. - to_common: Transforms native_data → TransformResult[common] (None if not configured). - from_common: Transforms common_data → TransformResult[native] (None if not configured). + source_schema: The source system's Python type (defaults to dict when not specified). + common_schema: The CommonGrants-format Pydantic model class produced by the generator. + If the plugin declares custom_fields, this is a generated subclass of + the base CG model (e.g. OpportunityBase) with those fields already + baked in as typed attributes. + to_common: Transforms source_data → TransformResult[common_schema] (None if not configured). + from_common: Transforms common_data → TransformResult[source_schema] (None if not configured). """ - native: type[TNative] - common: type[TCommon] - to_common: Callable[[TNative], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TNative]] | None = None + source_schema: type[TSource] + common_schema: type[TCommon] + to_common: Callable[[TSource], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TSource]] | None = None diff --git a/lib/python-sdk/examples/plugin_custom_fields.py b/lib/python-sdk/examples/plugin_custom_fields.py index 89d4a21ee..738dba2de 100644 --- a/lib/python-sdk/examples/plugin_custom_fields.py +++ b/lib/python-sdk/examples/plugin_custom_fields.py @@ -57,7 +57,9 @@ # Use the model returned via opportunity_extensions # --------------------------------------------------------------------------- -opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) +opp = opportunity_extensions.schemas.Opportunity.common_schema.model_validate( + api_response +) assert opp.custom_fields is not None assert opp.custom_fields.program_area is not None diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py index ae59f443f..4514e8f0e 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/__init__.py +++ b/lib/python-sdk/examples/plugins/grants_gov/__init__.py @@ -10,7 +10,6 @@ grants_gov = Plugin( schemas=schemas, - extensions=config.extensions, meta=config.meta, ) diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py index d78a3f9d9..c155cfc00 100644 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py @@ -13,8 +13,8 @@ from common_grants_sdk.extensions import ( CustomFieldSpec, - ObjectSchemasInput, PluginExtensionsMeta, + SchemaInput, build_transforms, define_plugin, ) @@ -144,7 +144,7 @@ def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: capabilities=["customFields", "transforms"], ), schemas={ - "Opportunity": ObjectSchemasInput( + "Opportunity": SchemaInput( custom_fields={ "legacyId": CustomFieldSpec( field_type=CustomFieldType.INTEGER, diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py index 736eb2390..c8f51a3ac 100644 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py +++ b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py @@ -8,7 +8,6 @@ opportunity_extensions = Plugin( schemas=schemas, - extensions=config.extensions, meta=config.meta, ) diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py index a926e7e9f..97a9d3765 100644 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py +++ b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py @@ -9,13 +9,13 @@ from typing import Any from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput +from common_grants_sdk.extensions import CustomFieldSpec, SchemaInput from common_grants_sdk.extensions.plugin import PluginConfig from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType config: PluginConfig[Any] = define_plugin( schemas={ - "Opportunity": ObjectSchemasInput( + "Opportunity": SchemaInput( custom_fields={ "programArea": CustomFieldSpec( field_type=CustomFieldType.STRING, diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py index 26002e5d9..637aa1ffb 100644 --- a/lib/python-sdk/examples/transforms.py +++ b/lib/python-sdk/examples/transforms.py @@ -89,7 +89,7 @@ def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: # Transform that uses the custom handlers and validates output against the generated -# Opportunity model. common_model=Opportunity (from generated/schemas.py) ensures +# Opportunity model. common_schema=Opportunity (from generated/schemas.py) ensures # model_validate runs against the extended class with typed custom fields # (legacyId, agencyName, applicantTypes), not just the base OpportunityBase. to_common_with_custom, from_common_with_custom = build_transforms( @@ -141,7 +141,7 @@ def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: } }, handlers={"join": join_fields, "split": split_field}, - common_model=Opportunity, + common_schema=Opportunity, ) diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index b8947a47c..072ee1306 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -1,11 +1,11 @@ -"""Tests for plugin.py — PluginExtensions-based API.""" +"""Tests for plugin.py — Plugin and PluginConfig API.""" + +import pytest from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin from common_grants_sdk.extensions.types import ( - ObjectSchemasInput, - PluginExtensions, + SchemaInput, PluginExtensionsMeta, - PluginExtensionsSchema, TransformResult, ) @@ -14,17 +14,10 @@ def test_define_plugin_no_args(): """define_plugin() with no args returns PluginConfig with all fields None.""" config = define_plugin() assert isinstance(config, PluginConfig) - assert config.extensions is None assert config.meta is None assert config.schemas is None -def test_define_plugin_with_extensions(): - ext = PluginExtensions() - config = define_plugin(extensions=ext) - assert config.extensions is ext - - def test_define_plugin_with_meta_and_schemas(): meta = PluginExtensionsMeta(name="test", source_system="test-system") @@ -32,9 +25,7 @@ def passthrough(x): return TransformResult(result=x, errors=[]) schemas = { - "Opportunity": ObjectSchemasInput( - to_common=passthrough, from_common=passthrough - ) + "Opportunity": SchemaInput(to_common=passthrough, from_common=passthrough) } config = define_plugin(meta=meta, schemas=schemas) assert config.meta is meta @@ -50,7 +41,7 @@ def always_transformed(_x): config = define_plugin( schemas={ - "Opportunity": ObjectSchemasInput( + "Opportunity": SchemaInput( to_common=always_transformed, from_common=always_transformed ) }, @@ -61,18 +52,15 @@ def always_transformed(_x): def test_plugin_fields_default_to_none(): - """Plugin.schemas holds the container; meta/extensions default to None.""" + """Plugin.schemas holds the container; meta defaults to None.""" base = Plugin(schemas=object()) assert base.meta is None - assert base.extensions is None def test_plugin_fields_populated(): meta = PluginExtensionsMeta(name="p", source_system="s") - ext = PluginExtensions(schemas={"Opportunity": PluginExtensionsSchema()}) - full = Plugin(schemas=object(), extensions=ext, meta=meta) + full = Plugin(schemas=object(), meta=meta) assert full.meta is meta - assert full.extensions is ext def test_plugin_schemas_is_attribute_container(): @@ -81,3 +69,86 @@ def test_plugin_schemas_is_attribute_container(): p = Plugin(schemas=s) assert p.schemas is s assert not hasattr(p, "generated_schemas") + + +# --------------------------------------------------------------------------- +# XOR constraint tests +# --------------------------------------------------------------------------- + + +def test_define_plugin_xor_mappings_and_both_callables_raises(): + """Providing mappings AND both explicit callables raises ValueError.""" + from common_grants_sdk.extensions.types import SchemaMappings + + def noop(x): + return TransformResult(result=x, errors=[]) + + with pytest.raises( + ValueError, + match="cannot specify both mappings and explicit to_common/from_common", + ): + define_plugin( + schemas={ + "Opportunity": SchemaInput( + mappings=SchemaMappings( + to_common={"title": {"field": "native_title"}}, + from_common={"native_title": {"field": "title"}}, + ), + to_common=noop, + from_common=noop, + ) + } + ) + + +def test_define_plugin_xor_mappings_and_one_callable_raises(): + """Providing mappings AND a single explicit callable raises ValueError.""" + from common_grants_sdk.extensions.types import SchemaMappings + + def noop(x): + return TransformResult(result=x, errors=[]) + + with pytest.raises( + ValueError, + match="cannot specify both mappings and explicit to_common/from_common", + ): + define_plugin( + schemas={ + "Opportunity": SchemaInput( + mappings=SchemaMappings( + to_common={"title": {"field": "native_title"}}, + from_common={"native_title": {"field": "title"}}, + ), + to_common=noop, + ) + } + ) + + +def test_define_plugin_mappings_without_callables_is_valid(): + """Providing mappings without explicit callables does not raise.""" + from common_grants_sdk.extensions.types import SchemaMappings + + config = define_plugin( + schemas={ + "Opportunity": SchemaInput( + mappings=SchemaMappings( + to_common={"title": {"field": "native_title"}}, + from_common={"native_title": {"field": "title"}}, + ) + ) + } + ) + assert config.schemas is not None + + +def test_define_plugin_callables_without_mappings_is_valid(): + """Providing explicit callables without mappings does not raise.""" + + def noop(x): + return TransformResult(result=x, errors=[]) + + config = define_plugin( + schemas={"Opportunity": SchemaInput(to_common=noop, from_common=noop)} + ) + assert config.schemas is not None diff --git a/lib/python-sdk/tests/extensions/test_transforms.py b/lib/python-sdk/tests/extensions/test_transforms.py index e049b7021..5ed37cf9e 100644 --- a/lib/python-sdk/tests/extensions/test_transforms.py +++ b/lib/python-sdk/tests/extensions/test_transforms.py @@ -5,7 +5,7 @@ import pytest from pydantic import BaseModel from common_grants_sdk.extensions.transforms import build_transforms -from common_grants_sdk.extensions.types import PluginError, TransformResult +from common_grants_sdk.extensions.types import TransformError, TransformResult # Shared model fixtures for output-path validation tests @@ -149,7 +149,7 @@ def test_from_common_roundtrip(): def test_exception_surfaces_as_plugin_error_not_raised(): - """Exceptions inside handlers surface as PluginError, not raised.""" + """Exceptions inside handlers surface as TransformError, not raised.""" def boom(data, _arg): raise RuntimeError("handler exploded") @@ -162,14 +162,14 @@ def boom(data, _arg): result = to_common(SOURCE_DATA) assert len(result.errors) == 1 err = result.errors[0] - assert isinstance(err, PluginError) + assert isinstance(err, TransformError) assert "handler exploded" in str(err) assert err.handler == "boom" assert isinstance(err.cause, RuntimeError) assert str(err.cause) == "handler exploded" -# --- model_validate via common_model --- +# --- model_validate via common_schema --- class _TitleModel(BaseModel): @@ -181,12 +181,12 @@ class _StrictModel(BaseModel): required_field: str # always missing from SOURCE_DATA transform output -def test_common_model_validates_result(): - """When common_model is provided, result is a model instance on success.""" +def test_common_schema_validates_result(): + """When common_schema is provided, result is a model instance on success.""" to_common, _ = build_transforms( {"title": {"field": "data.opportunity_title"}}, {}, - common_model=_TitleModel, + common_schema=_TitleModel, ) result = to_common(SOURCE_DATA) assert result.errors == [] @@ -194,22 +194,22 @@ def test_common_model_validates_result(): assert result.result.title == "Research into conservation techniques" -def test_common_model_validation_failure(): - """ValidationError surfaces as PluginError entries; raw dict is still returned.""" +def test_common_schema_validation_failure(): + """ValidationError surfaces as TransformError entries; raw dict is still returned.""" to_common, _ = build_transforms( {"title": {"field": "data.opportunity_title"}}, {}, - common_model=_StrictModel, + common_schema=_StrictModel, ) result = to_common(SOURCE_DATA) assert len(result.errors) >= 1 - assert all(isinstance(e, PluginError) for e in result.errors) + assert all(isinstance(e, TransformError) for e in result.errors) assert any("required_field" in (e.path or "") for e in result.errors) assert result.result["title"] == "Research into conservation techniques" -def test_common_model_non_validation_error_is_caught() -> None: - """Non-ValidationError exceptions from model_validate surface as PluginError (errors-as-values contract).""" +def test_common_schema_non_validation_error_is_caught() -> None: + """Non-ValidationError exceptions from model_validate surface as TransformError (errors-as-values contract).""" class _BrokenModel(BaseModel): title: str @@ -221,11 +221,11 @@ def model_validate(cls, obj: Any, **kwargs: Any) -> "_BrokenModel": to_common, _ = build_transforms( {"title": {"field": "data.opportunity_title"}}, {}, - common_model=_BrokenModel, + common_schema=_BrokenModel, ) result = to_common(SOURCE_DATA) assert len(result.errors) == 1 - assert isinstance(result.errors[0], PluginError) + assert isinstance(result.errors[0], TransformError) assert "misconfigured root validator" in str(result.errors[0]) # raw transformed dict is preserved so the caller can inspect it assert isinstance(result.result, dict) @@ -251,7 +251,7 @@ def handle_upper(data, path): assert result.result["title"] == "RESEARCH INTO CONSERVATION TECHNIQUES" -# --- Output field-path validation (when common_model is provided) --- +# --- Output field-path validation (when common_schema is provided) --- def test_unknown_output_key_raises_when_model_provided(): @@ -260,7 +260,7 @@ def test_unknown_output_key_raises_when_model_provided(): build_transforms( {"title": {"field": "data.opportunity_title"}, "unknown_xyz": "literal"}, {}, - common_model=_CommonModel, + common_schema=_CommonModel, ) @@ -272,7 +272,7 @@ def test_custom_fields_key_is_valid_output_path(): "custom_fields": {"legacyId": {"field": "data.opportunity_title"}}, }, {}, - common_model=_CommonModelWithCustomFields, + common_schema=_CommonModelWithCustomFields, ) result = to_common(SOURCE_DATA) assert result.errors == [] @@ -284,21 +284,21 @@ def test_custom_fields_key_is_valid_output_path(): ) -def test_output_path_validation_only_applies_to_to_common(): - """from_common output keys are not validated against common_model (they target native format).""" +def test_output_path_validation_only_applies_to_to_common_when_only_common_schema_given(): + """from_common output keys are not validated against common_schema (they target source system format).""" # from_common_mapping has keys that are NOT on _CommonModel — that's fine to_common, from_common = build_transforms( {"title": {"field": "data.opportunity_title"}}, {"data": {"opportunity_title": {"field": "title"}}}, - common_model=_CommonModel, + common_schema=_CommonModel, ) result = from_common({"title": "hello"}) assert result.errors == [] assert result.result["data"]["opportunity_title"] == "hello" -def test_no_output_validation_without_common_model(): - """Without common_model, unknown output keys are allowed (no schema to validate against).""" +def test_no_output_validation_without_common_schema(): + """Without common_schema, unknown output keys are allowed (no schema to validate against).""" to_common, _ = build_transforms( {"any_key_is_fine": {"field": "data.opportunity_title"}}, {}, @@ -306,3 +306,112 @@ def test_no_output_validation_without_common_model(): result = to_common(SOURCE_DATA) assert result.errors == [] assert result.result["any_key_is_fine"] == "Research into conservation techniques" + + +# --- source_schema validation --- + + +class _NativeModel(BaseModel): + native_title: str + native_id: str + + +class _StrictNativeModel(BaseModel): + native_title: str + native_id: str # always missing from the partial mapping tests below + + +def test_source_schema_validates_from_common_result(): + """When source_schema is provided, result is a model instance on success.""" + _, from_common = build_transforms( + {}, + { + "native_title": {"field": "title"}, + "native_id": {"field": "id"}, + }, + source_schema=_NativeModel, + ) + result = from_common({"title": "Test Opp", "id": "abc-123"}) + assert result.errors == [] + assert isinstance(result.result, _NativeModel) + assert result.result.native_title == "Test Opp" + assert result.result.native_id == "abc-123" + + +def test_source_schema_validation_failure_surfaces_as_plugin_errors(): + """ValidationError from source_schema surfaces as TransformError entries; raw dict still returned.""" + _, from_common = build_transforms( + {}, + # Only maps native_title — native_id will be missing, causing a ValidationError + {"native_title": {"field": "title"}}, + source_schema=_StrictNativeModel, + ) + result = from_common({"title": "Test Opp"}) + assert len(result.errors) >= 1 + assert all(isinstance(e, TransformError) for e in result.errors) + assert any("native_id" in (e.path or "") for e in result.errors) + # raw dict preserved alongside errors + assert result.result["native_title"] == "Test Opp" + + +def test_source_schema_aggregates_all_zod_issues(): + """All ValidationError issues are surfaced — not just the first.""" + + class _TwoFieldNative(BaseModel): + a: str + b: int + + _, from_common = build_transforms( + {}, + # Both fields will fail: a gets an int (expects str coercion ok) and b gets a non-int str + {"a": {"const": 123}, "b": {"const": "not-a-number"}}, + source_schema=_TwoFieldNative, + ) + result = from_common({}) + # Pydantic may coerce `a` (int→str), but `b` ("not-a-number"→int) must fail + assert len(result.errors) >= 1 + assert all(isinstance(e, TransformError) for e in result.errors) + + +def test_source_schema_validates_from_common_output_keys_at_call_time(): + """build_transforms raises at call time if a from_common output key is not on source_schema.""" + with pytest.raises(ValueError, match="unknown_native_field"): + build_transforms( + {}, + {"native_title": {"field": "title"}, "unknown_native_field": "literal"}, + source_schema=_NativeModel, + ) + + +def test_from_common_output_keys_not_validated_without_source_schema(): + """Without source_schema, from_common output keys are not validated.""" + _, from_common = build_transforms( + {}, + {"any_native_key": {"field": "title"}}, + ) + result = from_common({"title": "hello"}) + assert result.errors == [] + assert result.result["any_native_key"] == "hello" + + +def test_source_schema_non_validation_error_is_caught() -> None: + """Non-ValidationError exceptions from model_validate on source_schema surface as TransformError.""" + + class _BrokenNativeModel(BaseModel): + native_title: str + + @classmethod + def model_validate(cls, obj: Any, **kwargs: Any) -> "_BrokenNativeModel": + raise TypeError("broken native validator") + + _, from_common = build_transforms( + {}, + {"native_title": {"field": "title"}}, + source_schema=_BrokenNativeModel, + ) + result = from_common({"title": "Test"}) + assert len(result.errors) == 1 + assert isinstance(result.errors[0], TransformError) + assert "broken native validator" in str(result.errors[0]) + assert isinstance(result.result, dict) + assert result.result["native_title"] == "Test" diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index 9a6e90250..85e76ad71 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -2,22 +2,20 @@ from common_grants_sdk.extensions.specs import CustomFieldSpec from common_grants_sdk.extensions.types import ( - ObjectMappings, - ObjectSchemas, - ObjectSchemasInput, - PluginError, - PluginExtensions, PluginExtensionsMeta, - PluginExtensionsSchema, + SchemaConfig, + SchemaInput, + SchemaMappings, + TransformError, TransformResult, ) from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType -# --- PluginError --- +# --- TransformError --- -def test_plugin_error_is_exception_with_defaults(): - err = PluginError("something went wrong") +def test_transform_error_is_exception_with_defaults(): + err = TransformError("something went wrong") assert isinstance(err, Exception) assert str(err) == "something went wrong" assert err.path is None @@ -26,9 +24,9 @@ def test_plugin_error_is_exception_with_defaults(): assert err.cause is None -def test_plugin_error_structured_fields(): +def test_transform_error_structured_fields(): cause = ValueError("root cause") - err = PluginError( + err = TransformError( "msg", path="status.value", handler="switch", @@ -41,10 +39,10 @@ def test_plugin_error_structured_fields(): assert err.cause is cause -def test_plugin_error_source_value_excluded_from_str_and_repr(): +def test_transform_error_source_value_excluded_from_str_and_repr(): """source_value must not appear in str(err) or repr(err) — PII defence per ADR-0022 Decision #9.""" sensitive = {"ssn": "123-45-6789"} - err = PluginError("transform failed", source_value=sensitive) + err = TransformError("transform failed", source_value=sensitive) assert "123-45-6789" not in str(err) assert "123-45-6789" not in repr(err) @@ -57,19 +55,19 @@ def test_transform_result(): assert ok.result == {"title": "hello"} assert ok.errors == [] - err = PluginError("bad") + err = TransformError("bad") partial = TransformResult(result={}, errors=[err]) assert len(partial.errors) == 1 assert partial.errors[0] is err -# --- ObjectMappings --- +# --- SchemaMappings --- -def test_object_mappings(): - assert ObjectMappings().to_common is None - assert ObjectMappings().from_common is None - m = ObjectMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) +def test_schema_mappings(): + assert SchemaMappings().to_common is None + assert SchemaMappings().from_common is None + m = SchemaMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) assert m.to_common == {"title": "x"} assert m.from_common == {"x": "title"} @@ -78,65 +76,54 @@ def test_object_mappings(): def test_plugin_extensions_meta(): - assert PluginExtensionsMeta().name is None - assert PluginExtensionsMeta().source_system is None - m = PluginExtensionsMeta(sourceSystem="grants.gov") + m = PluginExtensionsMeta(name="grants.gov", sourceSystem="grants.gov") + assert m.name == "grants.gov" assert m.source_system == "grants.gov" + assert m.version is None + assert m.capabilities is None -# --- PluginExtensionsSchema --- +# --- SchemaInput --- -def test_plugin_extensions_schema(): - assert PluginExtensionsSchema().mappings is None - s = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) - assert s.mappings.to_common == {"a": "b"} - - -# --- PluginExtensions --- - - -def test_plugin_extensions(): - assert PluginExtensions().meta is None - assert PluginExtensions().schemas is None - schema = PluginExtensionsSchema(mappings=ObjectMappings(toCommon={"a": "b"})) - ext = PluginExtensions(schemas={"Opportunity": schema}) - assert ext.schemas["Opportunity"].mappings is not None - - -# --- ObjectSchemasInput --- - - -def test_object_schemas_input(): - assert ObjectSchemasInput().native is None - assert ObjectSchemasInput().custom_fields is None - assert ObjectSchemasInput().to_common is None +def test_schema_input(): + assert SchemaInput().source_schema is None + assert SchemaInput().custom_fields is None + assert SchemaInput().mappings is None + assert SchemaInput().to_common is None spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) - inp = ObjectSchemasInput(custom_fields={"legacyId": spec}) + inp = SchemaInput(custom_fields={"legacyId": spec}) assert inp.custom_fields == {"legacyId": spec} + m = SchemaMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) + inp_with_mappings = SchemaInput(mappings=m) + assert inp_with_mappings.mappings.to_common == {"title": "x"} + def passthrough(x): return TransformResult(result=x, errors=[]) - inp2 = ObjectSchemasInput(to_common=passthrough, from_common=passthrough) + inp2 = SchemaInput(to_common=passthrough, from_common=passthrough) assert inp2.to_common is passthrough -# --- ObjectSchemas --- +# --- SchemaConfig --- -def test_object_schemas(): +def test_schema_config(): def passthrough(x): return TransformResult(result=x, errors=[]) - schemas = ObjectSchemas( - native=dict, common=dict, to_common=passthrough, from_common=passthrough + schemas = SchemaConfig( + source_schema=dict, + common_schema=dict, + to_common=passthrough, + from_common=passthrough, ) - assert schemas.native is dict - assert schemas.common is dict + assert schemas.source_schema is dict + assert schemas.common_schema is dict # to_common and from_common are optional — omitting them is valid - minimal = ObjectSchemas(native=dict, common=dict) + minimal = SchemaConfig(source_schema=dict, common_schema=dict) assert minimal.to_common is None assert minimal.from_common is None diff --git a/lib/python-sdk/tests/schemas/test_plugin_registry.py b/lib/python-sdk/tests/schemas/test_plugin_registry.py index 6d4ae5b0b..0d0524be4 100644 --- a/lib/python-sdk/tests/schemas/test_plugin_registry.py +++ b/lib/python-sdk/tests/schemas/test_plugin_registry.py @@ -6,7 +6,7 @@ from common_grants_sdk.extensions import CustomFieldSpec from common_grants_sdk.extensions import Plugin -from common_grants_sdk.extensions.types import ObjectSchemas +from common_grants_sdk.extensions.types import SchemaConfig from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.schemas.pydantic.models.opp_base import OpportunityBase @@ -18,19 +18,19 @@ class _Schemas: """Minimal schemas container that mirrors the generated _Schemas class.""" - Opportunity: ObjectSchemas[Any, Any] + Opportunity: SchemaConfig[Any, Any] def _make_plugin( field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity" ) -> "Plugin[_Schemas]": - """Build a Plugin whose schemas.Opportunity.common is produced by with_custom_fields().""" + """Build a Plugin whose schemas.Opportunity.common_schema is produced by with_custom_fields().""" extended = OpportunityBase.with_custom_fields( custom_fields=field_specs, model_name=model_name, ) s = _Schemas() - s.Opportunity = ObjectSchemas(native=dict, common=extended) + s.Opportunity = SchemaConfig(source_schema=dict, common_schema=extended) return Plugin(schemas=s) @@ -85,7 +85,7 @@ def sample_payload() -> dict: def test_plugin_schema_is_subclass_of_opportunity_base(simple_plugin): - Opportunity = simple_plugin.schemas.Opportunity.common + Opportunity = simple_plugin.schemas.Opportunity.common_schema assert Opportunity is not OpportunityBase assert issubclass(Opportunity, OpportunityBase) @@ -103,8 +103,8 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): ) assert ( - simple_plugin.schemas.Opportunity.common - is not second_plugin.schemas.Opportunity.common + simple_plugin.schemas.Opportunity.common_schema + is not second_plugin.schemas.Opportunity.common_schema ) @@ -116,7 +116,7 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( simple_plugin, sample_payload ): - Opportunity = simple_plugin.schemas.Opportunity.common + Opportunity = simple_plugin.schemas.Opportunity.common_schema opp = Opportunity.model_validate(sample_payload) @@ -127,7 +127,7 @@ def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( def test_plugin_schema_validates_custom_fields(simple_plugin, sample_payload): - Opportunity = simple_plugin.schemas.Opportunity.common + Opportunity = simple_plugin.schemas.Opportunity.common_schema opp = Opportunity.model_validate(sample_payload) diff --git a/lib/python-sdk/tests/utils/test_merge_extensions.py b/lib/python-sdk/tests/utils/test_merge_extensions.py deleted file mode 100644 index 945b1f837..000000000 --- a/lib/python-sdk/tests/utils/test_merge_extensions.py +++ /dev/null @@ -1,140 +0,0 @@ -import pytest - -from common_grants_sdk.extensions import merge_extensions -from common_grants_sdk.extensions.types import ( - ObjectMappings, - PluginExtensions, - PluginExtensionsMeta, - PluginExtensionsSchema, -) - - -def test_merge_empty_returns_empty_plugin_extensions() -> None: - result = merge_extensions([]) - assert isinstance(result, PluginExtensions) - assert result.schemas is None - assert result.meta is None - - -def test_merge_single_source_passthrough() -> None: - source = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "name"}}, fromCommon={} - ) - ) - } - ) - merged = merge_extensions([source]) - assert merged is source - - -def test_merge_meta_raises_on_duplicate_name_by_default() -> None: - source_one = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-a")) - source_two = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-b")) - - with pytest.raises(ValueError, match="duplicate"): - merge_extensions([source_one, source_two]) - - -def test_merge_meta_last_wins() -> None: - source_one = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-a")) - source_two = PluginExtensions(meta=PluginExtensionsMeta(name="plugin-b")) - - merged = merge_extensions([source_one, source_two], on_conflict="last_wins") - assert merged.meta is not None - assert merged.meta.name == "plugin-b" - - -def test_merge_raises_on_duplicate_mappings_by_default() -> None: - source_one = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "name"}}, fromCommon={} - ) - ) - } - ) - source_two = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "other"}}, fromCommon={} - ) - ) - } - ) - - with pytest.raises(ValueError, match='duplicate mappings for object "Opportunity"'): - merge_extensions([source_one, source_two]) - - -def test_merge_meta_none_fields_do_not_overwrite() -> None: - """A None field in the second source does not erase a value from the first.""" - source_one = PluginExtensions( - meta=PluginExtensionsMeta(name="plugin-a", version="1.0") - ) - source_two = PluginExtensions(meta=PluginExtensionsMeta(name=None, version="2.0")) - - merged = merge_extensions([source_one, source_two], on_conflict="last_wins") - assert merged.meta is not None - assert merged.meta.name == "plugin-a" # preserved from source_one - assert merged.meta.version == "2.0" # overwritten by source_two - - -def test_merge_mappings_first_wins() -> None: - source_one = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "name"}}, fromCommon={} - ) - ) - } - ) - source_two = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "other"}}, fromCommon={} - ) - ) - } - ) - - merged = merge_extensions([source_one, source_two], on_conflict="first_wins") - assert merged.schemas is not None - assert merged.schemas["Opportunity"].mappings is not None - assert merged.schemas["Opportunity"].mappings.to_common == { - "title": {"field": "name"} - } - - -def test_merge_mappings_last_wins() -> None: - source_one = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "name"}}, fromCommon={} - ) - ) - } - ) - source_two = PluginExtensions( - schemas={ - "Opportunity": PluginExtensionsSchema( - mappings=ObjectMappings( - toCommon={"title": {"field": "other"}}, fromCommon={} - ) - ) - } - ) - - merged = merge_extensions([source_one, source_two], on_conflict="last_wins") - assert merged.schemas is not None - assert merged.schemas["Opportunity"].mappings is not None - assert merged.schemas["Opportunity"].mappings.to_common == { - "title": {"field": "other"} - } diff --git a/lib/python-sdk/tests/utils/test_plugin_generator.py b/lib/python-sdk/tests/utils/test_plugin_generator.py index a182475b0..8b39a829d 100644 --- a/lib/python-sdk/tests/utils/test_plugin_generator.py +++ b/lib/python-sdk/tests/utils/test_plugin_generator.py @@ -29,10 +29,10 @@ def _env_with_sdk_pythonpath() -> dict[str, str]: def test_define_plugin_returns_config_with_schemas(): - from common_grants_sdk.extensions.types import ObjectSchemasInput + from common_grants_sdk.extensions.types import SchemaInput schemas = { - "Opportunity": ObjectSchemasInput( + "Opportunity": SchemaInput( custom_fields={ "program_area": CustomFieldSpec( field_type=CustomFieldType.STRING, @@ -61,11 +61,11 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): [ "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import ObjectSchemasInput", + "from common_grants_sdk.extensions.types import SchemaInput", "", "config = define_plugin(", " schemas={", - ' "Opportunity": ObjectSchemasInput(', + ' "Opportunity": SchemaInput(', " custom_fields={", ' "program_area": CustomFieldSpec(', ' field_type="string",', @@ -105,7 +105,7 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): try: combined_module = importlib.import_module("plugins.combined") combined = getattr(combined_module, "combined") - opp_model = combined.schemas.Opportunity.common + opp_model = combined.schemas.Opportunity.common_schema type_hints = get_type_hints(opp_model, include_extras=False) assert "custom_fields" in type_hints @@ -139,7 +139,7 @@ def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): "nonprofit", "city_government", ] - assert combined.schemas.Opportunity.common is opp_model + assert combined.schemas.Opportunity.common_schema is opp_model finally: sys.path.remove(str(tmp_path)) @@ -155,7 +155,7 @@ def test_generate_emits_import_for_pydantic_model_in_cg_config(tmp_path: Path): "from pydantic import BaseModel", "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import ObjectSchemasInput", + "from common_grants_sdk.extensions.types import SchemaInput", "", "class AgentInfo(BaseModel):", " name: str", @@ -163,7 +163,7 @@ def test_generate_emits_import_for_pydantic_model_in_cg_config(tmp_path: Path): "", "config = define_plugin(", " schemas={", - ' "Opportunity": ObjectSchemasInput(', + ' "Opportunity": SchemaInput(', " custom_fields={", ' "point_of_contact": CustomFieldSpec(', ' field_type="object",', @@ -206,11 +206,11 @@ def test_generate_emits_import_for_external_module_type(tmp_path: Path): "from datetime import datetime", "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import ObjectSchemasInput", + "from common_grants_sdk.extensions.types import SchemaInput", "", "config = define_plugin(", " schemas={", - ' "Opportunity": ObjectSchemasInput(', + ' "Opportunity": SchemaInput(', " custom_fields={", ' "deadline": CustomFieldSpec(', ' field_type="string",', @@ -243,7 +243,7 @@ def test_generate_emits_import_for_external_module_type(tmp_path: Path): def test_generate_auto_builds_transforms_from_mappings(tmp_path): - """When cg_config has extensions.schemas[obj].mappings but no explicit schemas[obj], + """When cg_config has schemas[obj].mappings but no explicit to_common/from_common, the generated __init__.py calls build_transforms() automatically.""" plugin_dir = tmp_path / "plugins" / "auto_transform" plugin_dir.mkdir(parents=True) @@ -253,19 +253,17 @@ def test_generate_auto_builds_transforms_from_mappings(tmp_path): "\n".join( [ "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import PluginExtensions, PluginExtensionsSchema, ObjectMappings", + "from common_grants_sdk.extensions.types import SchemaInput, SchemaMappings", "", "config = define_plugin(", - " extensions=PluginExtensions(", - " schemas={", - ' "Opportunity": PluginExtensionsSchema(', - " mappings=ObjectMappings(", - ' to_common={"title": {"field": "data.title"}},', - " from_common={},", - " ),", - " )", - " }", - " ),", + " schemas={", + ' "Opportunity": SchemaInput(', + " mappings=SchemaMappings(", + ' to_common={"title": {"field": "data.title"}},', + " from_common={},", + " ),", + " )", + " },", ")", "", ] @@ -279,9 +277,9 @@ def test_generate_auto_builds_transforms_from_mappings(tmp_path): init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") assert "build_transforms" in init_content - assert 'config.extensions.schemas["Opportunity"].mappings.to_common' in init_content + assert 'config.schemas["Opportunity"].mappings.to_common' in init_content assert "_Opportunity_to_common" in init_content - assert "common_model=schemas.Opportunity.common" in init_content + assert "common_schema=schemas.Opportunity.common_schema" in init_content # Load the generated plugin and verify schemas are populated import importlib @@ -319,11 +317,11 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): [ "from common_grants_sdk import define_plugin", "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import ObjectSchemasInput", + "from common_grants_sdk.extensions.types import SchemaInput", "", "config = define_plugin(", " schemas={", - ' "Opportunity": ObjectSchemasInput(', + ' "Opportunity": SchemaInput(', " custom_fields={", ' "eligibility_type": CustomFieldSpec(field_type="array"),', " },", @@ -364,7 +362,7 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): ' "customFields": {"eligibility_type": {"fieldType": "array", "value": ["a"]}},', "}", "", - "opp = combined.schemas.Opportunity.common.model_validate(payload)", + "opp = combined.schemas.Opportunity.common_schema.model_validate(payload)", "if opp.custom_fields is not None and opp.custom_fields.eligibility_type is not None:", " values = opp.custom_fields.eligibility_type.value", " reveal_type(values)", @@ -390,7 +388,7 @@ def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): def test_generate_explicit_transforms(tmp_path): """When cg_config has config.schemas with explicit to_common/from_common, - the generated __init__.py emits ObjectSchemas with the supplied callables.""" + the generated __init__.py wires the supplied callables via inject_transforms.""" from common_grants_sdk.extensions.generate import generate_plugin plugin_dir = tmp_path / "plugins" / "explicit_tf" @@ -401,7 +399,7 @@ def test_generate_explicit_transforms(tmp_path): "\n".join( [ "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import ObjectSchemasInput, TransformResult", + "from common_grants_sdk.extensions.types import SchemaInput, TransformResult", "from common_grants_sdk.extensions import CustomFieldSpec", "", "def _to_common(native):", @@ -412,7 +410,7 @@ def test_generate_explicit_transforms(tmp_path): "", "config = define_plugin(", " schemas={", - ' "Opportunity": ObjectSchemasInput(', + ' "Opportunity": SchemaInput(', ' custom_fields={"legacyId": CustomFieldSpec(field_type="integer")},', " to_common=_to_common,", " from_common=_from_common,", @@ -469,7 +467,7 @@ def test_generate_transforms_only_no_custom_fields(tmp_path): "\n".join( [ "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import ObjectSchemasInput, TransformResult", + "from common_grants_sdk.extensions.types import SchemaInput, TransformResult", "", "def _to_common(native):", " return TransformResult(result={'title': native.get('name', '')}, errors=[])", @@ -479,7 +477,7 @@ def test_generate_transforms_only_no_custom_fields(tmp_path): "", "config = define_plugin(", " schemas={", - ' "Opportunity": ObjectSchemasInput(', + ' "Opportunity": SchemaInput(', " to_common=_to_common,", " from_common=_from_common,", " )", @@ -495,7 +493,7 @@ def test_generate_transforms_only_no_custom_fields(tmp_path): # schemas.py must assign self.Opportunity using the base SDK class schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "self.Opportunity = ObjectSchemas" in schemas_src + assert "self.Opportunity = SchemaConfig" in schemas_src assert "OpportunityBase" in schemas_src # __init__.py must use inject_transforms (not build_transforms) @@ -532,21 +530,17 @@ def test_generate_raises_on_missing_mapping_direction(tmp_path): "\n".join( [ "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import (", - " ObjectMappings, PluginExtensions, PluginExtensionsSchema,", - ")", + "from common_grants_sdk.extensions.types import SchemaInput, SchemaMappings", "", "config = define_plugin(", - " extensions=PluginExtensions(", - " schemas={", - ' "Opportunity": PluginExtensionsSchema(', - " mappings=ObjectMappings(", - ' to_common={"title": {"field": "data.title"}},', - " # from_common intentionally omitted (None)", - " ),", - " )", - " }", - " ),", + " schemas={", + ' "Opportunity": SchemaInput(', + " mappings=SchemaMappings(", + ' to_common={"title": {"field": "data.title"}},', + " # from_common intentionally omitted (None)", + " ),", + " )", + " },", ")", "", ] diff --git a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts index d462cf42c..a327d201d 100644 --- a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect } from "vitest"; import { z } from "zod"; import { buildTransforms, definePlugin, type PluginMeta, type TransformResult } from "@/extensions"; import { OpportunityBaseSchema } from "@/schemas/zod/models"; @@ -47,11 +47,11 @@ describe("definePlugin", () => { }); it("should preserve meta input by reference", () => { - const extensions = { meta: { name: "test", sourceSystem: "test" } }; + const meta: PluginMeta = { name: "test", sourceSystem: "test" }; - const plugin = definePlugin({ extensions }); + const plugin = definePlugin({ meta }); - expect(plugin.extensions).toBe(extensions); + expect(plugin.meta).toBe(meta); }); }); @@ -60,7 +60,7 @@ describe("definePlugin", () => { // ############################################################################ describe("extensible schemas", () => { - it("should parse payloads with custom fields via plugin.schemas.Opportunity.common", () => { + it("should parse payloads with custom fields via plugin.schemas.Opportunity.commonSchema", () => { const plugin = definePlugin({ schemas: { Opportunity: { @@ -78,7 +78,7 @@ describe("definePlugin", () => { }, } as const); - const result = plugin.schemas.Opportunity.common.parse({ + const result = plugin.schemas.Opportunity.commonSchema.parse({ ...validOpp, customFields: { legacyId: { @@ -113,7 +113,7 @@ describe("definePlugin", () => { }, } as const); - const result = plugin.schemas.Opportunity.common.safeParse({ + const result = plugin.schemas.Opportunity.commonSchema.safeParse({ ...validOpp, customFields: { legacyId: { @@ -136,7 +136,7 @@ describe("definePlugin", () => { it("should return base schema under .common when no customFields are provided", () => { const plugin = definePlugin({}); - expect(plugin.schemas.Opportunity.common).toBe(OpportunityBaseSchema); + expect(plugin.schemas.Opportunity.commonSchema).toBe(OpportunityBaseSchema); }); it("should return base schema under .common when customFields is empty", () => { @@ -144,7 +144,7 @@ describe("definePlugin", () => { schemas: { Opportunity: { customFields: {} } }, }); - expect(plugin.schemas.Opportunity.common).toBe(OpportunityBaseSchema); + expect(plugin.schemas.Opportunity.commonSchema).toBe(OpportunityBaseSchema); }); }); @@ -199,7 +199,7 @@ describe("definePlugin", () => { // ############################################################################ describe("schemas (transforms)", () => { - it("preserves toCommon and fromCommon on the returned plugin schemas", () => { + it("exposes toCommon and fromCommon callables on the returned plugin schemas", () => { const { toCommon, fromCommon } = buildTransforms( { title: { field: "data.opportunity_title" } }, { data: { opportunity_title: { field: "title" } } } @@ -211,14 +211,25 @@ describe("definePlugin", () => { }, }); - expect(plugin.schemas.Opportunity.toCommon).toBe(toCommon); - expect(plugin.schemas.Opportunity.fromCommon).toBe(fromCommon); + // definePlugin wraps callables with schema validation, so the references differ. + // Verify they are callable functions. + expect(typeof plugin.schemas.Opportunity.toCommon).toBe("function"); + expect(typeof plugin.schemas.Opportunity.fromCommon).toBe("function"); }); it("invokes the stored transform callable via plugin.schemas", () => { + // Mapping must cover all required OpportunityBaseSchema fields because + // definePlugin wraps toCommon with commonSchema validation. const { toCommon, fromCommon } = buildTransforms( - { title: { field: "data.opportunity_title" } }, - { data: { opportunity_title: { field: "title" } } } + { + id: { field: "native_id" }, + title: { field: "native_title" }, + description: { const: "Test opportunity" }, + createdAt: { const: "2025-01-01T00:00:00Z" }, + lastModifiedAt: { const: "2025-01-01T00:00:00Z" }, + status: { value: { const: "open" } }, + }, + { native_title: { field: "title" }, native_id: { field: "id" } } ); const plugin = definePlugin({ @@ -228,7 +239,8 @@ describe("definePlugin", () => { }); const out = plugin.schemas.Opportunity.toCommon?.({ - data: { opportunity_title: "Hello" }, + native_id: "573525f2-8e15-4405-83fb-e6523511d893", + native_title: "Hello", }) as TransformResult<{ title: string }>; expect(out.errors).toEqual([]); @@ -262,7 +274,7 @@ describe("definePlugin", () => { }, }); - const result = plugin.schemas.Opportunity.common.parse({ + const result = plugin.schemas.Opportunity.commonSchema.parse({ ...validOpp, customFields: { legacyId: { @@ -281,9 +293,8 @@ describe("definePlugin", () => { it("works without a schemas argument at all", () => { const plugin = definePlugin({}); - expect(plugin.extensions).toBeUndefined(); // Common schema defaults to base schema when no customFields provided. - const result = plugin.schemas.Opportunity.common.parse(validOpp); + const result = plugin.schemas.Opportunity.commonSchema.parse(validOpp); expect(result.title).toBe("Test Opportunity"); }); }); @@ -312,25 +323,23 @@ const autoWireFromCommonMapping = { }; describe("definePlugin — auto-wiring from mappings", () => { - it("auto-generates working toCommon/fromCommon from extensions.schemas.Opportunity.mappings", () => { + it("auto-generates working toCommon/fromCommon from schemas.Opportunity.mappings", () => { const plugin = definePlugin({ - extensions: { - schemas: { - Opportunity: { - mappings: { - toCommon: autoWireToCommonMapping, - fromCommon: autoWireFromCommonMapping, - }, + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + fromCommon: autoWireFromCommonMapping, }, }, }, }); - const nativeData = { + const sourceData = { native_id: "573525f2-8e15-4405-83fb-e6523511d893", native_title: "Test Opp", }; - const result = plugin.schemas.Opportunity.toCommon?.(nativeData); + const result = plugin.schemas.Opportunity.toCommon?.(sourceData); expect(result?.errors).toHaveLength(0); expect(result?.result).toMatchObject({ id: "573525f2-8e15-4405-83fb-e6523511d893", @@ -338,74 +347,64 @@ describe("definePlugin — auto-wiring from mappings", () => { }); }); - it("explicit callables take priority — any explicit callable disables auto-wiring for that object", () => { - const explicitToCommon = vi.fn( - (_: unknown): TransformResult<unknown> => ({ result: { id: "explicit" }, errors: [] }) - ); - const explicitFromCommon = vi.fn( - (_: unknown): TransformResult<unknown> => ({ result: {}, errors: [] }) - ); + it("throws when both mappings and explicit toCommon/fromCommon are provided (XOR constraint)", () => { + const explicitToCommon = (_: unknown): TransformResult<unknown> => ({ + result: { id: "explicit" }, + errors: [], + }); + const explicitFromCommon = (_: unknown): TransformResult<unknown> => ({ + result: {}, + errors: [], + }); - const plugin = definePlugin({ - extensions: { + expect(() => + definePlugin({ schemas: { Opportunity: { mappings: { toCommon: autoWireToCommonMapping, fromCommon: autoWireFromCommonMapping, }, + // @ts-expect-error — intentionally testing XOR constraint: mappings + explicit callables is invalid + toCommon: explicitToCommon, + // @ts-expect-error — intentionally testing XOR constraint: mappings + explicit callables is invalid + fromCommon: explicitFromCommon, }, }, - }, - schemas: { - Opportunity: { - toCommon: explicitToCommon, - fromCommon: explicitFromCommon, - }, - }, - }); - - plugin.schemas.Opportunity.toCommon?.({}); - expect(explicitToCommon).toHaveBeenCalledOnce(); + }) + ).toThrow(/cannot specify both mappings and explicit toCommon\/fromCommon/); }); - it("disables auto-wiring when only one explicit callable is provided", () => { - const explicitToCommon = vi.fn( - (_: unknown): TransformResult<unknown> => ({ result: {}, errors: [] }) - ); + it("throws when mappings and a single explicit callable are both provided (XOR constraint)", () => { + const explicitToCommon = (_: unknown): TransformResult<unknown> => ({ + result: {}, + errors: [], + }); - const plugin = definePlugin({ - extensions: { + expect(() => + definePlugin({ schemas: { Opportunity: { mappings: { toCommon: autoWireToCommonMapping, fromCommon: autoWireFromCommonMapping, }, + // @ts-expect-error — intentionally testing XOR constraint: mappings + explicit callable is invalid + toCommon: explicitToCommon, }, }, - }, - schemas: { - Opportunity: { toCommon: explicitToCommon }, - }, - }); - - // fromCommon is not auto-wired because explicitToCommon disables auto-wiring - expect(plugin.schemas.Opportunity.fromCommon).toBeUndefined(); - plugin.schemas.Opportunity.toCommon?.({}); - expect(explicitToCommon).toHaveBeenCalledOnce(); + }) + ).toThrow(/cannot specify both mappings and explicit toCommon\/fromCommon/); }); it("throws at definition time when mappings.fromCommon is absent", () => { expect(() => definePlugin({ - extensions: { - schemas: { - Opportunity: { - mappings: { - toCommon: autoWireToCommonMapping, - // fromCommon intentionally absent - }, + schemas: { + Opportunity: { + mappings: { + toCommon: autoWireToCommonMapping, + // fromCommon intentionally absent }, }, }, @@ -416,13 +415,11 @@ describe("definePlugin — auto-wiring from mappings", () => { it("throws at definition time when mappings.toCommon is absent", () => { expect(() => definePlugin({ - extensions: { - schemas: { - Opportunity: { - mappings: { - // toCommon intentionally absent - fromCommon: autoWireFromCommonMapping, - }, + schemas: { + Opportunity: { + mappings: { + // toCommon intentionally absent + fromCommon: autoWireFromCommonMapping, }, }, }, @@ -433,13 +430,11 @@ describe("definePlugin — auto-wiring from mappings", () => { it("auto-wired toCommon rejects unknown output fields via validateOutputPaths", () => { expect(() => definePlugin({ - extensions: { - schemas: { - Opportunity: { - mappings: { - toCommon: { unknownFieldXyz: { field: "data.x" } }, - fromCommon: autoWireFromCommonMapping, - }, + schemas: { + Opportunity: { + mappings: { + toCommon: { unknownFieldXyz: { field: "data.x" } }, + fromCommon: autoWireFromCommonMapping, }, }, }, @@ -447,44 +442,38 @@ describe("definePlugin — auto-wiring from mappings", () => { ).toThrow(/unknown output fields.*"unknownFieldXyz"/); }); - it("preserves native schema on schemas[Name] in the auto-wired branch", () => { - const nativeSchema = z.object({ native_id: z.string(), native_title: z.string() }); + it("preserves source schema on schemas[Name] in the auto-wired branch", () => { + const sourceSchema = z.object({ native_id: z.string(), native_title: z.string() }); const plugin = definePlugin({ - extensions: { - schemas: { - Opportunity: { - mappings: { - toCommon: autoWireToCommonMapping, - fromCommon: autoWireFromCommonMapping, - }, - }, - }, - }, schemas: { Opportunity: { - native: nativeSchema, + sourceSchema, + mappings: { + toCommon: autoWireToCommonMapping, + fromCommon: autoWireFromCommonMapping, + }, }, }, }); - expect(plugin.schemas.Opportunity.native).toBe(nativeSchema); + expect(plugin.schemas.Opportunity.sourceSchema).toBe(sourceSchema); }); - it("no-ops cleanly when extensions is absent entirely", () => { + it("no-ops cleanly when schemas is absent entirely", () => { const plugin = definePlugin({}); expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); expect(plugin.schemas.Opportunity.fromCommon).toBeUndefined(); }); - it("no-ops cleanly when extensions.schemas has no key for this model name", () => { - const plugin = definePlugin({ extensions: { schemas: {} } }); + it("no-ops cleanly when schemas has no key for this model name", () => { + const plugin = definePlugin({ schemas: {} }); expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); }); - it("no-ops cleanly when extensions.schemas[Name].mappings is undefined", () => { + it("no-ops cleanly when schemas[Name].mappings is undefined", () => { const plugin = definePlugin({ - extensions: { schemas: { Opportunity: { mappings: undefined } } }, + schemas: { Opportunity: { mappings: undefined } }, }); expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); }); diff --git a/lib/ts-sdk/__tests__/extensions/index.spec.ts b/lib/ts-sdk/__tests__/extensions/index.spec.ts index 3dac63802..165d725ac 100644 --- a/lib/ts-sdk/__tests__/extensions/index.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/index.spec.ts @@ -348,7 +348,7 @@ describe("plugin integration", () => { }, }, }; - const legacyResult = legacyPlugin.schemas.Opportunity.common.parse(legacyData); + const legacyResult = legacyPlugin.schemas.Opportunity.commonSchema.parse(legacyData); const legacyId = getCustomFieldValue(legacyResult, "legacyId", LegacyIdValueSchema); expect(legacyId).toEqual({ system: "grants-v1", id: 42 }); @@ -368,7 +368,8 @@ describe("plugin integration", () => { }, }, }; - const classResult = classificationPlugin.schemas.Opportunity.common.parse(classificationData); + const classResult = + classificationPlugin.schemas.Opportunity.commonSchema.parse(classificationData); const category = getCustomFieldValue(classResult, "category", z.string()); const priority = getCustomFieldValue(classResult, "priority", z.number().int()); expect(category).toBe("STEM Education"); @@ -419,7 +420,7 @@ describe("plugin integration", () => { }, }; - const parsed = combinedPlugin.schemas.Opportunity.common.parse(fullData); + const parsed = combinedPlugin.schemas.Opportunity.commonSchema.parse(fullData); // Extract all custom field values const legacyId = parsed.customFields?.legacyId?.value; @@ -458,7 +459,7 @@ describe("plugin integration", () => { }, }; - const result = combinedPlugin.schemas.Opportunity.common.safeParse(invalidData); + const result = combinedPlugin.schemas.Opportunity.commonSchema.safeParse(invalidData); expect(result.success).toBe(false); if (!result.success) { expect(result.error.issues[0].path).toContain("customFields"); diff --git a/lib/ts-sdk/__tests__/extensions/transformation.spec.ts b/lib/ts-sdk/__tests__/extensions/transformation.spec.ts index ae02eb0c3..2b2313bf0 100644 --- a/lib/ts-sdk/__tests__/extensions/transformation.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/transformation.spec.ts @@ -11,7 +11,7 @@ import { fieldValue, stringToNumber, switchOnValue, -} from "@/extensions/transformation"; +} from "@/utils/transformation"; // ############################################################################ // getFromPath diff --git a/lib/ts-sdk/__tests__/extensions/transforms.spec.ts b/lib/ts-sdk/__tests__/extensions/transforms.spec.ts index 145a6ff25..74c7b42a2 100644 --- a/lib/ts-sdk/__tests__/extensions/transforms.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/transforms.spec.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeEach } from "vitest"; import { z } from "zod"; -import { PluginError, buildTransforms, getFromPath, withCustomFields } from "@/extensions"; +import { TransformError, buildTransforms, getFromPath, withCustomFields } from "@/extensions"; import { OpportunityBaseSchema } from "@/schemas/zod/models"; import { CustomFieldType } from "@/constants"; @@ -112,14 +112,14 @@ describe("buildTransforms — toCommon", () => { }); }); - it("wraps a handler exception as a PluginError carrying handler + cause", () => { + it("wraps a handler exception as a TransformError carrying handler + cause", () => { const { toCommon } = buildTransforms({ amount: { stringToNumber: "data.bogus" } }, {}); // Use a source value that fails coercion so stringToNumber throws. const out = toCommon({ data: { bogus: "abc" } }); expect(out.errors).toHaveLength(1); const [err] = out.errors; - expect(err).toBeInstanceOf(PluginError); + expect(err).toBeInstanceOf(TransformError); expect(err.handler).toBe("stringToNumber"); expect(err.cause).toBeInstanceOf(Error); // On handler exception, result is an empty object — no partial fields. @@ -128,10 +128,10 @@ describe("buildTransforms — toCommon", () => { expect(out.result).toStrictEqual({}); }); - it("short-circuits on the first HandlerError — two failing fields produce a single PluginError", () => { + it("short-circuits on the first HandlerError — two failing fields produce a single TransformError", () => { // Locks in the documented asymmetry: handler-failure mode is first-error-stops, // while the Zod-validation path aggregates every issue. Two `stringToNumber` - // calls both fail on `"abc"`, but only one PluginError surfaces. + // calls both fail on `"abc"`, but only one TransformError surfaces. const { toCommon } = buildTransforms( { a: { stringToNumber: "data.bogus_a" }, @@ -143,11 +143,11 @@ describe("buildTransforms — toCommon", () => { const out = toCommon({ data: { bogus_a: "abc", bogus_b: "def" } }); expect(out.errors).toHaveLength(1); - expect(out.errors[0]).toBeInstanceOf(PluginError); + expect(out.errors[0]).toBeInstanceOf(TransformError); expect(out.errors[0].handler).toBe("stringToNumber"); }); - it("flattens Zod validation issues into PluginError[] when commonModel is provided", () => { + it("flattens Zod validation issues into TransformError[] when commonSchema is provided", () => { const { toCommon } = buildTransforms( // Output is intentionally missing required CG fields so Zod fails. { title: { field: "data.opportunity_title" } }, @@ -159,7 +159,7 @@ describe("buildTransforms — toCommon", () => { const out = toCommon(SOURCE_DATA); expect(out.errors.length).toBeGreaterThan(0); - expect(out.errors[0]).toBeInstanceOf(PluginError); + expect(out.errors[0]).toBeInstanceOf(TransformError); // Path is a non-empty dot-joined string for field-level issues, or // `undefined` for root-level issues from schema-wide `.refine()` calls. expect( @@ -171,9 +171,9 @@ describe("buildTransforms — toCommon", () => { expect(out.result).toEqual({ title: "Research into conservation techniques" }); }); - it("produces one PluginError per ZodIssue (aggregation contract — not first-issue-only)", () => { + it("produces one TransformError per ZodIssue (aggregation contract — not first-issue-only)", () => { // Pin the documented asymmetry from buildTransforms() JSDoc: handler - // failures short-circuit (one PluginError per call), Zod failures + // failures short-circuit (one TransformError per call), Zod failures // aggregate every issue. Build a small schema that produces a // deterministic two-issue failure so a regression to // `[parsed.error.issues[0]]` would fail this test. @@ -195,7 +195,7 @@ describe("buildTransforms — toCommon", () => { const out = toCommon({}); expect(out.errors).toHaveLength(2); - expect(out.errors.every(e => e instanceof PluginError)).toBe(true); + expect(out.errors.every(e => e instanceof TransformError)).toBe(true); const paths = out.errors.map(e => e.path).sort(); expect(paths).toEqual(["a", "b"]); }); @@ -268,19 +268,91 @@ describe("buildTransforms — fromCommon", () => { ); }); - it("wraps a handler exception as a PluginError carrying handler + cause", () => { + it("wraps a handler exception as a TransformError carrying handler + cause", () => { const { fromCommon } = buildTransforms({}, { data: { amount: { stringToNumber: "bogus" } } }); // Use a CG-shaped value where `bogus` fails coercion so stringToNumber throws. const out = fromCommon({ bogus: "abc" } as never); expect(out.errors).toHaveLength(1); const [err] = out.errors; - expect(err).toBeInstanceOf(PluginError); + expect(err).toBeInstanceOf(TransformError); expect(err.handler).toBe("stringToNumber"); expect(err.cause).toBeInstanceOf(Error); // Same contract as the toCommon side: literal `{}`, not null/undefined. expect(out.result).toStrictEqual({}); }); + + it("flattens Zod validation issues into TransformError[] when sourceSchema is provided", () => { + const SourceSchema = z.object({ + native_id: z.string(), + native_title: z.string(), + }); + + const { fromCommon } = buildTransforms( + {}, + // Only maps native_title — native_id will be missing, causing a Zod error. + { native_title: { field: "title" } }, + undefined, + undefined, + SourceSchema + ); + + const out = fromCommon({ title: "Test Opp" } as never); + + expect(out.errors.length).toBeGreaterThan(0); + expect(out.errors[0]).toBeInstanceOf(TransformError); + expect(out.errors.some(e => e.path === "native_id")).toBe(true); + // result preserves the raw transformed object so callers can inspect malformed data. + expect(out.result).toEqual({ native_title: "Test Opp" }); + }); + + it("passes through valid fromCommon output when sourceSchema is satisfied", () => { + const SourceSchema = z.object({ + native_id: z.string(), + native_title: z.string(), + }); + + const { fromCommon } = buildTransforms( + {}, + { + native_id: { field: "id" }, + native_title: { field: "title" }, + }, + undefined, + undefined, + SourceSchema + ); + + const out = fromCommon({ id: "abc-123", title: "Test Opp" } as never); + + expect(out.errors).toEqual([]); + expect(out.result).toEqual({ native_id: "abc-123", native_title: "Test Opp" }); + }); + + it("produces one TransformError per ZodIssue from sourceSchema (same aggregation contract as commonSchema)", () => { + const SourceSchema = z.object({ + a: z.string().min(5), + b: z.number().int(), + }); + + const { fromCommon } = buildTransforms( + {}, + { + a: { const: "x" }, + b: { const: "not-a-number" }, + }, + undefined, + undefined, + SourceSchema + ); + + const out = fromCommon({} as never); + + expect(out.errors).toHaveLength(2); + expect(out.errors.every(e => e instanceof TransformError)).toBe(true); + const paths = out.errors.map(e => e.path).sort(); + expect(paths).toEqual(["a", "b"]); + }); }); // ############################################################################ @@ -330,7 +402,7 @@ describe("buildTransforms — null preservation (three-state)", () => { // SDKs already accept null on `.nullish()` fields; this // pins it inside the buildTransforms() Zod path so a future schema // change that swapped `.nullish()` for plain `.optional()` would surface - // here as a PluginError. `source` is `.nullish()` on OpportunityBaseSchema; + // here as a TransformError. `source` is `.nullish()` on OpportunityBaseSchema; // `description` is required (z.string()) so it can't carry the null state. const { toCommon } = buildTransforms( { @@ -437,13 +509,13 @@ describe("buildTransforms — null preservation (three-state)", () => { // Custom handlers // ############################################################################ -describe("PluginError — serialization", () => { +describe("TransformError — serialization", () => { // The SDK does not redact by default. Both tests - // assert on the same PluginError instance — one without redaction (PII + // assert on the same TransformError instance — one without redaction (PII // flows), one with the adopter-supplied projection (PII contained). // Forcing one shared setup keeps "redacted vs. raw is the only delta" a // structural property of the test code rather than just narration. - let err: PluginError; + let err: TransformError; beforeEach(() => { const { toCommon } = buildTransforms({ amount: { stringToNumber: "data.bogus" } }, {}); @@ -451,7 +523,7 @@ describe("PluginError — serialization", () => { }); it("includes sourceValue and cause in JSON.stringify by default — adopters redact", () => { - expect(err).toBeInstanceOf(PluginError); + expect(err).toBeInstanceOf(TransformError); expect(err.sourceValue).toEqual({ data: { bogus: "abc", ssn: "PII_PAYLOAD_123" } }); expect(err.cause).toBeInstanceOf(Error); @@ -472,7 +544,7 @@ describe("PluginError — serialization", () => { const serialized = JSON.stringify(safe); expect(serialized).not.toContain("PII_PAYLOAD_123"); expect(JSON.parse(serialized)).toEqual({ - name: "PluginError", + name: "TransformError", message: expect.any(String), handler: "stringToNumber", }); @@ -510,10 +582,10 @@ describe("buildTransforms — custom handlers", () => { }); // ############################################################################ -// Output key validation (commonModel provided) +// Output key validation (commonSchema provided) // ############################################################################ -describe("buildTransforms — output key validation (commonModel provided)", () => { +describe("buildTransforms — output key validation (commonSchema provided)", () => { it("throws when a top-level mapping key is not a field on the schema", () => { expect(() => buildTransforms({ titlee: { field: "data.title" } }, {}, undefined, OpportunityBaseSchema) @@ -536,12 +608,12 @@ describe("buildTransforms — output key validation (commonModel provided)", () ).not.toThrow(); }); - it("does not throw when commonModel is absent", () => { - // No commonModel → output key validation is skipped entirely + it("does not throw when commonSchema is absent", () => { + // No commonSchema → output key validation is skipped entirely expect(() => buildTransforms({ notAField: { field: "data.x" } }, {})).not.toThrow(); }); - it("silently passes when commonModel is not a ZodObject", () => { + it("silently passes when commonSchema is not a ZodObject", () => { // ZodRecord is not instanceof ZodObject — permissive fallback, no throw const nonObjectModel = z.record(z.unknown()); expect(() => @@ -549,8 +621,8 @@ describe("buildTransforms — output key validation (commonModel provided)", () ).not.toThrow(); }); - it("does not validate fromCommonMapping output keys", () => { - // fromCommon maps to native — no commonModel shape to validate against + it("does not validate fromCommonMapping output keys when only commonSchema is provided", () => { + // commonSchema validates toCommon output — fromCommonMapping is not checked expect(() => buildTransforms( { title: { field: "data.title" } }, @@ -561,6 +633,23 @@ describe("buildTransforms — output key validation (commonModel provided)", () ).not.toThrow(); }); + it("validates fromCommonMapping output keys when sourceSchema is a ZodObject", () => { + const SourceSchema = z.object({ native_title: z.string() }); + expect(() => + buildTransforms( + {}, + { notANativeField: { field: "title" } }, + undefined, + undefined, + SourceSchema + ) + ).toThrow(/unknown output fields.*"notANativeField"/); + }); + + it("does not validate fromCommonMapping output keys when sourceSchema is absent", () => { + expect(() => buildTransforms({}, { notAField: { field: "title" } })).not.toThrow(); + }); + it("reports multiple unknown fields in the same error", () => { expect( () => diff --git a/lib/ts-sdk/examples/plugins.ts b/lib/ts-sdk/examples/plugins.ts index 9d18a3370..bb26b2ceb 100644 --- a/lib/ts-sdk/examples/plugins.ts +++ b/lib/ts-sdk/examples/plugins.ts @@ -73,7 +73,7 @@ function demonstrateStandalonePlugins() { console.log("--- Standalone plugins ---\n"); // Legacy plugin schema validates and types its own fields - const legacyOpp = legacyPlugin.schemas.Opportunity.common.parse({ + const legacyOpp = legacyPlugin.schemas.Opportunity.commonSchema.parse({ ...baseData, customFields: { legacyId: { @@ -90,7 +90,7 @@ function demonstrateStandalonePlugins() { console.log(` legacyId.id: ${legacyId?.id} (typed as number)\n`); // Classification plugin schema validates its fields - const classOpp = classificationPlugin.schemas.Opportunity.common.parse({ + const classOpp = classificationPlugin.schemas.Opportunity.commonSchema.parse({ ...baseData, customFields: { category: { @@ -145,7 +145,7 @@ function demonstrateValidation() { console.log(JSON.stringify(oppData, null, 2)); console.log(); - const result = combinedPlugin.schemas.Opportunity.common.safeParse(oppData); + const result = combinedPlugin.schemas.Opportunity.commonSchema.safeParse(oppData); if (!result.success) { const issue = result.error.issues[0]; diff --git a/lib/ts-sdk/examples/transforms.ts b/lib/ts-sdk/examples/transforms.ts index 55db2fb25..b25fb8e0f 100644 --- a/lib/ts-sdk/examples/transforms.ts +++ b/lib/ts-sdk/examples/transforms.ts @@ -227,8 +227,11 @@ const { toCommon, fromCommon } = buildTransforms( new Map([ ["join", joinFields], ["split", splitField], - ]), - ExtendedOpportunitySchema + ]) + // Note: do NOT pass ExtendedOpportunitySchema here — definePlugin wraps + // toCommon with wrapWithSchemaValidation, which applies the schema once. + // Passing it here too would cause double-validation and fail on date fields + // (Zod's .transform() produces Date objects; a second pass expects strings). ); // ############################################################################ @@ -236,7 +239,7 @@ const { toCommon, fromCommon } = buildTransforms( // ############################################################################ // All per-object input — customFields, toCommon, and fromCommon — lives on -// the same schemas[Opportunity] entry. See ObjectSchemasInput in +// the same schemas[Opportunity] entry. See SchemaInput in // extensions/types.ts for details. const grantsGovPlugin = definePlugin({ meta: { @@ -263,7 +266,7 @@ const toCommonResult = grantsGovPlugin.schemas.Opportunity.toCommon?.(SOURCE_DAT if (!toCommonResult) fail("schemas.Opportunity.toCommon missing"); if (toCommonResult.errors.length > 0) { // The source data in this example is fixed and PII-free, so embedding - // `e.message` here is safe. Production adopters: `PluginError.message` can + // `e.message` here is safe. Production adopters: `TransformError.message` can // carry source values on the Zod-validation path (Zod's default error map // embeds runtime values). See the README PII warning before copying this // logging shape. diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index b03b8c5c7..da7865124 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -45,11 +45,11 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor Here are some key concepts that are used to define custom fields and plugins that extend base schemas from the CommonGrants protocol. -| Concept | Description | -| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | -| **`CustomFieldSpec`** | A TypeScript object that _describes_ a custom field: its `fieldType`, optional `value` (a Zod schema for validating the custom field's value), and optional `name` and `description`. | -| **`Plugin`** | An object returned by `definePlugin()` with `.schemas` (per-object compiled output: `.common` Zod schema, `.native`, `.toCommon`, `.fromCommon`) and optional `.extensions` / `.meta`. | +| Concept | Description | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | +| **`CustomFieldSpec`** | A TypeScript object that _describes_ a custom field: its `fieldType`, optional `value` (a Zod schema for validating the custom field's value), and optional `name` and `description`. | +| **`Plugin`** | An object returned by `definePlugin()` with `.schemas` (per-object compiled output: `.commonSchema` Zod schema, `.sourceSchema`, `.toCommon`, `.fromCommon`) and optional `.meta`. | ## Extending base models with custom fields @@ -118,7 +118,7 @@ const legacyPlugin = definePlugin({ } as const); // The plugin exposes typed schemas for every extensible model -const opportunity = legacyPlugin.schemas.Opportunity.common.parse(data); +const opportunity = legacyPlugin.schemas.Opportunity.commonSchema.parse(data); opportunity.customFields?.legacyId?.value.id; // number ``` @@ -179,14 +179,13 @@ const missing = getCustomFieldValue(opp, "nonexistent", z.string()); A plugin is any object that satisfies the `Plugin` interface: ```typescript -interface Plugin<T extends SchemasInput = SchemasInput> { +interface Plugin<T extends PluginSchemasInput = PluginSchemasInput> { schemas: PluginSchemas<T>; - extensions?: PluginExtensions; meta?: PluginMeta; } ``` -The `Plugin` interface uses [structural typing](https://www.typescriptlang.org/docs/handbook/type-compatibility.html), so any object with the right shape qualifies as a plugin, whether it comes from a local file, a monorepo package, or an installed npm package. There is no base class to extend or registry to sign up for. In practice, you'll almost always create plugins with `definePlugin()`, which builds `.schemas` (including the `.common` Zod schema) from your `schemas` input automatically. +The `Plugin` interface uses [structural typing](https://www.typescriptlang.org/docs/handbook/type-compatibility.html), so any object with the right shape qualifies as a plugin, whether it comes from a local file, a monorepo package, or an installed npm package. There is no base class to extend or registry to sign up for. In practice, you'll almost always create plugins with `definePlugin()`, which builds `.schemas` (including the `.commonSchema` Zod schema) from your `schemas` input automatically. For the full interface definition, see [define-plugin.ts](./define-plugin.ts). @@ -227,12 +226,11 @@ const myPlugin = definePlugin({ > [!IMPORTANT] > Always pass `as const` to the options object for `definePlugin()` (and the specs object for `withCustomFields()`). Without it, TypeScript widens literal types like `"string"` to `string`, which prevents the type system from inferring the correct `value` type for each custom field. -The returned `Plugin` object has three properties: +The returned `Plugin` object has two properties: - **`myPlugin.schemas`**: a record of per-object compiled output, one entry per extensible model. Each entry has: - - `.common` — the Zod schema with typed `customFields` applied (use this to parse data). - - `.native`, `.toCommon`, `.fromCommon` — populated when transforms are configured. -- **`myPlugin.extensions`**: optional serializable config (mappings, meta) — safe to store as JSON. + - `.commonSchema` — the Zod schema with typed `customFields` applied (use this to parse data). + - `.sourceSchema`, `.toCommon`, `.fromCommon` — populated when transforms are configured. - **`myPlugin.meta`**: optional plugin identity (`name`, `version`, `sourceSystem`, `capabilities`). ### Publishing a plugin @@ -329,12 +327,12 @@ export default plugin; #### Pre-publish checklist 1. **Build** the package to generate `.js` and `.d.ts` files in `dist/`. -2. **Verify type inference**: import your plugin in a test file and confirm that `.extensions` keys and `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`: +2. **Verify type inference**: import your plugin in a test file and confirm that `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`: ```typescript import plugin from "./"; - plugin.schemas.Opportunity.common.parse({} as any); // fully typed result + plugin.schemas.Opportunity.commonSchema.parse({} as any); // fully typed result ``` 3. **Publish** with `npm publish` (or your preferred registry workflow). @@ -346,7 +344,7 @@ After installing the plugin (e.g. `npm install @commongrants/hhs-plugin`): ```typescript import hhs from "@commongrants/hhs-plugin"; -const opp = hhs.schemas.Opportunity.common.parse(data); +const opp = hhs.schemas.Opportunity.commonSchema.parse(data); opp.customFields?.programArea?.value.code; // string opp.customFields?.cfda?.value; // string ``` @@ -377,14 +375,14 @@ const client = new Client({ // Get a single opportunity with typed custom fields const opp = await client.opportunities.get(oppId, { - schema: myPlugin.schemas.Opportunity.common, + schema: myPlugin.schemas.Opportunity.commonSchema, }); opp.customFields?.legacyId?.value; // typed as number opp.customFields?.category?.value; // typed as string // List with the same schema const response = await client.opportunities.list({ - schema: myPlugin.schemas.Opportunity.common, + schema: myPlugin.schemas.Opportunity.commonSchema, }); for (const opp of response.items) { console.log(opp.customFields?.category?.value); @@ -394,7 +392,7 @@ for (const opp of response.items) { const results = await client.opportunities.search({ query: "health", statuses: ["open"], - schema: myPlugin.schemas.Opportunity.common, + schema: myPlugin.schemas.Opportunity.commonSchema, }); ``` @@ -446,7 +444,7 @@ if (result.errors.length === 0) { } ``` -Each callable returns a `TransformResult<T>` of `{ result, errors }` unconditionally. Partial failures surface as `PluginError[]` rather than thrown exceptions — consumers choose their own strict-vs-lenient rule. +Each callable returns a `TransformResult<T>` of `{ result, errors }` unconditionally. Partial failures surface as `TransformError[]` rather than thrown exceptions — consumers choose their own strict-vs-lenient rule. ### Built-in mapping handlers @@ -524,7 +522,7 @@ const { toCommon } = buildTransforms( ### Validating against the extended schema -Pass an optional `commonModel` to validate `toCommon` output. **Use the fully extended schema** (the result of `withCustomFields()`), not the base schema — passing the base silently weakens validation of typed custom fields: +Pass an optional `commonSchema` to validate `toCommon` output. **Use the fully extended schema** (the result of `withCustomFields()`), not the base schema — passing the base silently weakens validation of typed custom fields: ```typescript import { buildTransforms, withCustomFields } from "@common-grants/sdk/extensions"; @@ -575,11 +573,11 @@ const plugin = definePlugin({ const cg = plugin.schemas.Opportunity?.toCommon?.(sourceData); ``` -For a complete runnable round-trip with custom handlers and `commonModel` validation, see [`examples/transforms.ts`](../../examples/transforms.ts) (`pnpm example:transforms`). +For a complete runnable round-trip with custom handlers and `commonSchema` validation, see [`examples/transforms.ts`](../../examples/transforms.ts) (`pnpm example:transforms`). ### Error handling -`PluginError` carries structured context — `path`, `handler`, `sourceValue`, `cause` — so consumers can reason about failures programmatically without parsing error text: +`TransformError` carries structured context — `path`, `handler`, `sourceValue`, `cause` — so consumers can reason about failures programmatically without parsing error text: ```typescript const out = toCommon(sourceData); @@ -593,7 +591,7 @@ for (const err of out.errors) { } ``` -> **PII warning (ADR-0022 Decision #9):** The SDK does **not** redact by default. `PluginError.sourceValue` and `cause` are plain enumerable fields and flow through `JSON.stringify(err)`, `util.inspect(err)`, `console.log(err)`, and any logger that enumerates own properties. `sourceValue` is populated with the entire input record passed to `toCommon` / `fromCommon` — not just the value at the failing field. Log a redacted projection instead — e.g. `{ name: err.name, message: err.message, path: err.path, handler: err.handler }`. On the Zod-validation path (when `commonModel` is passed to `buildTransforms()`), `PluginError.message` is also data-bearing — Zod's default error map embeds the rejected value into `issue.message`, which flows verbatim into `PluginError.message`. Redact `message` alongside `sourceValue` and `cause`. Full-message sanitization is tracked under [#744](https://github.com/HHS/simpler-grants-protocol/issues/744). +> **PII warning (ADR-0022 Decision #9):** The SDK does **not** redact by default. `TransformError.sourceValue` and `cause` are plain enumerable fields and flow through `JSON.stringify(err)`, `util.inspect(err)`, `console.log(err)`, and any logger that enumerates own properties. `sourceValue` is populated with the entire input record passed to `toCommon` / `fromCommon` — not just the value at the failing field. Log a redacted projection instead — e.g. `{ name: err.name, message: err.message, path: err.path, handler: err.handler }`. On the Zod-validation path (when `commonSchema` is passed to `buildTransforms()`), `TransformError.message` is also data-bearing — Zod's default error map embeds the rejected value into `issue.message`, which flows verbatim into `TransformError.message`. Redact `message` alongside `sourceValue` and `cause`. Full-message sanitization is tracked under [#744](https://github.com/HHS/simpler-grants-protocol/issues/744). ## Best practices @@ -635,7 +633,7 @@ This allows consumers to use `getCustomFieldValue()` with the same schema the pl import hhs, { ProgramAreaValueSchema } from "@commongrants/hhs-plugin"; import { getCustomFieldValue } from "@common-grants/sdk/extensions"; -const opp = hhs.schemas.Opportunity.common.parse(data); +const opp = hhs.schemas.Opportunity.commonSchema.parse(data); // Extract the value with full type safety using the exported schema const area = getCustomFieldValue(opp, "programArea", ProgramAreaValueSchema); @@ -656,11 +654,11 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, ### Plugin creation -| Export | Kind | Description | Demonstrated in | -| ------------------------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| [`definePlugin()`](./define-plugin.ts) | function | Creates a `Plugin` from `DefinePluginOptions`. Returns an object with `.schemas` (per-object output: `.common`, `.native`, `.toCommon`, `.fromCommon`) and optional `.extensions` / `.meta`. | [Defining a plugin](#defining-a-plugin) | -| [`Plugin`](./define-plugin.ts) | interface | The object returned by `definePlugin()`. | [What is a plugin?](#what-is-a-plugin) | -| [`DefinePluginOptions`](./define-plugin.ts) | interface | Options for `definePlugin()`. `schemas` carries per-object input (custom fields, native schema, transforms); `extensions` is serializable-only config; `meta` is plugin identity. | [Defining a plugin](#defining-a-plugin) | +| Export | Kind | Description | Demonstrated in | +| ------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| [`definePlugin()`](./define-plugin.ts) | function | Creates a `Plugin` from `DefinePluginOptions`. Returns an object with `.schemas` (per-object output: `.commonSchema`, `.sourceSchema`, `.toCommon`, `.fromCommon`) and optional `.meta`. | [Defining a plugin](#defining-a-plugin) | +| [`Plugin`](./define-plugin.ts) | interface | The object returned by `definePlugin()`. | [What is a plugin?](#what-is-a-plugin) | +| [`DefinePluginOptions`](./define-plugin.ts) | interface | Options for `definePlugin()`. `schemas` carries all per-object input (custom fields, native schema, declarative mappings, transforms); `meta` is plugin identity. | [Defining a plugin](#defining-a-plugin) | ### Schema utilities @@ -672,26 +670,26 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, ### Transforms (PoC) -| Export | Kind | Description | Demonstrated in | -| ---------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | -| [`buildTransforms()`](./transforms.ts) | function | Compiles a pair of mapping objects into typed `(toCommon, fromCommon)` callables. Positional params: `(toCommonMapping, fromCommonMapping, handlers?, commonModel?)`. Validates mapping structure at call time; collisions with built-ins throw. | [Defining bidirectional transforms](#defining-bidirectional-transforms) | -| [`BuiltTransforms`](./transforms.ts) | interface | Return shape of `buildTransforms()` — `{ toCommon, fromCommon }`. | | -| [`transformFromMapping()`](./transformation.ts) | function | Low-level mapping walker used by `buildTransforms()`. Useful if you want to drive a single mapping pass without the call-time validation or error-wrapping layer. | | -| [`TransformFromMappingOptions`](./transformation.ts) | interface | Options for `transformFromMapping()`: optional `handlers` registry (`Map<string, Handler>`). | | -| [`DEFAULT_HANDLERS`](./transformation.ts) | const | `Map<string, Handler>` of built-in handlers: `const`, `field`, `match`, `numberToString`, `stringToNumber`, `switch`. | [Built-in mapping handlers](#built-in-mapping-handlers) | -| [`getFromPath()`](./transformation.ts) | function | Walks an object via dot-notation; returns `undefined` (or a provided default) when the path is missing or traverses a non-object. | | -| [`TransformResult`](./types.ts) | interface | Unconditional return shape `{ result, errors }` for `toCommon` / `fromCommon`. | [Defining bidirectional transforms](#defining-bidirectional-transforms) | -| [`PluginError`](./types.ts) | class | Structured transformation error carrying `path`, `handler`, `sourceValue`, `cause`. Extends `Error`. | [Error handling](#error-handling) | -| [`Handler`](./types.ts) | type | Signature for mapping handler functions: `(data, arg) => unknown`. | [Custom handlers](#custom-handlers) | -| [`PluginMeta`](./types.ts) | interface | Plugin identity: `name` (required), `sourceSystem` (required), optional `version` and `capabilities`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | -| [`PluginCapability`](./types.ts) | type | Literal union of capability names: `"customFields" \| "customFilters" \| "transforms" \| "client"`. | | -| [`SchemasInput`](./define-plugin.ts) | type | Map from extensible model name to `ObjectSchemasInput`. The shape of `DefinePluginOptions.schemas`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | -| [`ObjectSchemasInput`](./types.ts) | interface | Author-provided input per object: `{ native?, customFields?, toCommon?, fromCommon? }`. Passed inside `definePlugin({ schemas })`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | -| [`ObjectSchemas`](./types.ts) | interface | Compiled runtime shape: `{ native, common, toCommon, fromCommon }`. Accessed via `plugin.schemas.<Name>`. | | -| [`ObjectMappings`](./types.ts) | interface | Serializable `{ toCommon?, fromCommon? }` mapping dicts. Stored inside `PluginExtensionsObjectConfig.mappings`. | | -| [`PluginExtensionsObjectConfig`](./types.ts) | interface | Per-object slot inside `PluginExtensions.schemas`: `{ mappings? }`. | | -| [`PluginExtensions`](./types.ts) | interface | Serializable plugin config carrying `meta?: Partial<PluginMeta>` and per-object `schemas`. | | -| [`ClientConfig`](./types.ts) | type | The per-plugin client configuration shape. Concrete shape is deferred to the full SDK. | | +| Export | Kind | Description | Demonstrated in | +| ---------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| [`buildTransforms()`](./transforms.ts) | function | Compiles a pair of mapping objects into typed `(toCommon, fromCommon)` callables. Positional params: `(toCommonMapping, fromCommonMapping, handlers?, commonSchema?)`. Validates mapping structure at call time; collisions with built-ins throw. | [Defining bidirectional transforms](#defining-bidirectional-transforms) | +| [`BuiltTransforms`](./transforms.ts) | interface | Return shape of `buildTransforms()` — `{ toCommon, fromCommon }`. | | +| [`transformFromMapping()`](./transformation.ts) | function | Low-level mapping walker used by `buildTransforms()`. Useful if you want to drive a single mapping pass without the call-time validation or error-wrapping layer. | | +| [`TransformFromMappingOptions`](./transformation.ts) | interface | Options for `transformFromMapping()`: optional `handlers` registry (`Map<string, Handler>`). | | +| [`DEFAULT_HANDLERS`](./transformation.ts) | const | `Map<string, Handler>` of built-in handlers: `const`, `field`, `match`, `numberToString`, `stringToNumber`, `switch`. | [Built-in mapping handlers](#built-in-mapping-handlers) | +| [`getFromPath()`](./transformation.ts) | function | Walks an object via dot-notation; returns `undefined` (or a provided default) when the path is missing or traverses a non-object. | | +| [`TransformResult`](./types.ts) | interface | Unconditional return shape `{ result, errors }` for `toCommon` / `fromCommon`. | [Defining bidirectional transforms](#defining-bidirectional-transforms) | +| [`TransformError`](./types.ts) | class | Structured transformation error carrying `path`, `handler`, `sourceValue`, `cause`. Extends `Error`. | [Error handling](#error-handling) | +| [`Handler`](./types.ts) | type | Signature for mapping handler functions: `(data, arg) => unknown`. | [Custom handlers](#custom-handlers) | +| [`PluginMeta`](./types.ts) | interface | Plugin identity: `name` (required), `sourceSystem` (required), optional `version` and `capabilities`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | +| [`PluginCapability`](./types.ts) | type | Literal union of capability names: `"customFields" \| "customFilters" \| "transforms"`. | | +| [`PluginSchemasInput`](./define-plugin.ts) | type | Map from extensible model name to `SchemaInput`. The shape of `DefinePluginOptions.schemas`. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | +| [`SchemaInput`](./types.ts) | type | Author-provided input per object: `{ sourceSchema?, customFields?, mappings? }` or `{ sourceSchema?, customFields?, toCommon?, fromCommon? }`. XOR: `mappings` and explicit callables cannot both be present. | [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) | +| [`SchemaOnly`](./types.ts) | interface | Compiled output for schema-only entries: `{ commonSchema, sourceSchema? }`. Produced when no transforms are configured. | | +| [`SchemaWithTransforms`](./types.ts) | interface | Compiled output for entries with transforms: `{ commonSchema, sourceSchema?, toCommon, fromCommon }`. Produced when `mappings` or explicit callables are provided. | | +| [`SchemaMappings`](./types.ts) | interface | Declarative `{ toCommon?, fromCommon? }` mapping dicts. Stored inside `SchemaInput.mappings`. | | +| [`ToCommon`](./types.ts) | type | Helper alias for `(source: TSource) => TransformResult<TCommon>`. | | +| [`FromCommon`](./types.ts) | type | Helper alias for `(common: TCommon) => TransformResult<TSource>`. | | ### Shared types diff --git a/lib/ts-sdk/src/extensions/define-plugin.ts b/lib/ts-sdk/src/extensions/define-plugin.ts index cb0240b5d..18d10a44e 100644 --- a/lib/ts-sdk/src/extensions/define-plugin.ts +++ b/lib/ts-sdk/src/extensions/define-plugin.ts @@ -4,67 +4,62 @@ * @module @common-grants/sdk/extensions */ +import { z } from "zod"; import type { ExtensibleSchemaName, HasCustomFields, CustomFieldSpec, - ObjectSchemasInput, + SchemaInput, + SchemaMappings, PluginMeta, - PluginExtensions, + TransformResult, } from "./types"; -import { EXTENSIBLE_SCHEMA_MAP } from "./types"; +import { EXTENSIBLE_SCHEMA_MAP, TransformError } from "./types"; import { withCustomFields, type WithCustomFieldsResult } from "./with-custom-fields"; import { buildTransforms } from "./transforms"; // ############################################################################ -// Public types - SchemasInput, DefinePluginOptions, Plugin +// Public types - PluginSchemasInput, DefinePluginOptions, Plugin // ############################################################################ /** * Per-object schemas input keyed by extensible model name. * * Plugin authors populate this with hand-written or `buildTransforms()`-generated - * `toCommon` / `fromCommon` callables, an optional `native` schema, and optional + * `toCommon` / `fromCommon` callables, an optional `sourceSchema`, and optional * `customFields` specs. Passed as `DefinePluginOptions.schemas`. */ -// Per-entry (TNative, TCommon) pairs only meet at the `buildTransforms()` -// boundary. `unknown` would reject legitimate caller schemas at contravariant -// positions; the widening lives only at this dictionary storage layer. -export type SchemasInput = Partial< +// Each schema entry can have its own source/common type pair, which only gets +// checked when buildTransforms() is called. `any` lets the dictionary hold +// entries with different type parameters; a stricter type would reject valid +// plugin configs at the point they're stored here. +export type PluginSchemasInput = Partial< // eslint-disable-next-line @typescript-eslint/no-explicit-any - Record<ExtensibleSchemaName, ObjectSchemasInput<any, any>> + Record<ExtensibleSchemaName, SchemaInput<any, any>> >; /** * Options for `definePlugin()`. * - * `schemas` carries the consolidated per-object input (custom fields, - * native schema, transforms). `extensions` is the serializable-only surface - * for JSON-safe config (mappings, meta). Custom field declarations belong - * exclusively on `schemas[Object].customFields`. + * `schemas` is the single surface for all per-object declarations: custom + * fields, source schema, declarative `mappings`, and explicit transform + * callables. Inputs are declarative wherever possible; explicit callables are + * available when custom code-driven logic is needed. * * Structured as an options object for forward-compatibility with future * properties like `namespace`. */ -export interface DefinePluginOptions<T extends SchemasInput = SchemasInput> { - /** - * Serializable plugin config — mappings and meta, safe to store as JSON. - * - * Does not carry `customFields`; those belong on `schemas[Object].customFields`. - */ - extensions?: PluginExtensions; +export interface DefinePluginOptions<T extends PluginSchemasInput = PluginSchemasInput> { /** Optional plugin identity and capability declaration. */ meta?: PluginMeta; /** - * Per-object transform input — `native` schema, `customFields` specs, - * and `toCommon` / `fromCommon` callables — for each extensible model. - * - * This is the single surface for custom field declarations. + * Per-object input — `sourceSchema`, `customFields` specs, declarative + * `mappings`, and `toCommon` / `fromCommon` callables — for each extensible model. * - * `definePlugin()` compiles this into runtime schemas: `common` is built via + * `definePlugin()` compiles this into runtime schemas: `commonSchema` is built via * `withCustomFields()` when `customFields` are declared; `toCommon` / `fromCommon` - * are auto-wired from `extensions.schemas[Name].mappings` when no explicit - * callables are provided. Native input Zod-wrapping remains deferred. + * are auto-wired from `schemas[Name].mappings` when `mappings` is used. Providing + * both `mappings` and explicit callables is a runtime error. */ schemas?: T; } @@ -72,17 +67,48 @@ export interface DefinePluginOptions<T extends SchemasInput = SchemasInput> { /** * Configuration object returned by `definePlugin()`. * - * - `extensions` — serializable plugin config (mappings, meta), preserved by reference - * - `schemas` — per-object compiled output: `common` (extended Zod schema), `native`, - * `toCommon`, and `fromCommon` for each extensible model + * - `schemas` — per-object compiled output: `commonSchema` (extended Zod schema), + * `sourceSchema`, `toCommon`, and `fromCommon` for each extensible model * - `meta` — plugin identity passed through from options */ -export interface Plugin<T extends SchemasInput = SchemasInput> { - extensions?: PluginExtensions; +export interface Plugin<T extends PluginSchemasInput = PluginSchemasInput> { schemas: PluginSchemas<T>; meta?: PluginMeta; } +// ############################################################################ +// Internal helper - wrapWithSchemaValidation +// ############################################################################ + +/** + * Wrap a transform callable to validate its output against a Zod schema. + * + * If the callable returns errors, those are returned unchanged. Otherwise the + * output is passed through `schema.safeParse()`; any Zod issues are flattened + * into `TransformResult.errors`. + * + * @internal + */ +function wrapWithSchemaValidation<TIn, TOut>( + fn: (input: TIn) => TransformResult<TOut>, + // `any` lets this accept schemas with .transform() whose input type differs + // from the output type. `unknown` would reject those valid schemas here. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + schema: z.ZodType<TOut, z.ZodTypeDef, any> +): (input: TIn) => TransformResult<TOut> { + return (input: TIn): TransformResult<TOut> => { + const result = fn(input); + if (result.errors.length > 0) return result; + const parsed = schema.safeParse(result.result); + if (parsed.success) return { result: parsed.data, errors: [] }; + const errors = parsed.error.issues.map(issue => { + const path = issue.path.length > 0 ? issue.path.map(p => String(p)).join(".") : undefined; + return new TransformError(issue.message, { path }); + }); + return { result: result.result, errors }; + }; +} + // ############################################################################ // Public function - definePlugin() // ############################################################################ @@ -92,12 +118,12 @@ export interface Plugin<T extends SchemasInput = SchemasInput> { * * Iterates over extensible schemas. For each model, looks up `customFields` * specs from `schemas[name].customFields`. When specs are present, applies - * `withCustomFields()` to produce a typed `common` schema; otherwise the base + * `withCustomFields()` to produce a typed `commonSchema`; otherwise the base * schema passes through unchanged. The per-object result is wrapped under - * `.common` alongside any `native`, `toCommon`, and `fromCommon` provided. + * `.commonSchema` alongside any `sourceSchema`, `toCommon`, and `fromCommon` provided. * - * @param options - Options containing schemas and/or serializable extensions - * @returns A `Plugin` with `.extensions`, `.schemas`, and `.meta` + * @param options - Options containing `schemas` (per-object input) and optional `meta` + * @returns A `Plugin` with `.schemas` and optional `.meta` * * @example * ```typescript @@ -115,15 +141,15 @@ export interface Plugin<T extends SchemasInput = SchemasInput> { * } as const); * * // Access the extended Zod schema: - * const opp = plugin.schemas.Opportunity.common.parse(rawData); + * const opp = plugin.schemas.Opportunity.commonSchema.parse(rawData); * // Access the transform callables: - * const result = plugin.schemas.Opportunity.toCommon?.(nativeData); + * const result = plugin.schemas.Opportunity.toCommon?.(sourceData); * ``` */ -export function definePlugin<const T extends SchemasInput>( +export function definePlugin<const T extends PluginSchemasInput>( options: DefinePluginOptions<T> ): Plugin<T> { - const { extensions, meta, schemas: schemasInput } = options; + const { meta, schemas: schemasInput } = options; const schemas: Record<string, object> = {}; for (const [name, extensibleSchema] of Object.entries(EXTENSIBLE_SCHEMA_MAP) as [ @@ -131,57 +157,87 @@ export function definePlugin<const T extends SchemasInput>( HasCustomFields, ][]) { const specs = schemasInput?.[name]?.customFields; - const common = + const commonSchema = specs && Object.keys(specs).length > 0 ? withCustomFields(extensibleSchema, specs) : extensibleSchema; const explicitToCommon = schemasInput?.[name]?.toCommon; const explicitFromCommon = schemasInput?.[name]?.fromCommon; - const mappings = extensions?.schemas?.[name]?.mappings; - const nativeSchema = schemasInput?.[name]?.native; + const mappings = schemasInput?.[name]?.mappings; + const sourceSchema = schemasInput?.[name]?.sourceSchema; + + // XOR: providing both mappings AND explicit callables is an error. + const hasMappings = mappings !== undefined; + const hasCallables = explicitToCommon !== undefined || explicitFromCommon !== undefined; + if (hasMappings && hasCallables) { + throw new Error( + `definePlugin: ${name} cannot specify both mappings and explicit toCommon/fromCommon. ` + + `Use mappings for declarative transforms or provide explicit callables, not both.` + ); + } - // Auto-wire from declarative mappings when no explicit callables are supplied. - // All-or-nothing: any explicit callable disables auto-wiring for this object - // (mirrors Python's explicit_objs vs mappings_objs logic in generate.py). let toCommon = explicitToCommon; let fromCommon = explicitFromCommon; - if ( - mappings !== undefined && - explicitToCommon === undefined && - explicitFromCommon === undefined - ) { - if (mappings.toCommon === undefined) { + if (hasMappings) { + // Mappings path: validate both directions are present, then auto-wire. + if (mappings!.toCommon === undefined) { throw new Error( `definePlugin: ${name}.mappings.toCommon is required when auto-generating transforms. ` + `Either provide both mapping directions or pass explicit toCommon/fromCommon callables.` ); } - if (mappings.fromCommon === undefined) { + if (mappings!.fromCommon === undefined) { throw new Error( `definePlugin: ${name}.mappings.fromCommon is required when auto-generating transforms. ` + `Either provide both mapping directions or pass explicit toCommon/fromCommon callables.` ); } - // Pass `common` as commonModel so validateOutputPaths runs against the - // fully-resolved schema (base or extended). Key-existence checking is correct - // regardless of whether customFields were declared — this is not the same as - // the "base schema weakens custom-field type validation" warning in buildTransforms' - // JSDoc, which applies to Zod-type checking of custom field values at runtime. - const built = buildTransforms(mappings.toCommon, mappings.fromCommon, undefined, common); + // Pass `commonSchema` so validateOutputPaths runs against the fully-resolved + // schema (base or extended with customFields). This catches typo'd output keys + // at definition time rather than at first invocation. + const built = buildTransforms( + mappings!.toCommon, + mappings!.fromCommon, + undefined, + commonSchema + ); + // The specific (TSource, TCommon) types for this entry were erased when it + // was stored in PluginSchemasInput above. Cast needed so the compiled transform + // can be assigned back to the schema entry's callable slots. // eslint-disable-next-line @typescript-eslint/no-explicit-any toCommon = built.toCommon as any; // eslint-disable-next-line @typescript-eslint/no-explicit-any fromCommon = built.fromCommon as any; + } else if (hasCallables) { + // Explicit callables path: wrap each callable with schema validation so the + // runtime guarantee matches the mappings path (both directions get validated). + if (toCommon !== undefined) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + toCommon = wrapWithSchemaValidation(toCommon as any, commonSchema as any) as any; + } + if (fromCommon !== undefined && sourceSchema !== undefined) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fromCommon = wrapWithSchemaValidation(fromCommon as any, sourceSchema as any) as any; + } } - schemas[name] = { common, native: nativeSchema, toCommon, fromCommon }; + schemas[name] = { + commonSchema, + sourceSchema, + // Keep customFields and mappings on the compiled entry so consumers can + // inspect what was used to build the schema and transforms. + customFields: specs && Object.keys(specs).length > 0 ? specs : undefined, + mappings: hasMappings ? mappings : undefined, + toCommon, + fromCommon, + }; } // Cast is safe — the runtime loop mirrors the PluginSchemas<T> mapped type, // but TypeScript can't verify that from the dynamic Object.entries() iteration. - return { extensions, schemas, meta } as Plugin<T>; + return { schemas, meta } as Plugin<T>; } // ############################################################################ @@ -197,7 +253,10 @@ type BaseZodSchema<K extends ExtensibleSchemaName> = (typeof EXTENSIBLE_SCHEMA_M * Used to feed the custom-fields spec into `WithCustomFieldsResult` while * keeping the base schema as the fallback when no specs are declared. */ -type ExtractCustomFields<K extends ExtensibleSchemaName, T extends SchemasInput> = K extends keyof T +type ExtractCustomFields< + K extends ExtensibleSchemaName, + T extends PluginSchemasInput, +> = K extends keyof T ? NonNullable<T[K]> extends { customFields?: infer CF } ? CF extends Record<string, CustomFieldSpec> ? CF @@ -205,26 +264,72 @@ type ExtractCustomFields<K extends ExtensibleSchemaName, T extends SchemasInput> : never : never; -/** Resolves the `common` Zod schema for a single model. */ -type ResolveCommonSchema<K extends ExtensibleSchemaName, T extends SchemasInput> = [ +/** Resolves the `commonSchema` Zod schema for a single model. */ +type ResolveCommonSchema<K extends ExtensibleSchemaName, T extends PluginSchemasInput> = [ ExtractCustomFields<K, T>, ] extends [never] ? BaseZodSchema<K> : WithCustomFieldsResult<BaseZodSchema<K>, ExtractCustomFields<K, T>>; +/** + * Returns `true` when `T[K]` has transforms: either a `mappings` entry or an + * explicit `toCommon` callable. Used to produce the right callable type on the + * compiled output — the input's `toCommon` is `never` in the mappings branch + * of the `SchemaInput` XOR union, so we cannot just read `T[K]["toCommon"]` + * directly for mappings-based entries. + */ +type EntryHasTransforms< + K extends ExtensibleSchemaName, + T extends PluginSchemasInput, +> = K extends keyof T + ? NonNullable<T[K]> extends { mappings: SchemaMappings } + ? true + : // eslint-disable-next-line @typescript-eslint/no-explicit-any + NonNullable<T[K]> extends { toCommon: (...args: any[]) => any } + ? true + : false + : false; + +/** + * Extracts the source `TSource` from the entry's `sourceSchema`, or `unknown`. + */ +type ExtractSourceType< + K extends ExtensibleSchemaName, + T extends PluginSchemasInput, +> = K extends keyof T + ? NonNullable<T[K]>["sourceSchema"] extends z.ZodType<infer S> + ? S + : unknown + : unknown; + /** * Maps each extensible model to its compiled per-object output. * - * Each entry contains: - * - `common` — the fully extended Zod schema (base + custom fields) - * - `native` — the optional native-format Zod schema - * - `toCommon` / `fromCommon` — typed transform callables derived from `T[K]` + * When the entry has transforms (mappings or explicit callables), both + * `toCommon` and `fromCommon` are present and callable. When it is schema-only + * (no transforms configured) they are absent. `customFields` and `mappings` + * are kept for consumer inspection regardless of which path was used. */ -type PluginSchemas<T extends SchemasInput> = { - [K in ExtensibleSchemaName]: { - common: ResolveCommonSchema<K, T>; - native: K extends keyof T ? NonNullable<T[K]>["native"] : undefined; - toCommon: K extends keyof T ? NonNullable<T[K]>["toCommon"] : undefined; - fromCommon: K extends keyof T ? NonNullable<T[K]>["fromCommon"] : undefined; - }; +type PluginSchemas<T extends PluginSchemasInput> = { + [K in ExtensibleSchemaName]: EntryHasTransforms<K, T> extends true + ? { + commonSchema: ResolveCommonSchema<K, T>; + sourceSchema: K extends keyof T ? NonNullable<T[K]>["sourceSchema"] : undefined; + customFields: K extends keyof T ? NonNullable<T[K]>["customFields"] : undefined; + mappings: K extends keyof T ? NonNullable<T[K]>["mappings"] : undefined; + toCommon: ( + source: ExtractSourceType<K, T> + ) => TransformResult<z.infer<ResolveCommonSchema<K, T>>>; + fromCommon: ( + common: z.infer<ResolveCommonSchema<K, T>> + ) => TransformResult<ExtractSourceType<K, T>>; + } + : { + commonSchema: ResolveCommonSchema<K, T>; + sourceSchema?: undefined; + customFields: K extends keyof T ? NonNullable<T[K]>["customFields"] : undefined; + mappings?: undefined; + toCommon?: undefined; + fromCommon?: undefined; + }; }; diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index d016a461d..2820f29d7 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -9,7 +9,7 @@ */ // Plugin creation — define and compose plugins -export type { Plugin, DefinePluginOptions, SchemasInput } from "./define-plugin"; +export type { Plugin, DefinePluginOptions, PluginSchemasInput } from "./define-plugin"; export type { CustomFieldSpec, HasCustomFields, ExtensibleObject } from "./types"; export { definePlugin } from "./define-plugin"; @@ -21,21 +21,21 @@ export { getCustomFieldValue } from "./get-custom-field-value"; // Transforms — bidirectional plugin transformation contract export type { - ClientConfig, Handler, - ObjectMappings, - ObjectSchemas, - ObjectSchemasInput, + SchemaMappings, + SchemaInput, + SchemaOnly, + SchemaWithTransforms, + ToCommon, + FromCommon, PluginCapability, - PluginExtensions, - PluginExtensionsObjectConfig, PluginMeta, TransformResult, } from "./types"; -export { PluginError } from "./types"; +export { TransformError } from "./types"; export type { BuiltTransforms } from "./transforms"; export { buildTransforms } from "./transforms"; -export type { TransformFromMappingOptions } from "./transformation"; +export type { TransformFromMappingOptions } from "../utils/transformation"; // Individual handler functions (fieldValue, constValue, switchOnValue, etc.) // stay internal; reach them through DEFAULT_HANDLERS when needed. -export { DEFAULT_HANDLERS, getFromPath, transformFromMapping } from "./transformation"; +export { DEFAULT_HANDLERS, getFromPath, transformFromMapping } from "../utils/transformation"; diff --git a/lib/ts-sdk/src/extensions/transforms.ts b/lib/ts-sdk/src/extensions/transforms.ts index 73a012b9e..205e5f8d8 100644 --- a/lib/ts-sdk/src/extensions/transforms.ts +++ b/lib/ts-sdk/src/extensions/transforms.ts @@ -10,8 +10,8 @@ import { z } from "zod"; -import { PluginError, type Handler, type TransformResult } from "./types"; -import { DEFAULT_HANDLERS, HandlerError, transformFromMapping } from "./transformation"; +import { TransformError, type Handler, type TransformResult } from "./types"; +import { DEFAULT_HANDLERS, HandlerError, transformFromMapping } from "../utils/transformation"; // ############################################################################ // Internal - mapping structure validation @@ -33,9 +33,9 @@ import { DEFAULT_HANDLERS, HandlerError, transformFromMapping } from "./transfor * Sibling keys at a handler-dispatch node are rejected here. The runtime * walker is first-key-wins, so `{ field: "x", const: "fallback" }` would * silently drop `const` — almost always an author typo — so fail loud at - * build time instead (parity with the Python PoC's build-time validation). - * The low-level `transformFromMapping` walker stays lenient so programmatic - * callers composing partial mappings aren't forced into the strict shape. + * build time instead. The low-level `transformFromMapping` walker stays + * lenient so programmatic callers composing partial mappings aren't forced + * into the strict shape. * * @internal */ @@ -53,7 +53,7 @@ function validateMapping(mapping: unknown, knownHandlers: Set<string>, path = "" } // A handler invocation must be the sole key in its node — see the function - // docstring above for the first-key-wins rationale and cross-SDK parity. + // docstring above for the first-key-wins rationale. const nodeKeys = Object.keys(mapping as Record<string, unknown>); const handlerKeys = nodeKeys.filter(k => knownHandlers.has(k)); if (handlerKeys.length > 0 && nodeKeys.length > 1) { @@ -86,24 +86,19 @@ function validateMapping(mapping: unknown, knownHandlers: Set<string>, path = "" * `ZodObject`s (`.extend()` / `.merge()` preserve the concrete type), so the * fallback is a safety net, not an expected code path. * - * Called only for `toCommonMapping` — `fromCommonMapping` maps to native - * format whose shape is unknown to this layer. - * - * Cross-SDK note: Python raises `ValueError`; TypeScript uses `Error` for - * all structural build-time failures (consistent with `validateMapping`). - * * @internal */ function validateOutputPaths( mapping: Record<string, unknown>, knownHandlers: Set<string>, - // Bivariant `any` at the input position matches buildTransforms's own - // commonModel signature and avoids a contravariant `unknown` type error. + // `any` lets this accept Zod schemas whose input type differs from their + // output type (e.g. schemas that use .transform()). `unknown` would reject + // those valid schemas here. // eslint-disable-next-line @typescript-eslint/no-explicit-any - commonModel: z.ZodType<unknown, z.ZodTypeDef, any> + commonSchema: z.ZodType<unknown, z.ZodTypeDef, any> ): void { - if (!(commonModel instanceof z.ZodObject)) return; - const validNames = new Set(Object.keys(commonModel.shape)); + if (!(commonSchema instanceof z.ZodObject)) return; + const validNames = new Set(Object.keys(commonSchema.shape)); const outputKeys = Object.keys(mapping).filter(k => !knownHandlers.has(k)); const invalid = outputKeys.filter(k => !validNames.has(k)); if (invalid.length === 0) return; @@ -122,15 +117,20 @@ function validateOutputPaths( /** * Return shape of {@link buildTransforms}. */ -export interface BuiltTransforms<TNative, TCommon> { - toCommon: (native: TNative) => TransformResult<TCommon>; - fromCommon: (common: TCommon) => TransformResult<TNative>; +export interface BuiltTransforms<TSource, TCommon> { + toCommon: (source: TSource) => TransformResult<TCommon>; + fromCommon: (common: TCommon) => TransformResult<TSource>; } /** * Compile a pair of declarative mapping objects into typed * `(toCommon, fromCommon)` callables. * + * @internal Plugin authors should use `definePlugin({ schemas: { [Name]: { mappings } } })` + * instead of calling this directly. `definePlugin()` invokes `buildTransforms()` + * automatically from the `mappings` entry and wraps the result with schema validation. + * This function remains exported for unit testing and advanced use cases. + * * @example * ```ts * const { toCommon, fromCommon } = buildTransforms( @@ -160,39 +160,49 @@ export interface BuiltTransforms<TNative, TCommon> { * - Handler failures (a registered handler throws): the mapping walk * short-circuits on the first failure, so `errors` carries exactly one * `PluginError` even when several fields would have failed. - * - Zod-validation failures (`commonModel` provided): every `ZodIssue` is - * flattened into a separate `PluginError`, so `errors` carries the full - * set. + * - Zod-validation failures (`commonSchema` or `sourceSchema` provided): every + * `ZodIssue` is flattened into a separate `TransformError`, so `errors` carries + * the full set. * * Callers writing strict-mode handling should treat any non-empty `errors` * as failure regardless of length. * - * @param toCommonMapping - ADR-0017 mapping from native format → CommonGrants. - * @param fromCommonMapping - ADR-0017 mapping from CommonGrants → native format. + * @param toCommonMapping - Declarative mapping from source system format → CommonGrants. + * @param fromCommonMapping - Declarative mapping from CommonGrants → source system format. * @param handlers - Optional custom handlers registered for this call only. * Name collisions with {@link DEFAULT_HANDLERS} raise a `TypeError` at call * time rather than silently shadowing the default. - * @param commonModel - Optional Zod schema to validate `toCommon` output against. + * @param commonSchema - Optional Zod schema to validate `toCommon` output against. * Must be the fully extended schema (e.g. result of `withCustomFields(...)`) — * not the base schema. Passing the base schema silently weakens validation of * typed custom fields. When provided, `safeParse()` runs on the transform * result and Zod issues are flattened into `TransformResult.errors`. - * `PluginError.path` for Zod-flattened issues uses dot notation including + * `TransformError.path` for Zod-flattened issues uses dot notation including * numeric indices (e.g. `"customFields.items.0.value"`). + * @param sourceSchema - Optional Zod schema to validate `fromCommon` output + * against. Without this, `fromCommon` casts its result to `TSource` without + * any runtime check, so `TSource` provides no real safety guarantee. When + * provided, `safeParse()` runs on the transform result and Zod issues are + * flattened into `TransformResult.errors` using the same format as + * `commonSchema`. * * @throws TypeError when custom handler names collide with built-in defaults. * @throws Error when either mapping is structurally malformed (sibling keys * on a handler-dispatch node). */ -export function buildTransforms<TNative = unknown, TCommon = unknown>( +export function buildTransforms<TSource = unknown, TCommon = unknown>( toCommonMapping: Record<string, unknown>, fromCommonMapping: Record<string, unknown>, handlers?: Map<string, Handler>, - // Bivariant `any` accepts schemas with input/output asymmetry; `unknown` - // would reject them at the contravariant input position. + // `any` lets callers pass Zod schemas that transform their input into TCommon + // (e.g. schemas using .transform()). `unknown` would reject those valid schemas. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + commonSchema?: z.ZodType<TCommon, z.ZodTypeDef, any>, + // `any` lets callers pass Zod schemas that transform their input into TSource + // (e.g. schemas using .transform()). `unknown` would reject those valid schemas. // eslint-disable-next-line @typescript-eslint/no-explicit-any - commonModel?: z.ZodType<TCommon, z.ZodTypeDef, any> -): BuiltTransforms<TNative, TCommon> { + sourceSchema?: z.ZodType<TSource, z.ZodTypeDef, any> +): BuiltTransforms<TSource, TCommon> { if (handlers) { const collisions = [...handlers.keys()].filter(k => DEFAULT_HANDLERS.has(k)); if (collisions.length > 0) { @@ -209,14 +219,17 @@ export function buildTransforms<TNative = unknown, TCommon = unknown>( // fail at build time, not on first invocation. validateMapping(toCommonMapping, known); validateMapping(fromCommonMapping, known); - if (commonModel !== undefined) { - validateOutputPaths(toCommonMapping, known, commonModel); + if (commonSchema !== undefined) { + validateOutputPaths(toCommonMapping, known, commonSchema); + } + if (sourceSchema !== undefined) { + validateOutputPaths(fromCommonMapping, known, sourceSchema); } const runMapping = ( data: unknown, mapping: Record<string, unknown> - ): { ok: true; value: unknown } | { ok: false; error: PluginError } => { + ): { ok: true; value: unknown } | { ok: false; error: TransformError } => { try { return { ok: true, value: transformFromMapping(data, mapping, { handlers: merged }) }; } catch (exc) { @@ -224,7 +237,7 @@ export function buildTransforms<TNative = unknown, TCommon = unknown>( const cause = exc.cause; return { ok: false, - error: new PluginError(cause instanceof Error ? cause.message : String(cause), { + error: new TransformError(cause instanceof Error ? cause.message : String(cause), { handler: exc.handler, sourceValue: data, cause, @@ -233,7 +246,7 @@ export function buildTransforms<TNative = unknown, TCommon = unknown>( } return { ok: false, - error: new PluginError(exc instanceof Error ? exc.message : String(exc), { + error: new TransformError(exc instanceof Error ? exc.message : String(exc), { sourceValue: data, cause: exc, }), @@ -241,34 +254,47 @@ export function buildTransforms<TNative = unknown, TCommon = unknown>( } }; - const toCommon = (native: TNative): TransformResult<TCommon> => { - const ran = runMapping(native, toCommonMapping); + const toCommon = (source: TSource): TransformResult<TCommon> => { + const ran = runMapping(source, toCommonMapping); if (!ran.ok) return { result: {} as TCommon, errors: [ran.error] }; - if (commonModel === undefined) { + if (commonSchema === undefined) { return { result: ran.value as TCommon, errors: [] }; } - const parsed = commonModel.safeParse(ran.value); + const parsed = commonSchema.safeParse(ran.value); if (parsed.success) { return { result: parsed.data, errors: [] }; } const errors = parsed.error.issues.map(issue => { // Root-level issues (e.g. from `.refine()` on the schema itself) have an - // empty `path` — leave PluginError.path undefined so the "if known" contract + // empty `path` — leave TransformError.path undefined so the "if known" contract // in the docstring holds. const joined = issue.path.length > 0 ? issue.path.map(p => String(p)).join(".") : undefined; - return new PluginError(issue.message, { path: joined }); + return new TransformError(issue.message, { path: joined }); }); // Return the raw transformed object alongside errors so callers can // inspect malformed data. return { result: ran.value as TCommon, errors }; }; - const fromCommon = (common: TCommon): TransformResult<TNative> => { + const fromCommon = (common: TCommon): TransformResult<TSource> => { const ran = runMapping(common, fromCommonMapping); - if (!ran.ok) return { result: {} as TNative, errors: [ran.error] }; - return { result: ran.value as TNative, errors: [] }; + if (!ran.ok) return { result: {} as TSource, errors: [ran.error] }; + + if (sourceSchema === undefined) { + return { result: ran.value as TSource, errors: [] }; + } + + const parsed = sourceSchema.safeParse(ran.value); + if (parsed.success) { + return { result: parsed.data, errors: [] }; + } + const errors = parsed.error.issues.map(issue => { + const joined = issue.path.length > 0 ? issue.path.map(p => String(p)).join(".") : undefined; + return new TransformError(issue.message, { path: joined }); + }); + return { result: ran.value as TSource, errors }; }; return { toCommon, fromCommon }; diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index a3b02ff0c..df94c48f8 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -124,15 +124,7 @@ export const EXTENSIBLE_SCHEMA_MAP = { /** * Features a plugin can declare in `PluginMeta.capabilities`. */ -export type PluginCapability = "customFields" | "customFilters" | "transforms" | "client"; - -/** - * Loose configuration object for plugin-provided HTTP clients. - * - * Loose per-plugin client configuration. The PoC does not constrain the shape; - * the full SDK's `client` capability work decides the precise type. - */ -export type ClientConfig = Record<string, unknown>; +export type PluginCapability = "customFields" | "customFilters" | "transforms"; /** * Handler signature for transform mapping handlers. @@ -146,13 +138,13 @@ export type ClientConfig = Record<string, unknown>; * * **Do not throw `Error`s whose `.message` embeds source data when that * data may contain PII.** `buildTransforms()` wraps a handler exception's - * message verbatim into the resulting `PluginError.message`, which is + * message verbatim into the resulting `TransformError.message`, which is * enumerable on `Error.prototype` and rendered by `util.inspect` / * `console.log(err)`. The SDK does not redact by default — - * `PluginError.sourceValue` and `.cause` are enumerable, and + * `TransformError.sourceValue` and `.cause` are enumerable, and * `.message` flows through verbatim. The built-in `stringToNumber` handler * follows this rule by throwing a generic "cannot convert source value to a - * number" message; see the README's `PluginError` PII warning for the + * number" message; see the README's `TransformError` PII warning for the * adopter-side redaction pattern. */ export type Handler = (data: unknown, arg: unknown) => unknown; @@ -161,7 +153,7 @@ export type Handler = (data: unknown, arg: unknown) => unknown; * Unconditional return shape for `toCommon` / `fromCommon`. * * `result` is the transformed value (may be partial on handler error or validation - * failure). `errors` is the aggregated `PluginError` list, empty on full success. + * failure). `errors` is the aggregated `TransformError` list, empty on full success. * * Consumers apply their own strict-vs-lenient rule — strict adopters treat any * non-empty `errors` as failure; lenient adopters use `result` despite warnings @@ -169,7 +161,7 @@ export type Handler = (data: unknown, arg: unknown) => unknown; */ export interface TransformResult<T> { result: T; - errors: PluginError[]; + errors: TransformError[]; } /** @@ -200,14 +192,14 @@ export interface TransformResult<T> { * }; * ``` * - * `PluginError.message` is data-bearing on the Zod-validation path - * (`buildTransforms({ commonModel })`): Zod's default error map embeds the + * `TransformError.message` is data-bearing on the Zod-validation path + * (`buildTransforms({ commonSchema })`): Zod's default error map embeds the * received runtime value into `issue.message`, which flows verbatim into - * `PluginError.message`. Adopters whose source data may contain PII must redact + * `TransformError.message`. Adopters whose source data may contain PII must redact * `message` alongside `sourceValue` and `cause`. Full-message sanitization is * tracked under #744. */ -export class PluginError extends Error { +export class TransformError extends Error { /** Dot-notation field path where the error occurred, if known. */ path?: string; /** Name of the handler that raised, if applicable. */ @@ -227,7 +219,7 @@ export class PluginError extends Error { } ) { super(message); - this.name = "PluginError"; + this.name = "TransformError"; this.path = options?.path; this.handler = options?.handler; this.sourceValue = options?.sourceValue; @@ -235,51 +227,120 @@ export class PluginError extends Error { } } +// ############################################################################ +// Public types - SchemaInput (author-provided), SchemaOnly, SchemaWithTransforms +// ############################################################################ + /** * Author-provided input for a single extensible object, passed inside * `DefinePluginOptions.schemas`. * - * Plugin authors supply `toCommon` and `fromCommon` as plain callables — either - * hand-written or generated via `buildTransforms()`. `native` is the optional Zod - * schema for the source format. `customFields` declares any extra fields this - * object exposes beyond the base CommonGrants schema; `definePlugin()` applies - * them to the compiled schema via `withCustomFields()`. + * Exactly one of `mappings` or explicit callables (`toCommon` / `fromCommon`) + * may be present — providing both is a compile-time and runtime error. * - * @remarks - * `common` is intentionally absent here. It is injected by `definePlugin()` - * during compilation from `ObjectSchemasInput` → `ObjectSchemas`, resolved from - * the generated model classes produced by the code generator. Plugin config - * files cannot import from `generated/` (which is the input to generation). - * - * `customFields` is co-located with the transform callables so authors add a - * single per-object entry under `DefinePluginOptions.schemas` rather than - * splitting declarations across two top-level keys. This matches the Python SDK's - * `ObjectSchemasInput` shape, which also carries `custom_fields` alongside - * `to_common` / `from_common`. + * `sourceSchema` is the optional Zod schema for the source system format. + * `customFields` declares any extra fields this object exposes beyond the base + * CommonGrants schema; `definePlugin()` applies them via `withCustomFields()`. + * + * `commonSchema` is intentionally absent here. It is injected by `definePlugin()` + * during compilation, resolved from the generated model classes produced by the + * code generator. Plugin config files cannot import from `generated/` (it is the + * input to generation). + * + * When `mappings` is provided, `definePlugin()` auto-invokes `buildTransforms()` + * at call time and wraps the result with schema validation against the compiled + * common schema. When explicit callables are provided, `definePlugin()` wraps them + * with the same schema validation so both paths give the same runtime guarantee. */ -export interface ObjectSchemasInput<TNative = unknown, TCommon = unknown> { - native?: z.ZodType<TNative>; +export type SchemaInput<TSource = unknown, TCommon = unknown> = + | { + sourceSchema?: z.ZodType<TSource>; + customFields?: Record<string, CustomFieldSpec>; + mappings: SchemaMappings; + toCommon?: never; + fromCommon?: never; + } + | { + sourceSchema?: z.ZodType<TSource>; + customFields?: Record<string, CustomFieldSpec>; + mappings?: never; + toCommon?: (source: TSource) => TransformResult<TCommon>; + fromCommon?: (common: TCommon) => TransformResult<TSource>; + }; + +/** + * Compiled output for a schema-only entry — no transforms configured. + * + * Produced by `definePlugin()` for entries that declare only `customFields` + * (or nothing at all). The `commonSchema` is the fully extended Zod schema. + * `customFields` is kept on the entry so consumers can inspect the specs that + * were used to build it. + */ +export interface SchemaOnly<TCommon> { + commonSchema: z.ZodType<TCommon>; + /** Custom field specs that were used to extend the base schema, kept for inspection. */ customFields?: Record<string, CustomFieldSpec>; - toCommon?: (native: TNative) => TransformResult<TCommon>; - fromCommon?: (common: TCommon) => TransformResult<TNative>; } /** - * Runtime compiled type produced by `definePlugin()` — not provided directly by authors. + * Compiled output for a schema entry with bidirectional transforms. * - * `definePlugin()` now compiles `ObjectSchemasInput` into this shape: `common` is - * injected from the base CG model (extended via `withCustomFields()` when custom - * fields are declared), and `toCommon` / `fromCommon` are auto-wired from declarative - * mappings when no explicit callables are provided. Native input Zod-wrapping remains - * deferred. + * Produced by `definePlugin()` for entries that declare either `mappings` or + * explicit `toCommon` / `fromCommon` callables. Both transform directions are + * always present (non-optional) — `definePlugin()` validates both directions + * exist before producing this type. + * + * `customFields` and `mappings` are kept for consumer inspection: `customFields` + * shows the specs used to extend the common schema; `mappings` is present when + * the author used declarative mappings (absent when hand-written functions were used). */ -export interface ObjectSchemas<TNative, TCommon> { - native: z.ZodType<TNative>; - common: z.ZodType<TCommon>; - toCommon: (native: TNative) => TransformResult<TCommon>; - fromCommon: (common: TCommon) => TransformResult<TNative>; +export interface SchemaWithTransforms<TSource, TCommon> { + commonSchema: z.ZodType<TCommon>; + sourceSchema?: z.ZodType<TSource>; + /** Custom field specs that were used to extend the base schema, kept for inspection. */ + customFields?: Record<string, CustomFieldSpec>; + /** Declarative mappings kept for inspection; absent when hand-written functions were used. */ + mappings?: SchemaMappings; + toCommon: (source: TSource) => TransformResult<TCommon>; + fromCommon: (common: TCommon) => TransformResult<TSource>; } +/** + * Convenience type alias for a `toCommon` transform function. + * + * Annotate hand-written transform functions with this type to get full + * compile-time checking on both the source input and the common output. + * `TSource` comes from `z.infer<typeof sourceSchema>` and `TCommon` comes + * from `z.infer<typeof extendedCommonSchema>` (the schema produced by + * `withCustomFields()` when custom fields are declared). + * + * @example + * ```ts + * // Without custom fields — TCommon is z.infer<typeof OpportunityBaseSchema> + * const toCommon: ToCommon<z.infer<typeof SourceSchema>, z.infer<typeof OpportunityBaseSchema>> + * = source => ({ result: { ... }, errors: [] }); + * + * // With custom fields — TCommon is the extended schema type + * const toCommon: ToCommon<z.infer<typeof SourceSchema>, z.infer<typeof ExtendedSchema>> + * = source => ({ result: { ... }, errors: [] }); + * ``` + */ +export type ToCommon<TSource, TCommon> = (source: TSource) => TransformResult<TCommon>; + +/** + * Convenience type alias for a `fromCommon` transform function. + * + * Annotate hand-written transform functions with this type to get full + * compile-time checking on both the common input and the source output. + * + * @example + * ```ts + * const fromCommon: FromCommon<z.infer<typeof SourceSchema>, z.infer<typeof OpportunityBaseSchema>> + * = common => ({ result: { ... }, errors: [] }); + * ``` + */ +export type FromCommon<TSource, TCommon> = (common: TCommon) => TransformResult<TSource>; + /** * Plugin identity and capability declaration. * @@ -293,51 +354,20 @@ export interface PluginMeta { name: string; /** Plugin version (semver, e.g. `"1.0.0"`). */ version?: string; - /** Name of the native source system (e.g. `"grants.gov"`). */ + /** Name of the source system (e.g. `"grants.gov"`). */ sourceSystem: string; /** Features the plugin provides. */ capabilities?: PluginCapability[]; } /** - * Declarative mapping dicts for a single object, stored in the serializable - * extensions config. + * Declarative mapping dicts for a single object. * * Each direction is author-provided — `buildTransforms()` does not invert one * direction into the other, because many-to-one handlers like `switch` are not * reversible. */ -export interface ObjectMappings { +export interface SchemaMappings { toCommon?: Record<string, unknown>; fromCommon?: Record<string, unknown>; } - -/** - * Per-object config inside the serializable `PluginExtensions.schemas` dict. - * - * `mappings` carries optional declarative mappings; when present and no - * explicit `toCommon` / `fromCommon` is supplied in - * `DefinePluginOptions.schemas`, `definePlugin()` auto-invokes - * `buildTransforms()` on these at call time. - * - * @remarks - * `customFields` lives on {@link ObjectSchemasInput} (inside - * `DefinePluginOptions.schemas[obj]`) rather than here, so authors keep - * all per-object declarations — custom fields, native schema, and transforms - * — in one entry. This matches the Python SDK's `ObjectSchemasInput` shape. - * Cross-package composition of custom field declarations is done by defining - * a combined plugin with all fields under `schemas[Object].customFields`. - */ -export interface PluginExtensionsObjectConfig { - mappings?: ObjectMappings; -} - -/** - * Serializable portion of plugin config — safe to store as JSON. - * - * Used to store JSON-safe serializable config alongside a plugin. - */ -export interface PluginExtensions { - meta?: Partial<PluginMeta>; - schemas?: Partial<Record<ExtensibleSchemaName, PluginExtensionsObjectConfig>>; -} diff --git a/lib/ts-sdk/src/extensions/transformation.ts b/lib/ts-sdk/src/utils/transformation.ts similarity index 83% rename from lib/ts-sdk/src/extensions/transformation.ts rename to lib/ts-sdk/src/utils/transformation.ts index 2a8d551bd..b76d152d0 100644 --- a/lib/ts-sdk/src/extensions/transformation.ts +++ b/lib/ts-sdk/src/utils/transformation.ts @@ -1,9 +1,11 @@ /** * Mapping-runtime utilities for declarative bidirectional transforms. * - * Used by `buildTransforms()` in `./transforms`. Re-exported by `./index`. + * A pure, self-contained module — usable independently of the plugin or + * extensions framework. Re-exported by `@common-grants/sdk/extensions` for + * backward compatibility. * - * ## Null handling (three-state contract, ADR-0024) + * ## Null handling (three-state contract) * * Optional fields carry three distinct states, each preserved through every * built-in handler rather than collapsing `null` into `undefined`: @@ -18,10 +20,10 @@ * This is the canonical description — handlers below note only their own * departures from it. * - * @module @common-grants/sdk/extensions + * @module @common-grants/sdk/utils */ -import type { Handler } from "./types"; +import type { Handler } from "../extensions/types"; // ############################################################################ // Public utilities - getFromPath, handlers, transformFromMapping @@ -106,19 +108,10 @@ export function constValue(_data: unknown, value: unknown): unknown { * with no `field` is functionally equivalent to `const: <default>`; prefer * `const` for clarity when the constant case is what you want. * - * Cross-SDK divergence: Python's PoC uses bare `dict.get(val, default)`, which - * accepts non-string `val` natively and collapses `null` into `default`. This - * handler instead preserves `null` (three-state) and fails loud with a - * descriptive error on a malformed spec. + * `match` is the canonical handler name; `switch` is a convenience alias — + * both point at the same handler function. * - * `match` is the canonical handler name; `switch` is provided as a - * convenience alias — both point at the same handler function. (No prior SDK - * API defined `switch`; the alias is for ergonomic preference, not a - * backward-compatibility obligation.) - * - * @throws Error when `spec` is not a non-null object. The walker wraps this - * as a `HandlerError`; `buildTransforms` surfaces it as a `PluginError` - * with `handler: "match"`. + * @throws Error when `spec` is not a non-null object. */ export function switchOnValue(data: unknown, spec: unknown): unknown { if (typeof spec !== "object" || spec === null || Array.isArray(spec)) { @@ -170,24 +163,14 @@ export function numberToString(data: unknown, fieldPath: unknown): string | null * attempted); value → coerced via the integer / float / safe-integer rules * below. (See the module-level null-handling note.) * - * Divergences from Python's `int(s)` semantics, both intentional: - * - * - `int("42.0")` raises `ValueError` in Python; this handler falls through - * to `Number(s)` and returns `42`. Plugin authors porting a handler that - * relies on the Python behavior should add their own decimal-rejecting regex. - * - Python's `int()` is arbitrary precision; JavaScript numbers are IEEE 754 - * doubles with a safe-integer ceiling of `Number.MAX_SAFE_INTEGER` - * (2^53 − 1). An integer-shaped string outside that range cannot be - * represented without precision loss, so this handler throws rather than - * silently returning a corrupted value. Plugin authors round-tripping - * 64-bit IDs should declare the field as a string (and skip this handler) - * or write a custom handler that returns a `BigInt`. - * - * Empty and whitespace-only strings throw — `Number("")` and `Number(" ")` - * both coerce to `0` in JavaScript, which would silently turn an - * implicit-absent CSV cell into a real zero on the transformed side. - * Callers who want absent input to surface as `undefined` should null the - * field upstream. + * - Decimal strings like `"42.0"` fall through to `Number()` and return `42`. + * Add your own decimal-rejecting regex if you need strict integer-only behavior. + * - Integer strings outside `Number.MAX_SAFE_INTEGER` (2^53 − 1) throw rather + * than silently returning a corrupted value. Use a string field or a custom + * handler returning `BigInt` for 64-bit IDs. + * - Empty and whitespace-only strings throw — `Number("")` coerces to `0`, + * which would silently corrupt an absent field. Null the field upstream if + * you want `undefined` for absent input. */ export function stringToNumber(data: unknown, fieldPath: unknown): number | null | undefined { const val = getFromPath(data, String(fieldPath ?? "")); @@ -222,12 +205,7 @@ export function stringToNumber(data: unknown, fieldPath: unknown): number | null /** * Raised when a handler function throws. Carries the handler name for attribution. * - * `buildTransforms()` catches this and wraps it as a `PluginError`, so callers - * of the public `toCommon` / `fromCommon` pair will not see `HandlerError` - * directly. The class is **internal**: not re-exported from the package barrel, - * not part of the published `package.json` `exports` map, and not a supported - * import path for adopters. Tests that drive `transformFromMapping` itself - * import it from the source file for `instanceof` checks; consumers of + * Internal — not re-exported from the package barrel. Consumers of * `transformFromMapping()` through the public surface should treat thrown * values as plain `Error`s. */ diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index 5320e1b4e..b1dba40cc 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -13,16 +13,15 @@ _How should the Plugin object be structured to support both custom field declara ### Questions -- Should the top-level Plugin structure group by feature (`meta`, `client`, `schemas`, `extensions`) or by object (`Opportunity`, `Application`, …)? +- Should the top-level Plugin structure group by feature (`meta`, `client`, `schemas`) or by object (`Opportunity`, `Application`, …)? - Should client configuration (auth, transport, rate-limiting) sit alongside per-object schemas, or be lifted to the top level as a system-level concern? - Should custom fields and transforms be coupled in the same package, or allowed independently? ### Decision drivers - The framework must be implementable in both the Python and TypeScript SDKs with as consistent an interface as possible. -- The `extensions` config must be serializable and able to pass validation (JSON-safe). +- The inputs to `definePlugin()` should be declarative wherever possible, with optional overrides for custom code-driven transformation logic. - Existing plugin packages that declare only custom fields should remain valid with minimal changes. -- The SDK interface should support clean dependency injection — it must be possible to pass `client` or `schemas` as a coherent unit without reassembling them from per-object branches. - Auth, transport, and rate-limiting are system-level concerns that belong to a single client, not distributed across per-object branches. - The top-level Plugin surface should be short and stable — adding new protocol objects should not expand the top-level key set. @@ -32,17 +31,17 @@ We decided to: 1. **Keep "plugin" as the unified term** for both the published npm/PyPI packages in the website catalog and the runtime SDK object. No change to `PluginSourceEntry` or `src/content/plugins/index.json`. The existing `definePlugin()` function is expanded to accept the full set of top-level fields described below. -2. **Use functional grouping at the top level** with four keys — `meta`, `client`, `schemas`, and `extensions` — rather than grouping by object name at the root. Custom filters (`filters`) are an authoring-time input on `DefinePluginOptions` / `define_plugin()` and surface through the `Client` returned by `getClient()`, so they don't add a fifth top-level key on the runtime `Plugin` shape. +2. **Use functional grouping at the top level** with two keys — `meta` and `schemas` — rather than grouping by object name at the root. Client configuration, auth, and custom filters are deferred to future capabilities and are not part of the current Plugin shape. -3. **Use per-object grouping inside `schemas`** where it reflects real coupling: each object's native schema, CommonGrants schema, and bidirectional transforms are tightly coupled and change together. +3. **Use per-object grouping inside `schemas`** where it reflects real coupling: each object's source schema, CommonGrants schema, and bidirectional transforms are tightly coupled and change together. -4. **Expand `definePlugin()` to accept all top-level fields** — `meta`, `client`, `schemas`, and `extensions` — rather than only `extensions`. `extensions` holds serializable-only config (mappings, meta); all per-object declarations (custom fields, native schema, transforms) live under `schemas.<Object>`. +4. **Expand `definePlugin()` to accept all top-level fields** — `meta` and `schemas` — rather than only `customFields`. All per-object declarations (custom fields, source schema, transforms, and declarative mappings) live under `schemas.<Object>`. 5. **Make all top-level Plugin fields optional** so adopters can publish a plugin that provides only the features they need — for example, custom fields only — and expand to include transforms, client config, or additional schemas incrementally over time. -6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `PluginExtensions.schemas.<Object>` gains an optional `mappings` key carrying JSON-safe `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.<Object>`, the SDK invokes `buildTransforms()` automatically. In TypeScript this happens inside `definePlugin()`; in Python it happens inside the code generator (`generate.py`) at generation time, emitting a `build_transforms()` call into the generated `__init__.py`. Both mapping directions must be provided explicitly — `buildTransforms()` / `build_transforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. +6. **Plugin authors provide `toCommon` / `fromCommon` as functions; mappings are one way to generate them.** The SDK exposes `buildTransforms()` / `build_transforms()` as a public utility wrapping the existing mapping runtimes. `schemas.<Object>` gains an optional `mappings` key carrying `toCommon` / `fromCommon` mapping objects; when those are declared and no explicit transform is supplied in `schemas.<Object>`, the SDK invokes `buildTransforms()` automatically. In TypeScript this happens inside `definePlugin()`; in Python it happens inside the code generator (`generate.py`) at generation time, emitting a `build_transforms()` call into the generated `__init__.py`. Both mapping directions must be provided explicitly — `buildTransforms()` / `build_transforms()` does not invert one direction into the other, because many-to-one handlers like `switch` are not reversible. -7. **`toCommon` / `fromCommon` return a `TransformResult<T>` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. In the current PoC, this validation is opt-in at the `buildTransforms()` call site via the `commonModel` / `common_model` parameter — when supplied, validation runs inside `toCommon` against the fully extended generated schema. In the full SDK, `definePlugin()` will additionally inject validation when auto-generating transforms from `extensions.schemas.<Object>.mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. +7. **`toCommon` / `fromCommon` return a `TransformResult<T>` of `{ result, errors }` unconditionally; mapping definitions are validated at `buildTransforms()` call time.** Partial failure is routine for cross-schema transforms — field handlers can emit warnings that do not invalidate a record — so the transform surface is safe by default rather than throwing. Runtime schema validation (Zod `.parse()` / Pydantic `model_validate()`) surfaces as entries in `errors` rather than thrown exceptions. `buildTransforms()` accepts `commonModel` / `common_model` and `sourceModel` / `source_model` parameters — when supplied, validation runs inside `toCommon` and `fromCommon` respectively against the relevant schema. `definePlugin()` additionally injects validation when auto-generating transforms from `schemas.<Object>.mappings`. Plugin authors using hand-written transforms are responsible for their own validation. Consumers apply their own rule for what counts as success — strict adopters treat any non-empty `errors` as failure, lenient adopters tolerate warnings. Mappings passed to `buildTransforms()` are checked at the call site, failing fast on structural errors, unknown handlers, or unresolvable field paths. 8. **Custom handlers are registered per utility call, not globally.** `buildTransforms()` accepts an optional `handlers` argument (`Map<string, Handler>`) for registering additional handler names. Per-call scoping keeps behavior explicit and testable; name collisions with the default set raise at `buildTransforms()` call time rather than silently shadowing them. The registry is a `Map` rather than a plain object so that handler-name lookup uses `Map.has()` — which does not walk the prototype chain — rather than `in` or own-property checks on a plain object. @@ -52,16 +51,14 @@ The resulting Plugin shape: ``` plugin.meta // name, version, sourceSystem, capabilities -plugin.get_client // (config: ClientConfig) => Client; memoized by the code generator -plugin.extensions // serializable; used by merge_extensions() -plugin.schemas.<Name> // ObjectSchemas instance — unified access to model class and transforms -plugin.schemas.<Name>.common // generated Pydantic model class (includes any declared custom fields) -plugin.schemas.<Name>.native // source system type (defaults to dict) -plugin.schemas.<Name>.to_common // callable: native → TransformResult[common] (None if not configured) -plugin.schemas.<Name>.from_common // callable: common → TransformResult[native] (None if not configured) +plugin.schemas.<Name> // SchemaConfig instance — unified access to model class and transforms +plugin.schemas.<Name>.common // Zod schema / Pydantic model class (includes any declared custom fields) +plugin.schemas.<Name>.source // source system type (defaults to dict / Record<string, unknown>) +plugin.schemas.<Name>.to_common // callable: source → TransformResult[common] (None/undefined if not configured) +plugin.schemas.<Name>.from_common // callable: common → TransformResult[source] (None/undefined if not configured) ``` -**Python note:** In the Python SDK, `define_plugin()` returns a `PluginConfig` (build-time input) rather than a fully compiled `Plugin`. The code generator (`generate.py`) compiles `PluginConfig → Plugin` by injecting the generated model classes as the `common` schema, wrapping `get_client` with `functools.lru_cache`, and auto-generating `build_transforms()` calls for any objects that have `extensions.schemas[obj].mappings` but no explicit `to_common`/`from_common` in `schemas[obj]`. This split is necessary because `cg_config.py` cannot import from `generated/` — it is the input to code generation. Both SDKs declare `custom_fields` / `customFields` on `ObjectSchemasInput` (inside `schemas`), not on `PluginExtensionsSchema` / `PluginExtensionsObjectConfig` (inside `extensions`). +**Python note:** In the Python SDK, `define_plugin()` returns a `PluginConfig` (build-time input) rather than a fully compiled `Plugin`. The code generator (`generate.py`) compiles `PluginConfig → Plugin` by injecting the generated model classes as the `common` schema and auto-generating `build_transforms()` calls for any objects that have `schemas[obj].mappings` but no explicit `to_common`/`from_common` in `schemas[obj]`. This split is necessary because `cg_config.py` cannot import from `generated/` — it is the input to code generation. All per-object declarations (`custom_fields`, `source`, `mappings`, and transforms) live on `ObjectSchemasInput` inside `schemas`. ### Example interface @@ -72,8 +69,7 @@ plugin.schemas.<Name>.from_common // callable: common → TransformResult[nat type PluginCapability = | "customFields" // declares custom fields on CommonGrants schema objects | "customFilters" // declares custom filter parameters for resource methods - | "transforms" // provides toCommon/fromCommon transformation functions - | "client"; // provides a runtime client (auth, transport, resource methods) + | "transforms"; // provides toCommon/fromCommon transformation functions interface PluginMeta { name: string; @@ -100,24 +96,35 @@ interface TransformResult<T> { } // Runtime type — produced by definePlugin(), not provided directly by plugin authors -interface ObjectSchemas<TNative, TCommon> { - native: ZodType<TNative>; +interface SchemaConfig<TSource, TCommon> { + source: ZodType<TSource>; common: ZodType<TCommon>; - toCommon: (native: TNative) => TransformResult<TCommon>; - fromCommon: (common: TCommon) => TransformResult<TNative>; + toCommon: (source: TSource) => TransformResult<TCommon>; + fromCommon: (common: TCommon) => TransformResult<TSource>; } -// Input type — provided by plugin authors inside DefinePluginOptions.schemas +// Input type — provided by plugin authors inside DefinePluginOptions.schemas. +// All per-object declarations live here: custom fields, source schema, declarative +// mappings, and explicit transform callables. // common is intentionally absent: the plugin config file cannot import from generated/ // since it is the input to generation. definePlugin() injects common during compilation -// from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. +// from SchemaInput → SchemaConfig, resolved from the generated model classes. // customFields declares extra fields beyond the base CG schema; definePlugin() extends // the base schema with them to produce the typed common schema. -interface ObjectSchemasInput<TNative = unknown, TCommon = unknown> { - native?: ZodType<TNative>; // defaults to Record<string, unknown> if omitted +interface SchemaInput<TSource = unknown, TCommon = unknown> { + source?: ZodType<TSource>; // defaults to Record<string, unknown> if omitted customFields?: Record<string, CustomFieldSpec>; - toCommon?: (native: TNative) => TransformResult<TCommon>; - fromCommon?: (common: TCommon) => TransformResult<TNative>; + mappings?: SchemaMappings; // declarative ADR-0017 mappings; auto-wired by definePlugin() + toCommon?: (source: TSource) => TransformResult<TCommon>; + fromCommon?: (common: TCommon) => TransformResult<TSource>; +} + +// Declarative mapping objects in ADR-0017 format — held under SchemaInput.mappings. +// When present and no explicit toCommon/fromCommon is in SchemaInput, definePlugin() +// auto-invokes buildTransforms() on these. Each direction is author-provided; see Decision #6. +interface SchemaMappings { + toCommon?: Record<string, unknown>; // ADR-0017 mapping: source → CommonGrants + fromCommon?: Record<string, unknown>; // ADR-0017 mapping: CommonGrants → source } // Scalar types only — filters are query parameters, not schema fields @@ -128,59 +135,18 @@ interface CustomFilterSpec { description?: string; } -// Per-object config shape inside extensions.schemas — mirrors Python's PluginExtensionsSchema. -// customFields is intentionally absent here; it lives on ObjectSchemasInput so that -// all per-object declarations (custom fields, native schema, transforms) are co-located -// under schemas.<Object> rather than split across two top-level keys. -interface PluginExtensionsObjectConfig { - // Optional declarative mappings in ADR-0017 format. When present and no explicit - // toCommon / fromCommon is supplied in schemas.<Object>, definePlugin() auto-invokes - // buildTransforms() on these. Each direction is author-provided; see Decision #6. - mappings?: { - toCommon?: Record<string, unknown>; // ADR-0017 mapping: native → CommonGrants - fromCommon?: Record<string, unknown>; // ADR-0017 mapping: CommonGrants → native - }; -} - -// Serializable portion of the plugin config — safe to store as JSON. -// schemas keys are restricted to ExtensibleSchemaName (the known set of CommonGrants -// objects that support custom fields). -interface PluginExtensions { - meta?: Partial<PluginMeta>; - schemas?: Partial<Record<ExtensibleSchemaName, PluginExtensionsObjectConfig>>; -} - -// ClientConfig is defined by the plugin author to declare the system-specific inputs -// they require (e.g. auth token, base URL, max page size, timeouts). -interface ClientConfig { - [key: string]: unknown; -} - -// Client is a placeholder for the SDK's runtime client type (not shown here). -// No `filters` key on Plugin — handled by the Client returned by getClient() (see Decision #2). interface Plugin { meta?: PluginMeta; - getClient?: (config: ClientConfig) => Client; - extensions?: PluginExtensions; // serializable - schemas?: Partial< - Record<ExtensibleSchemaName, ObjectSchemas<unknown, unknown>> - >; + schemas: Partial<Record<ExtensibleSchemaName, SchemaConfig<unknown, unknown>>>; } // Input object for definePlugin(). Using a named-options object makes it easy to add // new inputs over time without breaking existing callers. interface DefinePluginOptions { meta?: PluginMeta; - // Plugin authors provide a factory function; definePlugin() wraps it with memoization - // so the same Client instance is returned for equivalent configs automatically. - getClient?: (config: ClientConfig & { auth?: AuthMethod }) => Client; - extensions?: PluginExtensions; // serializable // Plugin authors provide input schemas and transforms; definePlugin() compiles them - // into the full ObjectSchemas runtime type, merging any customFields from extensions. - schemas?: Partial<Record<ExtensibleSchemaName, ObjectSchemasInput>>; - filters?: Partial< - Record<ExtensibleSchemaName, Record<string, CustomFilterSpec>> - >; + // into the full SchemaConfig runtime type, extending the base schema with any customFields. + schemas?: Partial<Record<ExtensibleSchemaName, SchemaInput>>; } // Factory: all options are optional so adopters can start with only what they need @@ -188,14 +154,11 @@ interface DefinePluginOptions { // // definePlugin compiles DefinePluginOptions into a Plugin by: // - extending the base CommonGrants schema with any declared customFields → common -// - native defaults to Record<string, unknown> if omitted (extensions is JSON-safe; -// runtime Zod schemas cannot be included) -// - wrapping getClient with memoization so the same Client instance is returned -// for equivalent configs automatically +// - source defaults to Record<string, unknown> if omitted // // toCommon / fromCommon may be plain hand-written functions, generated via // buildTransforms() and passed in schemas, or auto-generated by definePlugin() -// itself — when extensions.schemas.<Object>.mappings is declared and schemas.<Object> +// itself — when schemas.<Object>.mappings is declared and schemas.<Object> // provides no explicit transform, definePlugin() invokes buildTransforms() internally. // All transforms return TransformResult<T>; definePlugin() validates the result field // at runtime with schema.parse / model_validate and appends any validation failures @@ -215,17 +178,19 @@ type Handler = (value: unknown, context: unknown) => unknown; // commonModel must be the fully extended generated schema (e.g. the generated // Opportunity with typed customFields), not the base schema — passing a base schema // silently weakens validation of typed custom fields. +// When sourceModel is provided, fromCommon similarly validates its output against that schema. // The underlying mapping runtime normalizes model/schema instances to plain objects // at the entry point, so fromCommon can receive the validated output of toCommon // and field paths still resolve correctly. -function buildTransforms<TNative, TCommon>( - toCommonMapping: Record<string, unknown>, // ADR-0017 mapping from native → CommonGrants - fromCommonMapping: Record<string, unknown>, // ADR-0017 mapping from CommonGrants → native +function buildTransforms<TSource, TCommon>( + toCommonMapping: Record<string, unknown>, // ADR-0017 mapping from source → CommonGrants + fromCommonMapping: Record<string, unknown>, // ADR-0017 mapping from CommonGrants → source handlers?: Map<string, Handler>, // Map (not plain object) — Map.has() is prototype-safe commonModel?: ZodType<TCommon>, // must be the generated extended schema, not the base + sourceModel?: ZodType<TSource>, // optional; validates fromCommon output ): { - toCommon: (native: TNative) => TransformResult<TCommon>; - fromCommon: (common: TCommon) => TransformResult<TNative>; + toCommon: (source: TSource) => TransformResult<TCommon>; + fromCommon: (common: TCommon) => TransformResult<TSource>; }; // Base class for SDK-emitted transformation errors (see Decision #9). @@ -244,9 +209,8 @@ interface PluginError extends Error { from dataclasses import dataclass from typing import Any, Callable, Generic, Literal, TypeVar from pydantic import BaseModel, ConfigDict, Field -# Client is a placeholder for the SDK's runtime client type (not shown here) -TNative = TypeVar('TNative') +TSource = TypeVar('TSource') TCommon = TypeVar('TCommon') T = TypeVar('T') @@ -273,24 +237,37 @@ class CustomFieldSpec: # common includes any custom fields declared by the plugin (it is a generated subclass of the # base CG model, e.g. OpportunityBase, with typed custom_fields baked in). @dataclass -class ObjectSchemas(Generic[TNative, TCommon]): - native: type[TNative] # source system type; defaults to dict +class ObjectSchemas(Generic[TSource, TCommon]): + source: type[TSource] # source system type; defaults to dict common: type[TCommon] # generated Pydantic model class (includes declared custom fields) - to_common: Callable[[TNative], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TNative]] | None = None + to_common: Callable[[TSource], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TSource]] | None = None -# Input type — provided by plugin authors inside define_plugin(schemas=...) +class ObjectMappings(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + # ADR-0017 mappings. Each direction is author-provided; see Decision #6. + to_common: dict[str, Any] | None = Field(default=None, alias='toCommon') # source → CommonGrants + from_common: dict[str, Any] | None = Field(default=None, alias='fromCommon') # CommonGrants → source + +# Input type — provided by plugin authors inside define_plugin(schemas=...). +# All per-object declarations live here: custom fields, source type, declarative +# mappings, and explicit transform callables. # common is intentionally absent: cg_config.py cannot import from generated/ since # it is the input to generation. define_plugin() injects common during compilation # from ObjectSchemasInput → ObjectSchemas, resolved from the generated model classes. # custom_fields declares extra fields beyond the base CG schema; the code generator # reads these and emits typed subclasses. +# mappings holds optional ADR-0017 declarative mappings. When present and no explicit +# to_common / from_common is supplied, the code generator auto-invokes build_transforms() +# on these. Explicit callables take priority and disable auto-wiring for that object. @dataclass -class ObjectSchemasInput(Generic[TNative, TCommon]): - native: type[TNative] | None = None # defaults to dict[str, Any] if omitted +class ObjectSchemasInput(Generic[TSource, TCommon]): + source: type[TSource] | None = None # defaults to dict[str, Any] if omitted custom_fields: dict[str, CustomFieldSpec] | None = None - to_common: Callable[[TNative], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TNative]] | None = None + mappings: ObjectMappings | None = None + to_common: Callable[[TSource], TransformResult[TCommon]] | None = None + from_common: Callable[[TCommon], TransformResult[TSource]] | None = None # Scalar types only — filters are query parameters, not schema fields CustomFilterType = Literal['string', 'number', 'integer', 'boolean'] @@ -300,11 +277,9 @@ class CustomFilterSpec: filter_type: CustomFilterType description: str = "" -PluginCapability = Literal['customFields', 'customFilters', 'transforms', 'client'] +PluginCapability = Literal['customFields', 'customFilters', 'transforms'] -# PluginExtensionsMeta is used for both Plugin.meta (top-level plugin identity) and -# PluginExtensions.meta (serializable meta in the extensions JSON object). All fields -# are optional at the type level; plugins should populate name and source_system. +# Plugin identity and capability declaration. All fields are optional. class PluginExtensionsMeta(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -313,27 +288,6 @@ class PluginExtensionsMeta(BaseModel): source_system: str | None = Field(default=None, alias='sourceSystem') capabilities: list[PluginCapability] | None = None -class ObjectMappings(BaseModel): - model_config = ConfigDict(populate_by_name=True) - - # ADR-0017 mappings. Each direction is author-provided; see Decision #6. - to_common: dict[str, Any] | None = Field(default=None, alias='toCommon') # native → CommonGrants - from_common: dict[str, Any] | None = Field(default=None, alias='fromCommon') # CommonGrants → native - -class PluginExtensionsSchema(BaseModel): - """Per-object config inside extensions.schemas. Holds declarative mappings only.""" - model_config = ConfigDict(populate_by_name=True) - - # Optional declarative mappings. When present and no explicit to_common / from_common - # is supplied in schemas[obj], define_plugin() auto-invokes build_transforms() on these. - mappings: ObjectMappings | None = None - -class PluginExtensions(BaseModel): - meta: PluginExtensionsMeta | None = None - schemas: dict[str, PluginExtensionsSchema] | None = None - -ClientConfig = dict[str, Any] # plugin authors define their own keys (auth, base_url, timeout, etc.) - # Runtime plugin container — assembled by the code generator (generate.py) from # the generated model classes and a compiled PluginConfig. Plugin authors do not # construct this directly; it is emitted into the plugin's __init__.py. @@ -343,57 +297,41 @@ ClientConfig = dict[str, Any] # plugin authors define their own keys (auth, bas # plugin.schemas.Opportunity.common → Pydantic model class (with custom fields) # plugin.schemas.Opportunity.to_common → transform callable (or None) # plugin.schemas.Opportunity.from_common → transform callable (or None) -# plugin.schemas.Opportunity.native → source system type (or dict) +# plugin.schemas.Opportunity.source → source system type (or dict) @dataclass class Plugin(Generic[T]): schemas: T - extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None - get_client: Callable[[ClientConfig], Any] | None = None - filters: dict[str, Any] | None = None # Build-time config — produced by define_plugin(), consumed by generate.py. # # Compilation from PluginConfig → Plugin (injecting the common model classes from -# generated/, wrapping get_client with functools.lru_cache) happens inside -# generate.py at code-generation time, not at define_plugin() call time. This split -# is necessary in Python because cg_config.py cannot import from generated/ — it is -# the input to code generation. +# generated/) happens inside generate.py at code-generation time, not at +# define_plugin() call time. This split is necessary in Python because cg_config.py +# cannot import from generated/ — it is the input to code generation. @dataclass(frozen=True) class PluginConfig: - extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None - get_client: Callable[[ClientConfig], Any] | None = None - filters: dict[str, Any] | None = None # All params are optional — adopters can start with only what they need and expand # incrementally. Unlike TypeScript, Python supports named optional params at the -# function root, so no DefinePluginOptions wrapper object is needed. +# function root, so no wrapper object is needed. # # define_plugin stores inputs as-is in a PluginConfig. The code generator # (generate.py) then compiles PluginConfig → Plugin by: # - extending the base CommonGrants model with any declared custom_fields → common -# - native defaults to dict[str, Any] if omitted -# - wrapping get_client with functools.lru_cache for memoization +# - source defaults to dict[str, Any] if omitted # # to_common / from_common may be plain hand-written callables, generated via # build_transforms() and passed in schemas, or auto-generated by the code generator -# itself — when extensions.schemas[obj].mappings is declared and schemas[obj] has no +# itself — when schemas[obj].mappings is declared and schemas[obj] has no # to_common/from_common, generate.py invokes build_transforms() in the emitted code. def define_plugin( meta: PluginExtensionsMeta | None = None, - get_client: Callable[[ClientConfig], Any] | None = None, - extensions: PluginExtensions | None = None, schemas: dict[str, ObjectSchemasInput[Any, Any]] | None = None, - filters: dict[str, Any] | None = None, ) -> PluginConfig: ... -def merge_extensions( - sources: list[PluginExtensions], - on_conflict: Literal["error", "first_wins", "last_wins"] = "error", -) -> PluginExtensions: ... - # Handler signature matches ADR-0017 runtime conventions. Handler = Callable[[Any, Any], Any] @@ -407,14 +345,16 @@ Handler = Callable[[Any, Any], Any] # common_model must be the fully extended generated model class (e.g. # generated/schemas.py's Opportunity), not the base class — passing a base class # silently weakens validation of typed custom fields. +# When source_model is provided, from_common similarly validates its output. # transform_from_mapping normalizes Pydantic model instances to plain dicts via # model_dump(mode="json") at the entry point, so from_common can receive the # validated model output of to_common and field paths still resolve correctly. def build_transforms( - to_common_mapping: dict[str, Any], # ADR-0017 mapping from native → CommonGrants - from_common_mapping: dict[str, Any], # ADR-0017 mapping from CommonGrants → native + to_common_mapping: dict[str, Any], # ADR-0017 mapping from source → CommonGrants + from_common_mapping: dict[str, Any], # ADR-0017 mapping from CommonGrants → source handlers: dict[str, Handler] | None = None, common_model: type[BaseModel] | None = None, # must be the generated extended model, not the base + source_model: type[BaseModel] | None = None, # optional; validates from_common output ) -> tuple[ Callable[[Any], TransformResult[Any]], Callable[[Any], TransformResult[Any]], @@ -440,7 +380,7 @@ class PluginError(Exception): // buildTransforms() is a utility that generates toCommon/fromCommon from declarative // mappings (ADR-0017). Using it is optional — plain functions work just as well. const { toCommon, fromCommon } = buildTransforms( - // toCommon: native grants.gov shape → CommonGrants Opportunity + // toCommon: grants.gov source shape → CommonGrants Opportunity { title: "data.opportunity_title", status: { @@ -453,7 +393,7 @@ const { toCommon, fromCommon } = buildTransforms( }, }, }, - // fromCommon: CommonGrants Opportunity → native grants.gov shape + // fromCommon: CommonGrants Opportunity → grants.gov source shape { "data.opportunity_title": "title", "data.opportunity_status": { @@ -474,20 +414,11 @@ const plugin = definePlugin({ version: "1.0.0", sourceSystem: "grants.gov", }, - // definePlugin memoizes getClient — the same Client is returned for equivalent configs. - getClient: (config: ClientConfig & { auth?: AuthMethod }) => - new Client({ - baseUrl: config.baseUrl ?? "https://api.grants.gov", - timeout: config.timeout, - pageSize: config.pageSize, - maxItems: config.maxItems, - auth: config.auth, - }), - // All per-object declarations — customFields, native schema, and transforms — live - // under schemas.<Object> so authors have one entry per object rather than two. + // All per-object declarations — customFields, source schema, and transforms — live + // under schemas.<Object> so authors have one entry per object. schemas: { Opportunity: { - native: GrantsGovOpportunitySchema, + source: GrantsGovOpportunitySchema, customFields: { programArea: { fieldType: CustomFieldType.String, @@ -503,9 +434,6 @@ const plugin = definePlugin({ }, }, }); - -// Calling getClient() with a config object — memoized, so repeated calls return the same instance -const client = plugin.getClient({ auth: Auth.bearer("token"), pageSize: 50 }); ``` </TabItem> @@ -515,7 +443,7 @@ const client = plugin.getClient({ auth: Auth.bearer("token"), pageSize: 50 }); # build_transforms() is a utility that generates to_common/from_common from declarative # mappings (ADR-0017). Using it is optional — plain callables work just as well. to_common, from_common = build_transforms( - # to_common: native grants.gov shape → CommonGrants Opportunity + # to_common: grants.gov source shape → CommonGrants Opportunity to_common_mapping={ 'title': 'data.opportunity_title', 'status': { @@ -528,7 +456,7 @@ to_common, from_common = build_transforms( }, }, }, - # from_common: CommonGrants Opportunity → native grants.gov shape + # from_common: CommonGrants Opportunity → grants.gov source shape from_common_mapping={ 'data.opportunity_title': 'title', 'data.opportunity_status': { @@ -545,18 +473,11 @@ to_common, from_common = build_transforms( plugin = define_plugin( meta=PluginExtensionsMeta(name='grants-gov-plugin', version='1.0.0', source_system='grants.gov'), # source_system serializes as 'sourceSystem' in JSON - # define_plugin memoizes get_client — the same Client is returned for equivalent configs. - get_client=lambda config: Client(config=Config( - base_url=config.get('base_url', 'https://api.grants.gov'), - api_key=config['api_key'], - timeout=config.get('timeout', 10.0), - page_size=config.get('page_size', 100), - list_items_limit=config.get('list_items_limit', 1000), - )), - # Python SDK: custom_fields lives on ObjectSchemasInput, not PluginExtensionsSchema + # All per-object declarations — custom_fields, source type, and transforms — live + # on ObjectSchemasInput so authors have one entry per object. schemas={ 'Opportunity': ObjectSchemasInput( - native=GrantsGovOpportunity, + source=GrantsGovOpportunity, custom_fields={ 'programArea': CustomFieldSpec(field_type=CustomFieldType.STRING, description='HHS program area code'), 'legacyGrantId': CustomFieldSpec(field_type=CustomFieldType.INTEGER, description='Numeric ID from legacy system'), @@ -566,14 +487,6 @@ plugin = define_plugin( ), }, ) - - -# Combine extensions from multiple packages before constructing the plugin -merged = merge_extensions([base_extensions, grants_gov_extensions]) -merged_plugin = define_plugin(extensions=merged) - -# Calling get_client() with a config dict — memoized, so repeated calls return the same instance -client = plugin.get_client({'api_key': 'abc123', 'page_size': 50}) ``` </TabItem> @@ -587,13 +500,7 @@ client = plugin.get_client({'api_key': 'abc123', 'page_size': 50}) ```ts import { grantsGovPlugin } from "grants-gov-plugin"; -// Get a configured client for this source system -const client = grantsGovPlugin.getClient({ - auth: Auth.bearer(process.env.GRANTS_GOV_API_KEY), - pageSize: 25, -}); - -// Use the compiled schemas to transform native data into CommonGrants shape. +// Use the compiled schemas to transform source data into CommonGrants shape. // toCommon / fromCommon return TransformResult<T> = { result, errors } — consumers // apply their own strict-vs-lenient rule for what counts as success. const { toCommon } = grantsGovPlugin.schemas.Opportunity; @@ -613,7 +520,7 @@ const successful = items // Inspect what the plugin declares about itself console.log(grantsGovPlugin.meta.sourceSystem); // "grants.gov" -console.log(grantsGovPlugin.meta.capabilities); // ["customFields", "transforms", "client"] +console.log(grantsGovPlugin.meta.capabilities); // ["customFields", "transforms"] ``` </TabItem> @@ -622,13 +529,7 @@ console.log(grantsGovPlugin.meta.capabilities); // ["customFields", "transforms" ```python from grants_gov_plugin import grants_gov_plugin -# Get a configured client for this source system -client = grants_gov_plugin.get_client({ - 'api_key': os.environ['GRANTS_GOV_API_KEY'], - 'page_size': 25, -}) - -# Use the compiled schemas to transform native data into CommonGrants shape. +# Use the compiled schemas to transform source data into CommonGrants shape. # to_common / from_common return TransformResult[T] = {result, errors} — consumers # apply their own strict-vs-lenient rule for what counts as success. to_common = grants_gov_plugin.schemas.Opportunity.to_common @@ -646,7 +547,7 @@ successful = [r.result for r in items if not r.errors] # Inspect what the plugin declares about itself print(grants_gov_plugin.meta.source_system) # "grants.gov" -print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms", "client"] +print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms"] ``` </TabItem> @@ -655,30 +556,26 @@ print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms", " ### Consequences - **Positive consequences** - - Client stays singular — `getClient()` / `get_client()` is memoized (in TypeScript by `definePlugin()`; in Python by the code generator wrapping it with `functools.lru_cache`), so one source system always produces one `Client` instance regardless of how many times it is called - - Top-level surface (`meta`, `client`, `schemas`, `extensions`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only - - Dependency injection works along functional lines: pass `getClient`, pass `Schemas`, pass `Extensions` as coherent units without needing to reassemble from per-object branches - - `extensions` (serializable config) is flat and JSON-safe — it can be stored, versioned, or passed across package boundaries without carrying runtime Zod/Pydantic schemas - - Per-object grouping inside `schemas` preserves the real coupling between native schema, CommonGrants schema, and bidirectional transforms — they share type signatures and change together - - Mirrors the SDK module structure (`client`, `schemas`, `extensions`), so Plugin reads as a system-specific version of the existing SDK rather than a different mental model - - `toCommon`/`fromCommon` can be plain hand-written functions, generated via `buildTransforms()` and passed in `schemas`, or auto-generated by `definePlugin()` from mappings declared in `extensions` — plugin authors are not required to use a declarative mapping format + - Top-level surface (`meta`, `schemas`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only + - Per-object grouping inside `schemas` preserves the real coupling between source schema, CommonGrants schema, and bidirectional transforms — they share type signatures and change together + - All per-object declarations (custom fields, source type, declarative mappings, and explicit callables) are co-located under `schemas.<Object>` — no split across multiple top-level keys + - `toCommon`/`fromCommon` can be plain hand-written functions, generated via `buildTransforms()` and passed in `schemas`, or auto-generated by `definePlugin()` from mappings declared in `schemas.<Object>.mappings` — plugin authors are not required to use a declarative mapping format - `buildTransforms()` accepts separate `toCommonMapping` and `fromCommonMapping` objects, reflecting that the two directions of a bidirectional transform are distinct - `toCommon` / `fromCommon` return `TransformResult<T>` so partial failure surfaces as data, batch processing is a plain `.map`, and consumers apply their own strict-vs-lenient rule for what counts as success; structured `PluginError` lets adopters reason about those failures programmatically without parsing error text - Custom handlers are registered per-call on `buildTransforms()`, not globally — behavior stays explicit and testable, and collisions with the default set raise at `buildTransforms()` call time - `customFields` is optional — the `customFields`-only config structure remains valid; existing plugin packages require only minimal code changes to adopt `definePlugin()` - All top-level Plugin fields are optional — adopters can start with only what they need and expand incrementally - **Negative consequences** - - `extensions` (serializable config) and `plugin` (runtime object including client) are distinct concepts that adopters must understand separately + - Client configuration, auth, and custom filters are deferred to future capabilities — current Plugin shape does not support them ### Criteria - **Backward compatible:** Existing custom-fields-only plugins remain valid without changes - **SDK-friendly:** Config shape maps naturally to Pydantic/Zod one-model-at-a-time usage inside `schemas` -- **Language-agnostic config:** The `extensions` JSON document uses camelCase keys (`customFields`, `fieldType`, `sourceSystem`) in both SDKs — Python source uses snake_case attributes with camelCase `alias` fields, matching the existing SDK convention +- **Language-agnostic config:** Both SDKs use camelCase keys (`customFields`, `fieldType`, `sourceSystem`) for serialized forms — Python source uses snake_case attributes with camelCase `alias` fields, matching the existing SDK convention - **Clear naming:** A single term — "plugin" — is used consistently across the registry catalog and SDK - **Supports both capabilities:** Custom field declarations and bidirectional transforms can coexist or be used independently; transforms may be hand-written or generated from declarative mappings - **Incremental adoption:** All top-level fields are optional, so adopters can start with only what they need -- **DI-friendly:** Functional top-level keys can be passed as coherent units without reassembly - **Stable surface:** New protocol objects do not expand the top-level key set ### Options considered @@ -794,18 +691,16 @@ Pure object-first / Plugin is best if: - Client, auth, and transport are system-level but must either be duplicated per object or kept implicit alongside object keys, creating an awkward mix of concerns - Filters attach to resource methods rather than schemas, and resource methods aren't consistent across objects (e.g. `opportunities.list/get/search` vs `applications.start/submit`), creating a poor fit - DI requires reassembling a flat view across all object branches (e.g. an `allSchemas` helper) — working against the grain of the structure - - `mergeExtensions()` must deeply merge nested per-object branches rather than operating on a flat serializable root + - `mergeExtensions()` must deeply merge nested per-object branches rather than operating on a flat declarative root ### Option 3: Functional top-level with per-object schema grouping _(chosen)_ -Top-level keys are functional (`meta`, `client`, `schemas`, `extensions`). Per-object grouping is used only inside `schemas`, where it reflects real coupling between native schemas, CommonGrants schemas, and bidirectional transforms. +Top-level keys are functional (`meta`, `schemas`). Per-object grouping is used only inside `schemas`, where it reflects real coupling between source schemas, CommonGrants schemas, and bidirectional transforms. All per-object declarations (custom fields, source type, declarative mappings, and explicit callables) live under `schemas.<Object>`. ```ts interface Plugin { meta?: PluginMeta; - getClient?: (config: ClientConfig) => Client; - extensions?: PluginExtensions; - schemas?: Partial<Record<ExtensibleSchemaName, ObjectSchemas>>; // per-object grouping only here + schemas: Partial<Record<ExtensibleSchemaName, SchemaConfig>>; // per-object grouping only here } ``` @@ -813,15 +708,12 @@ interface Plugin { Functional top-level / per-object schemas is best if: - we want a stable, short top-level surface that doesn't grow with the protocol's object list -- and we want a singular client and clean DI while still co-locating the tightly-coupled schema/transform pairs per object +- and we want all per-object declarations co-located under a single entry per object ::: - **Pros** - - Short, stable top-level surface — `meta`, `client`, `schemas`, `extensions` tracks a closed list regardless of how many protocol objects exist - - Client is singular — `getClient()` is memoized by `definePlugin()`, so one source system always produces one `Client` instance - - DI works along functional lines: pass `getClient`, pass `Schemas`, pass `Extensions` as units - - `mergeExtensions()` operates on flat, serializable data at the root, not deeply nested per-object branches - - Per-object grouping inside `schemas` preserves real coupling — native schema, CommonGrants schema, `toCommon`, and `fromCommon` share type signatures and change together - - Mirrors the SDK module structure (`client`, `schemas`, `extensions`) — Plugin is a system-specific version of the existing SDK, not a different mental model + - Short, stable top-level surface — `meta`, `schemas` tracks a closed list regardless of how many protocol objects exist + - Per-object grouping inside `schemas` preserves real coupling — source schema, CommonGrants schema, `toCommon`, and `fromCommon` share type signatures and change together + - All per-object declarations are co-located: no split across `schemas` and a separate `extensions` key - **Cons** - - `extensions` (serializable config) and `plugin` (runtime object including client) are distinct concepts that adopters must learn separately + - Client configuration and auth are deferred; the Plugin shape does not yet support them From e90b4bca5a61535100a14e631b399ca51c4e2f0d Mon Sep 17 00:00:00 2001 From: Billy Daly <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 11:21:27 -0400 Subject: [PATCH 18/51] refactor(ts-sdk): update transform helper types (#916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sdk): accept a Date as input on date schemas UTCDateTime, ISODate, and OffsetDateTime schemas now accept either an ISO string or a Date, normalizing a Date to a string before the existing strict validation. Output is still a Date, so hand-written transforms can return real Date fields and have them validate. * feat(sdk): model-based ToCommon/FromCommon helper types Take a named { model, sourceSchema, customFields } argument instead of positional <TSource, TCommon>. The common type is resolved from the model's base schema via EXTENSIBLE_SCHEMA_MAP, so authors pass their field specs rather than a prebuilt common schema, and one common type covers both directions. * docs(sdk): rework transforms example and refresh extensions docs Switch transforms.ts from calling buildTransforms() directly to using definePlugin(), which compiles the mappings internally and is the path plugin authors actually use. The example covers the mappings, hand-written-functions, no-custom-fields, and mappings-vs-functions cases with a runnable round-trip. In the README, move the "when you need as const" guidance into Best practices, restore the Important callout, and refresh the transform-helper API reference entries. Adds a test that a schema-only entry's toCommon is a compile error. * docs(ts-sdk): Removes unnecessary code comments * feat(sdk): require sourceSchema for transforms; always validate fromCommon Split SchemaInput into three variants — mappings (sourceSchema + mappings), functions (sourceSchema + both toCommon and fromCommon), and schema-only (customFields, no transforms). Providing a transform without a sourceSchema, or only one direction, is now a compile error. Since a transform entry always carries a sourceSchema, definePlugin wraps the hand-written fromCommon with safeParse validation against it unconditionally, mirroring toCommon against the common schema. * docs(ts-sdk): cleans up code comments --- .../extensions/define-plugin.spec.ts | 66 ++- .../__tests__/schemas/zod/types.spec.ts | 15 +- lib/ts-sdk/examples/transforms.ts | 530 +++++++++--------- lib/ts-sdk/src/extensions/README.md | 42 +- lib/ts-sdk/src/extensions/define-plugin.ts | 15 +- lib/ts-sdk/src/extensions/index.ts | 4 +- .../src/extensions/transform-helpers.ts | 98 ++++ lib/ts-sdk/src/extensions/types.ts | 130 +++-- lib/ts-sdk/src/schemas/zod/types.ts | 45 +- 9 files changed, 588 insertions(+), 357 deletions(-) create mode 100644 lib/ts-sdk/src/extensions/transform-helpers.ts diff --git a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts index a327d201d..da7284df4 100644 --- a/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/define-plugin.spec.ts @@ -207,7 +207,7 @@ describe("definePlugin", () => { const plugin = definePlugin({ schemas: { - Opportunity: { toCommon, fromCommon }, + Opportunity: { sourceSchema: z.object({}).passthrough(), toCommon, fromCommon }, }, }); @@ -234,7 +234,7 @@ describe("definePlugin", () => { const plugin = definePlugin({ schemas: { - Opportunity: { toCommon, fromCommon }, + Opportunity: { sourceSchema: z.object({}).passthrough(), toCommon, fromCommon }, }, }); @@ -253,6 +253,59 @@ describe("definePlugin", () => { expect(plugin.schemas.Opportunity.toCommon).toBeUndefined(); expect(plugin.schemas.Opportunity.fromCommon).toBeUndefined(); }); + + it("does not expose transform callables as callable on a schema-only entry", () => { + // A customFields-only entry resolves to the schema-only shape, so calling + // toCommon is a compile error (not just `undefined` at runtime). The guarded + // block never executes; the `@ts-expect-error` fails the build if the type + // ever starts exposing a callable transform here. + const plugin = definePlugin({ + schemas: { Opportunity: { customFields: { legacyId: { fieldType: "integer" } } } }, + }); + expect(plugin.schemas.Opportunity.commonSchema).toBeDefined(); + expect(plugin.schemas.Opportunity.customFields).toBeDefined(); + if (false as boolean) { + // @ts-expect-error — toCommon is not callable on a schema-only entry + plugin.schemas.Opportunity.toCommon({}); + } + }); + + it("rejects hand-written transforms without a sourceSchema (compile-time)", () => { + const noop = (): TransformResult<unknown> => ({ result: {}, errors: [] }); + definePlugin({ + schemas: { + // @ts-expect-error — the functions path requires a sourceSchema + Opportunity: { toCommon: noop, fromCommon: noop }, + }, + }); + }); + + it("rejects a single transform direction (compile-time)", () => { + const noop = (): TransformResult<unknown> => ({ result: {}, errors: [] }); + definePlugin({ + schemas: { + // @ts-expect-error — the functions path requires both toCommon and fromCommon + Opportunity: { sourceSchema: z.object({}).passthrough(), toCommon: noop }, + }, + }); + }); + + it("validates fromCommon output against the sourceSchema", () => { + const sourceSchema = z.object({ native_id: z.string() }); + const plugin = definePlugin({ + schemas: { + Opportunity: { + sourceSchema, + toCommon: (): TransformResult<unknown> => ({ result: validOpp, errors: [] }), + // Returns a source object missing the required native_id. + fromCommon: (): TransformResult<unknown> => ({ result: { wrong: true }, errors: [] }), + }, + }, + }); + + const out = plugin.schemas.Opportunity.fromCommon?.({} as never); + expect(out?.errors.length ?? 0).toBeGreaterThan(0); + }); }); // ############################################################################ @@ -321,12 +374,16 @@ const autoWireFromCommonMapping = { native_title: { field: "title" }, native_id: { field: "id" }, }; +// Transform entries require a sourceSchema. The mappings runtime here doesn't use +// it, so a permissive shape is fine for these tests. +const autoWireSourceSchema = z.object({}).passthrough(); describe("definePlugin — auto-wiring from mappings", () => { it("auto-generates working toCommon/fromCommon from schemas.Opportunity.mappings", () => { const plugin = definePlugin({ schemas: { Opportunity: { + sourceSchema: autoWireSourceSchema, mappings: { toCommon: autoWireToCommonMapping, fromCommon: autoWireFromCommonMapping, @@ -361,6 +418,7 @@ describe("definePlugin — auto-wiring from mappings", () => { definePlugin({ schemas: { Opportunity: { + sourceSchema: autoWireSourceSchema, mappings: { toCommon: autoWireToCommonMapping, fromCommon: autoWireFromCommonMapping, @@ -385,6 +443,7 @@ describe("definePlugin — auto-wiring from mappings", () => { definePlugin({ schemas: { Opportunity: { + sourceSchema: autoWireSourceSchema, mappings: { toCommon: autoWireToCommonMapping, fromCommon: autoWireFromCommonMapping, @@ -402,6 +461,7 @@ describe("definePlugin — auto-wiring from mappings", () => { definePlugin({ schemas: { Opportunity: { + sourceSchema: autoWireSourceSchema, mappings: { toCommon: autoWireToCommonMapping, // fromCommon intentionally absent @@ -417,6 +477,7 @@ describe("definePlugin — auto-wiring from mappings", () => { definePlugin({ schemas: { Opportunity: { + sourceSchema: autoWireSourceSchema, mappings: { // toCommon intentionally absent fromCommon: autoWireFromCommonMapping, @@ -432,6 +493,7 @@ describe("definePlugin — auto-wiring from mappings", () => { definePlugin({ schemas: { Opportunity: { + sourceSchema: autoWireSourceSchema, mappings: { toCommon: { unknownFieldXyz: { field: "data.x" } }, fromCommon: autoWireFromCommonMapping, diff --git a/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts b/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts index 9748bc2a8..96a80ca1e 100644 --- a/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts +++ b/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts @@ -163,11 +163,16 @@ describe("ISODate Schema", () => { await expectZodMatchesJsonSchema(ISODateSchema, jsonSchemaId); }); + it("accepts a Date object (normalized to YYYY-MM-DD, then parsed back to a Date)", () => { + const parsed = ISODateSchema.parse(new Date("2025-01-15T12:00:00Z")); + expect(parsed).toBeInstanceOf(Date); + expect(parsed.getUTCFullYear()).toBe(2025); + expect(parsed.getUTCDate()).toBe(15); + }); + it("should raise an error for an invalid ISODate", () => { // Must be in YYYY-MM-DD format expect(() => ISODateSchema.parse("01-01-2025")).toThrow(); - // Must be a string, not a Date object - expect(() => ISODateSchema.parse(new Date())).toThrow(); // Invalid date format expect(() => ISODateSchema.parse("not-a-date")).toThrow(); // Missing parts @@ -318,11 +323,13 @@ describe("OffsetDateTime Schema", () => { expect(withMsOffset2.getUTCHours()).toBe(20); // 12:00 + 8 hours = 20:00 UTC }); + it("accepts a Date object (normalized to an ISO string, then parsed back to a Date)", () => { + expect(OffsetDateTimeSchema.parse(new Date("2025-01-01T00:00:00Z"))).toBeInstanceOf(Date); + }); + it("should raise an error for an invalid OffsetDateTime", () => { // Must be a valid ISO datetime string expect(() => OffsetDateTimeSchema.parse("not-a-datetime")).toThrow(); - // Must be a string, not a Date object - expect(() => OffsetDateTimeSchema.parse(new Date())).toThrow(); // Invalid datetime format (missing time) expect(() => OffsetDateTimeSchema.parse("2025-01-01")).toThrow(); // Missing timezone diff --git a/lib/ts-sdk/examples/transforms.ts b/lib/ts-sdk/examples/transforms.ts index b25fb8e0f..dd3417150 100644 --- a/lib/ts-sdk/examples/transforms.ts +++ b/lib/ts-sdk/examples/transforms.ts @@ -1,260 +1,263 @@ /** - * Example script demonstrating bidirectional transforms. Shows: - * 1. Defining `toCommon` / `fromCommon` mappings. - * 2. Registering a custom mapping handler (`join`) for this call only. - * 3. Validating `toCommon` output against the fully extended Zod schema - * (`withCustomFields(OpportunityBaseSchema, ...)`) — passing the base - * schema would silently weaken validation of typed custom fields. - * 4. Exposing the compiled transform via `definePlugin({ schemas })`. - * 5. Round-tripping `native → common → native` and printing both directions. + * Example script demonstrating bidirectional transforms through `definePlugin()` + * following three common plugin author scenarios + * + * Scenarios shown: + * 1. Custom fields + declarative `mappings` → definePlugin compiles them + * 2. Custom fields + hand-written functions → `ToCommon` / `FromCommon` + * 3. Transforms with no custom fields → base CommonGrants schema + * X. `mappings` + functions on one entry → rejected (mappings XOR functions) * * Run with: `pnpm example:transforms` * * @remarks - * Zod's default `.parse()` strips unknown keys, so source-system fields that - * have no home in the CommonGrants schema must round-trip through - * `customFields` (declared on the extended schema). The example treats - * `opportunity_number` this way. + * The mappings scenario validates `toCommon` output against the extended common + * schema, so date strings are parsed into `Date` objects on the common side. + * The round-trip checks below therefore assert on fields that survive verbatim + * (ids, the three-state `source` null), not on the parsed date fields. */ import { z } from "zod"; import { CustomFieldType } from "../src/constants"; import { - buildTransforms, definePlugin, - getFromPath, - withCustomFields, - type Handler, + type FromCommon, + type ToCommon, + type TransformResult, } from "../src/extensions"; -import { OpportunityBaseSchema } from "../src/schemas/zod/models"; // ############################################################################ -// Step 1 — Sample grants.gov source data +// Shared setup — source schema, sample data, custom fields // ############################################################################ -// Note the `source_url: null` below — this is the publisher actively asserting -// "doesn't apply" for the source URL (three-state null). The transforms -// preserve it as `null` end-to-end rather than collapsing to absent, so a -// downstream consumer can distinguish "publisher said N/A" from "publisher -// didn't supply this." -const SOURCE_DATA = { +// Stands in for a real grants.gov payload schema. In a real plugin this would be +// imported from wherever the source-system types live. +const GrantsGovOpportunity = z.object({ + data: z.object({ + opportunity_uuid: z.string().uuid(), + opportunity_id: z.number().int(), + opportunity_title: z.string(), + opportunity_description: z.string(), + opportunity_status: z.string(), + // `source_url: null` below is the publisher actively asserting "doesn't + // apply" (three-state null). The transforms preserve it as `null` end to end + // rather than collapsing it to absent. + source_url: z.string().url().nullish(), + created_at: z.string(), + last_modified_at: z.string(), + }), +}); +type GrantsGovSource = z.infer<typeof GrantsGovOpportunity>; + +const SOURCE_DATA: GrantsGovSource = { data: { - agency_name: "Department of Examples", - created_at: "2025-01-15T09:00:00Z", - last_modified_at: "2025-04-01T12:30:00Z", - opportunity_description: - "Funding to advance research into conservation techniques for endangered ecosystems.", + opportunity_uuid: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", opportunity_id: 12345, - opportunity_number: "ABC-123-XYZ-001", - opportunity_status: "posted", opportunity_title: "Research into conservation techniques", - opportunity_uuid: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + opportunity_description: "Funding to advance conservation research.", + opportunity_status: "posted", source_url: null, - summary: { - applicant_types: ["state_governments"], - }, + created_at: "2025-01-15T09:00:00Z", + last_modified_at: "2025-04-01T12:30:00Z", }, }; -// ############################################################################ -// Step 2 — Custom handlers (joined-label round trip) -// ############################################################################ - -// A mapping leaf like `{ field: "data.x" }` dispatches to a *handler* — a -// `(data, spec) => value` function looked up by the leaf's key. `buildTransforms` -// ships built-ins (`field`, `const`, `match`/`switch`, `numberToString`, -// `stringToNumber`); anything beyond those is a custom handler you register on -// the `handlers` map in Step 4. A mapping invokes one by name — `{ join: {...} }` -// runs `joinFields` with `{...}` as its `spec`. `data` is always the whole object -// under transform, so handler paths are absolute from its root (hence the -// `data.`-prefixed paths below). A custom name that collides with a built-in is -// rejected when `buildTransforms` runs. -// -// `join` and `split` are an inverse pair, here to demo a *derived* custom field -// with no single source column: `join` composes `compositeLabel` -// ("<opportunity_number> — <title>") on the toCommon side, and `split` recovers -// `opportunity_number` back out of it on the fromCommon side. The round trip is -// lossless only while the separator never occurs inside a constituent value — -// see the NOTE on the `compositeLabel` mapping in Step 4. - -// join: concatenate the values at `spec.fields`, in order, joined by `spec.sep` -// (default " "). undefined/null parts are dropped; when nothing survives it -// returns undefined so the field is omitted rather than emitted as an empty "". -const joinFields: Handler = (data, spec) => { - const s = (spec ?? {}) as { fields?: string[]; sep?: string }; - const sep = s.sep ?? " "; - const parts = (s.fields ?? []) - .map(path => getFromPath(data, path)) - .filter(v => v !== undefined && v !== null) - .map(String); - return parts.length > 0 ? parts.join(sep) : undefined; -}; - -// split: the inverse of join. Read the string at `spec.field`, split it on -// `spec.sep` (default " "), and return the segment at `spec.index` (default 0). -// Returns undefined when the source value is absent or the index is out of range. -const splitField: Handler = (data, spec) => { - const s = (spec ?? {}) as { field?: string; sep?: string; index?: number }; - const value = getFromPath(data, s.field ?? ""); - if (value === undefined || value === null) return undefined; - const parts = String(value).split(s.sep ?? " "); - const idx = s.index ?? 0; - return idx < parts.length ? parts[idx] : undefined; -}; - -// ############################################################################ -// Step 3 — Custom field declarations + extended schema for validation -// ############################################################################ - -const customFieldSpecs = { +// Declared inline at each call below, but kept here too for the hand-written +// scenario, which needs `typeof customFields` for its helper-type annotations. +const customFields = { legacyId: { - name: "legacyId", fieldType: CustomFieldType.integer, value: z.number().int(), - description: "Numeric ID from the legacy database (round-trip preserved).", - }, - agencyName: { - name: "agencyName", - fieldType: CustomFieldType.string, - value: z.string(), - description: "Name of the agency hosting this opportunity.", - }, - applicantTypes: { - name: "applicantTypes", - fieldType: CustomFieldType.array, - value: z.array(z.string()), - description: "Types of applicants eligible for this opportunity.", - }, - // Derived field composed by the `join` handler; the `split` handler in - // fromCommon recovers opportunity_number from this value. Survives Zod - // validation because it's declared on the extended schema as a custom field. - compositeLabel: { - name: "compositeLabel", - fieldType: CustomFieldType.string, - value: z.string(), - description: "Composite label '<opportunity_number> — <opportunity_title>'.", + description: "Numeric ID from the legacy system (round-trip preserved).", }, } as const; -const ExtendedOpportunitySchema = withCustomFields(OpportunityBaseSchema, customFieldSpecs); - // ############################################################################ -// Step 4 — Compile bidirectional transforms +// Scenario 1 — custom fields + declarative mappings // ############################################################################ -const { toCommon, fromCommon } = buildTransforms( - { - id: { field: "data.opportunity_uuid" }, - title: { field: "data.opportunity_title" }, - description: { field: "data.opportunity_description" }, - createdAt: { field: "data.created_at" }, - lastModifiedAt: { field: "data.last_modified_at" }, - // Three-state demo: native `source_url: null` carries the - // publisher's "doesn't apply" assertion. The `field` handler preserves - // the terminal null; the walker places it on the output as a real null - // (distinct from an absent key). Zod's `.nullish()` accepts it. - source: { field: "data.source_url" }, - status: { - value: { - match: { - field: "data.opportunity_status", - case: { - posted: "open", - archived: "closed", - forecasted: "forecasted", +// definePlugin() compiles these mappings into toCommon / fromCommon using the +// built-in handlers (`field`, `match`, `const`). +const mappingsPlugin = definePlugin({ + meta: { name: "grants.gov (mappings)", sourceSystem: "grants.gov" }, + schemas: { + Opportunity: { + customFields, + sourceSchema: GrantsGovOpportunity, + mappings: { + toCommon: { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + // Three-state null: a terminal `null` is preserved as a real null. + source: { field: "data.source_url" }, + createdAt: { field: "data.created_at" }, + lastModifiedAt: { field: "data.last_modified_at" }, + status: { + value: { + match: { + field: "data.opportunity_status", + case: { posted: "open", archived: "closed" }, + default: "custom", + }, + }, + }, + customFields: { + legacyId: { + value: { field: "data.opportunity_id" }, + name: "legacyId", + fieldType: "integer", + }, }, - default: "custom", }, - }, - }, - customFields: { - legacyId: { - value: { field: "data.opportunity_id" }, - name: "legacyId", - fieldType: "integer", - }, - agencyName: { - value: { field: "data.agency_name" }, - name: "agencyName", - fieldType: "string", - }, - applicantTypes: { - value: { field: "data.summary.applicant_types" }, - name: "applicantTypes", - fieldType: "array", - }, - // Compose a derived label via the `join` custom handler; fromCommon - // recovers opportunity_number from it via `split`. - // NOTE: the separator must not appear inside any of the constituent - // field values, or `split` will produce a wrong result on the way back. - compositeLabel: { - value: { - join: { - fields: ["data.opportunity_number", "data.opportunity_title"], - sep: " — ", + fromCommon: { + data: { + opportunity_uuid: { field: "id" }, + opportunity_title: { field: "title" }, + opportunity_description: { field: "description" }, + source_url: { field: "source" }, + created_at: { field: "createdAt" }, + last_modified_at: { field: "lastModifiedAt" }, + opportunity_status: { const: "posted" }, + opportunity_id: { field: "customFields.legacyId.value" }, }, }, - name: "compositeLabel", - fieldType: "string", }, }, }, - { +}); + +// ############################################################################ +// Scenario 2 — custom fields + hand-written functions +// ############################################################################ + +// The author annotates the functions with `ToCommon` / `FromCommon`, passing the +// same inputs `definePlugin()` uses — `model`, `sourceSchema`, and the +// `customFields` specs they already have. The SDK resolves the common type from +// those (no need to build or borrow a `commonSchema`). One common type is used +// for both directions: the common date schemas accept either a string or a +// `Date`, so the author builds and returns real `Date` values here, and reads +// `Date` values back in `fromCommon`. +type OpportunityTransform = { + model: "Opportunity"; + sourceSchema: typeof GrantsGovOpportunity; + customFields: typeof customFields; +}; + +// The functions scenario is the override for logic the declarative mappings +// can't express — here, a status flag derived in code rather than via `match`. +const toCommon: ToCommon<OpportunityTransform> = source => ({ + result: { + id: source.data.opportunity_uuid, + title: source.data.opportunity_title, + description: source.data.opportunity_description, + source: source.data.source_url, + status: { value: source.data.opportunity_status === "posted" ? "open" : "custom" }, + createdAt: new Date(source.data.created_at), + lastModifiedAt: new Date(source.data.last_modified_at), + customFields: { + legacyId: { name: "legacyId", fieldType: "integer", value: source.data.opportunity_id }, + }, + }, + errors: [], +}); + +const fromCommon: FromCommon<OpportunityTransform> = common => ({ + result: { data: { - opportunity_uuid: { field: "id" }, - opportunity_title: { field: "title" }, - opportunity_description: { field: "description" }, - created_at: { field: "createdAt" }, - last_modified_at: { field: "lastModifiedAt" }, - // Recover opportunity_number from the joined label via `split`. The - // separator (` — `) must match what the `toCommon` side used to join, - // and it must not appear inside any constituent field value, or the - // split index will land on the wrong segment. See the join side above. - opportunity_number: { - split: { field: "customFields.compositeLabel.value", sep: " — ", index: 0 }, - }, - opportunity_id: { field: "customFields.legacyId.value" }, - agency_name: { field: "customFields.agencyName.value" }, - // Round-trip the "doesn't apply" assertion back to native: the null - // sourced from `source` on the CG side becomes `source_url: null` again. - source_url: { field: "source" }, - summary: { - applicant_types: { field: "customFields.applicantTypes.value" }, - }, + opportunity_uuid: common.id, + opportunity_id: common.customFields?.legacyId?.value ?? 0, + opportunity_title: common.title, + opportunity_description: common.description, + opportunity_status: "posted", + source_url: common.source ?? null, + // `createdAt` / `lastModifiedAt` arrive as `Date`; render back to strings. + created_at: common.createdAt.toISOString(), + last_modified_at: common.lastModifiedAt.toISOString(), }, }, - new Map([ - ["join", joinFields], - ["split", splitField], - ]) - // Note: do NOT pass ExtendedOpportunitySchema here — definePlugin wraps - // toCommon with wrapWithSchemaValidation, which applies the schema once. - // Passing it here too would cause double-validation and fail on date fields - // (Zod's .transform() produces Date objects; a second pass expects strings). -); + errors: [], +}); + +const functionsPlugin = definePlugin({ + meta: { name: "grants.gov (functions)", sourceSystem: "grants.gov" }, + schemas: { + Opportunity: { customFields, sourceSchema: GrantsGovOpportunity, toCommon, fromCommon }, + }, +}); // ############################################################################ -// Step 5 — Plug the compiled transforms into a plugin definition +// Scenario 3 — transforms with no custom fields // ############################################################################ -// All per-object input — customFields, toCommon, and fromCommon — lives on -// the same schemas[Opportunity] entry. See SchemaInput in -// extensions/types.ts for details. -const grantsGovPlugin = definePlugin({ - meta: { - name: "grants.gov", - version: "0.1.0", - sourceSystem: "grants.gov", - capabilities: ["customFields", "transforms"], - }, +// Omit `customFields`; the common schema is the base CommonGrants Opportunity. +const noCustomFieldsPlugin = definePlugin({ + meta: { name: "grants.gov (no custom fields)", sourceSystem: "grants.gov" }, schemas: { - Opportunity: { customFields: customFieldSpecs, toCommon, fromCommon }, + Opportunity: { + sourceSchema: GrantsGovOpportunity, + mappings: { + toCommon: { + id: { field: "data.opportunity_uuid" }, + title: { field: "data.opportunity_title" }, + description: { field: "data.opportunity_description" }, + source: { field: "data.source_url" }, + createdAt: { field: "data.created_at" }, + lastModifiedAt: { field: "data.last_modified_at" }, + status: { + value: { + match: { + field: "data.opportunity_status", + case: { posted: "open", archived: "closed" }, + default: "custom", + }, + }, + }, + }, + fromCommon: { + data: { + opportunity_uuid: { field: "id" }, + opportunity_title: { field: "title" }, + opportunity_description: { field: "description" }, + source_url: { field: "source" }, + created_at: { field: "createdAt" }, + last_modified_at: { field: "lastModifiedAt" }, + opportunity_status: { const: "posted" }, + }, + }, + }, + }, }, -} as const); +}); // ############################################################################ -// Step 6 — Run the round trip and report +// Scenario X — mappings + functions on one entry is rejected (XOR) +// ############################################################################ + +// Providing both `mappings` and explicit callables is a compile error and a +// runtime error. Wrapped here so the runtime backstop can be demonstrated +// without aborting the script. +function demonstrateXorIsRejected(): void { + try { + definePlugin({ + schemas: { + Opportunity: { + sourceSchema: GrantsGovOpportunity, + mappings: { toCommon: {}, fromCommon: {} }, + // @ts-expect-error — mappings XOR functions: cannot provide both on one entry + toCommon, + }, + }, + }); + fail("scenario X: expected definePlugin to reject mappings + functions"); + } catch { + console.log("scenario X (XOR): mappings + functions rejected at runtime, too"); + } +} + +// ############################################################################ +// Run the round trips and report // ############################################################################ function fail(message: string): never { @@ -262,63 +265,80 @@ function fail(message: string): never { process.exit(1); } -const toCommonResult = grantsGovPlugin.schemas.Opportunity.toCommon?.(SOURCE_DATA); -if (!toCommonResult) fail("schemas.Opportunity.toCommon missing"); -if (toCommonResult.errors.length > 0) { - // The source data in this example is fixed and PII-free, so embedding - // `e.message` here is safe. Production adopters: `TransformError.message` can - // carry source values on the Zod-validation path (Zod's default error map - // embeds runtime values). See the README PII warning before copying this - // logging shape. - fail( - `toCommon failed: ${toCommonResult.errors - .map(e => `[${e.path ?? "?"}] ${e.message}`) - .join("; ")}` - ); +function check(label: string, condition: boolean): void { + if (!condition) fail(`✗ ${label}`); + console.log(`✓ ${label}`); } -// The console output below dumps the entire transform result for demonstration. -// Production callers should not log `result` without a PII review — applicant -// records, EINs, and free-text fields routinely flow through `customFields`. -console.log("=== toCommon (native → CommonGrants) ==="); -console.log(JSON.stringify(toCommonResult.result, null, 2)); - -const fromCommonResult = grantsGovPlugin.schemas.Opportunity.fromCommon?.(toCommonResult.result); -if (!fromCommonResult) fail("schemas.Opportunity.fromCommon missing"); -if (fromCommonResult.errors.length > 0) { - // Same PII caveat as the toCommon error block above — `e.message` may carry - // source values on the Zod path. Safe here because the example data is fixed. - fail( - `fromCommon failed: ${fromCommonResult.errors - .map(e => `[${e.path ?? "?"}] ${e.message}`) - .join("; ")}` - ); +function reportErrors(label: string, result: TransformResult<unknown>): void { + // Source data here is fixed and PII-free, so printing messages is safe. + // Production adopters: `TransformError.message` can carry source values on the + // Zod-validation path — see the README PII warning before copying this shape. + if (result.errors.length > 0) { + fail( + `${label} produced errors: ${result.errors.map(e => `[${e.path ?? "?"}] ${e.message}`).join("; ")}` + ); + } } -console.log("\n=== fromCommon (CommonGrants → native) ==="); -console.log(JSON.stringify(fromCommonResult.result, null, 2)); +function roundTrip( + label: string, + plugin: { schemas: { Opportunity: { toCommon?: unknown; fromCommon?: unknown } } }, + opts: { expectLegacyId: boolean } +): void { + // The harness drives every plugin through one shape; the resolved common + // output type read off a built plugin is fine here (this is test plumbing, not + // the author-facing annotation the functions scenario above demonstrates). + type HarnessCommon = z.output<typeof mappingsPlugin.schemas.Opportunity.commonSchema>; + const opp = plugin.schemas.Opportunity; + const toCommonFn = opp.toCommon as (s: GrantsGovSource) => TransformResult<HarnessCommon>; + const fromCommonFn = opp.fromCommon as (c: HarnessCommon) => TransformResult<GrantsGovSource>; -// Spot-check a covered field round-trips. -const native = fromCommonResult.result as { - data: { opportunity_number: string; source_url: string | null | undefined }; -}; -if (native.data.opportunity_number !== SOURCE_DATA.data.opportunity_number) { - fail( - `round-trip mismatch on opportunity_number: ${native.data.opportunity_number} ≠ ${SOURCE_DATA.data.opportunity_number}` + const common = toCommonFn(SOURCE_DATA); + reportErrors(`${label} toCommon`, common); + + const back = fromCommonFn(common.result); + reportErrors(`${label} fromCommon`, back); + + // The uuid round-trips verbatim in every scenario (unlike the parsed date fields). + check( + `${label}: opportunity_uuid round-trips`, + back.result.data.opportunity_uuid === SOURCE_DATA.data.opportunity_uuid ); + // Three-state null survives both directions as a real null, not undefined. + check( + `${label}: source_url null ('doesn't apply') preserved`, + back.result.data.source_url === null + ); + // The legacy id only round-trips when a custom field carries it across. + if (opts.expectLegacyId) { + check(`${label}: legacy opportunity_id round-trips`, back.result.data.opportunity_id === 12345); + } } -// Three-state pin: an explicit `null` ("doesn't apply") on the -// source side must survive both transforms as a real `null`, not collapse -// to undefined ("not provided"). A future regression that put `undefined` -// here instead of `null` would fail this check. -if (native.data.source_url !== null) { - fail( - `three-state mismatch on source_url: expected explicit null ("doesn't apply"), got ${JSON.stringify( - native.data.source_url - )}` +function main(): void { + console.log("=== Transforms via definePlugin ===\n"); + + console.log("--- Scenario 1: declarative mappings ---"); + // `mappings` is kept on the entry for inspection (absent on the functions scenario). + check("mappings inspectable", mappingsPlugin.schemas.Opportunity.mappings !== undefined); + roundTrip("mappings", mappingsPlugin, { expectLegacyId: true }); + + console.log("\n--- Scenario 2: hand-written functions ---"); + check( + "mappings absent (functions scenario)", + functionsPlugin.schemas.Opportunity.mappings === undefined ); + roundTrip("functions", functionsPlugin, { expectLegacyId: true }); + + console.log("\n--- Scenario 3: no custom fields ---"); + check("customFields absent", noCustomFieldsPlugin.schemas.Opportunity.customFields === undefined); + roundTrip("base", noCustomFieldsPlugin, { expectLegacyId: false }); + + console.log("\n--- Scenario X: mappings XOR functions ---"); + demonstrateXorIsRejected(); + + console.log("\n=== Example complete ==="); } -console.log("\n✓ round-trip verified for fields covered by both mappings"); -console.log("✓ three-state null preserved: source_url null ('doesn't apply') round-tripped"); +main(); diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index da7865124..95fbac3d4 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -32,6 +32,7 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor - [Wiring transforms into a plugin](#wiring-transforms-into-a-plugin) - [Error handling](#error-handling) - [Best practices](#best-practices) + - [When you need `as const`](#when-you-need-as-const) - [Export value schemas alongside your plugin](#export-value-schemas-alongside-your-plugin) - [Use `peerDependencies` for `@common-grants/sdk`](#use-peerdependencies-for-common-grantssdk) - [Keep plugins focused](#keep-plugins-focused) @@ -92,7 +93,7 @@ opportunity.customFields?.category?.value; // string **Key points:** -- Pass `as const` to the specs object so TypeScript can infer literal `fieldType` values and preserve the specific keys. +- `as const` on the specs is only needed when you assign them to a variable before the call; inline specs are inferred correctly without it. Forgetting it on a hoisted variable is a compile error. See [When you need `as const`](#when-you-need-as-const) for details. - If a `value` Zod schema is provided in the spec, the custom field's `value` property is typed according to that schema. Otherwise, a default type is inferred from `fieldType` (e.g. `"string"` -> `string`, `"integer"` -> `number`). - Unregistered custom fields still pass through validation but are typed as the base `CustomField` type (with `value: unknown`). @@ -224,7 +225,7 @@ const myPlugin = definePlugin({ ``` > [!IMPORTANT] -> Always pass `as const` to the options object for `definePlugin()` (and the specs object for `withCustomFields()`). Without it, TypeScript widens literal types like `"string"` to `string`, which prevents the type system from inferring the correct `value` type for each custom field. +> Always pass `as const` to the options object for `definePlugin()` (and the specs object for `withCustomFields()`) as a safe default. Without it, TypeScript can widen literal types like `"string"` to `string`, which prevents the type system from inferring the correct `value` type for each custom field. It is strictly required only when you assign the specs to a variable before the call; see [When you need `as const`](#when-you-need-as-const) for the details. The returned `Plugin` object has two properties: @@ -595,6 +596,38 @@ for (const err of out.errors) { ## Best practices +### When you need `as const` + +**Shortcut:** if you write `fieldType` using the `CustomFieldType` enum (e.g. `CustomFieldType.integer`), you never need `as const`. Enum values are already fixed literals and don't widen. The rest of this section only matters if you type `fieldType` as a raw string literal like `"integer"`. + +For raw string literals, `as const` is only required when you assign the specs (or the whole options object) to a **variable** before passing them to `definePlugin()` or `withCustomFields()`. Written **inline** in the call, you don't need it. + +**Why:** each spec's `fieldType` needs to stay a literal (`"integer"`, `"string"`) for the type system to work. A raw string literal passed inline is kept literal for you by the function. But stored in a variable first, TypeScript infers the variable's type on its own and widens the raw literal `"integer"` to `string`. A widened `fieldType` can no longer determine the field's `value` type, so `value` falls back to `unknown`. (This only affects fields that rely on the default value type. Fields that pass an explicit `value` schema keep their type either way, but it's simplest to apply `as const` whenever you hoist raw-literal specs. Enum values, as noted above, never widen.) + +You typically hoist specs into a variable to reuse them, for example to feed `typeof customFields` into the `ToCommon` / `FromCommon` transform helper types. + +```ts +// OK (inline, no `as const` needed): +definePlugin({ + schemas: { Opportunity: { customFields: { legacyId: { fieldType: "integer" } } } }, +}); + +// Needs `as const` (specs hoisted into a variable): +const customFields = { legacyId: { fieldType: "integer" } } as const; +definePlugin({ schemas: { Opportunity: { customFields } } }); +``` + +If you hoist the specs and forget `as const`, the call still fails to compile. The message is TypeScript's built-in one: the widened `fieldType` (now `string`) is no longer a valid `CustomFieldType`. Adding `as const` resolves it. + +```ts +const customFields = { legacyId: { fieldType: "integer" } }; // no `as const` +definePlugin({ schemas: { Opportunity: { customFields } } }); +// Error: Type '{ fieldType: string; ... }' is not assignable to type 'CustomFieldSpec'. +// Type 'string' is not assignable to type '"string" | "number" | "integer" | ...'. +``` + +(Adding `as const` to an inline call is harmless, just unnecessary.) + ### Export value schemas alongside your plugin When you define Zod schemas for complex `value` fields, export them as named exports from your package. Downstream consumers may need these schemas for use with utilities like `getCustomFieldValue()`: @@ -688,8 +721,9 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, | [`SchemaOnly`](./types.ts) | interface | Compiled output for schema-only entries: `{ commonSchema, sourceSchema? }`. Produced when no transforms are configured. | | | [`SchemaWithTransforms`](./types.ts) | interface | Compiled output for entries with transforms: `{ commonSchema, sourceSchema?, toCommon, fromCommon }`. Produced when `mappings` or explicit callables are provided. | | | [`SchemaMappings`](./types.ts) | interface | Declarative `{ toCommon?, fromCommon? }` mapping dicts. Stored inside `SchemaInput.mappings`. | | -| [`ToCommon`](./types.ts) | type | Helper alias for `(source: TSource) => TransformResult<TCommon>`. | | -| [`FromCommon`](./types.ts) | type | Helper alias for `(common: TCommon) => TransformResult<TSource>`. | | +| [`TransformTypes`](./transform-helpers.ts) | interface | Named argument for `ToCommon` / `FromCommon`: `{ model, sourceSchema, customFields? }`. `model` selects the base schema; the common type is resolved from `customFields`. | | +| [`ToCommon`](./transform-helpers.ts) | type | Helper type for a hand-written `toCommon`. Takes a `TransformTypes` arg; `source` is typed from `sourceSchema`, the return checked against the resolved common **input** type. | | +| [`FromCommon`](./transform-helpers.ts) | type | Helper type for a hand-written `fromCommon`. Takes a `TransformTypes` arg; `common` is the resolved common **output** type, the return the source type. | | ### Shared types diff --git a/lib/ts-sdk/src/extensions/define-plugin.ts b/lib/ts-sdk/src/extensions/define-plugin.ts index 18d10a44e..6fe27a750 100644 --- a/lib/ts-sdk/src/extensions/define-plugin.ts +++ b/lib/ts-sdk/src/extensions/define-plugin.ts @@ -29,14 +29,7 @@ import { buildTransforms } from "./transforms"; * `toCommon` / `fromCommon` callables, an optional `sourceSchema`, and optional * `customFields` specs. Passed as `DefinePluginOptions.schemas`. */ -// Each schema entry can have its own source/common type pair, which only gets -// checked when buildTransforms() is called. `any` lets the dictionary hold -// entries with different type parameters; a stricter type would reject valid -// plugin configs at the point they're stored here. -export type PluginSchemasInput = Partial< - // eslint-disable-next-line @typescript-eslint/no-explicit-any - Record<ExtensibleSchemaName, SchemaInput<any, any>> ->; +export type PluginSchemasInput = Partial<Record<ExtensibleSchemaName, SchemaInput>>; /** * Options for `definePlugin()`. @@ -211,13 +204,13 @@ export function definePlugin<const T extends PluginSchemasInput>( // eslint-disable-next-line @typescript-eslint/no-explicit-any fromCommon = built.fromCommon as any; } else if (hasCallables) { - // Explicit callables path: wrap each callable with schema validation so the - // runtime guarantee matches the mappings path (both directions get validated). + // Explicit callables path: validate each direction's output. toCommon is + // checked against the commonSchema and fromCommon against the sourceSchema if (toCommon !== undefined) { // eslint-disable-next-line @typescript-eslint/no-explicit-any toCommon = wrapWithSchemaValidation(toCommon as any, commonSchema as any) as any; } - if (fromCommon !== undefined && sourceSchema !== undefined) { + if (fromCommon !== undefined) { // eslint-disable-next-line @typescript-eslint/no-explicit-any fromCommon = wrapWithSchemaValidation(fromCommon as any, sourceSchema as any) as any; } diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index 2820f29d7..2fce0424c 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -26,13 +26,13 @@ export type { SchemaInput, SchemaOnly, SchemaWithTransforms, - ToCommon, - FromCommon, PluginCapability, PluginMeta, TransformResult, } from "./types"; export { TransformError } from "./types"; +// Helper types for annotating hand-written toCommon / fromCommon functions. +export type { ToCommon, FromCommon, TransformTypes } from "./transform-helpers"; export type { BuiltTransforms } from "./transforms"; export { buildTransforms } from "./transforms"; export type { TransformFromMappingOptions } from "../utils/transformation"; diff --git a/lib/ts-sdk/src/extensions/transform-helpers.ts b/lib/ts-sdk/src/extensions/transform-helpers.ts new file mode 100644 index 000000000..dd98e22dc --- /dev/null +++ b/lib/ts-sdk/src/extensions/transform-helpers.ts @@ -0,0 +1,98 @@ +/** + * Helper types for annotating hand-written `toCommon` / `fromCommon` functions. + * + * A flat, multi-key `definePlugin({ schemas })` can't infer a per-entry common + * type to check an inline transform function, so authors annotate their + * functions with these helpers to recover full typing: `source` typed from + * `sourceSchema`, and the common side resolved from `model` + `customFields` — + * exactly the inputs `definePlugin()` itself uses. Authors pass the specs they + * already have, not a prebuilt common schema. + * + * @module @common-grants/sdk/extensions + */ + +import { z } from "zod"; +import { EXTENSIBLE_SCHEMA_MAP } from "./types"; +import type { CustomFieldSpec, ExtensibleSchemaName, TransformResult } from "./types"; +import type { WithCustomFieldsResult } from "./with-custom-fields"; + +/** + * The single named argument for {@link ToCommon} / {@link FromCommon}. + * + * `model` names the extensible schema being transformed; it selects the base + * schema from `EXTENSIBLE_SCHEMA_MAP`, so the common type is resolved per model + * rather than assuming one hardcoded base. `customFields` is the field specs + * (not a prebuilt common schema); omit it for the bare base schema. + * + * @example + * ```ts + * type OppTransform = { + * model: "Opportunity"; + * sourceSchema: typeof GrantsGovOpportunity; + * customFields: typeof customFields; + * }; + * const toCommon: ToCommon<OppTransform> = source => ({ result: { ... }, errors: [] }); + * ``` + */ +export interface TransformTypes { + /** The extensible model this transform targets (selects the base schema). */ + model: ExtensibleSchemaName; + /** The source-system Zod schema; `source` is typed as its inferred type. */ + sourceSchema: z.ZodTypeAny; + /** The custom field specs declared on the entry (NOT a prebuilt common schema). */ + customFields?: Record<string, CustomFieldSpec>; +} + +/** The base Zod schema for the model named by `T["model"]`. */ +type BaseSchemaOf<T extends TransformTypes> = (typeof EXTENSIBLE_SCHEMA_MAP)[T["model"]]; + +/** + * The common Zod schema for `T`, resolved exactly as `definePlugin()` builds it: + * the model's base schema, extended via `withCustomFields()` when `customFields` + * is present. + */ +type CommonSchemaOf<T extends TransformTypes> = + T["customFields"] extends Record<string, CustomFieldSpec> + ? WithCustomFieldsResult<BaseSchemaOf<T>, T["customFields"]> + : BaseSchemaOf<T>; + +/** + * Type for a hand-written `toCommon`. + * + * `source` is typed from `sourceSchema`; the return is checked against the + * resolved common type. Both directions use the same common type because the + * common date schemas accept either a string or a `Date` as input (they + * normalize internally), so the author can build and return real `Date` values. + * + * @example + * ```ts + * const toCommon: ToCommon<{ + * model: "Opportunity"; + * sourceSchema: typeof GrantsGovOpportunity; + * customFields: typeof customFields; + * }> = source => ({ result: { ... }, errors: [] }); + * ``` + */ +export type ToCommon<T extends TransformTypes> = ( + source: z.infer<T["sourceSchema"]> +) => TransformResult<z.infer<CommonSchemaOf<T>>>; + +/** + * Type for a hand-written `fromCommon`. + * + * `common` is the resolved common type (what a consumer holds after `toCommon` + * ran — e.g. date fields as `Date`). The return is the source type inferred + * from `sourceSchema`. + * + * @example + * ```ts + * const fromCommon: FromCommon<{ + * model: "Opportunity"; + * sourceSchema: typeof GrantsGovOpportunity; + * customFields: typeof customFields; + * }> = common => ({ result: { ... }, errors: [] }); + * ``` + */ +export type FromCommon<T extends TransformTypes> = ( + common: z.infer<CommonSchemaOf<T>> +) => TransformResult<z.infer<T["sourceSchema"]>>; diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index df94c48f8..265e2397e 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -231,42 +231,74 @@ export class TransformError extends Error { // Public types - SchemaInput (author-provided), SchemaOnly, SchemaWithTransforms // ############################################################################ +/** + * Mappings authoring path: declarative `mappings` compiled by `buildTransforms()` + * inside `definePlugin()`. Requires a `sourceSchema`; forbids hand-written + * `toCommon` / `fromCommon`. + */ +export interface MappingsSchemaInput { + /** Custom fields to attach via `withCustomFields()`. */ + customFields?: Record<string, CustomFieldSpec>; + /** Source-system Zod schema (the shape a source system returns). */ + sourceSchema: z.ZodTypeAny; + /** Declarative mappings compiled into transforms by `definePlugin()`. */ + mappings: SchemaMappings; + toCommon?: never; + fromCommon?: never; +} + +/** + * Functions authoring path: hand-written `toCommon` / `fromCommon`. Requires a + * `sourceSchema` and both directions; forbids declarative `mappings`. + * + * The function slots are loose on the input side (`source: any`): a flat, + * multi-key `definePlugin()` cannot infer a per-entry common type to check an + * inline function, so the parameter falls to `any`. The slot still pins the + * `TransformResult` envelope (a function returning a non-`TransformResult` is + * rejected). Authors recover full typing with the `ToCommon` / `FromCommon` + * helper types, and the resolved consumer-facing types are always correct. + */ +export interface FunctionsSchemaInput { + /** Custom fields to attach via `withCustomFields()`. */ + customFields?: Record<string, CustomFieldSpec>; + /** Source-system Zod schema (the shape a source system returns). */ + sourceSchema: z.ZodTypeAny; + mappings?: never; + /** Map a source record to the common-schema shape. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + toCommon: (source: any) => TransformResult<unknown>; + /** Map a common-schema record back to the source shape. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fromCommon: (common: any) => TransformResult<unknown>; +} + +/** Schema-only path: custom fields, no transforms. Forbids the other two paths. */ +export interface SchemaOnlyInput { + /** Custom fields to attach via `withCustomFields()`. */ + customFields?: Record<string, CustomFieldSpec>; + sourceSchema?: never; + mappings?: never; + toCommon?: never; + fromCommon?: never; +} + /** * Author-provided input for a single extensible object, passed inside * `DefinePluginOptions.schemas`. * - * Exactly one of `mappings` or explicit callables (`toCommon` / `fromCommon`) - * may be present — providing both is a compile-time and runtime error. - * - * `sourceSchema` is the optional Zod schema for the source system format. - * `customFields` declares any extra fields this object exposes beyond the base - * CommonGrants schema; `definePlugin()` applies them via `withCustomFields()`. - * - * `commonSchema` is intentionally absent here. It is injected by `definePlugin()` - * during compilation, resolved from the generated model classes produced by the - * code generator. Plugin config files cannot import from `generated/` (it is the - * input to generation). - * - * When `mappings` is provided, `definePlugin()` auto-invokes `buildTransforms()` - * at call time and wraps the result with schema validation against the compiled - * common schema. When explicit callables are provided, `definePlugin()` wraps them - * with the same schema validation so both paths give the same runtime guarantee. + * An exclusive choice between three paths: + * - {@link MappingsSchemaInput}: `sourceSchema` + declarative `mappings`. + * - {@link FunctionsSchemaInput}: `sourceSchema` + hand-written `toCommon` and + * `fromCommon` (both required). + * - {@link SchemaOnlyInput}: `customFields` only, no transforms. + * + * Both transform paths require a `sourceSchema`, so a transform can always be + * validated against the source shape. Supplying both `mappings` and functions, + * a single transform direction, or a transform without a `sourceSchema`, is a + * compile error (the `?: never` slots). `commonSchema` is intentionally absent; + * `definePlugin()` derives it from `customFields` during compilation. */ -export type SchemaInput<TSource = unknown, TCommon = unknown> = - | { - sourceSchema?: z.ZodType<TSource>; - customFields?: Record<string, CustomFieldSpec>; - mappings: SchemaMappings; - toCommon?: never; - fromCommon?: never; - } - | { - sourceSchema?: z.ZodType<TSource>; - customFields?: Record<string, CustomFieldSpec>; - mappings?: never; - toCommon?: (source: TSource) => TransformResult<TCommon>; - fromCommon?: (common: TCommon) => TransformResult<TSource>; - }; +export type SchemaInput = MappingsSchemaInput | FunctionsSchemaInput | SchemaOnlyInput; /** * Compiled output for a schema-only entry — no transforms configured. @@ -305,42 +337,6 @@ export interface SchemaWithTransforms<TSource, TCommon> { fromCommon: (common: TCommon) => TransformResult<TSource>; } -/** - * Convenience type alias for a `toCommon` transform function. - * - * Annotate hand-written transform functions with this type to get full - * compile-time checking on both the source input and the common output. - * `TSource` comes from `z.infer<typeof sourceSchema>` and `TCommon` comes - * from `z.infer<typeof extendedCommonSchema>` (the schema produced by - * `withCustomFields()` when custom fields are declared). - * - * @example - * ```ts - * // Without custom fields — TCommon is z.infer<typeof OpportunityBaseSchema> - * const toCommon: ToCommon<z.infer<typeof SourceSchema>, z.infer<typeof OpportunityBaseSchema>> - * = source => ({ result: { ... }, errors: [] }); - * - * // With custom fields — TCommon is the extended schema type - * const toCommon: ToCommon<z.infer<typeof SourceSchema>, z.infer<typeof ExtendedSchema>> - * = source => ({ result: { ... }, errors: [] }); - * ``` - */ -export type ToCommon<TSource, TCommon> = (source: TSource) => TransformResult<TCommon>; - -/** - * Convenience type alias for a `fromCommon` transform function. - * - * Annotate hand-written transform functions with this type to get full - * compile-time checking on both the common input and the source output. - * - * @example - * ```ts - * const fromCommon: FromCommon<z.infer<typeof SourceSchema>, z.infer<typeof OpportunityBaseSchema>> - * = common => ({ result: { ... }, errors: [] }); - * ``` - */ -export type FromCommon<TSource, TCommon> = (common: TCommon) => TransformResult<TSource>; - /** * Plugin identity and capability declaration. * diff --git a/lib/ts-sdk/src/schemas/zod/types.ts b/lib/ts-sdk/src/schemas/zod/types.ts index dad5cb35c..17689fc14 100644 --- a/lib/ts-sdk/src/schemas/zod/types.ts +++ b/lib/ts-sdk/src/schemas/zod/types.ts @@ -42,14 +42,32 @@ const ensureUTC = (date: string) => { ); }; -/** Schema for UTC datetime fields */ -export const UTCDateTimeSchema = z.string().datetime().transform(ensureUTC); +/** + * Accept a `Date` as input by normalizing it to a string before the string + * validators run, so a caller can pass either an ISO string or a `Date`. A + * string goes straight to the inner schema's strict validation; a `Date` is + * rendered by `toStr` into the string format the inner schema expects. Output + * is a `Date`. + */ +const acceptDate = + (toStr: (d: Date) => string) => + (val: unknown): unknown => + val instanceof Date ? toStr(val) : val; -/** Schema for ISO date format: YYYY-MM-DD (parsed into a Date object at midnight UTC) */ -export const ISODateSchema = z - .string() - .date() - .transform(str => new Date(str)); +/** Schema for UTC datetime fields (accepts an ISO string or a `Date`; outputs a `Date`) */ +export const UTCDateTimeSchema = z.preprocess( + acceptDate(d => d.toISOString()), + z.string().datetime().transform(ensureUTC) +); + +/** Schema for ISO date format: YYYY-MM-DD (accepts a YYYY-MM-DD string or a `Date`; outputs a `Date`) */ +export const ISODateSchema = z.preprocess( + acceptDate(d => d.toISOString().slice(0, 10)), + z + .string() + .date() + .transform(str => new Date(str)) +); /** Schema for ISO time format: HH:MM:SS with optional fractional seconds and timezone (RFC 3339 partial-time) */ export const ISOTimeSchema = z.preprocess(val => { @@ -61,8 +79,11 @@ export const ISOTimeSchema = z.preprocess(val => { return val; }, z.string().time()); -/** Schema for offset datetime fields (ISO 8601 with timezone offset, parsed into a Date object) */ -export const OffsetDateTimeSchema = z - .string() - .datetime({ offset: true }) - .transform(str => new Date(str)); +/** Schema for offset datetime fields (accepts an ISO 8601 offset string or a `Date`; outputs a `Date`) */ +export const OffsetDateTimeSchema = z.preprocess( + acceptDate(d => d.toISOString()), + z + .string() + .datetime({ offset: true }) + .transform(str => new Date(str)) +); From 8aed1465a44737e7d332814f255752bba1f33467 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Tue, 16 Jun 2026 08:31:20 -0700 Subject: [PATCH 19/51] Custom filters Python PoC (#869) (#885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(01-01): add failing tests for BooleanComparisonFilter - Import BooleanComparisonFilter from filters.boolean (RED — module does not exist yet) - Tests: constructs with eq/neq + bool, coerces string operators, rejects invalid operators, rejects non-bool values * feat(01-01): implement BooleanComparisonFilter Pydantic model - BooleanComparisonFilter(operator, value: bool) with EquivalenceOperator (eq|neq) - @field_validator coerces string operators to EquivalenceOperator enum - Rejects operators outside eq|neq (ValidationError); rejects non-bool values - SDK-level only — no lib/core TypeSpec change (DP-14, CFP-02) * feat(01-01): export BooleanComparisonFilter from filters barrel - Add '# Boolean Filters' section to __all__ (between Numeric and Opportunity) - Add 'from .boolean import BooleanComparisonFilter' to import block - BooleanComparisonFilter now importable from common_grants_sdk.schemas.pydantic.filters * feat(01-02): add CustomFilterType enum and CustomFilterSpec dataclass - 11-value StrEnum (stringComparison / stringArray / *Range / *Array) per DP-14 - CustomFilterSpec(filter_type, description=None) mirrors CustomFieldSpec but intentionally omits `value` — operators derived at classify time (DP-07) * feat(01-02): add PluginRoutes / RouteDeclarations / RouteMethodFilters aliases Route-keyed type aliases for custom-filter declaration (DP-06). CustomFilterSpec referenced under TYPE_CHECKING to prevent circular imports at runtime; existing runtime import from .specs is unchanged. * feat(01-02): add routes slot to PluginConfig and define_plugin - Import PluginRoutes from .types - Add routes: PluginRoutes | None = None as last field on frozen PluginConfig - Add routes kwarg to both @overload signatures (2-overload approach; A1 verified green — mypy accepts routes independently of schemas narrowing) - Add routes to implementation signature and pass routes=routes to PluginConfig() - inject_transforms is byte-for-byte unchanged (iterates config.schemas only) - Positional-construction safety confirmed: only one PluginConfig( call site, using keyword args * test(01-03): add failing pytest suite for filter helpers, classification, and validation - Covers f.* helpers (eq, neq, gt, like, in_, not_in, between, outside) - Covers three-bucket classify_filters (default snake/camel, registered custom, ad-hoc) - Includes camelCase-alias landmine test (closeDateRange -> named field, not customFilters) - Includes test_oppfilters_mixed_case_roundtrip (Strategy A proof, RESEARCH A4) - Covers validate_routes raising on unknown filter_type and DEFAULT_FILTER_NAMES collision - Covers validate_filter_call raising on registered mismatch and ad-hoc invalid shape - No custom-vs-custom duplicate-name test (vacuous with dict semantics, DP-15 note) * feat(01-03): implement extensions/filters.py — f helpers, classifier, validators - _FHelpers class with all 12 methods (eq/neq/gt/gte/lt/lte/in_/not_in/like/not_like/between/outside) - FILTER_TYPE_SCHEMAS maps all 11 CustomFilterType members (integerComparison reuses NumberComparisonFilter) - DEFAULT_FILTER_NAMES frozenset includes both snake_case and camelCase aliases (landmine fix) - Strategy A (RESEARCH A4): normalize snake_case keys to alias form for OppFilters construction (OppFilters requires alias keys without populate_by_name; passing snake_case silently yields None) - validate_routes raises PluginError on unknown filter_type and DEFAULT_FILTER_NAMES collision - validate_filter_call raises PluginError on registered mismatch and ad-hoc invalid shape - classify_filters routes default keys (snake+camel) to named fields, custom/ad-hoc to customFilters - All 23 pytest tests pass including test_oppfilters_mixed_case_roundtrip (Strategy A proof) * refactor(01-03): wire filters public API + fix enum types for mypy - extensions/__init__.py: export classify_filters, f, validate_routes, CustomFilterSpec, CustomFilterType, PluginRoutes (six new public symbols) - filters.py: use typed enum values in _FHelpers (EquivalenceOperator, ComparisonOperator, ArrayOperator, StringOperator, RangeOperator) instead of string literals to satisfy mypy arg-type checks - black-format both modified files; make checks + make test pass (pre-existing test_generate_models_typecheck_with_pyright_strict failure excluded per success criteria) * feat(01-04): add codegen-free custom_filters.py example - Demonstrates route-keyed plugin declaration with STRING_ARRAY and STRING_COMPARISON custom filter specs - Classifies five consumer filters (2 default, 2 registered, 1 ad-hoc) into the ADR-0012 three-bucket wire body - Includes camelCase alias case (closeDateRange) to exercise the DEFAULT_FILTER_NAMES landmine path - Demonstrates PluginError raised and specifically caught on bad operator - Uses mode='json' on model_dump to produce JSON-serializable output - SDK-direct imports only, no codegen dependency * fix(01): unify PluginError contract across all three filter buckets Wrong-shaped default-bucket filters (e.g. an equivalence filter for the array-typed "status" field) previously leaked a raw pydantic ValidationError from the terminal OppFilters construction, while the registered and ad-hoc buckets wrapped failures in PluginError — so a consumer following the documented `except PluginError` pattern missed default-bucket errors. - Wrap the OppFilters construction in the same PluginError translation, making it the single validation point for default values (validated against each named field's real type, stricter than the permissive DefaultFilter check the dropped per-key call performed). - Narrow the ad-hoc branch's `except (ValidationError, Exception)` to `except ValidationError` so unexpected non-validation bugs propagate instead of being mislabeled as invalid filter shapes. - Document the uniform contract in the classify_filters docstring and cover both default-key forms (snake_case and camelCase alias) with regression tests. * docs(01): scrub internal planning shorthand from committed source Replace planning-doc IDs and working labels in comments/docstrings with the technical reasoning they pointed at, matching the TS custom-filters code style. Real references (ADR-0012, issues #646/#869, TS SDK parallels) are kept. No code changes. * feat: enforce int values for integerComparison via dedicated filter model Add IntegerComparisonFilter (StrictInt value) and point the integerComparison filter type at it instead of reusing the looser NumberComparisonFilter, mirroring the TS SDK's IntegerComparisonFilterSchema (z.number().int()). Fractional values and numeric strings now fail call-time validation; the strict annotation disables pydantic's lax str -> int coercion to keep cross-SDK behavior aligned. * refactor: drop unreachable duplicate-name guard from validate_routes The seen-set check could never fire: routes is dict-keyed, so duplicate filter names cannot exist by construction. Removing it also drops the noqa: C901 it forced. Parallel to the TS SDK dropping the same unreachable check from validateRoutes. * docs: use request-body framing for the classifier output Rename wire-body wording to request body in docstrings, comments, test names, and the example output label — the classifier result is the JSON body of the ADR-0012 search request sent to the receiver. Operator serialization notes (wire operator values for in/notIn) keep their wording. Matches the TS SDK's request-body wording. * test: cover money filter types in validate_filter_call moneyComparison and moneyRange had no extensions-level coverage — the money types are where the TS compile-time filter map drifted (range operators and a numeric amount against a schema taking gt/gte/lt/lte and a decimal-string Money). Lock the runtime layer here: valid Money shapes pass, array/comparison operator mismatches and numeric amounts raise PluginError. * style: apply black to integer filter and money tests * fix: use StrictBool in BooleanComparisonFilter for TS parity Lax pydantic bool coerces 1/0 and "true"/"false", which the TS SDK's z.boolean() rejects. Matches the StrictInt treatment already applied to IntegerComparisonFilter. * docs: correct routes validation claim in define_plugin docstring validate_routes() is an explicit registration-time check, not invoked by classify_filters() — matching the TS SDK where validateRoutes is a standalone function. * refactor: align PluginError paths with the TS filters.<name> convention Buckets 2/3 now raise path="filters.<name>" (TS uses filters.${filterName}); the bucket-1 OppFilters-construction wrap names the failing field(s) instead of the misleading routes.<resource>.<method> path. Also document mode="json" in the classify_filters return docstring — the python-mode dump is not JSON-serializable (dates, operator enums). * test: construct bogus filter spec directly CustomFilterSpec is not a frozen dataclass, so the __new__ + object.__setattr__ dance is unnecessary — dataclasses don't validate field types at runtime. * fix: conform DefaultFilter.value to the core spec's `unknown` The narrowed union (str | int | float | list | dict) diverged from filters/base.tsp (`value: unknown`) and the TS SDK's z.unknown(), and lax union coercion mutated inputs: True -> 1, so boolean custom filters serialized as integers on the wire. `Any` preserves inputs as-is; strict per-type checking stays with the typed filter models. * fix: accept eq/neq on numeric comparison filters per core spec NumberComparisonFilter.operator in filters/numeric.tsp widened to ComparisonOperators | EquivalenceOperators in protocol v0.3; the Pydantic model never tracked the change (the TS SDK did). IntegerComparisonFilter inherits the same surface so the numeric family stays consistent across SDKs. * fix: ship validated filter values and unify the PluginError contract validate_filter_call now returns the validated filter as a DefaultFilter and classify_filters places that in the wire body, so the payload is exactly what passed validation (lax coercion could previously diverge: a value validated as 42.0 shipped as "42"). - ad-hoc DefaultFilter instances are re-validated via model_dump instead of trusted; mutated instances now raise instead of serializing invalid operators with only a stderr warning - unknown spec.filter_type raises PluginError (uniform contract) instead of a bare KeyError when validate_routes was skipped - OppFilters construction errors name the failing filter from nested customFilters locs instead of misattributing them as default-filter failures * docs: correct filter-engine docstrings and demonstrate alias normalization - classify_filters bucket-1 description said keys normalize to snake_case; construction requires the camelCase alias form (the code and the docstring's own construction paragraph already said so) - mode="json" rationale named operator enums as non-serializable; StrEnum members serialize fine — only coerced date objects need it - the alias-detection map's comment described a lookup that never happens (only key membership is used) - DEFAULT_FILTER_NAMES comment now names both consumers of the dual-form set (validate_routes collision check AND bucket-1 routing) - the example passed closeDateRange (the form that needs no normalization) while calling it the landmine case; it now passes close_date_range and shows the snake->alias rewrite in the wire body - drop the hardcoded enum count, diff-relative "new" framing, and the codegen-only framing on define_plugin's docstring * test: lock wire-body integrity, normalization, and error-contract behavior - boolean true survives to the wire as JSON true (not 1) - registered filters ship the validated/coerced value, not raw input - mutated ad-hoc instances are re-validated and raise - unknown filter_type raises PluginError, not KeyError - snake_case form of an aliased default key normalizes to the alias (the _SNAKE_TO_ALIAS hit branch had no coverage) - model_dump(mode="json") round-trips through json.dumps with ISO dates - FILTER_TYPE_SCHEMAS covers every CustomFilterType member (catalog drift now fails CI instead of raising at runtime) - PluginError path is filters.<name> across all three buckets, and the multi-failure collective path is "filters" - eq/neq accepted on Number/Integer comparison per filters/numeric.tsp; integer filters reject True (bool subclasses int) and 100.0 (recorded divergence: JS cannot represent the distinction, StrictInt rejects) - all 12 f helpers parametrized over their wire operator/value shapes - the camelCase-alias landmine test's conditional assert could never fail on the path it guarded; assert the empty bucket outright * fix(py-sdk): reject bool values in numeric filter models bool subclasses int, so the int|float unions in NumberComparisonFilter, NumberArrayFilter, and NumberRange lax-coerced True -> 1 and shipped a number for a boolean — the same wire corruption the DefaultFilter.value widening fixed, surviving one layer down in the typed models. Also an internal inconsistency (IntegerComparisonFilter's StrictInt already rejects bool) and a divergence from z.number(), which rejects booleans. Pinned at both layers: schema-level rejection tests and a classify-time wire-integrity test (f.eq(True) on a numberComparison filter raises instead of shipping value 1). * fix(py-sdk): raise on duplicate snake/camel forms of a default filter Both forms normalize to the same OppFilters key, so plain dict assignment silently dropped whichever value the consumer's dict ordered first — a quiet wrong-results bug when merging filter dicts that use different naming conventions. Now raises PluginError, keeping the uniform error contract. Also collapses the bucket-1 normalization to a single _SNAKE_TO_ALIAS lookup (alias-form and unaliased keys fall through .get(key, key) unchanged, so the _ALIAS_TO_SNAKE membership branch was redundant) and documents that registered-spec lookup is exact on (resource, method): a non-matching pair downgrades every non-default filter to permissive ad-hoc validation with no signal. Route-scoping now pinned by tests at both lookup levels. * feat(py-sdk): surface first validation failure in PluginError messages str(PluginError) carried an error count but no error content, so the logged message a consumer sees ('2 error(s)') said nothing actionable; the detail lived only on .cause. Each wrapping site now appends a one-line summary of the first pydantic error (message + loc), and the example demonstrates the structured fields (.path, .cause) rather than teaching the lossy print(exc)-only pattern. Also collapses the customFilters arm of the bucket-1 field extractor: custom-bucket values are validated before OppFilters construction, so only default fields can fail there — the nested-loc branch was unreachable defensive code. * refactor(py-sdk): enforce filter-engine invariants at the type level Convert documented invariants into enforced ones: - CustomFilterSpec is frozen: a spec is an immutable declaration, and mutating one after registration would bypass validate_routes. - FILTER_TYPE_SCHEMAS is a MappingProxyType: the catalog is closed, so runtime mutation is not an extension point. - classify_filters takes Mapping[str, DefaultFilter | dict[str, Any]] instead of dict[str, Any], matching its documented contract so type checkers can catch e.g. a bare string value at the call site (Mapping keeps narrower caller dicts assignable where dict's invariance would reject them). - CustomFilterSpec joins the existing module-level specs import in types.py; the TYPE_CHECKING guard and forward-string reference were workarounds for a cycle that doesn't exist (specs.py imports .types only lazily). * docs(py-sdk): fix three inaccurate filter comments; ground TS parity claims Accuracy fixes, each verified by execution: - DEFAULT_FILTER_NAMES comment had the failure mode inverted: the aliases ARE the camelCase forms, so an alias-only set would drop snake_case keys into customFilters, not camelCase ones. - The example's alias-landmine note claimed an unnormalized snake key would land in customFilters; pydantic actually ignores the kwarg (extra='ignore', no populate_by_name) so the filter is silently dropped from the wire body entirely — now consistent with filters.py. - validate_filter_call's coercion example claimed "42" -> 42.0; the int|float smart-union resolves int first, so it coerces to 42. Grounding fixes: - TS parity notes cited TS SDK symbols (F object, validateRoutes, *FilterSchema names) that exist only on the unmerged TS PoC branch; rephrased as the planned/PoC surface so the claims can't silently rot if the TS shape changes before merge. - validate_routes' gov.<system>@<filterName> wording read as a settled protocol mandate; no ADR defines it yet, so it now describes the escape-hatch mechanism instead. - 'validate_routes catches this at registration time' overstated an opt-in check; reworded as guidance to call it. - 'three-bucket ADR-0012 classifier' implied bucket language in ADR-0012; the buckets are this implementation's framing, the ADR defines the request-body shape. - CustomFilterSpec docstring said operators are 'derived' from filter_type; nothing derives — the filter_type's validation model enforces the allowed set at call time. * test(py-sdk): dedupe filter tests, close coverage gaps Deletions (tests must discriminate — no test that a parametrized table already covers): - test_f_eq..test_f_outside were an exact subset of the test_f_helper_wire_values table; the table moves up into the f.* section and keeps the reserved-word cases. - test_boolean_comparison_filter_coerces_string_operator was a strict subset of test_boolean_comparison_filter_constructs. Additions: - Invalid-operator rejection parametrized across NumberComparisonFilter AND IntegerComparisonFilter: both share the rewritten two-enum validate_operator dispatch whose fallthrough relies on the enum-union field to reject unknown strings, but only the integer model was pinned. - define_plugin(routes=...) -> PluginConfig.routes passthrough. Tightening: - test_oppfilters_mixed_case_roundtrip's disjunctive customFilters assertion could not fail on the path it guards; now asserts the bucket is absent outright, matching the sibling test's stated standard. - test_registered_filter_ships_validated_value_not_raw_input now asserts the coerced type (int), pinning the documented "42" -> 42 behavior. * fix(py-sdk): revert integerComparison to NumberComparisonFilter The spec defines no integer filter model (lib/core numeric filters are NumberComparisonFilter / NumberRangeFilter / NumberArrayFilter only), so the SDK-side IntegerComparisonFilter added in 70b1f97 had no spec counterpart. Mirrors the TS-side revert on #884: the spec is the source of truth and changes flow from the spec to the SDK; an int-enforcing model can follow a spec change that introduces one. integerComparison validates as a plain number again. * refactor(py-sdk): align PluginRoutes shape with the TypeScript SDK The TS SDK nests per-method filter declarations under a RouteDeclarations wrapper ({resource: {method: {filters: {name: spec}}}}); the Python SDK mapped filter names directly at the method level. Cross-SDK parity testing showed the asymmetry is hazardous for consumers porting between SDKs: a Python-shaped routes object passed to the TS classifier silently downgrades every registered filter to ad-hoc shape-only validation. PluginRoutes is now {resource: {method: RouteDeclarations}} where RouteDeclarations is a TypedDict with an optional filters key, mirroring the TS interface (optional so a method can be declared without filters and so future declaration kinds can ride alongside). validate_routes skips methods with no filters key and its error path gains the filters segment to match the TS path string; classify_filters resolves registered specs through the filters key. Tests and the runnable example are updated to the new shape. * Address review feedback on custom-filters PoC - Export validate_filter_call from extensions __init__ (parity with TS SDK) - Note classify_filters' opportunity-bound shape; link resource-agnostic follow-up (#896) - Rename opp_key to alias_key in classify_filters - Drop the future-declaration-kinds clause from RouteDeclarations docstring * docs(py-sdk): keep user-facing docstrings free of ADR refs and cross-SDK notes Docstrings rendered to SDK users should describe behavior in plain language; ADR numbers and TS-SDK implementation details belong in contributor-facing comments. The tracking link for the classifier's Opportunity binding moves from the docstring to a code comment above classify_filters. --- .../common_grants_sdk/extensions/__init__.py | 19 +- .../common_grants_sdk/extensions/filters.py | 462 +++++++++++++ .../common_grants_sdk/extensions/plugin.py | 18 +- .../common_grants_sdk/extensions/specs.py | 30 + .../common_grants_sdk/extensions/types.py | 23 +- .../schemas/pydantic/filters/__init__.py | 3 + .../schemas/pydantic/filters/base.py | 8 +- .../schemas/pydantic/filters/boolean.py | 31 + .../schemas/pydantic/filters/numeric.py | 48 +- lib/python-sdk/examples/custom_filters.py | 132 ++++ .../tests/extensions/test_filters.py | 636 ++++++++++++++++++ .../tests/extensions/test_plugin.py | 18 + lib/python-sdk/tests/schemas/test_filters.py | 111 +++ 13 files changed, 1530 insertions(+), 9 deletions(-) create mode 100644 lib/python-sdk/common_grants_sdk/extensions/filters.py create mode 100644 lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/boolean.py create mode 100644 lib/python-sdk/examples/custom_filters.py create mode 100644 lib/python-sdk/tests/extensions/test_filters.py diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 8589e1fa8..4d91ef1f7 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,7 +1,15 @@ """Public extension APIs for the CommonGrants Python SDK.""" +from .filters import classify_filters, f, validate_filter_call, validate_routes from .plugin import Plugin, PluginConfig, define_plugin, inject_transforms -from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions, merge_extensions +from .specs import ( + ConflictStrategy, + CustomFieldSpec, + CustomFilterSpec, + CustomFilterType, + SchemaExtensions, + merge_extensions, +) from .transforms import build_transforms from .types import ( Handler, @@ -13,6 +21,7 @@ PluginExtensions, PluginExtensionsMeta, PluginExtensionsSchema, + PluginRoutes, TransformResult, ) @@ -39,4 +48,12 @@ "PluginExtensionsMeta", "PluginExtensionsSchema", "TransformResult", + # Custom filters + "classify_filters", + "CustomFilterSpec", + "CustomFilterType", + "f", + "PluginRoutes", + "validate_filter_call", + "validate_routes", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py new file mode 100644 index 000000000..6abdea026 --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -0,0 +1,462 @@ +"""Pure-runtime filter engine for the CommonGrants Python SDK. + +Provides: +- ``f`` helper singleton for building DefaultFilter instances. +- ``FILTER_TYPE_SCHEMAS`` — map from CustomFilterType to the Pydantic validation model. +- ``DEFAULT_FILTER_NAMES`` — frozenset of all core default-filter field names (snake + alias). +- ``validate_routes(routes)`` — registration-time validator; raises PluginError. +- ``validate_filter_call(spec, filter_name, value)`` — call-time validator; returns the + wire-ready DefaultFilter; raises PluginError. +- ``classify_filters(routes, resource, method, consumer_filters)`` — classifier producing + the ``OppFilters`` search request body (default named fields + ``customFilters``). + +No generate.py / codegen dependency. Correctness is enforced at runtime by Pydantic v2. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Any, Optional + +from pydantic import BaseModel, ValidationError + +from common_grants_sdk.schemas.pydantic.filters.base import ( + ArrayOperator, + ComparisonOperator, + DefaultFilter, + EquivalenceOperator, + RangeOperator, + StringOperator, +) +from common_grants_sdk.schemas.pydantic.filters.boolean import BooleanComparisonFilter +from common_grants_sdk.schemas.pydantic.filters.date import ( + DateComparisonFilter, + DateRangeFilter, +) +from common_grants_sdk.schemas.pydantic.filters.money import ( + MoneyComparisonFilter, + MoneyRangeFilter, +) +from common_grants_sdk.schemas.pydantic.filters.numeric import ( + NumberArrayFilter, + NumberComparisonFilter, + NumberRangeFilter, +) +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OppDefaultFilters, + OppFilters, +) +from common_grants_sdk.schemas.pydantic.filters.string import ( + StringArrayFilter, + StringComparisonFilter, +) + +from .specs import CustomFilterSpec, CustomFilterType +from .types import PluginError, PluginRoutes + +# --------------------------------------------------------------------------- +# f.* helpers +# --------------------------------------------------------------------------- + + +class _FHelpers: + """Helper namespace for building ``DefaultFilter`` instances. + + Import as ``from common_grants_sdk.extensions import f`` and use as + ``f.eq("open")``, ``f.in_([...])``, etc. + + ``in_`` and ``not_in`` avoid Python reserved words; the wire operator + values they emit are still ``"in"`` / ``"notIn"``. + """ + + def eq(self, value: Any) -> DefaultFilter: + """Return DefaultFilter(operator="eq", value=value).""" + return DefaultFilter(operator=EquivalenceOperator.EQUAL, value=value) + + def neq(self, value: Any) -> DefaultFilter: + """Return DefaultFilter(operator="neq", value=value).""" + return DefaultFilter(operator=EquivalenceOperator.NOT_EQUAL, value=value) + + def gt(self, value: Any) -> DefaultFilter: + """Return DefaultFilter(operator="gt", value=value).""" + return DefaultFilter(operator=ComparisonOperator.GREATER_THAN, value=value) + + def gte(self, value: Any) -> DefaultFilter: + """Return DefaultFilter(operator="gte", value=value).""" + return DefaultFilter( + operator=ComparisonOperator.GREATER_THAN_OR_EQUAL, value=value + ) + + def lt(self, value: Any) -> DefaultFilter: + """Return DefaultFilter(operator="lt", value=value).""" + return DefaultFilter(operator=ComparisonOperator.LESS_THAN, value=value) + + def lte(self, value: Any) -> DefaultFilter: + """Return DefaultFilter(operator="lte", value=value).""" + return DefaultFilter( + operator=ComparisonOperator.LESS_THAN_OR_EQUAL, value=value + ) + + def in_(self, value: list[Any]) -> DefaultFilter: + """Return DefaultFilter with wire operator "in" (Python keyword workaround: f.in_).""" + return DefaultFilter(operator=ArrayOperator.IN, value=value) + + def not_in(self, value: list[Any]) -> DefaultFilter: + """Return DefaultFilter with wire operator "notIn".""" + return DefaultFilter(operator=ArrayOperator.NOT_IN, value=value) + + def like(self, value: str) -> DefaultFilter: + """Return DefaultFilter(operator="like", value=value).""" + return DefaultFilter(operator=StringOperator.LIKE, value=value) + + def not_like(self, value: str) -> DefaultFilter: + """Return DefaultFilter(operator="notLike", value=value).""" + return DefaultFilter(operator=StringOperator.NOT_LIKE, value=value) + + def between(self, min: Any, max: Any) -> DefaultFilter: + """Return DefaultFilter(operator="between", value={"min": min, "max": max}).""" + return DefaultFilter( + operator=RangeOperator.BETWEEN, value={"min": min, "max": max} + ) + + def outside(self, min: Any, max: Any) -> DefaultFilter: + """Return DefaultFilter(operator="outside", value={"min": min, "max": max}).""" + return DefaultFilter( + operator=RangeOperator.OUTSIDE, value={"min": min, "max": max} + ) + + +#: Module-level singleton — use as ``f.eq(...)``, ``f.in_([...])``, etc. +f = _FHelpers() + +# --------------------------------------------------------------------------- +# FILTER_TYPE_SCHEMAS — call-time validation map +# --------------------------------------------------------------------------- + +#: Maps each CustomFilterType to the Pydantic model used to validate operator/value shape. +#: ``booleanComparison`` uses the SDK-level ``BooleanComparisonFilter`` model (the spec +#: defines no boolean filter model). +#: Read-only: the catalog is closed — registering new filter types is a spec/SDK +#: change (extend CustomFilterType + this map together), not a runtime extension point. +FILTER_TYPE_SCHEMAS: Mapping[CustomFilterType, type[BaseModel]] = MappingProxyType( + { + CustomFilterType.STRING_COMPARISON: StringComparisonFilter, + CustomFilterType.STRING_ARRAY: StringArrayFilter, + CustomFilterType.NUMBER_COMPARISON: NumberComparisonFilter, + CustomFilterType.NUMBER_ARRAY: NumberArrayFilter, + CustomFilterType.NUMBER_RANGE: NumberRangeFilter, + # integerComparison reuses NumberComparisonFilter — the spec defines no + # integer filter model, so the int constraint is not schema-enforced + CustomFilterType.INTEGER_COMPARISON: NumberComparisonFilter, + CustomFilterType.BOOLEAN_COMPARISON: BooleanComparisonFilter, + CustomFilterType.DATE_COMPARISON: DateComparisonFilter, + CustomFilterType.DATE_RANGE: DateRangeFilter, + CustomFilterType.MONEY_COMPARISON: MoneyComparisonFilter, + CustomFilterType.MONEY_RANGE: MoneyRangeFilter, + } +) + +# --------------------------------------------------------------------------- +# DEFAULT_FILTER_NAMES — must include BOTH snake_case field names AND camelCase aliases +# --------------------------------------------------------------------------- + +#: All core default-filter names from OppDefaultFilters: snake_case field names PLUS their +#: camelCase ``alias`` values. Both forms are needed by BOTH consumers of this set: +#: ``validate_routes`` must catch a custom filter whose name shadows either form, and the +#: bucket-1 membership test in ``classify_filters`` must route either key form to a named +#: field (an alias-only set would silently drop snake_case keys into ``customFilters``). +DEFAULT_FILTER_NAMES: frozenset[str] = frozenset( + list(OppDefaultFilters.model_fields.keys()) + + [v.alias for v in OppDefaultFilters.model_fields.values() if v.alias] +) + +# --------------------------------------------------------------------------- +# Alias-normalization maps for classify_filters +# +# OppDefaultFilters uses snake_case field names with camelCase aliases but does NOT +# set populate_by_name=True. Pydantic v2 therefore requires the alias form when +# constructing OppFilters via **kwargs — passing the snake_case field name silently +# results in None (the alias is the required construction key). +# +# classify_filters normalizes consumer keys to the alias (or field-name for fields +# without an alias) before passing them to OppFilters(**...): +# - snake_case keys with a camelCase alias → converted to the alias (closeDateRange) +# - camelCase alias keys → kept as-is (already the alias) +# - keys with no alias (e.g. "status") → kept as-is (snake == request key) +# --------------------------------------------------------------------------- + +# Map from snake_case field name → camelCase alias (used for OppFilters construction). +# Only fields that declare an alias are included; alias-form keys and fields without +# aliases fall through ``_SNAKE_TO_ALIAS.get(key, key)`` unchanged — one lookup +# normalizes all three key classes. +_SNAKE_TO_ALIAS: dict[str, str] = { + field_name: field_info.alias + for field_name, field_info in OppDefaultFilters.model_fields.items() + if field_info.alias +} + +# --------------------------------------------------------------------------- +# validate_routes — registration-time validator +# --------------------------------------------------------------------------- + + +def validate_routes(routes: PluginRoutes) -> None: + """Registration-time validator for a plugin's route filter declarations. + + Iterates every filter spec in ``routes`` and raises ``PluginError`` on: + 1. Unknown ``filter_type`` (not in ``FILTER_TYPE_SCHEMAS``). + 2. Filter name that collides with a core default-filter name in + ``DEFAULT_FILTER_NAMES`` (the escape-hatch collision check; a namespaced + key such as ``gov.<system>@<filterName>`` passes through as ad-hoc instead). + + Duplicate custom-filter names need no check: ``routes`` is dict-keyed, so a + duplicate name cannot reach this validator (a duplicated literal key collapses + silently to its last occurrence at dict construction, before this runs). + + Methods whose ``RouteDeclarations`` carry no ``filters`` key are skipped — + declaring a method with no filters is valid. + + Args: + routes: Route-keyed filter declarations as ``PluginRoutes``. + + Raises: + PluginError: On the first invalid declaration found. + """ + for resource, methods in routes.items(): + for method, declarations in methods.items(): + filter_specs = declarations.get("filters") + if not filter_specs: + continue + for filter_name, spec in filter_specs.items(): + path = f"routes.{resource}.{method}.filters.{filter_name}" + if spec.filter_type not in FILTER_TYPE_SCHEMAS: + raise PluginError( + f'Unknown filter_type "{spec.filter_type}" for filter "{filter_name}"', + path=path, + source_value=spec, + ) + if filter_name in DEFAULT_FILTER_NAMES: + raise PluginError( + f'Filter name "{filter_name}" collides with a default filter name', + path=path, + source_value=filter_name, + ) + + +def _first_error_detail(exc: ValidationError) -> str: + """One-line summary of the first pydantic error, for PluginError messages. + + The full ValidationError stays on ``cause`` for programmatic access; this + puts the most useful line in ``str(exc)`` so a logged message says what + failed, not just how many things did. + """ + first = exc.errors()[0] + loc = ".".join(str(part) for part in first["loc"]) or "<root>" + return f"{first['msg']} (at {loc})" + + +# --------------------------------------------------------------------------- +# validate_filter_call — call-time validator +# --------------------------------------------------------------------------- + + +def validate_filter_call( + spec: Optional[CustomFilterSpec], + filter_name: str, + value: Any, +) -> DefaultFilter: + """Call-time validator for a single filter value. + + Validates ``value`` against the Pydantic model for ``spec.filter_type`` when + ``spec`` is provided (registered filter), or against ``DefaultFilter`` when + ``spec`` is ``None`` (ad-hoc / escape-hatch filter). + + Returns the validated filter as a ``DefaultFilter`` carrying the coerced + operator/value — the wire payload is exactly what passed validation, never + the raw input (lax coercion can differ from the input, e.g. ``"42"`` → ``42``). + Model instances are re-validated via ``model_dump()`` rather than trusted: + the filter models are mutable, so an instance that was valid at construction + may not be valid now. + + Args: + spec: The registered ``CustomFilterSpec`` for this filter, or ``None`` for ad-hoc. + filter_name: The filter name (used in PluginError path). + value: The filter value to validate (typically a ``DefaultFilter`` instance). + + Returns: + The validated filter as a ``DefaultFilter`` with coerced operator/value. + + Raises: + PluginError: On operator/value-shape mismatch (wrapping the pydantic + ``ValidationError`` as ``cause``), or on a ``spec.filter_type`` not + present in ``FILTER_TYPE_SCHEMAS`` — call ``validate_routes(routes)`` + at registration time to catch the latter earlier. + """ + payload = value.model_dump() if isinstance(value, BaseModel) else value + if spec is not None: + model_cls = FILTER_TYPE_SCHEMAS.get(spec.filter_type) + if model_cls is None: + raise PluginError( + f'Unknown filter_type "{spec.filter_type}" for filter ' + f'"{filter_name}" — call validate_routes(routes) at ' + "registration time to catch this earlier", + path=f"filters.{filter_name}", + source_value=spec, + ) + try: + validated = model_cls.model_validate(payload) + except ValidationError as exc: + raise PluginError( + f'Filter "{filter_name}" failed validation: ' + f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", + path=f"filters.{filter_name}", + source_value=value, + cause=exc, + ) from exc + # Re-shape to DefaultFilter so the wire bucket carries the coerced + # operator/value. DefaultFilter.value is Any per the core spec, so + # nothing the typed model accepted can fail here. + return DefaultFilter.model_validate(validated.model_dump()) + # Ad-hoc / escape-hatch: validate against DefaultFilter shape only + try: + return DefaultFilter.model_validate(payload) + except ValidationError as exc: + raise PluginError( + f'Ad-hoc filter "{filter_name}" has an invalid DefaultFilter shape: ' + f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", + path=f"filters.{filter_name}", + source_value=value, + cause=exc, + ) from exc + + +# --------------------------------------------------------------------------- +# classify_filters — ADR-0012 request-body classifier +# --------------------------------------------------------------------------- + + +# The Opportunity binding (OppDefaultFilters / OppFilters) is a known limitation; +# deriving both from the declared resource is tracked in +# https://github.com/HHS/simpler-grants-protocol/issues/896. +def classify_filters( + routes: PluginRoutes, + resource: str, + method: str, + consumer_filters: Mapping[str, DefaultFilter | dict[str, Any]], +) -> OppFilters: + """Classify consumer filter dict into the ``OppFilters`` search request body. + + Three-bucket classification: + - Bucket 1 (default): key is in ``DEFAULT_FILTER_NAMES`` (snake or camelCase alias) → + normalize to the camelCase alias form, land in a named OppFilters field. + - Bucket 2 (registered custom): key matches a registered ``CustomFilterSpec`` for the + given resource/method → land in ``custom_filters``. + - Bucket 3 (ad-hoc): any other key → land in ``custom_filters`` passthrough. + + The classifier is opportunity-bound today: default names come from + ``OppDefaultFilters`` and the wire body is ``OppFilters``. A future + revision will derive both from the declared resource. + + Registered specs are looked up by the exact ``(resource, method)`` strings + declared in ``routes``. A non-matching pair (e.g. a pluralization typo in + ``resource``) yields no registered bucket at all: every non-default filter is + then validated only against the permissive ``DefaultFilter`` shape, exactly + like ad-hoc input, with no error raised. Call sites must pass the same + resource/method strings the plugin declared. + + Construction normalizes all default consumer keys to the form that + ``OppFilters(**kwargs)`` accepts. Because ``OppDefaultFilters`` does NOT set + ``populate_by_name=True``, Pydantic v2 requires the alias form (e.g. + ``closeDateRange``) for aliased fields. Snake_case keys (e.g. + ``close_date_range``) are therefore mapped to their alias via ``_SNAKE_TO_ALIAS`` + before construction. Fields without aliases (e.g. ``status``) pass through + unchanged. The alternative — enabling ``populate_by_name=True`` on + ``OppFilters.model_config`` — is deliberately avoided: the classifier must not + modify core schema model config. + + Args: + routes: Plugin route declarations (used to identify registered custom filters). + resource: Resource name (e.g. ``"opportunities"``). + method: Method name (e.g. ``"search"``). + consumer_filters: Flat ``{name: DefaultFilter}`` mapping from the consumer + call site (raw ``{"operator": ..., "value": ...}`` dicts also accepted). + + Returns: + ``OppFilters`` request body. Call + ``.model_dump(by_alias=True, exclude_none=True, mode="json")`` for the JSON + body of the search request — ``mode="json"`` is required because + coerced ``date`` objects are not JSON-serializable in the default python + mode (operator enums are ``StrEnum`` and serialize fine either way). + + Raises: + PluginError: When any filter value fails validation — registered and ad-hoc + values at classification time, default values at ``OppFilters`` + construction — or when the snake_case and camelCase forms of the same + default filter are both supplied. The error surface is uniform across + all three buckets. + """ + route_declarations = routes.get(resource, {}).get(method, {}) + registered_specs: dict[str, CustomFilterSpec] = route_declarations.get( + "filters", {} + ) + + default_fields: dict[str, Any] = {} + custom_buckets: dict[str, DefaultFilter] = {} + + for key, value in consumer_filters.items(): + if key in DEFAULT_FILTER_NAMES: + # Bucket 1: core default filter. + # OppFilters requires the alias form (e.g. "closeDateRange") when constructing + # via **kwargs because populate_by_name is not set on OppDefaultFilters. + # Normalize: camelCase aliases stay as-is; snake_case keys are mapped to their + # alias; keys with no alias (e.g. "status") are passed through unchanged. + # No validate_filter_call here: default values are validated at the wrapped + # OppFilters construction below, against the named field's REAL type (e.g. + # status → StringArrayFilter) — stricter than the permissive DefaultFilter + # check, and the single validation point for this bucket. + alias_key = _SNAKE_TO_ALIAS.get(key, key) + if alias_key in default_fields: + # Snake and camel forms of the same field normalize to one key; + # without this guard, plain dict assignment would silently drop + # whichever form the consumer's dict ordered first. + raise PluginError( + f'Default filter "{alias_key}" was supplied more than once ' + "(snake_case and camelCase forms of the same filter)", + path=f"filters.{alias_key}", + source_value=value, + ) + default_fields[alias_key] = value + elif key in registered_specs: + # Bucket 2: registered custom filter — ship the validated value, + # never the raw input (see validate_filter_call) + spec = registered_specs[key] + custom_buckets[key] = validate_filter_call(spec, key, value) + else: + # Bucket 3: ad-hoc / escape-hatch passthrough — ship the validated value + custom_buckets[key] = validate_filter_call(None, key, value) + + # OppFilters requires the alias form for construction (populate_by_name is not set). + # Use "customFilters" (the alias) rather than "custom_filters" (the field name). + # This construction is the validation point for bucket-1 (default) values — wrap + # pydantic failures in PluginError so the error contract is uniform across all + # three buckets (consumers catch `except PluginError` regardless of bucket). + try: + return OppFilters( + **default_fields, + customFilters=custom_buckets if custom_buckets else None, + ) + except ValidationError as exc: + # Name the failing field(s) so bucket-1 errors are as pinpointed as the + # filters.<name> paths raised for buckets 2/3. The loc values are the + # alias keys used at construction (e.g. "closeDateRange"); customFilters + # values were already validated above, so only default fields fail here. + failed = sorted({str(err["loc"][0]) for err in exc.errors() if err.get("loc")}) + field_list = ", ".join(failed) if failed else "<unknown>" + raise PluginError( + f"Filter(s) {field_list} failed validation: " + f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", + path=f"filters.{failed[0]}" if len(failed) == 1 else "filters", + source_value=default_fields, + cause=exc, + ) from exc diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 7f8c037fa..5e7118c96 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -8,6 +8,7 @@ from .types import ( PluginExtensions, PluginExtensionsMeta, + PluginRoutes, ) T = TypeVar("T") @@ -36,6 +37,7 @@ class PluginConfig(Generic[TSchemas]): extensions: PluginExtensions | None = None meta: PluginExtensionsMeta | None = None schemas: TSchemas | None = None + routes: PluginRoutes | None = None @dataclass @@ -62,6 +64,7 @@ def define_plugin( meta: PluginExtensionsMeta | None = ..., extensions: PluginExtensions | None = ..., schemas: None = ..., + routes: PluginRoutes | None = ..., ) -> PluginConfig[None]: ... @@ -70,6 +73,7 @@ def define_plugin( meta: PluginExtensionsMeta | None = ..., extensions: PluginExtensions | None = ..., schemas: TSchemas = ..., + routes: PluginRoutes | None = ..., ) -> PluginConfig[TSchemas]: ... @@ -77,8 +81,12 @@ def define_plugin( meta: PluginExtensionsMeta | None = None, extensions: PluginExtensions | None = None, schemas: Any = None, + routes: PluginRoutes | None = None, ) -> PluginConfig[Any]: - """Create a PluginConfig consumed by the code generator. + """Create a PluginConfig from plugin declarations. + + Schema inputs are consumed by the code generator; route-keyed filter + declarations are consumed by the runtime filter engine (extensions/filters.py). No compilation occurs here — inputs are stored as-is. The code generator (generate.py) compiles ObjectSchemasInput → ObjectSchemas by injecting @@ -88,11 +96,19 @@ def define_plugin( (e.g. {"Opportunity": ObjectSchemasInput[MyNative, MyCg](...) }) preserves those per-object generics on the returned PluginConfig rather than widening them to Any. + + routes: optional route-keyed custom-filter declarations. + Shape: {resourceName: {methodName: {filterName: CustomFilterSpec}}}. + Passed through to PluginConfig.routes unvalidated. Registration-time + validation is explicit: call validate_routes() (extensions/filters.py) + on the declarations, e.g. at plugin startup. classify_filters() does + not invoke it; route validation is a standalone check. """ return PluginConfig( extensions=extensions, meta=meta, schemas=schemas, + routes=routes, ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index d6ef97852..279cd7a64 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from enum import StrEnum from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict from ..schemas.pydantic.fields.custom import CustomFieldType @@ -16,6 +17,35 @@ ConflictStrategy = Literal["error", "first_wins", "last_wins"] +class CustomFilterType(StrEnum): + """Catalog of registerable filter types. Uses *Comparison/*Array/*Range wire values.""" + + STRING_COMPARISON = "stringComparison" + STRING_ARRAY = "stringArray" + NUMBER_COMPARISON = "numberComparison" + NUMBER_ARRAY = "numberArray" + NUMBER_RANGE = "numberRange" + INTEGER_COMPARISON = "integerComparison" + BOOLEAN_COMPARISON = "booleanComparison" + DATE_COMPARISON = "dateComparison" + DATE_RANGE = "dateRange" + MONEY_COMPARISON = "moneyComparison" + MONEY_RANGE = "moneyRange" + + +@dataclass(frozen=True) +class CustomFilterSpec: + """Per-filter declaration: filter_type constrains to a known type; description is optional. + + Frozen: a spec is an immutable declaration; mutating one after registration + would bypass validate_routes. No ``value`` field — the allowed operator set + is enforced by the filter_type's validation model at call time. + """ + + filter_type: CustomFilterType + description: Optional[str] = None + + @dataclass class CustomFieldSpec: """Custom Field spec class to support adding custom fields""" diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index eccacc519..2cbed9d8e 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -3,11 +3,11 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Callable, Generic, Literal, TypeVar +from typing import Any, Callable, Generic, Literal, NotRequired, TypedDict, TypeVar from pydantic import BaseModel, ConfigDict, Field -from .specs import CustomFieldSpec +from .specs import CustomFieldSpec, CustomFilterSpec TNative = TypeVar("TNative") TCommon = TypeVar("TCommon") @@ -19,6 +19,25 @@ # Type aliases Handler = Callable[[Any, Any], Any] +# Route-keyed custom-filter declaration types, mirroring the TS SDK shape: +# PluginRoutes = {resourceName: {methodName: RouteDeclarations}}. +RouteMethodFilters = dict[str, CustomFilterSpec] # {filterName: spec} + + +class RouteDeclarations(TypedDict): + """Filter declarations for a single route method (e.g. ``search``). + + ``filters`` maps filter name → ``CustomFilterSpec`` and is optional: a + method may appear in the route map with no declarations. + """ + + filters: NotRequired[RouteMethodFilters] + + +PluginRoutes = dict[ + str, dict[str, RouteDeclarations] +] # {resource: {method: declarations}} + class PluginError(Exception): """Structured transformation error per ADR-0022 Decision #9. diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py index 617edc3c3..9ea43cf06 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py @@ -18,6 +18,8 @@ "MoneyComparisonFilter", "MoneyRange", "MoneyRangeFilter", + # Boolean Filters + "BooleanComparisonFilter", # Numeric Filters "NumberArrayFilter", "NumberComparisonFilter", @@ -39,6 +41,7 @@ RangeOperator, StringOperator, ) +from .boolean import BooleanComparisonFilter from .date import ( DateComparisonFilter, DateRange, diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/base.py index f04e06601..ae78fdc24 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/base.py @@ -1,7 +1,7 @@ """Filter models for the CommonGrants API.""" from enum import StrEnum -from typing import Union +from typing import Any, Union from pydantic import Field, field_validator @@ -64,7 +64,11 @@ class DefaultFilter(CommonGrantsBaseModel): StringOperator, RangeOperator, ] = Field(..., description="The operator to apply to the filter value") - value: Union[str, int, float, list, dict] = Field( + # Core spec (filters/base.tsp) declares `value: unknown` — any narrowing here + # diverges from the contract and mutates inputs (e.g. a union without `bool` + # lax-coerces True → 1). Strict per-type checking belongs to the typed filter + # models (StringArrayFilter, NumberComparisonFilter, ...), not this base. + value: Any = Field( ..., description="The value to use for the filter operation", ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/boolean.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/boolean.py new file mode 100644 index 000000000..59067d335 --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/boolean.py @@ -0,0 +1,31 @@ +"""Boolean filter schemas.""" + +from pydantic import Field, StrictBool, field_validator + +from ..base import CommonGrantsBaseModel +from .base import EquivalenceOperator + + +class BooleanComparisonFilter(CommonGrantsBaseModel): + """Filter that matches a boolean value for equality (eq | neq). + + Strict boolean: 1/0 and "true"/"false" are rejected, matching the Zod + ``z.boolean()`` semantics of the TS SDK's ``BooleanComparisonFilterSchema``. + + No spec model backs this filter yet: the core spec defines no boolean + filter, so both SDKs carry an SDK-level model until one lands. + """ + + operator: EquivalenceOperator = Field( + ..., + description="The operator to apply to the filter value", + ) + value: StrictBool = Field(..., description="The boolean value to compare against") + + @field_validator("operator", mode="before") + @classmethod + def validate_operator(cls, v): + """Convert string to enum if needed.""" + if isinstance(v, str): + return EquivalenceOperator(v) + return v diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/numeric.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/numeric.py index 318c2e6d3..7a22f868f 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/numeric.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/numeric.py @@ -8,6 +8,7 @@ from .base import ( ArrayOperator, ComparisonOperator, + EquivalenceOperator, RangeOperator, ) @@ -16,17 +17,40 @@ # ############################################################ +def _ensure_not_bool(v: object) -> object: + """Reject bool before union coercion. + + bool subclasses int in Python, so a ``Union[int, float]`` field + lax-coerces ``True`` -> ``1`` and a boolean silently ships as a number — + the same wire corruption ``DefaultFilter.value: Any`` guards against. + ``z.number()`` rejects booleans, so rejection keeps the SDKs aligned. + """ + if isinstance(v, bool): + raise ValueError("value must be a number, not a bool") + return v + + class NumberRange(CommonGrantsBaseModel): """Represents a range between two numeric values.""" min: Union[int, float] = Field(..., description="The minimum value in the range") max: Union[int, float] = Field(..., description="The maximum value in the range") + @field_validator("min", "max", mode="before") + @classmethod + def reject_bool(cls, v): + """Reject bool min/max (would lax-coerce to 1/0; see _ensure_not_bool).""" + return _ensure_not_bool(v) + class NumberComparisonFilter(CommonGrantsBaseModel): - """Filter that matches numbers against a specific value.""" + """Filter that matches numbers against a specific value. + + Accepts equivalence operators (``eq``/``neq``) in addition to comparison + operators, per the core spec (filters/numeric.tsp, since protocol v0.3). + """ - operator: ComparisonOperator = Field( + operator: ComparisonOperator | EquivalenceOperator = Field( ..., description="The comparison operator to apply to the filter value", ) @@ -39,9 +63,18 @@ class NumberComparisonFilter(CommonGrantsBaseModel): def validate_operator(cls, v): """Convert string to enum if needed.""" if isinstance(v, str): - return ComparisonOperator(v) + if v in [op.value for op in ComparisonOperator]: + return ComparisonOperator(v) + elif v in [op.value for op in EquivalenceOperator]: + return EquivalenceOperator(v) return v + @field_validator("value", mode="before") + @classmethod + def reject_bool(cls, v): + """Reject bool values (would lax-coerce to 1/0; see _ensure_not_bool).""" + return _ensure_not_bool(v) + class NumberRangeFilter(CommonGrantsBaseModel): """Filter that matches numbers within a specified range.""" @@ -79,3 +112,12 @@ def validate_operator(cls, v): if isinstance(v, str): return ArrayOperator(v) return v + + @field_validator("value", mode="before") + @classmethod + def reject_bool_items(cls, v): + """Reject bool items (each would lax-coerce to 1/0; see _ensure_not_bool).""" + if isinstance(v, list): + for item in v: + _ensure_not_bool(item) + return v diff --git a/lib/python-sdk/examples/custom_filters.py b/lib/python-sdk/examples/custom_filters.py new file mode 100644 index 000000000..ec1d508ab --- /dev/null +++ b/lib/python-sdk/examples/custom_filters.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Custom filters example — codegen-free request-body demo. + +Demonstrates route-keyed custom filter declaration, flat call-site classification, +and the three-bucket ``OppFilters`` request body (default named fields + customFilters record) +using the grants.gov canonical example from #646/#869. + +No code generation (custom filters are a pure-runtime classifier). The example +imports directly from ``common_grants_sdk.extensions`` — no codegen schemas needed. + +Run (from lib/python-sdk/): + poetry run python examples/custom_filters.py +""" + +from __future__ import annotations + +import json + +from common_grants_sdk.extensions import classify_filters, define_plugin, f +from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType +from common_grants_sdk.extensions.types import PluginError, PluginExtensionsMeta + +# --------------------------------------------------------------------------- +# Plugin declaration — route-keyed custom filter specs +# --------------------------------------------------------------------------- + +grants_gov = define_plugin( + meta=PluginExtensionsMeta( + name="grants-gov", + version="0.1.0", + sourceSystem="grants.gov", + capabilities=["customFilters"], + ), + routes={ + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ), + "fundingProgram": CustomFilterSpec( + filter_type=CustomFilterType.STRING_COMPARISON, + description="Program name filter", + ), + } + } + } + }, +) + + +def _section(title: str) -> None: + print(f"\n{'=' * 60}") + print(title) + print("=" * 60) + + +def main() -> None: + # --- Declared routes --- + _section("PLUGIN ROUTES (declared)") + if grants_gov.routes: + for resource, methods in grants_gov.routes.items(): + for method, declarations in methods.items(): + print(f" {resource}.{method}:") + for name, spec in declarations.get("filters", {}).items(): + desc = f" — {spec.description}" if spec.description else "" + print(f" {name}: {spec.filter_type.value}{desc}") + + # --- Classify: default + registered custom + ad-hoc --- + _section("CLASSIFY FILTERS — default + custom + ad-hoc (canonical grants.gov demo)") + + # Mixing three kinds of filters in a single flat dict: + # "status" — default core filter (snake_case key, no alias) + # "close_date_range" — default core filter, snake_case key for an ALIASED field: + # the classifier must normalize it to "closeDateRange" in the + # wire body (an unnormalized snake key would be silently + # dropped from the wire body — the alias landmine) + # "agency" — registered custom filter (routes.opportunities.search.agency) + # "fundingProgram" — registered custom filter (routes.opportunities.search.fundingProgram) + # "legacyTag" — ad-hoc passthrough (not registered, not a core default) + consumer_filters = { + "status": f.in_(["open", "forecasted"]), + "close_date_range": f.between("2026-01-01", "2026-12-31"), + "agency": f.in_(["NSF", "NIH"]), + "fundingProgram": f.eq("research-grants"), + "legacyTag": f.eq("priority"), + } + + assert grants_gov.routes is not None + request_body = classify_filters( + grants_gov.routes, "opportunities", "search", consumer_filters + ) + + print("\nRequest body (by_alias=True, exclude_none=True, mode='json'):") + print( + json.dumps( + request_body.model_dump(by_alias=True, exclude_none=True, mode="json"), + indent=2, + ) + ) + + # --- PluginError demo — bad call raises and is caught --- + _section("VALIDATION — bad operator raises PluginError (runtime guarantee)") + + # agency is registered as STRING_ARRAY (expects ArrayOperator: in/notIn). + # Passing f.eq(...) (EquivalenceOperator.EQUAL) triggers call-time validation failure. + bad_filters = { + "agency": f.eq( + "wrong-operator-for-array-type" + ), # eq is not a valid STRING_ARRAY op + } + try: + classify_filters(grants_gov.routes, "opportunities", "search", bad_filters) + except PluginError as exc: + # str(exc) summarizes the first failure; the structured fields carry + # the full detail — exc.path names the failing filter, exc.cause is + # the underlying pydantic ValidationError for programmatic access. + print(f"PluginError caught: {exc}") + print(f" path: {exc.path}") + print(f" cause: {type(exc.cause).__name__}") + + # --- Plugin metadata --- + _section("PLUGIN METADATA") + if grants_gov.meta: + print(f"name: {grants_gov.meta.name}") + print(f"version: {grants_gov.meta.version}") + print(f"sourceSystem: {grants_gov.meta.source_system}") + print(f"capabilities: {grants_gov.meta.capabilities}") + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py new file mode 100644 index 000000000..c7c602556 --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -0,0 +1,636 @@ +"""Tests for classify_filters, f helpers, validate_routes, and validate_filter_call +in common_grants_sdk.extensions.filters.""" + +from __future__ import annotations + +import json +from datetime import date + +import pytest +from pydantic import ValidationError + +from common_grants_sdk.extensions.filters import ( + FILTER_TYPE_SCHEMAS, + classify_filters, + f, + validate_filter_call, + validate_routes, +) +from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType +from common_grants_sdk.extensions.types import PluginError +from common_grants_sdk.schemas.pydantic.filters.opportunity import OppFilters + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +AGENCY_SPEC = CustomFilterSpec(filter_type=CustomFilterType.STRING_ARRAY) +FUNDING_PROGRAM_SPEC = CustomFilterSpec( + filter_type=CustomFilterType.STRING_COMPARISON, + description="Program name filter", +) + +SAMPLE_ROUTES = { + "opportunities": { + "search": { + "filters": { + "agency": AGENCY_SPEC, + "fundingProgram": FUNDING_PROGRAM_SPEC, + } + } + } +} + + +# --------------------------------------------------------------------------- +# f.* helpers +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("helper", "args", "operator", "value"), + [ + ("eq", ("open",), "eq", "open"), + ("neq", ("x",), "neq", "x"), + ("gt", (1,), "gt", 1), + ("gte", (1,), "gte", 1), + ("lt", (1,), "lt", 1), + ("lte", (1,), "lte", 1), + ("in_", (["a"],), "in", ["a"]), + ("not_in", (["a"],), "notIn", ["a"]), + ("like", ("%x%",), "like", "%x%"), + ("not_like", ("%x%",), "notLike", "%x%"), + ("between", (0, 1), "between", {"min": 0, "max": 1}), + ("outside", (0, 1), "outside", {"min": 0, "max": 1}), + ], +) +def test_f_helper_wire_values(helper, args, operator, value): + """Every f helper produces its documented wire operator and value shape. + + Includes the reserved-word workarounds: Python f.in_ / f.not_in produce + wire operators "in" / "notIn". + """ + flt = getattr(f, helper)(*args) + assert flt.operator == operator + assert flt.value == value + + +# --------------------------------------------------------------------------- +# classify_filters: three-bucket classification +# --------------------------------------------------------------------------- + + +def test_classify_default_snake_key_lands_in_named_field(): + """Default snake_case key (e.g. "status") lands in a named OppFilters field, not customFilters.""" + consumer_filters = {"status": f.in_(["open"])} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert isinstance(result, OppFilters) + assert result.status is not None + assert result.custom_filters is None + + +def test_classify_default_camel_alias_lands_in_named_field(): + """THE LANDMINE: camelCase alias "closeDateRange" must land in named field, NOT customFilters.""" + consumer_filters = {"closeDateRange": f.between("2026-01-01", "2026-12-31")} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert isinstance(result, OppFilters) + # The camelCase alias must normalize to the snake_case field + assert result.close_date_range is not None + # It must NOT appear in customFilters (a conditional check here could + # never fail on the path it guards — assert the bucket is empty outright) + assert result.custom_filters is None + + +def test_classify_registered_custom_filter_lands_in_custom_filters(): + """A registered custom filter (e.g. "agency") lands in OppFilters.custom_filters.""" + consumer_filters = {"agency": f.in_(["NSF", "NIH"])} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert isinstance(result, OppFilters) + assert result.custom_filters is not None + assert "agency" in result.custom_filters + + +def test_classify_adhoc_unregistered_filter_lands_in_custom_filters(): + """An unregistered ad-hoc key (e.g. "legacyTag") passes through to customFilters.""" + consumer_filters = {"legacyTag": f.eq("priority")} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert isinstance(result, OppFilters) + assert result.custom_filters is not None + assert "legacyTag" in result.custom_filters + + +def test_classify_escape_hatch_key_lands_in_custom_filters(): + """gov.<system>@<filter> escape-hatch keys pass through to customFilters.""" + consumer_filters = {"gov.someSystem@someFilter": f.eq("test")} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert isinstance(result, OppFilters) + assert result.custom_filters is not None + assert "gov.someSystem@someFilter" in result.custom_filters + + +# --------------------------------------------------------------------------- +# Request-body shape +# --------------------------------------------------------------------------- + + +def test_request_body_has_default_fields_at_top_level_and_custom_filters_nested(): + """model_dump(by_alias=True, exclude_none=True) yields the ADR-0012 request-body shape. + + Default filters appear at top level; custom/ad-hoc appear under "customFilters". + """ + consumer_filters = { + "status": f.in_(["open"]), + "agency": f.in_(["NSF"]), + "legacyTag": f.eq("priority"), + } + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + request_body = result.model_dump(by_alias=True, exclude_none=True) + + # Default field appears at top level + assert "status" in request_body + # Custom and ad-hoc filters are nested under customFilters + assert "customFilters" in request_body + assert "agency" in request_body["customFilters"] + assert "legacyTag" in request_body["customFilters"] + # customFilters is NOT a top-level key for a default filter + assert "status" not in request_body.get("customFilters", {}) + + +def test_request_body_no_custom_filters_key_when_all_defaults(): + """customFilters key is absent from the request body when all filters are default fields.""" + consumer_filters = {"status": f.in_(["open"])} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + request_body = result.model_dump(by_alias=True, exclude_none=True) + assert "customFilters" not in request_body + + +def test_oppfilters_mixed_case_roundtrip(): + """Alias-normalization round-trip proof. + + A mix of snake_case and camelCase default keys — including 'closeDateRange' as + a camelCase alias — must normalize to the correct snake_case field names and + produce the correct ADR-0012 request JSON via model_dump(by_alias=True, exclude_none=True). + + Asserts: + - closeDateRange lands in its named default field (close_date_range), NOT in customFilters. + - request-body output uses camelCase aliases (by_alias=True). + """ + consumer_filters = { + "status": f.in_(["open", "forecasted"]), # snake_case default + "closeDateRange": f.between( + "2026-01-01", "2026-12-31" + ), # camelCase alias default + } + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert isinstance(result, OppFilters) + + # camelCase alias must land in named field + assert result.close_date_range is not None + + request_body = result.model_dump(by_alias=True, exclude_none=True) + + # Both named fields appear at top level with their alias names + assert "status" in request_body + assert "closeDateRange" in request_body + + # Nothing landed in customFilters — assert the bucket is absent outright + # (a conditional check could never fail on the path it guards) + assert "customFilters" not in request_body + + +# --------------------------------------------------------------------------- +# Registration-time validation (validate_routes) — RAISES PluginError +# --------------------------------------------------------------------------- + + +def test_validate_routes_unknown_filter_type_raises(): + """validate_routes raises PluginError when filter_type is not in FILTER_TYPE_SCHEMAS.""" + # Dataclasses don't validate field types at runtime, so a bogus filter_type + # can be passed directly (the annotation is for type checkers only). + bad_spec = CustomFilterSpec(filter_type="unknownType") # type: ignore[arg-type] + + routes = {"opportunities": {"search": {"filters": {"myFilter": bad_spec}}}} + with pytest.raises(PluginError, match="Unknown filter_type"): + validate_routes(routes) + + +def test_validate_routes_collision_with_default_filter_name_raises(): + """validate_routes raises PluginError when a custom filter name collides with a CORE DEFAULT name. + + This is the escape-hatch collision check. E.g. naming a custom filter "status" + would shadow the core default "status" field — must be caught at registration time. + """ + routes = { + "opportunities": { + "search": { + "filters": { + "status": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY, + description="Should collide with default", + ), + } + } + } + } + with pytest.raises(PluginError, match="collides"): + validate_routes(routes) + + +def test_validate_routes_collision_with_camel_alias_raises(): + """validate_routes raises PluginError for camelCase alias collision (e.g. "closeDateRange").""" + routes = { + "opportunities": { + "search": { + "filters": { + "closeDateRange": CustomFilterSpec( + filter_type=CustomFilterType.DATE_RANGE, + description="Should collide with default alias", + ), + } + } + } + } + with pytest.raises(PluginError, match="collides"): + validate_routes(routes) + + +def test_validate_routes_valid_routes_do_not_raise(): + """validate_routes does not raise for a fully valid routes dict.""" + # Should not raise + validate_routes(SAMPLE_ROUTES) + + +# --------------------------------------------------------------------------- +# Call-time validation (validate_filter_call) — RAISES PluginError +# --------------------------------------------------------------------------- + + +def test_validate_filter_call_registered_bad_operator_raises(): + """validate_filter_call raises PluginError when a registered filter has an operator/value mismatch.""" + # AGENCY_SPEC is STRING_ARRAY — an "eq" with a scalar value is wrong for StringArrayFilter + bad_filter = f.eq("not-an-array") + with pytest.raises(PluginError): + validate_filter_call(AGENCY_SPEC, "agency", bad_filter) + + +def test_validate_filter_call_adhoc_invalid_shape_raises(): + """validate_filter_call raises PluginError when an ad-hoc filter has an invalid DefaultFilter shape.""" + + # Pass None as spec (ad-hoc), with something that isn't a DefaultFilter + class _BadShape: + operator = "not_a_real_operator" + value = object() # not a valid value type + + with pytest.raises(PluginError): + validate_filter_call(None, "legacyTag", _BadShape()) # type: ignore[arg-type] + + +def test_validate_filter_call_valid_registered_does_not_raise(): + """validate_filter_call does not raise for a valid registered filter call.""" + valid_filter = f.in_(["NSF", "NIH"]) + # Should not raise — agency is STRING_ARRAY, in_ with list is valid + validate_filter_call(AGENCY_SPEC, "agency", valid_filter) + + +def test_validate_filter_call_money_comparison_passes_valid_money(): + """A registered moneyComparison filter accepts a comparison operator and Money value. + + Money.amount is a decimal STRING ("1000000"), not a number — the shape that + drifted in the TS compile-time filter map and was locked there with + compile-error tests; covered here at the runtime layer. + """ + spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) + validate_filter_call( + spec, "awardFloor", f.gt({"amount": "1000000", "currency": "USD"}) + ) + + +def test_validate_filter_call_money_comparison_rejects_array_operator(): + """A registered moneyComparison filter raises PluginError for an array operator.""" + spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) + with pytest.raises(PluginError): + validate_filter_call( + spec, "awardFloor", f.in_([{"amount": "1000000", "currency": "USD"}]) + ) + + +def test_validate_filter_call_money_comparison_rejects_numeric_amount(): + """A registered moneyComparison filter raises PluginError for a numeric amount. + + Money.amount is a DecimalString — a raw number is the wrong shape. + """ + spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) + with pytest.raises(PluginError): + validate_filter_call( + spec, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) + ) + + +def test_validate_filter_call_money_range_passes_valid_range(): + """A registered moneyRange filter accepts between with {min, max} Money values.""" + spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) + validate_filter_call( + spec, + "awardRange", + f.between( + {"amount": "10000", "currency": "USD"}, + {"amount": "500000", "currency": "USD"}, + ), + ) + + +def test_validate_filter_call_money_range_rejects_comparison_operator(): + """A registered moneyRange filter raises PluginError for a comparison operator.""" + spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) + with pytest.raises(PluginError): + validate_filter_call( + spec, "awardRange", f.gt({"amount": "10000", "currency": "USD"}) + ) + + +def test_classify_default_wrong_shape_raises_plugin_error(): + """A wrong-shaped DEFAULT filter raises PluginError, not a raw pydantic ValidationError. + + "status" is a StringArrayFilter (ArrayOperator + list[str]); f.eq("open") is an + equivalence filter. The error contract must be uniform across all three buckets: + consumers following the documented `except PluginError` pattern must catch this. + """ + with pytest.raises(PluginError) as exc_info: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} + ) + # The underlying pydantic error is preserved as cause for programmatic access + assert isinstance(exc_info.value.__cause__, ValidationError) + + +def test_validate_filter_call_integer_comparison_validates_as_number(): + """A registered integerComparison filter validates against NumberComparisonFilter. + + The spec defines no integer filter model, so the int constraint is not + schema-enforced (same as the TS SDK); a numeric value passes, a non-numeric + value fails. + """ + spec = CustomFilterSpec(filter_type=CustomFilterType.INTEGER_COMPARISON) + validate_filter_call(spec, "awardCount", f.gt(100)) + with pytest.raises(PluginError): + validate_filter_call(spec, "awardCount", f.gt("not a number")) + + +def test_classify_default_camel_alias_wrong_shape_raises_plugin_error(): + """A wrong-shaped default filter via its camelCase alias also raises PluginError. + + "closeDateRange" is a DateRangeFilter; f.eq("2026-01-01") is an equivalence + filter — the alias normalization path must surface the same PluginError. + """ + with pytest.raises(PluginError): + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"closeDateRange": f.eq("2026-01-01")}, + ) + + +# --------------------------------------------------------------------------- +# Wire-body integrity: the value that passed validation is the value shipped +# --------------------------------------------------------------------------- + +WIRE_ROUTES = { + "opportunities": { + "search": { + "filters": { + "isOpen": CustomFilterSpec( + filter_type=CustomFilterType.BOOLEAN_COMPARISON + ), + "awardCount": CustomFilterSpec( + filter_type=CustomFilterType.NUMBER_COMPARISON + ), + } + } + } +} + + +def test_boolean_filter_value_survives_to_wire_as_json_true(): + """f.eq(True) serializes as JSON true, not 1. + + DefaultFilter.value is Any per the core spec (filters/base.tsp `unknown`); + a narrowed union without bool lax-coerced True -> 1 and corrupted the wire. + """ + result = classify_filters( + WIRE_ROUTES, "opportunities", "search", {"isOpen": f.eq(True)} + ) + body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert body["customFilters"]["isOpen"]["value"] is True + + +def test_registered_filter_ships_validated_value_not_raw_input(): + """The wire body carries the value that passed validation, not the raw input. + + NumberComparisonFilter lax-coerces "42" -> 42 (smart-union resolves the + int|float union to int first); shipping the raw string would mean the + payload differs from what validation approved. + """ + result = classify_filters( + WIRE_ROUTES, "opportunities", "search", {"awardCount": f.gt("42")} + ) + body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert body["customFilters"]["awardCount"]["value"] == 42 + assert isinstance(body["customFilters"]["awardCount"]["value"], int) + + +def test_number_comparison_registered_filter_rejects_bool(): + """f.eq(True) on a numberComparison-registered filter raises, never ships 1. + + bool subclasses int; without an explicit rejection the int|float union + lax-coerces True -> 1 and the wire silently carries a number for a + boolean — the corruption class the DefaultFilter.value widening fixed. + """ + with pytest.raises(PluginError): + classify_filters( + WIRE_ROUTES, "opportunities", "search", {"awardCount": f.eq(True)} + ) + + +@pytest.mark.filterwarnings("ignore::UserWarning") # pydantic warns during the +# model_dump of the mutated instance, before re-validation raises +def test_mutated_adhoc_instance_is_revalidated_and_raises(): + """An ad-hoc DefaultFilter mutated after construction raises instead of shipping. + + The filter models are mutable; the ad-hoc branch must re-validate instances + rather than trust isinstance. + """ + flt = f.eq("x") + flt.operator = "bogus" # type: ignore[assignment] + with pytest.raises(PluginError): + classify_filters(SAMPLE_ROUTES, "opportunities", "search", {"legacy": flt}) + + +def test_unknown_filter_type_raises_plugin_error_not_key_error(): + """A spec whose filter_type never passed validate_routes raises PluginError. + + The uniform error contract holds even when registration-time validation was + skipped — consumers catching `except PluginError` must not see a KeyError. + """ + spec = CustomFilterSpec(filter_type="bogusType") # type: ignore[arg-type] + with pytest.raises(PluginError): + validate_filter_call(spec, "x", f.eq(1)) + + +def test_validate_filter_call_adhoc_accepts_raw_dict(): + """Ad-hoc validation accepts a raw operator/value dict and returns a DefaultFilter.""" + validated = validate_filter_call(None, "x", {"operator": "eq", "value": "v"}) + assert validated.operator == "eq" + assert validated.value == "v" + + +# --------------------------------------------------------------------------- +# Alias normalization (snake form), serialization contract, error paths +# --------------------------------------------------------------------------- + + +def test_classify_default_snake_form_of_aliased_key_normalizes_to_alias(): + """Snake_case key for an ALIASED field lands in the named field, not customFilters. + + Exercises the _SNAKE_TO_ALIAS hit branch: without normalization, + OppFilters(close_date_range=...) is silently dropped by pydantic + (populate_by_name is not set) and the field stays None. + """ + consumer_filters = {"close_date_range": f.between("2026-01-01", "2026-12-31")} + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + assert result.close_date_range is not None + assert result.custom_filters is None + + +def test_classify_both_forms_of_same_default_filter_raises(): + """Supplying snake AND camel forms of one default filter raises PluginError. + + Both keys normalize to "closeDateRange"; without the guard, dict + assignment silently drops whichever range the consumer's dict ordered + first (plausible when merging filter dicts from two naming conventions). + """ + consumer_filters = { + "close_date_range": f.between("2026-01-01", "2026-06-30"), + "closeDateRange": f.between("2026-07-01", "2026-12-31"), + } + with pytest.raises(PluginError, match="more than once") as exc_info: + classify_filters(SAMPLE_ROUTES, "opportunities", "search", consumer_filters) + assert exc_info.value.path == "filters.closeDateRange" + + +@pytest.mark.parametrize( + ("resource", "method"), + [ + ("opportunities", "list"), # method not declared in routes + ("opportunity", "search"), # resource near-miss (pluralization) + ], +) +def test_classify_unmatched_route_treats_registered_name_as_adhoc(resource, method): + """A (resource, method) pair with no routes entry has NO registered bucket. + + "agency" is registered as STRING_ARRAY under opportunities.search only; + via any other route pair it is validated as permissive ad-hoc, so + f.eq("NSF") (invalid for STRING_ARRAY) passes through to customFilters. + Discriminates both levels of the routes[resource][method] lookup — a + regression that flattens or mis-keys it either wrongly applies the spec + or wrongly skips it. + """ + result = classify_filters(SAMPLE_ROUTES, resource, method, {"agency": f.eq("NSF")}) + assert result.custom_filters is not None + assert result.custom_filters["agency"].value == "NSF" + + # ...and the same filter via the declared pair IS spec-validated and rejected + with pytest.raises(PluginError): + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} + ) + + +def test_request_body_mode_json_round_trip(): + """The documented model_dump(mode="json") call yields a json.dumps-able body. + + Coerced date objects only serialize in json mode — this is the ADR-0012 + wire body the classifier exists to produce. + """ + consumer_filters = { + "close_date_range": f.between(date(2026, 1, 1), date(2026, 12, 31)), + "agency": f.in_(["NSF"]), + } + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + body = json.loads( + json.dumps(result.model_dump(by_alias=True, exclude_none=True, mode="json")) + ) + assert body["closeDateRange"]["operator"] == "between" + assert body["closeDateRange"]["value"]["min"] == "2026-01-01" + assert body["customFilters"]["agency"]["operator"] == "in" + + +def test_classify_empty_filters_dict_yields_empty_body(): + """An empty consumer dict produces an OppFilters with no customFilters entry.""" + result = classify_filters(SAMPLE_ROUTES, "opportunities", "search", {}) + assert result.custom_filters is None + body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert "customFilters" not in body + + +def test_filter_type_schemas_covers_every_custom_filter_type(): + """Every CustomFilterType member has a validation model. + + A catalog member without a FILTER_TYPE_SCHEMAS entry would reject valid + registrations in validate_routes — this assert turns that drift into a + CI failure at the moment the enum and the map diverge. + """ + assert set(FILTER_TYPE_SCHEMAS) == set(CustomFilterType) + + +def test_plugin_error_path_is_uniform_across_buckets(): + """All three buckets raise PluginError with a filters.<name> path.""" + with pytest.raises(PluginError) as exc1: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} + ) + assert exc1.value.path == "filters.status" + + with pytest.raises(PluginError) as exc2: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} + ) + assert exc2.value.path == "filters.agency" + + with pytest.raises(PluginError) as exc3: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"adhoc": {"operator": "bogus"}} + ) + assert exc3.value.path == "filters.adhoc" + + +def test_multiple_failing_defaults_use_collective_path(): + """Two failing default filters produce the collective path "filters".""" + with pytest.raises(PluginError) as exc_info: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"status": f.eq("open"), "closeDateRange": f.eq("x")}, + ) + assert exc_info.value.path == "filters" diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index b8947a47c..ca6fe6285 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -1,6 +1,7 @@ """Tests for plugin.py — PluginExtensions-based API.""" from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin +from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType from common_grants_sdk.extensions.types import ( ObjectSchemasInput, PluginExtensions, @@ -25,6 +26,23 @@ def test_define_plugin_with_extensions(): assert config.extensions is ext +def test_define_plugin_routes_passthrough(): + """define_plugin stores routes as-is on PluginConfig.routes (no validation).""" + routes = { + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + config = define_plugin(routes=routes) + assert config.routes is routes + + def test_define_plugin_with_meta_and_schemas(): meta = PluginExtensionsMeta(name="test", source_system="test-system") diff --git a/lib/python-sdk/tests/schemas/test_filters.py b/lib/python-sdk/tests/schemas/test_filters.py index 23ec84cf7..a2d462738 100644 --- a/lib/python-sdk/tests/schemas/test_filters.py +++ b/lib/python-sdk/tests/schemas/test_filters.py @@ -2,7 +2,9 @@ from datetime import date import pytest +from pydantic import ValidationError +from common_grants_sdk.schemas.pydantic.filters.boolean import BooleanComparisonFilter from common_grants_sdk.schemas.pydantic.filters.base import ( ArrayOperator, ComparisonOperator, @@ -508,3 +510,112 @@ def test_number_array_filter(): filter_obj = NumberArrayFilter(operator=ArrayOperator.IN, value=[1, 2.5, 3]) assert filter_obj.operator == ArrayOperator.IN assert filter_obj.value == [1, 2.5, 3] + + +def test_boolean_comparison_filter_constructs(): + """BooleanComparisonFilter constructs with valid operator and value.""" + filter_obj = BooleanComparisonFilter(operator="eq", value=True) + assert filter_obj.operator == EquivalenceOperator.EQUAL + assert filter_obj.value is True + + filter_obj = BooleanComparisonFilter(operator="neq", value=False) + assert filter_obj.operator == EquivalenceOperator.NOT_EQUAL + assert filter_obj.value is False + + +def test_boolean_comparison_filter_rejects_invalid_operator(): + """BooleanComparisonFilter raises ValidationError for operator not in EquivalenceOperator.""" + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="gt", value=True) + + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="like", value=True) + + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="in", value=True) + + +def test_boolean_comparison_filter_rejects_invalid_value(): + """BooleanComparisonFilter raises ValidationError for non-bool value.""" + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="eq", value="not-a-bool") + + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="eq", value=42) + + +def test_boolean_comparison_filter_rejects_coercible_values(): + """BooleanComparisonFilter raises ValidationError for bool-coercible non-bool values. + + Mirrors the TS BooleanComparisonFilterSchema, where z.boolean() rejects + 1/0 and "true"/"false"; the strict annotation disables pydantic's lax + int/str -> bool coercion. + """ + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="eq", value=1) + + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="eq", value=0) + + with pytest.raises(ValidationError): + BooleanComparisonFilter(operator="eq", value="true") + + +@pytest.mark.parametrize("operator", ["in", "between", "like"]) +def test_number_comparison_filter_rejects_invalid_operator(operator): + """NumberComparisonFilter raises ValidationError for operators outside its surface. + + The two-enum validate_operator dispatch's fallthrough returns unknown + strings unchanged, relying on the enum-union field to reject them — the + rejection must hold. + """ + with pytest.raises(ValidationError): + NumberComparisonFilter(operator=operator, value=100) + + +def test_number_comparison_filter_accepts_equivalence_operators(): + """NumberComparisonFilter accepts eq/neq. + + The core spec (filters/numeric.tsp) widened the operator surface to + ComparisonOperators | EquivalenceOperators in protocol v0.3; the TS SDK + tracks it and this model must match. + """ + assert NumberComparisonFilter(operator="eq", value=3).operator == "eq" + assert NumberComparisonFilter(operator="neq", value=3.5).operator == "neq" + + +def test_number_comparison_filter_rejects_bool(): + """NumberComparisonFilter raises ValidationError for True/False. + + bool subclasses int, so the int|float union would lax-coerce True -> 1 + and ship a number for a boolean — the same wire corruption the + DefaultFilter.value widening fixed, one layer down. z.number() rejects + booleans, so rejection also keeps the SDKs aligned. + """ + with pytest.raises(ValidationError): + NumberComparisonFilter(operator="eq", value=True) + + with pytest.raises(ValidationError): + NumberComparisonFilter(operator="gt", value=False) + + +def test_number_array_filter_rejects_bool_items(): + """NumberArrayFilter raises ValidationError when any item is a bool. + + Same lax-coercion vector as NumberComparisonFilter: [True] would ship + as [1] without an explicit bool rejection. + """ + with pytest.raises(ValidationError): + NumberArrayFilter(operator=ArrayOperator.IN, value=[1, True, 3]) + + +def test_number_range_rejects_bool_bounds(): + """NumberRange raises ValidationError for a bool min or max. + + Same lax-coercion vector: {"min": True, "max": 10} would ship min=1. + """ + with pytest.raises(ValidationError): + NumberRange(min=True, max=10) + + with pytest.raises(ValidationError): + NumberRange(min=0, max=False) From d5b322b5e43ba66c4e994ed8a54010dbc1a3d36f Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 13:04:47 -0400 Subject: [PATCH 20/51] feat(py-sdk): add codegen-free schema extension framework Add the building blocks for declaring plugins without a build step: - schema(...) factory returning discriminated SchemaWithTransforms / SchemaOnly extensions (mappings XOR functions XOR schema-only), with CustomField[V] as the single source of truth for custom fields (field_type and inspectable value derived from V), CustomFieldSet / NoCustomFields, and the EXTENSIBLE_SCHEMA_MAP registry. - Generic Opportunity[CF] common model over its custom-fields container. - PluginCustomFieldSpec: the resolved/inspection view of a custom field. --- .../common_grants_sdk/extensions/schema.py | 393 ++++++++++++++++++ .../common_grants_sdk/extensions/specs.py | 31 +- .../schemas/pydantic/models/__init__.py | 2 + .../schemas/pydantic/models/opportunity.py | 36 ++ 4 files changed, 461 insertions(+), 1 deletion(-) create mode 100644 lib/python-sdk/common_grants_sdk/extensions/schema.py create mode 100644 lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py new file mode 100644 index 000000000..04e8033e4 --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -0,0 +1,393 @@ +"""Schema extensions: the ``schema(...)`` factory and its building blocks. + +Authors build each schema extension with the overloaded ``schema(...)`` factory, +which returns a discriminated ``SchemaWithTransforms`` or ``SchemaOnly``. The +overloads enforce, statically: mappings XOR hand-written transforms, a source when +transforms are present, and no ``to_common`` on schema-only entries. Registry +membership, custom-field consistency, and mapping output keys are validated when the +extension is built, aggregated into one error. + +``CustomField[V]`` is the single source of truth for a custom field: ``field_type`` +and the inspectable value type are derived from ``V``, so they cannot drift from the +typed declaration. The common models are generics over their custom-fields container +(``Opportunity[OpportunityFields]``), so consumers get concrete, non-optional types. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + Any, + Callable, + Generic, + Optional, + TypeGuard, + TypeVar, + get_args, + get_origin, + overload, +) + +from pydantic import ( + AliasGenerator, + BaseModel, + ConfigDict, + ValidationError, +) +from pydantic.alias_generators import to_camel + +from ..schemas.pydantic.fields.custom import CustomFieldType +from ..schemas.pydantic.models import Opportunity +from .specs import PluginCustomFieldSpec +from .transforms import build_transforms +from .types import TransformError, TransformResult + +__all__ = [ + "EXTENSIBLE_SCHEMA_MAP", + "CustomField", + "CustomFieldSet", + "NoCustomFields", + "PluginDefinitionError", + "SchemaOnly", + "SchemaWithTransforms", + "resolve_custom_field_specs", + "schema", + "validate_into", +] + +V = TypeVar("V") +TSource = TypeVar("TSource", bound=BaseModel) +TCommon = TypeVar("TCommon", bound=BaseModel) +T = TypeVar("T", bound=BaseModel) + + +class _CamelModel(BaseModel): + """Base model that is camelCase on the wire and snake_case in code. + + Authors and consumers use snake_case field names (``field_type``); JSON I/O uses + camelCase (``fieldType``). Construction by field name type-checks and validation + accepts either casing. + """ + + model_config = ConfigDict( + alias_generator=AliasGenerator( + validation_alias=to_camel, + serialization_alias=to_camel, + ), + populate_by_name=True, + ) + + +class CustomField(_CamelModel, Generic[V]): + """A single custom field: a typed ``value`` plus its metadata. + + ``CustomField[V]`` is the single source of truth -- ``field_type`` and the + inspectable value type are derived from ``V`` -- so authors declare a field as + ``Optional[CustomField[V]] = Field(default=None, description=...)`` and never + restate the type. + """ + + name: str + field_type: CustomFieldType + value: V + + +class CustomFieldSet(_CamelModel): + """Base class an author subclasses to declare a schema's custom fields.""" + + +class NoCustomFields(CustomFieldSet): + """Marker for schema entries that declare no custom fields.""" + + +# The extensible-schema registry: the closed set of models a plugin may extend, +# mapped to their generic base model. Add an entry when a new model gains +# custom-field support. +EXTENSIBLE_SCHEMA_MAP: dict[str, type[BaseModel]] = { + "Opportunity": Opportunity, +} +_BASE_TO_NAME: dict[type[BaseModel], str] = { + v: k for k, v in EXTENSIBLE_SCHEMA_MAP.items() +} + + +class PluginDefinitionError(Exception): + """Raised at definition (import) time, listing every problem at once.""" + + def __init__(self, cls_name: str, errors: list[str]) -> None: + self.errors = errors + body = "\n".join(f" - {e}" for e in errors) + super().__init__(f"{cls_name} is not a valid plugin definition:\n{body}") + + +def _is_model_class(obj: Any) -> TypeGuard[type[BaseModel]]: + return isinstance(obj, type) and issubclass(obj, BaseModel) + + +def _infer_field_type(value_type: Any) -> Optional[CustomFieldType]: + """Derive the JSON-schema ``field_type`` tag from a ``CustomField[V]`` value type. + + Returns ``None`` for value types that do not map to a ``CustomFieldType``. + """ + origin = get_origin(value_type) + if value_type is bool: # before int: bool is a subclass of int + return CustomFieldType.BOOLEAN + if value_type is int: + return CustomFieldType.INTEGER + if value_type is float: + return CustomFieldType.NUMBER + if value_type is str: + return CustomFieldType.STRING + if isinstance(value_type, type) and issubclass(value_type, BaseModel): + return CustomFieldType.OBJECT + if value_type is dict or origin is dict: + return CustomFieldType.OBJECT + if value_type is list or origin is list: + return CustomFieldType.ARRAY + return None + + +def _value_type(annotation: Any) -> Any: + """Pull ``V`` out of an ``Optional[CustomField[V]]`` annotation. + + ``CustomField[V]`` is a parameterized Pydantic generic (a concrete subclass), so + its argument lives in ``__pydantic_generic_metadata__`` rather than via + ``get_origin``/``get_args``. + """ + candidates = get_args(annotation) or (annotation,) + for cand in candidates: + meta = getattr(cand, "__pydantic_generic_metadata__", None) + if meta and meta.get("origin") is CustomField: + args = meta.get("args", ()) + return args[0] if args else None + if get_origin(cand) is CustomField: # non-concrete fallback + args = get_args(cand) + return args[0] if args else None + return None + + +def _check_custom_fields(container: type[BaseModel]) -> list[str]: + """Ensure each custom field's value type ``V`` maps to a known ``field_type``. + + Nothing is reconciled: ``field_type`` and the inspectable value type are derived + from ``CustomField[V]``, so the only failure is a ``V`` we cannot map. + """ + errors: list[str] = [] + for name, info in container.model_fields.items(): + value_type = _value_type(info.annotation) + if _infer_field_type(value_type) is None: + label = getattr(value_type, "__name__", value_type) + errors.append( + f"custom field {name!r}: cannot derive a field_type from value type " + f"{label}; use a supported CustomField[V]" + ) + return errors + + +def resolve_custom_field_specs( + container: Optional[type[BaseModel]], +) -> dict[str, PluginCustomFieldSpec]: + """Build the resolved spec for each custom field, for consumer inspection. + + Everything is derived from the single source of truth, ``CustomField[V]``: + ``field_type`` and the inspectable value type come from ``V``, ``name`` is the + attribute name, and ``description`` is the Pydantic field description. This is + what ``extension.custom_fields`` exposes; because nothing is authored twice, it + cannot drift. + """ + if not (isinstance(container, type) and issubclass(container, CustomFieldSet)): + return {} + out: dict[str, PluginCustomFieldSpec] = {} + for name, info in container.model_fields.items(): + value_type = _value_type(info.annotation) + out[name] = PluginCustomFieldSpec( + field_type=_infer_field_type(value_type), + value=value_type, + name=name, + description=info.description or "", + ) + return out + + +def validate_into(model: type[T], data: Any) -> TransformResult[T]: + """Validate ``data`` into ``model``, routing failures to ``errors``. + + The helper hand-written transform authors use so their ``to_common`` / + ``from_common`` return the validated model on success, or the raw data alongside + structured ``TransformError`` entries on failure. + """ + try: + return TransformResult(result=model.model_validate(data), errors=[]) + except ValidationError as exc: + errors = [ + TransformError(e["msg"], path=".".join(str(loc) for loc in e["loc"])) + for e in exc.errors() + ] + return TransformResult(result=data, errors=errors) + + +def _output_field_names(model: type[BaseModel]) -> set[str]: + """Valid top-level output keys for a model: field names plus their aliases.""" + names: set[str] = set(model.model_fields.keys()) + for info in model.model_fields.values(): + if info.alias: + names.add(info.alias) + return names + + +def _accepts_arbitrary_keys(model: type[BaseModel]) -> bool: + """True for permissive sources (PassthroughModel) where path checks are moot.""" + return model.model_config.get("extra") == "allow" + + +Mappings = dict[str, Any] + + +def _resolve_common(common: Any) -> tuple[Any, Any]: + """Return ``(origin, custom_fields_model)`` for a common type. + + A parameterized Pydantic generic (``Opportunity[OpportunityFields]``) is a + concrete subclass, so its origin/args live in ``__pydantic_generic_metadata__``. + """ + pyd_meta = getattr(common, "__pydantic_generic_metadata__", None) + if pyd_meta and pyd_meta.get("origin"): + args = pyd_meta.get("args", ()) + return pyd_meta["origin"], (args[0] if args else None) + return (get_origin(common) or common), (get_args(common)[:1] or (None,))[0] + + +@dataclass +class SchemaWithTransforms(Generic[TSource, TCommon]): + """A schema extension with transforms. Built by ``schema(...)``, never by hand. + + ``custom_fields`` exposes the resolved specs (field_type, value, name, + description) so consumers can inspect each declared custom field. + """ + + schema_name: str + common_schema: type[TCommon] + custom_fields: dict[str, PluginCustomFieldSpec] + source_schema: type[TSource] + to_common: Callable[[TSource], TransformResult[TCommon]] + from_common: Callable[[TCommon], TransformResult[TSource]] + + def parse(self, data: Any) -> TCommon: + return self.common_schema.model_validate(data) + + +@dataclass +class SchemaOnly(Generic[TCommon]): + """A schema extension with custom fields but no transforms. Built by ``schema(...)``. + + It deliberately has no ``to_common`` / ``from_common``, so a consumer cannot call + a transform on a schema-only entry -- that is a static error. ``custom_fields`` + exposes the resolved specs for consumer inspection. + """ + + schema_name: str + common_schema: type[TCommon] + custom_fields: dict[str, PluginCustomFieldSpec] + + def parse(self, data: Any) -> TCommon: + return self.common_schema.model_validate(data) + + +@overload +def schema( + *, source: type[TSource], common: type[TCommon], mappings: Mappings +) -> SchemaWithTransforms[TSource, TCommon]: ... +@overload +def schema( + *, + source: type[TSource], + common: type[TCommon], + to_common: Callable[[TSource], TransformResult[TCommon]], + from_common: Callable[[TCommon], TransformResult[TSource]], +) -> SchemaWithTransforms[TSource, TCommon]: ... +@overload +def schema(*, common: type[TCommon]) -> SchemaOnly[TCommon]: ... +def schema( + *, + source: Any = None, + common: Any, + mappings: Optional[Mappings] = None, + to_common: Any = None, + from_common: Any = None, +) -> Any: + """Build a schema extension. The overloads enforce, statically: + + - ``mappings`` XOR hand-written ``to_common`` / ``from_common`` (both = no match), + - a ``source`` is required whenever transforms are present, + - a schema-only entry (``common`` only) returns a ``SchemaOnly`` whose + type has no ``to_common`` (so consumers cannot transform it). + + Registry membership, custom-field consistency, and mapping output keys are + validated here at call (import) time, aggregated into one ``PluginDefinitionError``. + """ + errors: list[str] = [] + common_origin, custom_fields_model = _resolve_common(common) + + schema_name = _BASE_TO_NAME.get(common_origin) + if schema_name is None: + base_label = getattr(common_origin, "__name__", repr(common_origin)) + raise PluginDefinitionError( + "schema", + [ + f"common base {base_label} is not a registered extensible schema " + f"(expected one of {sorted(EXTENSIBLE_SCHEMA_MAP)})" + ], + ) + + if ( + custom_fields_model is not None + and isinstance(custom_fields_model, type) + and issubclass(custom_fields_model, CustomFieldSet) + ): + errors.extend(_check_custom_fields(custom_fields_model)) + + if mappings is not None: + for direction in ("to_common", "from_common"): + if direction not in mappings: + errors.append(f"mappings: missing `{direction}` direction") + if "to_common" in mappings and _is_model_class(common): + unknown = set(mappings["to_common"]) - _output_field_names(common) + if unknown: + errors.append( + f"mappings.to_common: unknown output field(s) {sorted(unknown)} " + f"for {common.__name__}" + ) + if ( + "from_common" in mappings + and _is_model_class(source) + and not _accepts_arbitrary_keys(source) + ): + unknown_src = set(mappings["from_common"]) - _output_field_names(source) + if unknown_src: + errors.append( + f"mappings.from_common: unknown source field(s) " + f"{sorted(unknown_src)} for {source.__name__}" + ) + + if errors: + raise PluginDefinitionError(schema_name, errors) + + custom_fields = resolve_custom_field_specs(custom_fields_model) + + if mappings is not None: + # Passing both schemas makes the compiled callables validate their output: + # to_common into the common model, from_common into the source model. So + # both directions return a validated instance (not a raw dict) on success. + to_fn, from_fn = build_transforms( + mappings["to_common"], + mappings["from_common"], + common_schema=common, + source_schema=source, + ) + return SchemaWithTransforms( + schema_name, common, custom_fields, source, to_fn, from_fn + ) + if to_common is not None: + return SchemaWithTransforms( + schema_name, common, custom_fields, source, to_common, from_common + ) + return SchemaOnly(schema_name, common, custom_fields) diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index 4b4124b57..a91dd925c 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -12,7 +12,13 @@ @dataclass class CustomFieldSpec: - """Custom Field spec class to support adding custom fields""" + """Runtime custom-field declaration consumed by the registration path. + + Used by ``utils.custom_fields.add_custom_fields`` (and + ``OpportunityBase.with_custom_fields``) to build a typed custom-fields container + at runtime. ``field_type`` is a required input here: it selects the value type + when ``value`` is not given, and is pinned on the resulting ``CustomField``. + """ field_type: CustomFieldType value: Optional[Any] = None @@ -20,6 +26,29 @@ class CustomFieldSpec: description: str = "" +@dataclass +class PluginCustomFieldSpec: + """Resolved, inspection-only view of a single custom field. + + Mirrors the field shape of :class:`CustomFieldSpec`, but authors never construct + it. The ``schema(...)`` factory produces it via ``resolve_custom_field_specs`` + from a ``CustomField[V]`` declaration on a ``CustomFieldSet``, and exposes it to + consumers through ``extension.custom_fields`` so they can introspect each field + without it ever drifting from the typed declaration: + + - ``field_type`` -- the JSON-schema tag derived from ``V`` (``str -> string``, + a Pydantic model -> ``object``, ...); ``None`` when ``V`` cannot be mapped. + - ``value`` -- the static value type ``V`` itself, for runtime inspection. + - ``name`` -- the attribute name on the container. + - ``description`` -- the Pydantic field description. + """ + + field_type: Optional[CustomFieldType] = None + value: Optional[Any] = None + name: str = "" + description: str = "" + + class SchemaExtensions(TypedDict, total=False): """Maps extensible model names to custom field specifications.""" diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py index 9c2ae819e..4fac17cda 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py @@ -4,8 +4,10 @@ from .opp_funding import OppFunding from .opp_status import OppStatus, OppStatusOptions from .opp_timeline import OppTimeline +from .opportunity import Opportunity __all__ = [ + "Opportunity", "OpportunityBase", "OppFunding", "OppStatus", diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py new file mode 100644 index 000000000..7cc9dcc8c --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py @@ -0,0 +1,36 @@ +"""Generic Opportunity model parameterized by its custom-fields container.""" + +from __future__ import annotations + +from typing import Generic, Optional, TypeVar + +from pydantic import ConfigDict, Field + +from .opp_base import OpportunityBase + +CF = TypeVar("CF") + + +class Opportunity(OpportunityBase, Generic[CF]): + """The common Opportunity model, generic over its custom-fields container. + + ``OpportunityBase`` carries every core field; this subclass overrides the + ``custom_fields`` slot with the author's typed container ``CF`` so consumers + get concrete, non-optional dot access (e.g. ``opp.custom_fields.agency_code``). + Unextended plugins use ``Opportunity[NoCustomFields]``. + + ``populate_by_name=True`` lets callers construct with snake_case field names + (``created_at=...``) while JSON I/O stays camelCase via the inherited aliases. + """ + + model_config = ConfigDict( + from_attributes=True, + strict=False, + populate_by_name=True, + ) + + custom_fields: Optional[CF] = Field( # type: ignore[assignment] + default=None, + alias="customFields", + description="Additional custom fields specific to this opportunity", + ) From e82858bb938afacfbd2167da359354ab4d18d8d8 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 13:05:15 -0400 Subject: [PATCH 21/51] refactor(py-sdk): rewire the public surface onto the new framework - Rewrite plugin.py around PluginSchemas / Plugin / define_plugin / PluginMeta, replacing PluginConfig / inject_transforms. - Trim the codegen-only types (SchemaInput / SchemaConfig / SchemaMappings) and rename PluginExtensionsMeta to PluginMeta for parity with the TS SDK. - build_transforms validates from_common output into the source model and skips output-path checks for permissive (extra=allow) sources. - Update extensions and top-level package exports. - Break the schemas->extensions import cycle via TYPE_CHECKING guards. --- lib/python-sdk/common_grants_sdk/__init__.py | 5 +- .../common_grants_sdk/extensions/__init__.py | 59 ++++-- .../common_grants_sdk/extensions/plugin.py | 194 ++++++------------ .../extensions/transforms.py | 24 ++- .../common_grants_sdk/extensions/types.py | 88 ++------ .../schemas/pydantic/models/opp_base.py | 8 +- .../common_grants_sdk/utils/custom_fields.py | 8 +- 7 files changed, 154 insertions(+), 232 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/__init__.py b/lib/python-sdk/common_grants_sdk/__init__.py index f70067807..9d2b9bad3 100644 --- a/lib/python-sdk/common_grants_sdk/__init__.py +++ b/lib/python-sdk/common_grants_sdk/__init__.py @@ -8,7 +8,7 @@ from . import schemas from .client import Auth, Client, Config -from .extensions import Plugin, PluginConfig, define_plugin +from .extensions import Plugin, PluginSchemas, define_plugin, schema __all__ = [ "schemas", @@ -16,6 +16,7 @@ "Auth", "Config", "Plugin", - "PluginConfig", + "PluginSchemas", "define_plugin", + "schema", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 0fb2c8148..c4d81f17a 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,34 +1,61 @@ """Public extension APIs for the CommonGrants Python SDK.""" -from .plugin import Plugin, PluginConfig, define_plugin, inject_transforms -from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions +from .plugin import ( + Plugin, + PluginMeta, + PluginSchemas, + define_plugin, +) +from .schema import ( + EXTENSIBLE_SCHEMA_MAP, + CustomField, + CustomFieldSet, + NoCustomFields, + PluginDefinitionError, + SchemaOnly, + SchemaWithTransforms, + resolve_custom_field_specs, + schema, + validate_into, +) +from .specs import ( + ConflictStrategy, + CustomFieldSpec, + PluginCustomFieldSpec, + SchemaExtensions, +) from .transforms import build_transforms from .types import ( Handler, + PassthroughModel, PluginCapability, - PluginExtensionsMeta, - SchemaConfig, - SchemaInput, - SchemaMappings, TransformError, TransformResult, ) __all__ = [ + "EXTENSIBLE_SCHEMA_MAP", "ConflictStrategy", + "CustomField", + "CustomFieldSet", "CustomFieldSpec", - "Plugin", - "PluginConfig", - "SchemaExtensions", - "define_plugin", - "inject_transforms", - "build_transforms", "Handler", + "NoCustomFields", + "PassthroughModel", + "Plugin", "PluginCapability", - "PluginExtensionsMeta", - "SchemaConfig", - "SchemaInput", - "SchemaMappings", + "PluginCustomFieldSpec", + "PluginDefinitionError", + "PluginMeta", + "PluginSchemas", + "SchemaExtensions", + "SchemaOnly", "TransformError", + "SchemaWithTransforms", "TransformResult", + "build_transforms", + "define_plugin", + "resolve_custom_field_specs", + "schema", + "validate_into", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 49cd62d26..da4b01484 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -1,156 +1,92 @@ -"""Plugin configuration and composition APIs.""" +"""Plugin assembly: ``PluginSchemas``, ``Plugin``, and ``define_plugin``. -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Generic, TypeVar, overload - -from .types import PluginExtensionsMeta +A plugin maps the schema extensions an author builds with ``schema(...)`` onto the +registered extensible schemas, keyed by registry name. Schemas a plugin does not +extend fall back to the base schema (a ``SchemaOnly``), never ``None``, so +consumers get fully-typed, non-optional dot access: ``plugin.schemas.Opportunity``. +""" -T = TypeVar("T") -TSchemas = TypeVar("TSchemas") -_TSchemasContainer = TypeVar("_TSchemasContainer") +from __future__ import annotations +from dataclasses import dataclass, field, fields +from typing import Any, Generic, TypeVar, cast -@dataclass(frozen=True) -class PluginConfig(Generic[TSchemas]): - """Build-time plugin config produced by define_plugin() and consumed by generate.py. +import typing_extensions as te - Generic on TSchemas so the precise type of the schemas dict is preserved — e.g. - PluginConfig[dict[str, SchemaInput[MyNative, MyCg]]] — rather than being - widened to SchemaInput[Any, Any] at the storage boundary. +from ..schemas.pydantic.models import Opportunity +from .schema import ( + NoCustomFields, + PluginDefinitionError, + SchemaOnly, + SchemaWithTransforms, + schema, +) +from .types import PluginMeta - Stores inputs as-is — no compilation occurs at define_plugin() call time. - generate.py compiles this into a fully resolved Plugin by injecting the generated - Pydantic model class as the common schema for each SchemaInput entry, and - auto-generating build_transforms() calls for any object that has - schemas[obj].mappings but no explicit schemas[obj].to_common / from_common. +SchemasT = TypeVar("SchemasT") - All fields are optional so adopters can start with only what they need. - """ +# The fallback for a schema a plugin does not extend: the base schema, no custom +# fields, no transforms. A SchemaOnly type, so unextended slots have no +# to_common either. +DefaultOpportunity = SchemaOnly[Opportunity[NoCustomFields]] - meta: PluginExtensionsMeta | None = None - schemas: TSchemas | None = None +_TOpportunity = te.TypeVar("_TOpportunity", default=DefaultOpportunity) @dataclass -class Plugin(Generic[T]): - """Runtime plugin container assembled by generate.py after code generation. - - schemas: the _Schemas object from generated/schemas.py. Each attribute is a - SchemaConfig instance providing unified access to the model class and - transforms for that object: - plugin.schemas.Opportunity.common_schema → the Pydantic model class (includes - any custom fields declared by the plugin) - plugin.schemas.Opportunity.to_common → transform callable (or None) - plugin.schemas.Opportunity.from_common → transform callable (or None) - plugin.schemas.Opportunity.source_schema → the source system's type (or dict) - """ - - schemas: T - meta: PluginExtensionsMeta | None = None - - -@overload -def define_plugin( - meta: PluginExtensionsMeta | None = ..., - schemas: None = ..., -) -> PluginConfig[None]: ... - +class PluginSchemas(Generic[_TOpportunity]): + """Maps your extensions to the extensible schemas. Construct it directly. -@overload -def define_plugin( - meta: PluginExtensionsMeta | None = ..., - schemas: TSchemas = ..., -) -> PluginConfig[TSchemas]: ... + Pass one extension per schema you extend, keyed by the registered schema name. + Schemas you omit fall back to the base schema (a ``SchemaOnly``), never + ``None``. Unknown schema names are a *static* error, and each slot's type is + inferred concretely, so consumers get non-optional dot access:: + plugin = define_plugin(PluginSchemas(Opportunity=opp_ext), meta=...) + plugin.schemas.Opportunity # the extension you passed -def define_plugin( - meta: PluginExtensionsMeta | None = None, - schemas: Any = None, -) -> PluginConfig[Any]: - """Create a PluginConfig consumed by the code generator. - - No compilation occurs here — inputs are stored as-is. The code generator - (generate.py) compiles SchemaInput → SchemaConfig by injecting - the common model from the generated schemas, and auto-wires build_transforms() - for any object that has schemas[obj].mappings but no explicit callables. - - The return type is generic on the schemas argument: passing a typed dict - (e.g. {"Opportunity": SchemaInput[MyNative, MyCg](...) }) preserves - those per-object generics on the returned PluginConfig rather than widening - them to Any. - - Raises: - ValueError: If any schema entry specifies both mappings and explicit - to_common/from_common callables (XOR constraint). + There is one field per registered extensible schema. """ - if schemas: - for obj_name, schema_input in schemas.items(): - has_mappings = schema_input.mappings is not None - has_callables = ( - schema_input.to_common is not None - or schema_input.from_common is not None - ) - if has_mappings and has_callables: - raise ValueError( - f"define_plugin: {obj_name} cannot specify both mappings and explicit " - f"to_common/from_common. " - f"Use mappings for declarative transforms or provide explicit callables, not both." - ) - return PluginConfig( - meta=meta, - schemas=schemas, + + Opportunity: _TOpportunity = field( + default_factory=lambda: cast( + _TOpportunity, schema(common=Opportunity[NoCustomFields]) + ) ) -def inject_transforms( - config: PluginConfig[Any], schemas: _TSchemasContainer -) -> _TSchemasContainer: - """Wire transform callables from plugin config into the generated schemas container. +@dataclass(frozen=True) +class Plugin(Generic[SchemasT]): + """The plugin singleton consumers import. - Called by the generated plugin __init__.py to inject to_common/from_common - callables (and the native type) from cg_config into the SchemaConfig instances - produced by the code generator. + ``schemas`` is a typed frozen dataclass, so ``plugin.schemas.Opportunity`` is + fully typed (dot access). + """ - Iterates over all entries in config.schemas that have at least one callable, - validates that both directions are present, then sets the attributes on the - matching schemas container attribute (e.g. schemas.Opportunity). + schemas: SchemasT + meta: PluginMeta - Returns the same schemas container (mutated in place) so callers can write - ``schemas = inject_transforms(config, schemas)`` and retain the concrete - generated type rather than widening to Any. - Args: - config: The PluginConfig produced by define_plugin(). - schemas: The generated _Schemas container from generated/schemas.py. +def define_plugin(schemas: SchemasT, *, meta: PluginMeta) -> Plugin[SchemasT]: + """Assemble the plugin from a ``PluginSchemas`` instance and metadata. - Returns: - The same schemas container, with transform callables injected. + Each attribute name must equal the entry's ``schema_name``, so + ``schemas.Opportunity`` really holds the Opportunity extensible schema. Raises: - ValueError: If a schema with any callable is missing its counterpart, - or if the object name is not found in the schemas container. + PluginDefinitionError: If any slot does not hold a schema extension, or holds + one whose ``schema_name`` does not match its attribute name. """ - if not config.schemas: - return schemas - for obj_name, schema_input in config.schemas.items(): - if schema_input.to_common is None and schema_input.from_common is None: - continue - obj_schemas = getattr(schemas, obj_name, None) - if obj_schemas is None: - raise ValueError( - f"Plugin object {obj_name!r}: not found in generated schemas" - ) - if schema_input.to_common is None: - raise ValueError( - f"Plugin object {obj_name!r}: to_common callable is required" - ) - if schema_input.from_common is None: - raise ValueError( - f"Plugin object {obj_name!r}: from_common callable is required" + errors: list[str] = [] + for fld in fields(cast(Any, schemas)): + entry = getattr(schemas, fld.name) + if not isinstance(entry, (SchemaWithTransforms, SchemaOnly)): + errors.append(f"schemas.{fld.name}: not a schema extension") + elif entry.schema_name != fld.name: + errors.append( + f"schemas.{fld.name}: holds the {entry.schema_name!r} extensible " + f"schema; the attribute name must match the schema name" ) - obj_schemas.source_schema = schema_input.source_schema or dict - obj_schemas.to_common = schema_input.to_common - obj_schemas.from_common = schema_input.from_common - return schemas + if errors: + raise PluginDefinitionError("plugin", errors) + return Plugin(schemas=schemas, meta=meta) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index b4099a1ad..c63151047 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -34,12 +34,18 @@ def _validate_output_paths( ) -> None: """Validate that top-level output keys in mapping are valid fields on model. - Called when common_schema is supplied to build_transforms(). Custom fields - declared by the plugin appear as regular model fields on the generated common + Called when common_schema or source_schema is supplied to build_transforms(). + Custom fields declared by the plugin appear as regular model fields on the common model and are therefore treated as valid output paths automatically. + Models configured with ``extra="allow"`` (e.g. ``PassthroughModel``) accept + arbitrary keys, so every output key is valid and the check is skipped. + Raises ValueError if any top-level key is not a field name or alias on model. """ + if model.model_config.get("extra") == "allow": + return + valid_names: set[str] = set(model.model_fields.keys()) for field_info in model.model_fields.values(): if field_info.alias: @@ -53,7 +59,7 @@ def _validate_output_paths( raise ValueError( f"build_transforms ({direction}_mapping): unknown output {noun} " f"{sorted(invalid)!r} for model {model.__name__}. " - f"Declare them as custom_fields in SchemaInput or check the field name." + f"Declare them on the schema's CustomFieldSet or check the field name." ) @@ -189,13 +195,11 @@ def handle_upper(data, path): handlers={"upper": handle_upper}, ) common_schema: Optional Pydantic model class to validate the to_common output - against. Must be the fully extended generated model class (e.g. the - generated Opportunity from generated/schemas.py), NOT the base class - (e.g. OpportunityBase). Passing a base class will silently weaken - validation — custom_fields will only be checked against - dict[str, CustomField] rather than the typed container produced by the - plugin's custom field declarations. When provided, model_validate is - called on the transform result and any ValidationErrors are appended to + against. Pass the model that carries the plugin's custom fields (e.g. + ``Opportunity[OpportunityFields]``), not the base ``OpportunityBase``: + a base class only checks custom_fields against dict[str, CustomField] + rather than the typed container. When provided, model_validate is called + on the transform result and any ValidationErrors are appended to TransformResult.errors rather than raised. Note on result shape: when common_schema is set, TransformResult.result diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 493bba1eb..3d88d6f47 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -7,10 +7,6 @@ from pydantic import BaseModel, ConfigDict, Field -from .specs import CustomFieldSpec - -TSource = TypeVar("TSource") -TCommon = TypeVar("TCommon") T = TypeVar("T") # Capability enum — Literal rather than StrEnum to stay JSON-safe. @@ -20,6 +16,19 @@ Handler = Callable[[Any, Any], Any] +class PassthroughModel(BaseModel): + """Permissive source schema, which preserves the transformation result dict as is. + + Validates only that the input is a mapping and preserves arbitrary keys + (``extra="allow"``) without constraining any field. Use it as the + ``source_schema`` on a transform entry to satisfy the source-schema + requirement without modeling the source-system shape (e.g. in tests or + early development). + """ + + model_config = ConfigDict(extra="allow") + + class TransformError(Exception): """Structured transformation error per ADR-0022 Decision #9. @@ -63,20 +72,7 @@ class TransformResult(Generic[T]): errors: list[TransformError] -class SchemaMappings(BaseModel): - """ADR-0017 declarative mapping dicts for a single object. - - Each direction is author-provided — build_transforms() does not invert one into - the other because many-to-one handlers like switch are not reversible (Decision #6). - """ - - model_config = ConfigDict(populate_by_name=True) - - to_common: dict[str, Any] | None = Field(default=None, alias="toCommon") - from_common: dict[str, Any] | None = Field(default=None, alias="fromCommon") - - -class PluginExtensionsMeta(BaseModel): +class PluginMeta(BaseModel): """Plugin identity and capability declaration. name and source_system are required so that plugin registries and @@ -88,58 +84,8 @@ class PluginExtensionsMeta(BaseModel): model_config = ConfigDict(populate_by_name=True) name: str - source_system: str = Field(alias="sourceSystem") + source_system: str = Field( + validation_alias="sourceSystem", serialization_alias="sourceSystem" + ) version: str | None = None capabilities: list[PluginCapability] | None = None - - -@dataclass -class SchemaInput(Generic[TSource, TCommon]): - """Input type provided by plugin authors inside define_plugin(schemas=...). - - This is the single surface for all per-object declarations. Plugin authors supply - to_common and from_common as plain callables — either hand-written or generated - via build_transforms(). source_schema defaults to dict[str, Any] if omitted. - - custom_fields declares any extra fields this object exposes beyond the base - CommonGrants schema. The code generator reads these and emits typed subclasses. - - mappings holds optional declarative mappings. When present and no - explicit to_common / from_common is supplied, the code generator auto-invokes - build_transforms() on these. Explicit callables take priority and disable - auto-wiring for that object. - - common_schema is intentionally absent here. It is injected by define_plugin() during - compilation from SchemaInput → SchemaConfig, resolved from the generated - model classes produced by the code generator. Plugin authors never set it directly — - cg_config.py cannot import from generated/ (it is the input to generation). - """ - - source_schema: type[TSource] | None = None - custom_fields: dict[str, CustomFieldSpec] | None = None - mappings: SchemaMappings | None = None - to_common: Callable[[TSource], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TSource]] | None = None - - -@dataclass -class SchemaConfig(Generic[TSource, TCommon]): - """Runtime compiled schema container for a single object (ADR-0022). - - Bundles the type information and transform callables for one schema object - (e.g. Opportunity). Accessed via attribute lookup on the plugin's schemas - container: plugin.schemas.Opportunity. - - source_schema: The source system's Python type (defaults to dict when not specified). - common_schema: The CommonGrants-format Pydantic model class produced by the generator. - If the plugin declares custom_fields, this is a generated subclass of - the base CG model (e.g. OpportunityBase) with those fields already - baked in as typed attributes. - to_common: Transforms source_data → TransformResult[common_schema] (None if not configured). - from_common: Transforms common_data → TransformResult[source_schema] (None if not configured). - """ - - source_schema: type[TSource] - common_schema: type[TCommon] - to_common: Callable[[TSource], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TSource]] | None = None diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py index eac3bc183..689357eb2 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py @@ -1,6 +1,8 @@ """Base model for funding opportunities.""" -from typing import Any, Optional, Type, TypeVar +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar from uuid import UUID from pydantic import Field, HttpUrl @@ -14,7 +16,9 @@ add_custom_fields, get_custom_field_value, ) -from common_grants_sdk.extensions.specs import CustomFieldSpec + +if TYPE_CHECKING: + from common_grants_sdk.extensions.specs import CustomFieldSpec V = TypeVar("V") # Unbound to support both BaseModel subclasses and primitives diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index 05e5ba6c6..c98db17ef 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -1,9 +1,13 @@ -from typing import Optional, Any, Type, TypeVar +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional, Any, Type, TypeVar from pydantic import BaseModel, Field, create_model, ConfigDict, model_validator from ..schemas.pydantic.fields import CustomField, CustomFieldType from ..schemas.pydantic.base import CommonGrantsBaseModel from common_grants_sdk.utils.json import snake -from common_grants_sdk.extensions.specs import CustomFieldSpec + +if TYPE_CHECKING: + from common_grants_sdk.extensions.specs import CustomFieldSpec T = TypeVar("T", bound=BaseModel) # For add_custom_fields V = TypeVar("V") # For get_custom_field_value (unbound to support primitives) From a8fdb7eccafbb2eaca9b4305c6fb263af8279553 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 13:05:16 -0400 Subject: [PATCH 22/51] refactor(py-sdk): port examples and tests to the new framework - Add examples/plugins.py: one file that defines every authoring scenario (mappings, hand-written functions, schema-only, and a combined grants.gov plugin) and validates them, with assert_type consumer lines. - Add test_schema.py covering the schema(...) factory, custom-field resolution, negative cases, and the camelCase round-trip. - Port test_plugin / test_types / test_plugin_registry to the new surface. --- lib/python-sdk/examples/plugins.py | 540 ++++++++++++++++++ .../tests/extensions/test_plugin.py | 199 +++---- .../tests/extensions/test_schema.py | 299 ++++++++++ lib/python-sdk/tests/extensions/test_types.py | 76 +-- .../tests/schemas/test_plugin_registry.py | 75 +-- 5 files changed, 935 insertions(+), 254 deletions(-) create mode 100644 lib/python-sdk/examples/plugins.py create mode 100644 lib/python-sdk/tests/extensions/test_schema.py diff --git a/lib/python-sdk/examples/plugins.py b/lib/python-sdk/examples/plugins.py new file mode 100644 index 000000000..ce2f3dce1 --- /dev/null +++ b/lib/python-sdk/examples/plugins.py @@ -0,0 +1,540 @@ +"""Plugin authoring examples -- every scenario in one module. + +Each scenario builds a schema extension with the overloaded ``schema(...)`` factory +(mappings XOR hand-written functions XOR schema-only, enforced statically), then +assembles a ``Plugin`` with ``define_plugin``. ``CustomField[V]`` is the single +source of truth for custom fields: ``field_type`` is derived from ``V``. A plugin is +just the ``Plugin`` ``define_plugin`` returns, so each scenario is a self-contained +plugin in this one file. + +Run the consumer side with ``poetry run python examples/scenarios.py``. +""" + +from __future__ import annotations + +from typing import Any, Callable, Optional, cast + +from pydantic import BaseModel, Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + NoCustomFields, + PassthroughModel, + PluginMeta, + PluginSchemas, + TransformResult, + build_transforms, + define_plugin, + schema, + validate_into, +) +from common_grants_sdk.schemas.pydantic.models import Opportunity +from common_grants_sdk.utils.transformation import get_from_path + +# --- Author-declared custom-field containers ---------------------------------- + + +class AgencyFields(CustomFieldSet): + """The single custom field shared by the mappings and functions scenarios.""" + + agency_code: Optional[CustomField[str]] = Field( + default=None, description="Agency code carried from grants.gov." + ) + + +class ExtensionFields(CustomFieldSet): + """HHS- and locality-specific custom fields (schema-only scenario).""" + + program_area: Optional[CustomField[str]] = Field( + default=None, description="HHS program area code (e.g. 'CFDA-93.243')" + ) + legacy_grant_id: Optional[CustomField[int]] = Field( + default=None, description="Numeric ID from the legacy grants management system" + ) + eligibility_types: Optional[CustomField[list[str]]] = Field( + default=None, description="Types of organizations eligible to apply" + ) + award_ceiling: Optional[CustomField[float]] = Field( + default=None, description="Maximum award amount in USD" + ) + + +class GrantsGovFields(CustomFieldSet): + """Custom fields grants.gov carries on the Opportunity schema.""" + + legacy_id: Optional[CustomField[int]] = Field( + default=None, description="Unique identifier in legacy database" + ) + legacy_id_str: Optional[CustomField[str]] = Field( + default=None, description="Legacy ID coerced to a string via numberToString" + ) + agency_name: Optional[CustomField[str]] = Field( + default=None, description="Agency hosting the opportunity" + ) + applicant_types: Optional[CustomField[list[str]]] = Field( + default=None, description="Types of applicants eligible to apply" + ) + priority_score: Optional[CustomField[float]] = Field( + default=None, description="Numeric priority score coerced from a string" + ) + composite_label: Optional[CustomField[str]] = Field( + default=None, + description="Composite '<opportunity_number> -- <opportunity_title>' label", + ) + + +# --- Scenario 1: custom fields + declarative mappings ------------------------- + +mappings_plugin = define_plugin( + PluginSchemas( + Opportunity=schema( + source=PassthroughModel, + common=Opportunity[AgencyFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "description": {"field": "opportunity_description"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } + } + }, + "customFields": { + "agencyCode": { + "value": {"field": "agency_code"}, + "name": {"const": "agencyCode"}, + "fieldType": {"const": "string"}, + } + }, + }, + "from_common": { + "opportunity_uuid": {"field": "id"}, + "opportunity_title": {"field": "title"}, + "opportunity_status": {"const": "posted"}, + "agency_code": {"field": "customFields.agencyCode.value"}, + }, + }, + ) + ), + meta=PluginMeta(name="grants.gov (mappings)", source_system="grants.gov"), +) + + +# --- Scenario 2: custom fields + hand-written functions ----------------------- + + +class GrantsGovOpportunity(BaseModel): + """The grants.gov source shape (validated on the from_common output).""" + + opportunity_uuid: str + opportunity_title: str + opportunity_description: str + opportunity_status: str + created_at: str + last_modified_at: str + agency_code: str + + +def gadget_to_common( + source: GrantsGovOpportunity, +) -> TransformResult[Opportunity[AgencyFields]]: + return validate_into( + Opportunity[AgencyFields], + { + "id": source.opportunity_uuid, + "title": source.opportunity_title, + "description": source.opportunity_description, + "status": { + "value": "open" if source.opportunity_status == "posted" else "custom" + }, + "createdAt": source.created_at, + "lastModifiedAt": source.last_modified_at, + "customFields": { + "agencyCode": { + "name": "agencyCode", + "fieldType": "string", + "value": source.agency_code, + } + }, + }, + ) + + +def gadget_from_common( + common: Opportunity[AgencyFields], +) -> TransformResult[GrantsGovOpportunity]: + # The make-or-break path: `common` is fully typed (agency_code.value -> str). + agency = "" + if common.custom_fields and common.custom_fields.agency_code: + agency = common.custom_fields.agency_code.value + return validate_into( + GrantsGovOpportunity, + { + "opportunity_uuid": str(common.id), + "opportunity_title": common.title, + "opportunity_description": common.description, + "opportunity_status": "posted", + "created_at": common.created_at.isoformat(), + "last_modified_at": common.last_modified_at.isoformat(), + "agency_code": agency, + }, + ) + + +functions_plugin = define_plugin( + PluginSchemas( + Opportunity=schema( + source=GrantsGovOpportunity, + common=Opportunity[AgencyFields], + to_common=gadget_to_common, + from_common=gadget_from_common, + ) + ), + meta=PluginMeta(name="grants.gov (functions)", source_system="grants.gov"), +) + + +# --- Scenario 3: declarative mappings, no custom fields ----------------------- + +base_plugin = define_plugin( + PluginSchemas( + Opportunity=schema( + source=PassthroughModel, + common=Opportunity[NoCustomFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "description": {"field": "opportunity_description"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } + } + }, + }, + "from_common": { + "opportunity_uuid": {"field": "id"}, + "opportunity_title": {"field": "title"}, + "opportunity_status": {"const": "posted"}, + }, + }, + ) + ), + meta=PluginMeta(name="grants.gov (no custom fields)", source_system="grants.gov"), +) + + +# --- Scenario 4: custom fields only, no transforms ---------------------------- + +opportunity_extensions = define_plugin( + PluginSchemas(Opportunity=schema(common=Opportunity[ExtensionFields])), + meta=PluginMeta(name="opportunity extensions", source_system="hhs"), +) + + +# --- Realistic combined plugin: grants.gov ------------------------------------ + + +def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + sep = spec.get("sep", " ") + parts = [get_from_path(data, path) for path in spec.get("fields", [])] + values = [str(p) for p in parts if p is not None] + return sep.join(values) if values else None + + +# Both directions are author-provided -- build_transforms() does not invert one into +# the other because many-to-one handlers like ``match`` are not reversible. +_gg_to_common, _gg_from_common = build_transforms( + handlers={"join": _join_fields}, + common_schema=Opportunity[GrantsGovFields], + source_schema=PassthroughModel, + to_common_mapping={ + "id": {"field": "data.opportunity_uuid"}, + "title": {"field": "data.opportunity_title"}, + "description": {"field": "data.opportunity_description"}, + "createdAt": {"field": "data.created_at"}, + "lastModifiedAt": {"field": "data.last_modified_at"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": { + "forecasted": "forecasted", + "posted": "open", + "archived": "closed", + }, + "default": "custom", + } + }, + "description": { + "const": "The opportunity is currently accepting applications" + }, + }, + "funding": { + "minAwardAmount": { + "amount": {"numberToString": "data.summary.award_floor"}, + "currency": {"const": "USD"}, + }, + "maxAwardAmount": { + "amount": {"numberToString": "data.summary.award_ceiling"}, + "currency": {"const": "USD"}, + }, + }, + "keyDates": { + "appOpens": { + "name": {"const": "Open Date"}, + "date": {"field": "data.summary.forecasted_post_date"}, + "description": {"const": "Applications begin being accepted"}, + }, + "appDeadline": { + "name": {"const": "Application Deadline"}, + "date": {"field": "data.summary.forecasted_close_date"}, + "description": { + "const": "Final submission deadline for all grant applications" + }, + }, + }, + "customFields": { + "legacyIdStr": { + "value": {"numberToString": "data.opportunity_id"}, + "name": {"const": "legacyIdStr"}, + "fieldType": {"const": "string"}, + }, + "priorityScore": { + "value": {"stringToNumber": "data.priority_score_str"}, + "name": {"const": "priorityScore"}, + "fieldType": {"const": "number"}, + }, + "compositeLabel": { + "value": { + "join": { + "fields": ["data.opportunity_number", "data.opportunity_title"], + "sep": " -- ", + } + }, + "name": {"const": "compositeLabel"}, + "fieldType": {"const": "string"}, + }, + }, + }, + from_common_mapping={ + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "match": { + "field": "status.value", + "case": { + "open": "posted", + "closed": "archived", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "summary": { + "award_floor": {"stringToNumber": "funding.minAwardAmount.amount"}, + "award_ceiling": {"stringToNumber": "funding.maxAwardAmount.amount"}, + "forecasted_post_date": {"field": "keyDates.appOpens.date"}, + "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, + }, + "priority_score_str": { + "numberToString": "customFields.priorityScore.value" + }, + } + }, +) + +# build_transforms returns loosely-typed callables (its output may be a dict on a +# validation error). Restate the precise signatures for the typed consumer surface. +gg_to_common = cast( + Callable[[PassthroughModel], TransformResult[Opportunity[GrantsGovFields]]], + _gg_to_common, +) +gg_from_common = cast( + Callable[[Opportunity[GrantsGovFields]], TransformResult[PassthroughModel]], + _gg_from_common, +) + +grants_gov = define_plugin( + PluginSchemas( + Opportunity=schema( + source=PassthroughModel, + common=Opportunity[GrantsGovFields], + to_common=gg_to_common, + from_common=gg_from_common, + ) + ), + meta=PluginMeta( + name="grants-gov", + version="0.1.0", + source_system="grants.gov", + capabilities=["customFields", "transforms"], + ), +) + + +# --- Consumer side: validate each plugin in this same file -------------------- +# +# Consumers use non-optional dot access on ``plugin.schemas.Opportunity``. The +# ``assert_type`` lines document the concrete static types each plugin yields. +# Run with: poetry run python examples/plugins.py + +from typing import assert_type # noqa: E402 + +from common_grants_sdk.extensions import SchemaOnly # noqa: E402 + +_FLAT_SOURCE = { + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "opportunity_title": "Conservation research", + "opportunity_description": "Funding for conservation.", + "opportunity_status": "posted", + "created_at": "2025-01-01T00:00:00Z", + "last_modified_at": "2025-01-01T00:00:00Z", + "agency_code": "HHS-123", +} + +_GRANTS_GOV_SOURCE = { + "data": { + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "opportunity_id": 12345, + "opportunity_number": "ABC-123-XYZ-001", + "opportunity_title": "Research into conservation techniques", + "opportunity_description": "Funding to advance conservation research.", + "opportunity_status": "posted", + "created_at": "2025-01-15T09:00:00Z", + "last_modified_at": "2025-04-01T12:30:00Z", + "priority_score_str": "75", + "summary": { + "award_floor": 10000, + "award_ceiling": 100000, + "forecasted_post_date": "2025-05-01", + "forecasted_close_date": "2025-07-15", + }, + } +} + +_SCHEMA_ONLY_RECORD = { + "id": "573525f2-8e15-4405-83fb-e6523511d893", + "title": "Community Health Innovation Grant", + "status": {"value": "open"}, + "description": "Funding for community-led health initiatives", + "createdAt": "2025-03-01T00:00:00Z", + "lastModifiedAt": "2025-03-15T00:00:00Z", + "customFields": { + "legacyGrantId": { + "name": "legacyGrantId", + "fieldType": "integer", + "value": 98765, + } + }, +} + + +def _check(label: str, ok: bool) -> None: + print(f" [{'PASS' if ok else 'FAIL'}] {label}") + + +def main() -> None: + # Scenario 1 -- custom fields + declarative mappings + print("Scenario 1 -- custom fields + mappings") + res1 = mappings_plugin.schemas.Opportunity.to_common( + PassthroughModel.model_validate(_FLAT_SOURCE) + ) + assert_type(res1, TransformResult[Opportunity[AgencyFields]]) + opp1 = res1.result + _check("no transform errors", res1.errors == []) + _check("title mapped", opp1.title == "Conservation research") + cf1 = opp1.custom_fields + if cf1 and cf1.agency_code: + assert_type(cf1.agency_code.value, str) + _check( + "agency_code.value typed str == 'HHS-123'", + cf1.agency_code.value == "HHS-123", + ) + specs = mappings_plugin.schemas.Opportunity.custom_fields + _check( + "inspect: agency_code field_type derived STRING", + specs["agency_code"].field_type is not None + and specs["agency_code"].field_type.value == "string", + ) + back1 = mappings_plugin.schemas.Opportunity.from_common(opp1) + _check("round-trips (validated source instance)", not back1.errors) + + # Scenario 2 -- custom fields + hand-written functions + print("Scenario 2 -- custom fields + hand-written functions") + res2 = functions_plugin.schemas.Opportunity.to_common( + GrantsGovOpportunity(**_FLAT_SOURCE) + ) + g = res2.result + _check("title mapped", g.title == "Conservation research") + if g.custom_fields and g.custom_fields.agency_code: + assert_type(g.custom_fields.agency_code.value, str) + _check( + "agency_code.value == 'HHS-123'", + g.custom_fields.agency_code.value == "HHS-123", + ) + back2 = functions_plugin.schemas.Opportunity.from_common(g) + assert_type(back2, TransformResult[GrantsGovOpportunity]) + _check("from_common -> typed source", back2.result.agency_code == "HHS-123") + + # Scenario 3 -- declarative mappings, no custom fields + print("Scenario 3 -- mappings, no custom fields") + res3 = base_plugin.schemas.Opportunity.to_common( + PassthroughModel.model_validate(_FLAT_SOURCE) + ) + _check("title mapped", res3.result.title == "Conservation research") + + # Scenario 4 -- custom fields only, no transforms (schema-only) + print("Scenario 4 -- custom fields only, no transforms") + ext = opportunity_extensions.schemas.Opportunity + assert_type(ext, SchemaOnly[Opportunity[ExtensionFields]]) + parsed = ext.parse(_SCHEMA_ONLY_RECORD) + if parsed.custom_fields and parsed.custom_fields.legacy_grant_id: + assert_type(parsed.custom_fields.legacy_grant_id.value, int) + _check( + "schema-only legacy_grant_id.value typed int == 98765", + parsed.custom_fields.legacy_grant_id.value == 98765, + ) + print(" inspect: resolved specs") + for name, spec in ext.custom_fields.items(): + print(f" {name}: {spec.field_type} -- {spec.description}") + + # Realistic combined plugin -- grants.gov (custom handler + round-trip) + print("grants.gov -- custom fields + transform with a custom handler") + res5 = grants_gov.schemas.Opportunity.to_common( + PassthroughModel.model_validate(_GRANTS_GOV_SOURCE) + ) + _check("no transform errors", not res5.errors) + gg = res5.result + _check("title mapped", gg.title == "Research into conservation techniques") + if gg.custom_fields and gg.custom_fields.composite_label: + _check( + "compositeLabel joined via custom handler", + gg.custom_fields.composite_label.value + == "ABC-123-XYZ-001 -- Research into conservation techniques", + ) + back5 = grants_gov.schemas.Opportunity.from_common(gg) + assert_type(back5, TransformResult[PassthroughModel]) + native = back5.result.model_dump(by_alias=True) + _check( + "from_common -> validated source instance", + native["data"]["opportunity_title"] == "Research into conservation techniques", + ) + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index 072ee1306..211dc26c5 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -1,154 +1,87 @@ -"""Tests for plugin.py — Plugin and PluginConfig API.""" +"""Tests for plugin.py -- PluginSchemas / Plugin / define_plugin assembly.""" -import pytest +from typing import Optional -from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin -from common_grants_sdk.extensions.types import ( - SchemaInput, - PluginExtensionsMeta, - TransformResult, +import pytest +from pydantic import Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + NoCustomFields, + PassthroughModel, + Plugin, + PluginMeta, + PluginSchemas, + SchemaOnly, + SchemaWithTransforms, + define_plugin, + schema, ) +from common_grants_sdk.extensions.schema import PluginDefinitionError +from common_grants_sdk.schemas.pydantic.models import Opportunity -def test_define_plugin_no_args(): - """define_plugin() with no args returns PluginConfig with all fields None.""" - config = define_plugin() - assert isinstance(config, PluginConfig) - assert config.meta is None - assert config.schemas is None - - -def test_define_plugin_with_meta_and_schemas(): - meta = PluginExtensionsMeta(name="test", source_system="test-system") +class OpportunityFields(CustomFieldSet): + agency_code: Optional[CustomField[str]] = Field( + default=None, description="Agency code" + ) - def passthrough(x): - return TransformResult(result=x, errors=[]) - schemas = { - "Opportunity": SchemaInput(to_common=passthrough, from_common=passthrough) - } - config = define_plugin(meta=meta, schemas=schemas) - assert config.meta is meta - assert config.meta.name == "test" - assert config.schemas is schemas +def _meta() -> PluginMeta: + return PluginMeta(name="test", source_system="test-system") -def test_define_plugin_schemas_callable_roundtrip(): - """config.schemas["Opportunity"].to_common(data) works.""" +# --------------------------------------------------------------------------- +# define_plugin assembly +# --------------------------------------------------------------------------- - def always_transformed(_x): - return TransformResult(result={"transformed": True}, errors=[]) - config = define_plugin( - schemas={ - "Opportunity": SchemaInput( - to_common=always_transformed, from_common=always_transformed - ) +def test_define_plugin_returns_plugin_with_schemas_and_meta(): + ext = schema(common=Opportunity[OpportunityFields]) + plugin = define_plugin(PluginSchemas(Opportunity=ext), meta=_meta()) + assert isinstance(plugin, Plugin) + assert plugin.schemas.Opportunity is ext + assert plugin.meta.name == "test" + assert plugin.meta.source_system == "test-system" + + +def test_omitted_schema_falls_back_to_base_schema_only_extension(): + """An unextended PluginSchemas slot is a SchemaOnly over the base, never None.""" + plugin = define_plugin(PluginSchemas(), meta=_meta()) + entry = plugin.schemas.Opportunity + assert isinstance(entry, SchemaOnly) + assert entry.schema_name == "Opportunity" + assert entry.custom_fields == {} + # The base schema has no custom fields declared. + assert entry.common_schema is Opportunity[NoCustomFields] + + +def test_mappings_entry_is_a_transform_extension(): + ext = schema( + source=PassthroughModel, + common=Opportunity[NoCustomFields], + mappings={ + "to_common": {"title": {"field": "opportunity_title"}}, + "from_common": {"opportunity_title": {"field": "title"}}, }, ) - result = config.schemas["Opportunity"].to_common({"raw": "data"}) - assert result.result == {"transformed": True} - assert result.errors == [] - - -def test_plugin_fields_default_to_none(): - """Plugin.schemas holds the container; meta defaults to None.""" - base = Plugin(schemas=object()) - assert base.meta is None + plugin = define_plugin(PluginSchemas(Opportunity=ext), meta=_meta()) + assert isinstance(plugin.schemas.Opportunity, SchemaWithTransforms) -def test_plugin_fields_populated(): - meta = PluginExtensionsMeta(name="p", source_system="s") - full = Plugin(schemas=object(), meta=meta) - assert full.meta is meta - - -def test_plugin_schemas_is_attribute_container(): - """Plugin.schemas holds the _Schemas object (no generated_schemas field).""" - s = object() - p = Plugin(schemas=s) - assert p.schemas is s - assert not hasattr(p, "generated_schemas") +def test_define_plugin_rejects_non_extension_in_slot(): + bad = PluginSchemas(Opportunity="not an extension") # type: ignore[arg-type] + with pytest.raises(PluginDefinitionError, match="not a schema extension"): + define_plugin(bad, meta=_meta()) # --------------------------------------------------------------------------- -# XOR constraint tests +# Plugin container # --------------------------------------------------------------------------- -def test_define_plugin_xor_mappings_and_both_callables_raises(): - """Providing mappings AND both explicit callables raises ValueError.""" - from common_grants_sdk.extensions.types import SchemaMappings - - def noop(x): - return TransformResult(result=x, errors=[]) - - with pytest.raises( - ValueError, - match="cannot specify both mappings and explicit to_common/from_common", - ): - define_plugin( - schemas={ - "Opportunity": SchemaInput( - mappings=SchemaMappings( - to_common={"title": {"field": "native_title"}}, - from_common={"native_title": {"field": "title"}}, - ), - to_common=noop, - from_common=noop, - ) - } - ) - - -def test_define_plugin_xor_mappings_and_one_callable_raises(): - """Providing mappings AND a single explicit callable raises ValueError.""" - from common_grants_sdk.extensions.types import SchemaMappings - - def noop(x): - return TransformResult(result=x, errors=[]) - - with pytest.raises( - ValueError, - match="cannot specify both mappings and explicit to_common/from_common", - ): - define_plugin( - schemas={ - "Opportunity": SchemaInput( - mappings=SchemaMappings( - to_common={"title": {"field": "native_title"}}, - from_common={"native_title": {"field": "title"}}, - ), - to_common=noop, - ) - } - ) - - -def test_define_plugin_mappings_without_callables_is_valid(): - """Providing mappings without explicit callables does not raise.""" - from common_grants_sdk.extensions.types import SchemaMappings - - config = define_plugin( - schemas={ - "Opportunity": SchemaInput( - mappings=SchemaMappings( - to_common={"title": {"field": "native_title"}}, - from_common={"native_title": {"field": "title"}}, - ) - ) - } - ) - assert config.schemas is not None - - -def test_define_plugin_callables_without_mappings_is_valid(): - """Providing explicit callables without mappings does not raise.""" - - def noop(x): - return TransformResult(result=x, errors=[]) - - config = define_plugin( - schemas={"Opportunity": SchemaInput(to_common=noop, from_common=noop)} - ) - assert config.schemas is not None +def test_plugin_is_frozen(): + plugin = define_plugin(PluginSchemas(), meta=_meta()) + with pytest.raises((AttributeError, TypeError)): + plugin.meta = _meta() # type: ignore[misc] diff --git a/lib/python-sdk/tests/extensions/test_schema.py b/lib/python-sdk/tests/extensions/test_schema.py new file mode 100644 index 000000000..3ad684e15 --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_schema.py @@ -0,0 +1,299 @@ +"""Tests for the codegen-free schema(...) factory and its building blocks.""" + +from typing import Optional, assert_type + +import pytest +from pydantic import BaseModel, Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + NoCustomFields, + PassthroughModel, + SchemaOnly, + SchemaWithTransforms, + TransformResult, + resolve_custom_field_specs, + schema, + validate_into, +) +from common_grants_sdk.extensions.schema import PluginDefinitionError, _infer_field_type +from common_grants_sdk.schemas.pydantic.fields import CustomFieldType +from common_grants_sdk.schemas.pydantic.models import Opportunity, OpportunityBase + + +class LegacyRef(BaseModel): + system: str + id: int + + +class OpportunityFields(CustomFieldSet): + agency_code: Optional[CustomField[str]] = Field( + default=None, description="Agency code" + ) + legacy_id: Optional[CustomField[int]] = Field(default=None) + legacy_ref: Optional[CustomField[LegacyRef]] = Field(default=None) + tags: Optional[CustomField[list[str]]] = Field(default=None) + + +FLAT_SOURCE = { + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "opportunity_title": "Conservation research", + "opportunity_description": "Funding.", + "opportunity_status": "posted", + "created_at": "2025-01-01T00:00:00Z", + "last_modified_at": "2025-01-01T00:00:00Z", + "agency_code": "HHS-123", +} + + +def _mappings_extension() -> ( + SchemaWithTransforms[PassthroughModel, Opportunity[OpportunityFields]] +): + return schema( + source=PassthroughModel, + common=Opportunity[OpportunityFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "description": {"field": "opportunity_description"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open"}, + "default": "custom", + } + } + }, + "customFields": { + "agencyCode": { + "value": {"field": "agency_code"}, + "name": {"const": "agencyCode"}, + "fieldType": {"const": "string"}, + } + }, + }, + "from_common": { + "opportunity_uuid": {"field": "id"}, + "opportunity_title": {"field": "title"}, + "agency_code": {"field": "customFields.agencyCode.value"}, + }, + }, + ) + + +# --------------------------------------------------------------------------- +# _infer_field_type / resolve_custom_field_specs +# --------------------------------------------------------------------------- + + +def test_infer_field_type_mapping(): + assert _infer_field_type(str) == CustomFieldType.STRING + assert _infer_field_type(int) == CustomFieldType.INTEGER + assert _infer_field_type(float) == CustomFieldType.NUMBER + assert _infer_field_type(bool) == CustomFieldType.BOOLEAN + assert _infer_field_type(LegacyRef) == CustomFieldType.OBJECT + assert _infer_field_type(list[str]) == CustomFieldType.ARRAY + assert _infer_field_type(dict[str, int]) == CustomFieldType.OBJECT + + +def test_resolve_custom_field_specs_derives_from_value_type(): + specs = resolve_custom_field_specs(OpportunityFields) + assert specs["agency_code"].field_type == CustomFieldType.STRING + assert specs["agency_code"].value is str + assert specs["agency_code"].name == "agency_code" + assert specs["agency_code"].description == "Agency code" + assert specs["legacy_id"].field_type == CustomFieldType.INTEGER + assert specs["legacy_ref"].field_type == CustomFieldType.OBJECT + assert specs["legacy_ref"].value is LegacyRef + assert specs["tags"].field_type == CustomFieldType.ARRAY + + +def test_resolve_custom_field_specs_empty_for_no_container(): + assert resolve_custom_field_specs(None) == {} + assert resolve_custom_field_specs(NoCustomFields) == {} + + +# --------------------------------------------------------------------------- +# schema(...) discrimination +# --------------------------------------------------------------------------- + + +def test_schema_only_returns_schema_only_extension(): + ext = schema(common=Opportunity[OpportunityFields]) + assert isinstance(ext, SchemaOnly) + assert ext.schema_name == "Opportunity" + assert not hasattr(ext, "to_common") + + +def test_mappings_returns_transform_extension(): + ext = _mappings_extension() + assert isinstance(ext, SchemaWithTransforms) + assert ext.source_schema is PassthroughModel + assert ext.custom_fields["agency_code"].field_type == CustomFieldType.STRING + + +def test_functions_returns_transform_extension(): + def to_common( + _src: PassthroughModel, + ) -> TransformResult[Opportunity[NoCustomFields]]: + return TransformResult( + result=Opportunity[NoCustomFields].model_validate( + FLAT_SOURCE + | { + "id": FLAT_SOURCE["opportunity_uuid"], + "title": FLAT_SOURCE["opportunity_title"], + "status": {"value": "open"}, + "createdAt": FLAT_SOURCE["created_at"], + "lastModifiedAt": FLAT_SOURCE["last_modified_at"], + } + ), + errors=[], + ) + + def from_common( + _c: Opportunity[NoCustomFields], + ) -> TransformResult[PassthroughModel]: + return TransformResult(result=PassthroughModel(), errors=[]) + + ext = schema( + source=PassthroughModel, + common=Opportunity[NoCustomFields], + to_common=to_common, + from_common=from_common, + ) + assert isinstance(ext, SchemaWithTransforms) + + +# --------------------------------------------------------------------------- +# Negative cases (validated at schema(...) call time) +# --------------------------------------------------------------------------- + + +def test_unregistered_base_raises(): + class NotRegistered(BaseModel): + x: int + + with pytest.raises( + PluginDefinitionError, match="not a registered extensible schema" + ): + schema(common=NotRegistered) + + +def test_unextended_base_class_is_not_registered(): + """Only the generic Opportunity is registered, not OpportunityBase.""" + with pytest.raises( + PluginDefinitionError, match="not a registered extensible schema" + ): + schema(common=OpportunityBase) + + +def test_unknown_to_common_output_field_raises(): + with pytest.raises(PluginDefinitionError, match="unknown output field"): + schema( + source=PassthroughModel, + common=Opportunity[NoCustomFields], + mappings={ + "to_common": {"nope": {"const": 1}}, + "from_common": {}, + }, + ) + + +def test_missing_mapping_direction_raises(): + with pytest.raises(PluginDefinitionError, match="missing `from_common`"): + schema( + source=PassthroughModel, + common=Opportunity[NoCustomFields], + mappings={"to_common": {"title": {"field": "x"}}}, # type: ignore[typeddict-item] + ) + + +# --------------------------------------------------------------------------- +# Consumer typing + behavior (the make-or-break path) +# --------------------------------------------------------------------------- + + +def test_mappings_consumer_typed_and_round_trips() -> None: + ext = _mappings_extension() + res = ext.to_common(PassthroughModel.model_validate(FLAT_SOURCE)) + assert_type(res, TransformResult[Opportunity[OpportunityFields]]) + assert res.errors == [] + opp = res.result + assert opp.title == "Conservation research" + assert opp.custom_fields is not None + assert opp.custom_fields.agency_code is not None + assert_type(opp.custom_fields.agency_code.value, str) + assert opp.custom_fields.agency_code.value == "HHS-123" + back = ext.from_common(opp) + assert_type(back, TransformResult[PassthroughModel]) + assert back.errors == [] + + +def test_schema_only_parse_typed() -> None: + ext = schema(common=Opportunity[OpportunityFields]) + parsed = ext.parse( + { + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "title": "T", + "status": {"value": "open"}, + "description": "d", + "createdAt": "2025-01-01T00:00:00Z", + "lastModifiedAt": "2025-01-01T00:00:00Z", + "customFields": { + "legacyRef": { + "name": "legacyRef", + "fieldType": "object", + "value": {"system": "legacy", "id": 7}, + } + }, + } + ) + assert_type(parsed, Opportunity[OpportunityFields]) + assert parsed.custom_fields is not None + assert parsed.custom_fields.legacy_ref is not None + assert_type(parsed.custom_fields.legacy_ref.value.id, int) + assert parsed.custom_fields.legacy_ref.value.id == 7 + + +# --------------------------------------------------------------------------- +# camelCase round-trip +# --------------------------------------------------------------------------- + + +def test_camel_case_round_trip(): + camel = { + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "title": "T", + "status": {"value": "open"}, + "description": "d", + "createdAt": "2025-01-01T00:00:00Z", + "lastModifiedAt": "2025-01-01T00:00:00Z", + "customFields": { + "agencyCode": { + "name": "agencyCode", + "fieldType": "string", + "value": "HHS-123", + } + }, + } + opp = Opportunity[OpportunityFields].model_validate(camel) + # snake_case typed access + assert opp.custom_fields is not None + assert opp.custom_fields.agency_code is not None + assert opp.custom_fields.agency_code.value == "HHS-123" + # serializes back to camelCase + dumped = opp.model_dump(by_alias=True, mode="json") + assert "customFields" in dumped + assert dumped["customFields"]["agencyCode"]["fieldType"] == "string" + assert "createdAt" in dumped + + +def test_validate_into_routes_errors(): + res = validate_into(LegacyRef, {"system": "x"}) # missing id + assert res.errors + assert any("id" in (e.path or "") for e in res.errors) diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index 85e76ad71..7b18132b1 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -1,15 +1,11 @@ -"""Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" +"""Tests for the plugin framework types in common_grants_sdk.extensions.types.""" -from common_grants_sdk.extensions.specs import CustomFieldSpec from common_grants_sdk.extensions.types import ( - PluginExtensionsMeta, - SchemaConfig, - SchemaInput, - SchemaMappings, + PassthroughModel, + PluginMeta, TransformError, TransformResult, ) -from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType # --- TransformError --- @@ -40,7 +36,7 @@ def test_transform_error_structured_fields(): def test_transform_error_source_value_excluded_from_str_and_repr(): - """source_value must not appear in str(err) or repr(err) — PII defence per ADR-0022 Decision #9.""" + """source_value must not appear in str(err) or repr(err) -- PII defence.""" sensitive = {"ssn": "123-45-6789"} err = TransformError("transform failed", source_value=sensitive) assert "123-45-6789" not in str(err) @@ -61,69 +57,25 @@ def test_transform_result(): assert partial.errors[0] is err -# --- SchemaMappings --- +# --- PluginMeta --- -def test_schema_mappings(): - assert SchemaMappings().to_common is None - assert SchemaMappings().from_common is None - m = SchemaMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) - assert m.to_common == {"title": "x"} - assert m.from_common == {"x": "title"} - - -# --- PluginExtensionsMeta --- - - -def test_plugin_extensions_meta(): - m = PluginExtensionsMeta(name="grants.gov", sourceSystem="grants.gov") +def test_plugin_extensions_meta_camel_alias(): + m = PluginMeta(name="grants.gov", sourceSystem="grants.gov") assert m.name == "grants.gov" assert m.source_system == "grants.gov" assert m.version is None assert m.capabilities is None -# --- SchemaInput --- - - -def test_schema_input(): - assert SchemaInput().source_schema is None - assert SchemaInput().custom_fields is None - assert SchemaInput().mappings is None - assert SchemaInput().to_common is None - - spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) - inp = SchemaInput(custom_fields={"legacyId": spec}) - assert inp.custom_fields == {"legacyId": spec} - - m = SchemaMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) - inp_with_mappings = SchemaInput(mappings=m) - assert inp_with_mappings.mappings.to_common == {"title": "x"} - - def passthrough(x): - return TransformResult(result=x, errors=[]) - - inp2 = SchemaInput(to_common=passthrough, from_common=passthrough) - assert inp2.to_common is passthrough - - -# --- SchemaConfig --- +def test_plugin_extensions_meta_populate_by_name(): + m = PluginMeta(name="grants.gov", source_system="grants.gov") + assert m.source_system == "grants.gov" -def test_schema_config(): - def passthrough(x): - return TransformResult(result=x, errors=[]) +# --- PassthroughModel --- - schemas = SchemaConfig( - source_schema=dict, - common_schema=dict, - to_common=passthrough, - from_common=passthrough, - ) - assert schemas.source_schema is dict - assert schemas.common_schema is dict - # to_common and from_common are optional — omitting them is valid - minimal = SchemaConfig(source_schema=dict, common_schema=dict) - assert minimal.to_common is None - assert minimal.from_common is None +def test_passthrough_model_accepts_arbitrary_keys(): + instance = PassthroughModel.model_validate({"data": {"any": "shape"}, "n": 1}) + assert instance.model_dump() == {"data": {"any": "shape"}, "n": 1} diff --git a/lib/python-sdk/tests/schemas/test_plugin_registry.py b/lib/python-sdk/tests/schemas/test_plugin_registry.py index 0d0524be4..5c3fdb9e8 100644 --- a/lib/python-sdk/tests/schemas/test_plugin_registry.py +++ b/lib/python-sdk/tests/schemas/test_plugin_registry.py @@ -1,12 +1,8 @@ -"""Tests for OpportunityBase.with_custom_fields().""" - -from typing import Any +"""Tests for OpportunityBase.with_custom_fields() (runtime custom-field registration).""" import pytest from common_grants_sdk.extensions import CustomFieldSpec -from common_grants_sdk.extensions import Plugin -from common_grants_sdk.extensions.types import SchemaConfig from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.schemas.pydantic.models.opp_base import OpportunityBase @@ -15,28 +11,12 @@ # --------------------------------------------------------------------------- -class _Schemas: - """Minimal schemas container that mirrors the generated _Schemas class.""" - - Opportunity: SchemaConfig[Any, Any] - - -def _make_plugin( - field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity" -) -> "Plugin[_Schemas]": - """Build a Plugin whose schemas.Opportunity.common_schema is produced by with_custom_fields().""" - extended = OpportunityBase.with_custom_fields( +def _extended(field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity"): + """Return an OpportunityBase subclass produced by with_custom_fields().""" + return OpportunityBase.with_custom_fields( custom_fields=field_specs, model_name=model_name, ) - s = _Schemas() - s.Opportunity = SchemaConfig(source_schema=dict, common_schema=extended) - return Plugin(schemas=s) - - -def _make_plugin_without_opportunity() -> "Plugin[_Schemas]": - """Build a Plugin that has no Opportunity schema.""" - return Plugin(schemas=_Schemas()) # --------------------------------------------------------------------------- @@ -45,8 +25,8 @@ def _make_plugin_without_opportunity() -> "Plugin[_Schemas]": @pytest.fixture -def simple_plugin() -> "Plugin[_Schemas]": - return _make_plugin( +def simple_opportunity(): + return _extended( { "program_area": CustomFieldSpec( field_type=CustomFieldType.STRING, @@ -80,32 +60,20 @@ def sample_payload() -> dict: # --------------------------------------------------------------------------- -# Plugin schema tests +# Schema construction tests # --------------------------------------------------------------------------- -def test_plugin_schema_is_subclass_of_opportunity_base(simple_plugin): - Opportunity = simple_plugin.schemas.Opportunity.common_schema - - assert Opportunity is not OpportunityBase - assert issubclass(Opportunity, OpportunityBase) - - -def test_plugin_without_opportunity_has_no_opportunity_schema(): - plugin = _make_plugin_without_opportunity() - - assert not hasattr(plugin.schemas, "Opportunity") +def test_with_custom_fields_is_subclass_of_opportunity_base(simple_opportunity): + assert simple_opportunity is not OpportunityBase + assert issubclass(simple_opportunity, OpportunityBase) -def test_two_plugins_produce_distinct_schemas(simple_plugin): - second_plugin = _make_plugin( +def test_two_calls_produce_distinct_schemas(simple_opportunity): + second = _extended( {"award_ceiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER)} ) - - assert ( - simple_plugin.schemas.Opportunity.common_schema - is not second_plugin.schemas.Opportunity.common_schema - ) + assert simple_opportunity is not second # --------------------------------------------------------------------------- @@ -113,23 +81,12 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): # --------------------------------------------------------------------------- -def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( - simple_plugin, sample_payload +def test_validates_payload_and_exposes_typed_custom_fields( + simple_opportunity, sample_payload ): - Opportunity = simple_plugin.schemas.Opportunity.common_schema - - opp = Opportunity.model_validate(sample_payload) + opp = simple_opportunity.model_validate(sample_payload) assert opp.title == "Community Health Grant" assert opp.custom_fields is not None assert opp.custom_fields.program_area.value == "Health" assert opp.custom_fields.eligibility_types.value == ["nonprofit", "tribal"] - - -def test_plugin_schema_validates_custom_fields(simple_plugin, sample_payload): - Opportunity = simple_plugin.schemas.Opportunity.common_schema - - opp = Opportunity.model_validate(sample_payload) - - assert opp.custom_fields.program_area.value == "Health" - assert opp.custom_fields.eligibility_types.value == ["nonprofit", "tribal"] From b56c0dc4273d641eea441c1e4c99741bfc8e455c Mon Sep 17 00:00:00 2001 From: Bryan Thompson <bryan@snowboardtechie.com> Date: Tue, 16 Jun 2026 10:08:11 -0700 Subject: [PATCH 23/51] fix(sdk): add custom-filters FilterError class Merging the transforms refactor (#916) into the custom-filters work left custom-filters.ts and filters.py importing a structured error class that the merged extensions types modules no longer defined. Add it as FilterError. #916 renamed the framework's shared structured error to TransformError for the transforms path; the filters path gets its own domain-named error here, so each domain is clearly named: TransformError for transform failures, FilterError for filter-validation failures. Covers the filters surface: class in types.ts/types.py, exports in index.ts and extensions/__init__.py, custom-filters.ts, filters.py, tests, examples, README. --- .../common_grants_sdk/extensions/__init__.py | 2 + .../common_grants_sdk/extensions/filters.py | 36 ++++----- .../common_grants_sdk/extensions/types.py | 28 +++++++ lib/python-sdk/examples/custom_filters.py | 10 +-- .../tests/extensions/test_filters.py | 78 +++++++++---------- .../extensions/custom-filters.spec.ts | 50 ++++++------ lib/ts-sdk/src/extensions/README.md | 12 +-- lib/ts-sdk/src/extensions/custom-filters.ts | 18 ++--- lib/ts-sdk/src/extensions/index.ts | 1 + lib/ts-sdk/src/extensions/types.ts | 36 +++++++++ 10 files changed, 169 insertions(+), 102 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 40b21672f..ca585c21e 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -13,6 +13,7 @@ from .types import ( Handler, PluginCapability, + FilterError, PluginExtensionsMeta, PluginRoutes, SchemaConfig, @@ -33,6 +34,7 @@ "build_transforms", "Handler", "PluginCapability", + "FilterError", "PluginExtensionsMeta", "SchemaConfig", "SchemaInput", diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index 6abdea026..4aa93da1c 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -4,9 +4,9 @@ - ``f`` helper singleton for building DefaultFilter instances. - ``FILTER_TYPE_SCHEMAS`` — map from CustomFilterType to the Pydantic validation model. - ``DEFAULT_FILTER_NAMES`` — frozenset of all core default-filter field names (snake + alias). -- ``validate_routes(routes)`` — registration-time validator; raises PluginError. +- ``validate_routes(routes)`` — registration-time validator; raises FilterError. - ``validate_filter_call(spec, filter_name, value)`` — call-time validator; returns the - wire-ready DefaultFilter; raises PluginError. + wire-ready DefaultFilter; raises FilterError. - ``classify_filters(routes, resource, method, consumer_filters)`` — classifier producing the ``OppFilters`` search request body (default named fields + ``customFilters``). @@ -53,7 +53,7 @@ ) from .specs import CustomFilterSpec, CustomFilterType -from .types import PluginError, PluginRoutes +from .types import FilterError, PluginRoutes # --------------------------------------------------------------------------- # f.* helpers @@ -204,7 +204,7 @@ def outside(self, min: Any, max: Any) -> DefaultFilter: def validate_routes(routes: PluginRoutes) -> None: """Registration-time validator for a plugin's route filter declarations. - Iterates every filter spec in ``routes`` and raises ``PluginError`` on: + Iterates every filter spec in ``routes`` and raises ``FilterError`` on: 1. Unknown ``filter_type`` (not in ``FILTER_TYPE_SCHEMAS``). 2. Filter name that collides with a core default-filter name in ``DEFAULT_FILTER_NAMES`` (the escape-hatch collision check; a namespaced @@ -221,7 +221,7 @@ def validate_routes(routes: PluginRoutes) -> None: routes: Route-keyed filter declarations as ``PluginRoutes``. Raises: - PluginError: On the first invalid declaration found. + FilterError: On the first invalid declaration found. """ for resource, methods in routes.items(): for method, declarations in methods.items(): @@ -231,13 +231,13 @@ def validate_routes(routes: PluginRoutes) -> None: for filter_name, spec in filter_specs.items(): path = f"routes.{resource}.{method}.filters.{filter_name}" if spec.filter_type not in FILTER_TYPE_SCHEMAS: - raise PluginError( + raise FilterError( f'Unknown filter_type "{spec.filter_type}" for filter "{filter_name}"', path=path, source_value=spec, ) if filter_name in DEFAULT_FILTER_NAMES: - raise PluginError( + raise FilterError( f'Filter name "{filter_name}" collides with a default filter name', path=path, source_value=filter_name, @@ -245,7 +245,7 @@ def validate_routes(routes: PluginRoutes) -> None: def _first_error_detail(exc: ValidationError) -> str: - """One-line summary of the first pydantic error, for PluginError messages. + """One-line summary of the first pydantic error, for FilterError messages. The full ValidationError stays on ``cause`` for programmatic access; this puts the most useful line in ``str(exc)`` so a logged message says what @@ -281,14 +281,14 @@ def validate_filter_call( Args: spec: The registered ``CustomFilterSpec`` for this filter, or ``None`` for ad-hoc. - filter_name: The filter name (used in PluginError path). + filter_name: The filter name (used in FilterError path). value: The filter value to validate (typically a ``DefaultFilter`` instance). Returns: The validated filter as a ``DefaultFilter`` with coerced operator/value. Raises: - PluginError: On operator/value-shape mismatch (wrapping the pydantic + FilterError: On operator/value-shape mismatch (wrapping the pydantic ``ValidationError`` as ``cause``), or on a ``spec.filter_type`` not present in ``FILTER_TYPE_SCHEMAS`` — call ``validate_routes(routes)`` at registration time to catch the latter earlier. @@ -297,7 +297,7 @@ def validate_filter_call( if spec is not None: model_cls = FILTER_TYPE_SCHEMAS.get(spec.filter_type) if model_cls is None: - raise PluginError( + raise FilterError( f'Unknown filter_type "{spec.filter_type}" for filter ' f'"{filter_name}" — call validate_routes(routes) at ' "registration time to catch this earlier", @@ -307,7 +307,7 @@ def validate_filter_call( try: validated = model_cls.model_validate(payload) except ValidationError as exc: - raise PluginError( + raise FilterError( f'Filter "{filter_name}" failed validation: ' f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", path=f"filters.{filter_name}", @@ -322,7 +322,7 @@ def validate_filter_call( try: return DefaultFilter.model_validate(payload) except ValidationError as exc: - raise PluginError( + raise FilterError( f'Ad-hoc filter "{filter_name}" has an invalid DefaultFilter shape: ' f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", path=f"filters.{filter_name}", @@ -390,7 +390,7 @@ def classify_filters( mode (operator enums are ``StrEnum`` and serialize fine either way). Raises: - PluginError: When any filter value fails validation — registered and ad-hoc + FilterError: When any filter value fails validation — registered and ad-hoc values at classification time, default values at ``OppFilters`` construction — or when the snake_case and camelCase forms of the same default filter are both supplied. The error surface is uniform across @@ -420,7 +420,7 @@ def classify_filters( # Snake and camel forms of the same field normalize to one key; # without this guard, plain dict assignment would silently drop # whichever form the consumer's dict ordered first. - raise PluginError( + raise FilterError( f'Default filter "{alias_key}" was supplied more than once ' "(snake_case and camelCase forms of the same filter)", path=f"filters.{alias_key}", @@ -439,8 +439,8 @@ def classify_filters( # OppFilters requires the alias form for construction (populate_by_name is not set). # Use "customFilters" (the alias) rather than "custom_filters" (the field name). # This construction is the validation point for bucket-1 (default) values — wrap - # pydantic failures in PluginError so the error contract is uniform across all - # three buckets (consumers catch `except PluginError` regardless of bucket). + # pydantic failures in FilterError so the error contract is uniform across all + # three buckets (consumers catch `except FilterError` regardless of bucket). try: return OppFilters( **default_fields, @@ -453,7 +453,7 @@ def classify_filters( # values were already validated above, so only default fields fail here. failed = sorted({str(err["loc"][0]) for err in exc.errors() if err.get("loc")}) field_list = ", ".join(failed) if failed else "<unknown>" - raise PluginError( + raise FilterError( f"Filter(s) {field_list} failed validation: " f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", path=f"filters.{failed[0]}" if len(failed) == 1 else "filters", diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index ff37ba4ef..5f4ba7ac3 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -66,6 +66,34 @@ def __init__( self.cause = cause +class FilterError(Exception): + """Structured error raised by custom-filter validation. + + Raised by validate_routes (registration time) and validate_filter_call / + classify_filters (call time). Carries field path, handler name, source + value, and underlying cause so consumers can reason about failures + programmatically without parsing error text. + + Note: source_value may contain PII. Adopters are responsible for redacting + it before logging or re-raising. The SDK does not redact by default. + """ + + def __init__( + self, + message: str, + *, + path: str | None = None, + handler: str | None = None, + source_value: Any = None, + cause: BaseException | None = None, + ) -> None: + super().__init__(message) + self.path = path + self.handler = handler + self.source_value = source_value + self.cause = cause + + @dataclass class TransformResult(Generic[T]): """Unconditional return shape for to_common / from_common (ADR-0022 Decision #7). diff --git a/lib/python-sdk/examples/custom_filters.py b/lib/python-sdk/examples/custom_filters.py index ec1d508ab..2cff4e3ef 100644 --- a/lib/python-sdk/examples/custom_filters.py +++ b/lib/python-sdk/examples/custom_filters.py @@ -18,7 +18,7 @@ from common_grants_sdk.extensions import classify_filters, define_plugin, f from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType -from common_grants_sdk.extensions.types import PluginError, PluginExtensionsMeta +from common_grants_sdk.extensions.types import FilterError, PluginExtensionsMeta # --------------------------------------------------------------------------- # Plugin declaration — route-keyed custom filter specs @@ -99,8 +99,8 @@ def main() -> None: ) ) - # --- PluginError demo — bad call raises and is caught --- - _section("VALIDATION — bad operator raises PluginError (runtime guarantee)") + # --- FilterError demo — bad call raises and is caught --- + _section("VALIDATION — bad operator raises FilterError (runtime guarantee)") # agency is registered as STRING_ARRAY (expects ArrayOperator: in/notIn). # Passing f.eq(...) (EquivalenceOperator.EQUAL) triggers call-time validation failure. @@ -111,11 +111,11 @@ def main() -> None: } try: classify_filters(grants_gov.routes, "opportunities", "search", bad_filters) - except PluginError as exc: + except FilterError as exc: # str(exc) summarizes the first failure; the structured fields carry # the full detail — exc.path names the failing filter, exc.cause is # the underlying pydantic ValidationError for programmatic access. - print(f"PluginError caught: {exc}") + print(f"FilterError caught: {exc}") print(f" path: {exc.path}") print(f" cause: {type(exc.cause).__name__}") diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py index c7c602556..ac410f13e 100644 --- a/lib/python-sdk/tests/extensions/test_filters.py +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -17,7 +17,7 @@ validate_routes, ) from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType -from common_grants_sdk.extensions.types import PluginError +from common_grants_sdk.extensions.types import FilterError from common_grants_sdk.schemas.pydantic.filters.opportunity import OppFilters # --------------------------------------------------------------------------- @@ -215,23 +215,23 @@ def test_oppfilters_mixed_case_roundtrip(): # --------------------------------------------------------------------------- -# Registration-time validation (validate_routes) — RAISES PluginError +# Registration-time validation (validate_routes) — RAISES FilterError # --------------------------------------------------------------------------- def test_validate_routes_unknown_filter_type_raises(): - """validate_routes raises PluginError when filter_type is not in FILTER_TYPE_SCHEMAS.""" + """validate_routes raises FilterError when filter_type is not in FILTER_TYPE_SCHEMAS.""" # Dataclasses don't validate field types at runtime, so a bogus filter_type # can be passed directly (the annotation is for type checkers only). bad_spec = CustomFilterSpec(filter_type="unknownType") # type: ignore[arg-type] routes = {"opportunities": {"search": {"filters": {"myFilter": bad_spec}}}} - with pytest.raises(PluginError, match="Unknown filter_type"): + with pytest.raises(FilterError, match="Unknown filter_type"): validate_routes(routes) def test_validate_routes_collision_with_default_filter_name_raises(): - """validate_routes raises PluginError when a custom filter name collides with a CORE DEFAULT name. + """validate_routes raises FilterError when a custom filter name collides with a CORE DEFAULT name. This is the escape-hatch collision check. E.g. naming a custom filter "status" would shadow the core default "status" field — must be caught at registration time. @@ -248,12 +248,12 @@ def test_validate_routes_collision_with_default_filter_name_raises(): } } } - with pytest.raises(PluginError, match="collides"): + with pytest.raises(FilterError, match="collides"): validate_routes(routes) def test_validate_routes_collision_with_camel_alias_raises(): - """validate_routes raises PluginError for camelCase alias collision (e.g. "closeDateRange").""" + """validate_routes raises FilterError for camelCase alias collision (e.g. "closeDateRange").""" routes = { "opportunities": { "search": { @@ -266,7 +266,7 @@ def test_validate_routes_collision_with_camel_alias_raises(): } } } - with pytest.raises(PluginError, match="collides"): + with pytest.raises(FilterError, match="collides"): validate_routes(routes) @@ -277,27 +277,27 @@ def test_validate_routes_valid_routes_do_not_raise(): # --------------------------------------------------------------------------- -# Call-time validation (validate_filter_call) — RAISES PluginError +# Call-time validation (validate_filter_call) — RAISES FilterError # --------------------------------------------------------------------------- def test_validate_filter_call_registered_bad_operator_raises(): - """validate_filter_call raises PluginError when a registered filter has an operator/value mismatch.""" + """validate_filter_call raises FilterError when a registered filter has an operator/value mismatch.""" # AGENCY_SPEC is STRING_ARRAY — an "eq" with a scalar value is wrong for StringArrayFilter bad_filter = f.eq("not-an-array") - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call(AGENCY_SPEC, "agency", bad_filter) def test_validate_filter_call_adhoc_invalid_shape_raises(): - """validate_filter_call raises PluginError when an ad-hoc filter has an invalid DefaultFilter shape.""" + """validate_filter_call raises FilterError when an ad-hoc filter has an invalid DefaultFilter shape.""" # Pass None as spec (ad-hoc), with something that isn't a DefaultFilter class _BadShape: operator = "not_a_real_operator" value = object() # not a valid value type - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call(None, "legacyTag", _BadShape()) # type: ignore[arg-type] @@ -322,21 +322,21 @@ def test_validate_filter_call_money_comparison_passes_valid_money(): def test_validate_filter_call_money_comparison_rejects_array_operator(): - """A registered moneyComparison filter raises PluginError for an array operator.""" + """A registered moneyComparison filter raises FilterError for an array operator.""" spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call( spec, "awardFloor", f.in_([{"amount": "1000000", "currency": "USD"}]) ) def test_validate_filter_call_money_comparison_rejects_numeric_amount(): - """A registered moneyComparison filter raises PluginError for a numeric amount. + """A registered moneyComparison filter raises FilterError for a numeric amount. Money.amount is a DecimalString — a raw number is the wrong shape. """ spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call( spec, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) ) @@ -356,22 +356,22 @@ def test_validate_filter_call_money_range_passes_valid_range(): def test_validate_filter_call_money_range_rejects_comparison_operator(): - """A registered moneyRange filter raises PluginError for a comparison operator.""" + """A registered moneyRange filter raises FilterError for a comparison operator.""" spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call( spec, "awardRange", f.gt({"amount": "10000", "currency": "USD"}) ) def test_classify_default_wrong_shape_raises_plugin_error(): - """A wrong-shaped DEFAULT filter raises PluginError, not a raw pydantic ValidationError. + """A wrong-shaped DEFAULT filter raises FilterError, not a raw pydantic ValidationError. "status" is a StringArrayFilter (ArrayOperator + list[str]); f.eq("open") is an equivalence filter. The error contract must be uniform across all three buckets: - consumers following the documented `except PluginError` pattern must catch this. + consumers following the documented `except FilterError` pattern must catch this. """ - with pytest.raises(PluginError) as exc_info: + with pytest.raises(FilterError) as exc_info: classify_filters( SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} ) @@ -388,17 +388,17 @@ def test_validate_filter_call_integer_comparison_validates_as_number(): """ spec = CustomFilterSpec(filter_type=CustomFilterType.INTEGER_COMPARISON) validate_filter_call(spec, "awardCount", f.gt(100)) - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call(spec, "awardCount", f.gt("not a number")) def test_classify_default_camel_alias_wrong_shape_raises_plugin_error(): - """A wrong-shaped default filter via its camelCase alias also raises PluginError. + """A wrong-shaped default filter via its camelCase alias also raises FilterError. "closeDateRange" is a DateRangeFilter; f.eq("2026-01-01") is an equivalence - filter — the alias normalization path must surface the same PluginError. + filter — the alias normalization path must surface the same FilterError. """ - with pytest.raises(PluginError): + with pytest.raises(FilterError): classify_filters( SAMPLE_ROUTES, "opportunities", @@ -462,7 +462,7 @@ def test_number_comparison_registered_filter_rejects_bool(): lax-coerces True -> 1 and the wire silently carries a number for a boolean — the corruption class the DefaultFilter.value widening fixed. """ - with pytest.raises(PluginError): + with pytest.raises(FilterError): classify_filters( WIRE_ROUTES, "opportunities", "search", {"awardCount": f.eq(True)} ) @@ -478,18 +478,18 @@ def test_mutated_adhoc_instance_is_revalidated_and_raises(): """ flt = f.eq("x") flt.operator = "bogus" # type: ignore[assignment] - with pytest.raises(PluginError): + with pytest.raises(FilterError): classify_filters(SAMPLE_ROUTES, "opportunities", "search", {"legacy": flt}) def test_unknown_filter_type_raises_plugin_error_not_key_error(): - """A spec whose filter_type never passed validate_routes raises PluginError. + """A spec whose filter_type never passed validate_routes raises FilterError. The uniform error contract holds even when registration-time validation was - skipped — consumers catching `except PluginError` must not see a KeyError. + skipped — consumers catching `except FilterError` must not see a KeyError. """ spec = CustomFilterSpec(filter_type="bogusType") # type: ignore[arg-type] - with pytest.raises(PluginError): + with pytest.raises(FilterError): validate_filter_call(spec, "x", f.eq(1)) @@ -521,7 +521,7 @@ def test_classify_default_snake_form_of_aliased_key_normalizes_to_alias(): def test_classify_both_forms_of_same_default_filter_raises(): - """Supplying snake AND camel forms of one default filter raises PluginError. + """Supplying snake AND camel forms of one default filter raises FilterError. Both keys normalize to "closeDateRange"; without the guard, dict assignment silently drops whichever range the consumer's dict ordered @@ -531,7 +531,7 @@ def test_classify_both_forms_of_same_default_filter_raises(): "close_date_range": f.between("2026-01-01", "2026-06-30"), "closeDateRange": f.between("2026-07-01", "2026-12-31"), } - with pytest.raises(PluginError, match="more than once") as exc_info: + with pytest.raises(FilterError, match="more than once") as exc_info: classify_filters(SAMPLE_ROUTES, "opportunities", "search", consumer_filters) assert exc_info.value.path == "filters.closeDateRange" @@ -558,7 +558,7 @@ def test_classify_unmatched_route_treats_registered_name_as_adhoc(resource, meth assert result.custom_filters["agency"].value == "NSF" # ...and the same filter via the declared pair IS spec-validated and rejected - with pytest.raises(PluginError): + with pytest.raises(FilterError): classify_filters( SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} ) @@ -604,20 +604,20 @@ def test_filter_type_schemas_covers_every_custom_filter_type(): def test_plugin_error_path_is_uniform_across_buckets(): - """All three buckets raise PluginError with a filters.<name> path.""" - with pytest.raises(PluginError) as exc1: + """All three buckets raise FilterError with a filters.<name> path.""" + with pytest.raises(FilterError) as exc1: classify_filters( SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} ) assert exc1.value.path == "filters.status" - with pytest.raises(PluginError) as exc2: + with pytest.raises(FilterError) as exc2: classify_filters( SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} ) assert exc2.value.path == "filters.agency" - with pytest.raises(PluginError) as exc3: + with pytest.raises(FilterError) as exc3: classify_filters( SAMPLE_ROUTES, "opportunities", "search", {"adhoc": {"operator": "bogus"}} ) @@ -626,7 +626,7 @@ def test_plugin_error_path_is_uniform_across_buckets(): def test_multiple_failing_defaults_use_collective_path(): """Two failing default filters produce the collective path "filters".""" - with pytest.raises(PluginError) as exc_info: + with pytest.raises(FilterError) as exc_info: classify_filters( SAMPLE_ROUTES, "opportunities", diff --git a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts index 07c88d2fa..1f2227001 100644 --- a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { classifyFilters, F, validateFilterCall, validateRoutes } from "@/extensions"; -import { PluginError } from "@/extensions"; +import { FilterError } from "@/extensions"; import type { PluginRoutes } from "@/extensions"; import { OppFiltersSchema } from "@/schemas/zod/models"; import { DefaultFilterSchema } from "@/schemas/zod/filters"; @@ -129,7 +129,7 @@ describe("classifyFilters", () => { // ############################################################################ describe("registration-time validation", () => { - it("throws PluginError on unknown filterType", () => { + it("throws FilterError on unknown filterType", () => { const badRoutes: PluginRoutes = { opportunities: { search: { @@ -141,10 +141,10 @@ describe("classifyFilters", () => { }, }; - expect(() => validateRoutes(badRoutes)).toThrow(PluginError); + expect(() => validateRoutes(badRoutes)).toThrow(FilterError); }); - it("PluginError for unknown filterType includes the path and sourceValue", () => { + it("FilterError for unknown filterType includes the path and sourceValue", () => { const badRoutes: PluginRoutes = { opportunities: { search: { @@ -158,16 +158,16 @@ describe("classifyFilters", () => { try { validateRoutes(badRoutes); - expect.fail("Expected PluginError to be thrown"); + expect.fail("Expected FilterError to be thrown"); } catch (err) { - expect(err).toBeInstanceOf(PluginError); - const pluginErr = err as PluginError; + expect(err).toBeInstanceOf(FilterError); + const pluginErr = err as FilterError; expect(pluginErr.path).toBe("routes.opportunities.search.filters.myFilter"); expect(pluginErr.sourceValue).toMatchObject({ filterType: "unknownType" }); } }); - it("throws PluginError when custom filter name collides with a default filter name (status)", () => { + it("throws FilterError when custom filter name collides with a default filter name (status)", () => { const collidingRoutes: PluginRoutes = { opportunities: { search: { @@ -179,10 +179,10 @@ describe("classifyFilters", () => { }, }; - expect(() => validateRoutes(collidingRoutes)).toThrow(PluginError); + expect(() => validateRoutes(collidingRoutes)).toThrow(FilterError); }); - it("throws PluginError when custom filter name collides with closeDateRange", () => { + it("throws FilterError when custom filter name collides with closeDateRange", () => { const collidingRoutes: PluginRoutes = { opportunities: { search: { @@ -193,7 +193,7 @@ describe("classifyFilters", () => { }, }; - expect(() => validateRoutes(collidingRoutes)).toThrow(PluginError); + expect(() => validateRoutes(collidingRoutes)).toThrow(FilterError); }); it("does not throw for valid routes", () => { @@ -206,31 +206,31 @@ describe("classifyFilters", () => { // ############################################################################ describe("call-time validation", () => { - it("throws PluginError on operator/filterType mismatch for a registered filter", () => { + it("throws FilterError on operator/filterType mismatch for a registered filter", () => { // `like` operator is not valid for numberComparison (only gt/gte/lt/lte/eq/neq) const spec = { filterType: "numberComparison" } as const; expect(() => validateFilterCall(spec, "amount", { operator: "like", value: "100" })).toThrow( - PluginError + FilterError ); }); - it("throws PluginError on value-shape mismatch for a registered stringArray filter", () => { + it("throws FilterError on value-shape mismatch for a registered stringArray filter", () => { // stringArray requires value to be string[]; passing a plain string fails const spec = { filterType: "stringArray" } as const; expect(() => validateFilterCall(spec, "agency", { operator: "in", value: "not-an-array" }) - ).toThrow(PluginError); + ).toThrow(FilterError); }); - it("throws PluginError on value-shape mismatch for a registered numberComparison filter", () => { + it("throws FilterError on value-shape mismatch for a registered numberComparison filter", () => { // numberComparison requires value to be a number; passing a string fails const spec = { filterType: "numberComparison" } as const; expect(() => validateFilterCall(spec, "amount", { operator: "eq", value: "not-a-number" }) - ).toThrow(PluginError); + ).toThrow(FilterError); }); it("passes a valid integerComparison filter without throwing", () => { @@ -241,15 +241,15 @@ describe("classifyFilters", () => { ).not.toThrow(); }); - it("PluginError path includes the filter name", () => { + it("FilterError path includes the filter name", () => { const spec = { filterType: "stringArray" } as const; try { validateFilterCall(spec, "agency", { operator: "in", value: "wrong" }); - expect.fail("Expected PluginError to be thrown"); + expect.fail("Expected FilterError to be thrown"); } catch (err) { - expect(err).toBeInstanceOf(PluginError); - expect((err as PluginError).path).toBe("filters.agency"); + expect(err).toBeInstanceOf(FilterError); + expect((err as FilterError).path).toBe("filters.agency"); } }); @@ -268,21 +268,21 @@ describe("classifyFilters", () => { ).not.toThrow(); }); - it("throws PluginError for an ad-hoc filter with an invalid shape", () => { + it("throws FilterError for an ad-hoc filter with an invalid shape", () => { // Missing `operator` key — fails DefaultFilterSchema shape check (operator is required/enum) expect(() => validateFilterCall(undefined, "badFilter", { value: "something" })).toThrow( - PluginError + FilterError ); }); - it("throws PluginError for an ad-hoc filter with an unknown operator", () => { + it("throws FilterError for an ad-hoc filter with an unknown operator", () => { // `superCustomOp` is not in AllOperatorsEnum — fails DefaultFilterSchema expect(() => validateFilterCall(undefined, "badFilter", { operator: "superCustomOp", value: "x", }) - ).toThrow(PluginError); + ).toThrow(FilterError); }); }); }); diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index c8e31078e..3e861f617 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -735,7 +735,7 @@ For a complete runnable example with assertions, see [`examples/custom-filters.t ### Validation — registration-time and call-time -`validateRoutes()` is called at registration time. It throws `PluginError` if: +`validateRoutes()` is called at registration time. It throws `FilterError` if: - A filter spec uses an unknown `filterType` value. - A custom filter name collides with a default filter field name (e.g. registering `"status"` would shadow the protocol's standard `status` filter). @@ -744,15 +744,15 @@ For a complete runnable example with assertions, see [`examples/custom-filters.t In the PoC, `definePlugin()` passes `routes` through **unvalidated** — the full SDK client will run `validateRoutes()` when a plugin is attached and call-time validation inside each search. Until then, call `validateRoutes()` yourself after defining a plugin if you want registration-time errors surfaced early. -> **PII note:** as with transforms, `PluginError.sourceValue` carries the raw input — here, the consumer's filter value. The [PII warning](#error-handling) above applies equally; log a redacted projection. +> **PII note:** as with transforms, `FilterError.sourceValue` carries the raw input — here, the consumer's filter value. The [PII warning](#error-handling) above applies equally; log a redacted projection. ```typescript import { validateRoutes, validateFilterCall } from "@common-grants/sdk/extensions"; -// Registration-time — throws PluginError on unknown filterType or collision +// Registration-time — throws FilterError on unknown filterType or collision validateRoutes(grantsGovPlugin.routes!); -// Call-time — throws PluginError on operator/value mismatch for registered filters. +// Call-time — throws FilterError on operator/value mismatch for registered filters. // Pass the filter's CustomFilterSpec (looked up from the route-method's filters), // its name, and the value. const agencySpec = grantsGovPlugin.routes!.opportunities.search.filters!.agency; @@ -902,8 +902,8 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, | Export | Kind | Description | Demonstrated in | | --------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | | [`classifyFilters()`](./custom-filters.ts) | function | Three-bucket classifier. Maps a flat consumer `filters` object to the ADR-0012 `OppFilters` request body: default fields → top-level named fields; registered custom + ad-hoc → `customFilters` record. | [Classifying consumer filters](#classifying-consumer-filters-into-the-request-body) | -| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `PluginError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | -| [`validateFilterCall()`](./custom-filters.ts) | function | Call-time validator. Validates a single filter against its declared `filterType` schema (registered) or shape-only (ad-hoc). Throws `PluginError` on mismatch. | [Validation](#validation--registration-time-and-call-time) | +| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `FilterError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | +| [`validateFilterCall()`](./custom-filters.ts) | function | Call-time validator. Validates a single filter against its declared `filterType` schema (registered) or shape-only (ad-hoc). Throws `FilterError` on mismatch. | [Validation](#validation--registration-time-and-call-time) | | [`F`](./custom-filters.ts) | namespace | Helper namespace. `F.eq`, `F.neq`, `F.gt`, `F.gte`, `F.lt`, `F.lte`, `F.in`, `F.notIn`, `F.like`, `F.notLike`, `F.between`, `F.outside` — each compiles to `{ operator, value }`. Note: `F.in` is `"in"` as an object property key. | [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) | | [`CustomFilterSpec`](./types.ts) | interface | Per-filter declaration: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType`; no `value` field. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | | [`CustomFilterType`](./types.ts) | type | 11-value literal union: `stringComparison \| stringArray \| numberComparison \| numberArray \| numberRange \| integerComparison \| booleanComparison \| dateComparison \| dateRange \| moneyComparison \| moneyRange`. | [Filter-type catalog](#filter-type-catalog-and-the-f-helpers) | diff --git a/lib/ts-sdk/src/extensions/custom-filters.ts b/lib/ts-sdk/src/extensions/custom-filters.ts index b63073580..8e37014c7 100644 --- a/lib/ts-sdk/src/extensions/custom-filters.ts +++ b/lib/ts-sdk/src/extensions/custom-filters.ts @@ -35,7 +35,7 @@ import { } from "../schemas/zod/filters"; import { OppDefaultFiltersSchema, OppFiltersSchema } from "../schemas/zod/models"; import type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; -import { PluginError } from "./types"; +import { FilterError } from "./types"; // ############################################################################ // Internal — filter-type schema map @@ -131,7 +131,7 @@ export const F = { /** * Registration-time validation for a `PluginRoutes` declaration. * - * Throws `PluginError` on: + * Throws `FilterError` on: * 1. Unknown `filterType` (not one of the 11 `CustomFilterType` values) * 2. A custom filter name that collides with a default-filter field name * (`status`, `closeDateRange`, `totalFundingAvailableRange`, @@ -143,7 +143,7 @@ export const F = { * Implements ASVS L1 input validation at the plugin-author trust boundary. * * @param routes - The PluginRoutes declaration to validate - * @throws {PluginError} on any constraint violation + * @throws {FilterError} on any constraint violation */ export function validateRoutes(routes: PluginRoutes): void { for (const [resourceKey, methods] of Object.entries(routes)) { @@ -156,7 +156,7 @@ export function validateRoutes(routes: PluginRoutes): void { // Check for unknown filterType if (!VALID_FILTER_TYPES.has(spec.filterType)) { - throw new PluginError( + throw new FilterError( `Unknown filterType "${spec.filterType}" for filter "${filterName}". ` + `Must be one of: ${[...VALID_FILTER_TYPES].join(", ")}`, { path, sourceValue: spec } @@ -165,7 +165,7 @@ export function validateRoutes(routes: PluginRoutes): void { // Check for collision with default-filter field names if (DEFAULT_FILTER_NAMES.has(filterName)) { - throw new PluginError( + throw new FilterError( `Custom filter name "${filterName}" collides with a default filter field. ` + `Default filter names are reserved: ${[...DEFAULT_FILTER_NAMES].join(", ")}`, { path, sourceValue: spec } @@ -192,7 +192,7 @@ export function validateRoutes(routes: PluginRoutes): void { * @param spec - The registered `CustomFilterSpec` for this filter, or `undefined` for ad-hoc * @param filterName - The filter key (used in error `path`) * @param filterValue - The raw filter value from the consumer `filters` object - * @throws {PluginError} on operator/filterType mismatch or value-shape mismatch + * @throws {FilterError} on operator/filterType mismatch or value-shape mismatch */ export function validateFilterCall( spec: CustomFilterSpec | undefined, @@ -205,7 +205,7 @@ export function validateFilterCall( // Ad-hoc filter — shape-only check against DefaultFilterSchema const result = DefaultFilterSchema.safeParse(filterValue); if (!result.success) { - throw new PluginError( + throw new FilterError( `Ad-hoc filter "${filterName}" has an invalid shape: ${result.error.message}`, { path, sourceValue: filterValue } ); @@ -217,7 +217,7 @@ export function validateFilterCall( const schema = FILTER_TYPE_SCHEMAS[spec.filterType]; if (!schema) { // Should not reach here if validateRoutes was called first, but guard anyway - throw new PluginError( + throw new FilterError( `Unknown filterType "${spec.filterType}" for registered filter "${filterName}"`, { path, sourceValue: filterValue } ); @@ -226,7 +226,7 @@ export function validateFilterCall( // One parse validates both the operator enum and the value shape const result = schema.safeParse(filterValue); if (!result.success) { - throw new PluginError( + throw new FilterError( `Filter "${filterName}" (filterType: "${spec.filterType}") failed validation: ${result.error.message}`, { path, sourceValue: filterValue } ); diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index e9dce8b71..ef6edeadd 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -15,6 +15,7 @@ export { definePlugin } from "./define-plugin"; // Custom filters — route-keyed filter registration + classification export type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; +export { FilterError } from "./types"; export { classifyFilters, validateRoutes, validateFilterCall, F } from "./custom-filters"; // Schema utilities — lower-level tools for working with custom fields diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index 1a9390a96..d616c863e 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -308,6 +308,42 @@ export class TransformError extends Error { } } +/** + * Structured error raised by custom-filter validation (`validateRoutes`, + * `validateFilterCall`, `classifyFilters`). + * + * Carries field path, handler name, source value, and underlying cause for + * programmatic handling. `sourceValue` and `cause` may carry PII — redact + * before logging. + */ +export class FilterError extends Error { + /** Dot-notation field path where the error occurred, if known. */ + path?: string; + /** Name of the handler that raised, if applicable. */ + handler?: string; + /** The source value that triggered the error (may contain PII — redact before logging). */ + sourceValue?: unknown; + /** Underlying cause of the error, if any (may contain PII — redact before logging). */ + cause?: unknown; + + constructor( + message: string, + options?: { + path?: string; + handler?: string; + sourceValue?: unknown; + cause?: unknown; + } + ) { + super(message); + this.name = "FilterError"; + this.path = options?.path; + this.handler = options?.handler; + this.sourceValue = options?.sourceValue; + this.cause = options?.cause; + } +} + // ############################################################################ // Public types - SchemaInput (author-provided), SchemaOnly, SchemaWithTransforms // ############################################################################ From 41144f4b57dfbf5dc46629004f44e5a6c73b1993 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 13:24:46 -0400 Subject: [PATCH 24/51] chore(py-sdk): remove the codegen build path Delete the codegen extensions pipeline now that plugins are plain Python: - extensions/generate.py and the Makefile generate target. - The per-plugin codegen example directories (cg_config.py + generated/) and the old transforms / plugin_custom_fields scripts. - tests/utils/test_plugin_generator.py (covered the deleted generator). --- lib/python-sdk/Makefile | 5 +- .../common_grants_sdk/extensions/generate.py | 659 ------------------ .../examples/plugin_custom_fields.py | 87 --- lib/python-sdk/examples/plugins/__init__.py | 0 .../examples/plugins/grants_gov/__init__.py | 16 - .../examples/plugins/grants_gov/cg_config.py | 184 ----- .../opportunity_extensions/__init__.py | 14 - .../opportunity_extensions/cg_config.py | 39 -- lib/python-sdk/examples/transforms.py | 293 -------- .../tests/utils/test_plugin_generator.py | 552 --------------- 10 files changed, 1 insertion(+), 1848 deletions(-) delete mode 100644 lib/python-sdk/common_grants_sdk/extensions/generate.py delete mode 100644 lib/python-sdk/examples/plugin_custom_fields.py delete mode 100644 lib/python-sdk/examples/plugins/__init__.py delete mode 100644 lib/python-sdk/examples/plugins/grants_gov/__init__.py delete mode 100644 lib/python-sdk/examples/plugins/grants_gov/cg_config.py delete mode 100644 lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py delete mode 100644 lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py delete mode 100644 lib/python-sdk/examples/transforms.py delete mode 100644 lib/python-sdk/tests/utils/test_plugin_generator.py diff --git a/lib/python-sdk/Makefile b/lib/python-sdk/Makefile index bd9d4614e..043a6fdbf 100644 --- a/lib/python-sdk/Makefile +++ b/lib/python-sdk/Makefile @@ -22,10 +22,7 @@ check-format: check-lint: $(RUNTIME_PREFIX) ruff check . -check-types: plugins +check-types: $(RUNTIME_PREFIX) mypy . checks: check-format check-lint check-types - -plugins: - $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions examples/plugins/grants_gov diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py deleted file mode 100644 index 5b6b222ba..000000000 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ /dev/null @@ -1,659 +0,0 @@ -"""Generate typed plugin schema models from a plugin config.""" - -from __future__ import annotations - -import argparse -import importlib.util -import keyword -import re -from pathlib import Path -from typing import Any, Iterable - -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -from common_grants_sdk.utils.json import snake -from .plugin import PluginConfig -from .specs import CustomFieldSpec - -# Maps extensible model names to the SDK base class they extend in generated code. -# Add an entry here when a new model gains customFields support. -MODEL_BASE_CLASS: dict[str, str] = { - "Opportunity": "OpportunityBase", -} - -# Default Python type annotation strings written into generated source for each field type. -# ARRAY and OBJECT use parameterized forms (list[Any], dict[str, Any]) as practical defaults; -# callers can override these by setting spec.value to a more specific type. -FIELD_TYPE_DEFAULT_ANNOTATION: dict[CustomFieldType, str] = { - CustomFieldType.STRING: "str", - CustomFieldType.NUMBER: "float", - CustomFieldType.INTEGER: "int", - CustomFieldType.BOOLEAN: "bool", - CustomFieldType.OBJECT: "dict[str, Any]", - CustomFieldType.ARRAY: "list[Any]", -} - - -def _load_config(config_path: Path) -> PluginConfig[Any]: - """Load and validate a plugin config file, returning the PluginConfig object. - - Uses importlib to load cg_config.py as an isolated module so it doesn't - pollute sys.modules and can be loaded from any directory at runtime. - - Args: - config_path: Absolute path to the cg_config.py file. - - Returns: - The ``PluginConfig`` bound to the module-level ``config`` variable. - - Raises: - RuntimeError: If the file cannot be loaded or does not expose a valid - ``config`` variable created by ``define_plugin()``. - """ - spec = importlib.util.spec_from_file_location( - f"cg_plugin_config_{config_path.parent.name}", config_path - ) - if spec is None or spec.loader is None: - raise RuntimeError(f"Unable to load plugin config from {config_path}") - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - config = getattr(module, "config", None) - if not isinstance(config, PluginConfig): - raise RuntimeError( - 'Plugin config must expose a "config" variable created by define_plugin()' - ) - return config - - -def _extract_custom_fields( - config: PluginConfig[Any], -) -> dict[str, dict[str, CustomFieldSpec]]: - """Extract custom field specs from config.schemas into the flat shape used by generators. - - Returns an empty dict if config.schemas is None or has no schemas with custom_fields. - """ - if config.schemas is None: - return {} - return { - obj: schema.custom_fields - for obj, schema in config.schemas.items() - if schema.custom_fields is not None - } - - -def _normalize_identifier(name: str) -> str: - """Convert an arbitrary string into a valid Python identifier. - - Args: - name: The raw string to normalise (e.g. a directory name or field key). - - Returns: - A non-empty string that is a legal Python identifier and not a keyword. - - Example:: - - _normalize_identifier("my-plugin") # "my_plugin" - _normalize_identifier("123abc") # "plugin_123abc" - _normalize_identifier("class") # "class_plugin" - """ - # Replace any non-word characters with underscores to produce a valid Python identifier. - ident = re.sub(r"\W+", "_", name.strip()) - if not ident: - ident = "plugin" # empty string edge case - if ident[0].isdigit(): - ident = f"plugin_{ident}" # identifiers can't start with a digit - if keyword.iskeyword(ident): - ident = f"{ident}_plugin" # e.g. "class" -> "class_plugin" - return ident - - -def _to_pascal(value: str) -> str: - """Convert a snake_case or kebab-case string to PascalCase. - - Args: - value: The string to convert (e.g. ``"eligibility_type"``). - - Returns: - PascalCase string (e.g. ``"EligibilityType"``). Falls back to - ``"Field"`` if the input contains no alphanumeric characters. - """ - parts = re.split(r"[^A-Za-z0-9]+", value) - return "".join(p[:1].upper() + p[1:] for p in parts if p) or "Field" - - -def _resolve_field_type(field_type: CustomFieldType | str) -> CustomFieldType: - """Normalise a field type value to a ``CustomFieldType`` enum member. - - Accepts either an already-resolved enum member or a plain string (e.g. - ``"string"``, ``"array"``) as used in ``cg_config.py`` shorthand. - - Args: - field_type: A ``CustomFieldType`` member or its string value. - - Returns: - The corresponding ``CustomFieldType`` enum member. - - Raises: - ValueError: If the string does not match any ``CustomFieldType`` value. - """ - if isinstance(field_type, CustomFieldType): - return field_type - return CustomFieldType(field_type) - - -def _annotation_for_spec(spec: CustomFieldSpec, resolved_type: CustomFieldType) -> str: - """Determine the Python type annotation string for a custom field's ``value`` property. - - Handles three cases based on what ``spec.value`` contains: - - - ``None``: looks up a default annotation from ``FIELD_TYPE_DEFAULT_ANNOTATION`` - (e.g. ``field_type="array"`` → ``"list[Any]"``). - - A plain ``type`` object (e.g. ``int``, ``MyModel``): uses ``__name__``. Builtin - and SDK types are already importable in the generated file; external types (e.g. - Pydantic models from ``cg_config.py`` or third-party packages) will have their - import emitted by :func:`_collect_extra_imports`. - - A generic alias or complex type (e.g. ``list[str]``, ``Optional[int]``): converts - to string, strips the ``typing.`` prefix added by older Python versions, and unwraps - ``<class 'str'>``-style repr strings produced by ``str()`` in some contexts. - - Args: - spec: The ``CustomFieldSpec`` for the field being generated. - resolved_type: The normalised ``CustomFieldType`` for the field. - - Returns: - A string suitable for use as a type annotation in generated Python source, - e.g. ``"str"``, ``"list[str]"``, ``"dict[str, Any]"``, or ``"MyModel"``. - """ - if spec.value is None: - return FIELD_TYPE_DEFAULT_ANNOTATION[resolved_type] - - value = spec.value - if isinstance(value, type): - # Always use the bare name; _collect_extra_imports handles the import - # statement for any type that isn't already available in the generated file. - return value.__name__ - - # spec.value is a generic alias or other non-type (e.g. list[str], Optional[int]). - # Strip the "typing." prefix so the annotation is valid in generated source that - # uses `from __future__ import annotations`. - rendered = str(value).replace("typing.", "") - # str(type_obj) for builtins renders as "<class 'str'>" rather than "str" — unwrap it. - if rendered in { - "<class 'str'>", - "<class 'int'>", - "<class 'float'>", - "<class 'bool'>", - }: - return rendered.split("'")[1] - return rendered or "Any" - - -def _collect_extra_imports( - custom_fields: dict[str, dict[str, CustomFieldSpec]], -) -> list[str]: - """Collect import lines needed for external types used as ``spec.value``. - - Walks all specs and returns one ``import`` line per distinct external type. - Types from ``builtins`` or ``common_grants_sdk`` are skipped because they - are already available in the generated file without an explicit import. - - Types loaded from a ``cg_config.py`` module (identified by the synthetic - ``cg_plugin_config_*`` module name that :func:`_load_config` assigns) are - imported via a relative ``from ..cg_config import`` statement. All other - types are imported using their ``__module__`` path directly. - - Args: - custom_fields: The flat custom fields mapping extracted from the plugin config. - - Returns: - A deduplicated list of import-statement strings in the order they were - first encountered, ready to be inserted into the generated source file. - """ - seen: set[tuple[str, str]] = set() - imports: list[str] = [] - - for fields in custom_fields.values(): - for spec in fields.values(): - if not isinstance(spec.value, type): - continue - module = spec.value.__module__ - name = spec.value.__name__ - if module == "builtins" or module.startswith("common_grants_sdk"): - continue - key = (module, name) - if key in seen: - continue - seen.add(key) - if module.startswith("cg_plugin_config_"): - # Type was defined in cg_config.py, loaded via importlib with a - # synthetic module name — import it relative to the generated/ dir. - imports.append(f"from ..cg_config import {name}") - else: - imports.append(f"from {module} import {name}") - - return imports - - -def _model_blocks( - custom_fields: dict[str, dict[str, CustomFieldSpec]], -) -> Iterable[str]: - """Yield source-code blocks for every model defined in the extensions mapping. - - For each model, yields three blocks in dependency order: - - 1. One ``CustomField`` subclass per field key (typed ``value`` property). - 2. A ``CustomFields`` container model grouping all fields for the model. - 3. The extended model class that adds a ``custom_fields`` attribute typed to - the container. - - Args: - custom_fields: The flat custom fields mapping extracted from the plugin config. - - Yields: - Source-code strings to be joined and written into ``schemas.py``. - - Raises: - ValueError: If a model name is not present in ``MODEL_BASE_CLASS``. - """ - for model_name, fields in custom_fields.items(): - if model_name not in MODEL_BASE_CLASS: - raise ValueError( - f'Generator does not support model "{model_name}". ' - f"Supported models: {sorted(MODEL_BASE_CLASS)}" - ) - - custom_field_classes: list[str] = [] - custom_fields_attrs: list[str] = [] - - for field_key, spec in fields.items(): - resolved_type = _resolve_field_type(spec.field_type) - # Derive generated class names and the snake_case attribute name for this field. - field_cls_name = f"{model_name}{_to_pascal(field_key)}CustomField" - attr_name = _normalize_identifier(snake(field_key)) - value_annotation = _annotation_for_spec( - spec=spec, resolved_type=resolved_type - ) - # Use spec.name as the runtime display name if provided, otherwise fall back - # to the field key (the dict key in SchemaInput.custom_fields). - field_name_default = spec.name or field_key - # repr() produces a quoted string literal safe to embed directly in source code. - description_default = repr(spec.description) if spec.description else "None" - - custom_field_classes.append( - "\n".join( - [ - f"class {field_cls_name}(CustomField):", - " model_config = ConfigDict(populate_by_name=True)", - " field_type: CustomFieldType = Field(", - f" default=CustomFieldType.{resolved_type.name},", - ' alias="fieldType",', - " )", - f" name: str = Field(default={field_name_default!r})", - f" description: Optional[str] = Field(default={description_default})", - f" value: Optional[{value_annotation}] = None", - ] - ) - ) - custom_fields_attrs.append( - "\n".join( - [ - f" {attr_name}: Optional[{field_cls_name}] = Field(", - " default=None,", - f" alias={field_key!r},", - " )", - ] - ) - ) - - custom_fields_model_name = f"{model_name}CustomFields" - model_name_with_extensions = model_name - base_class = MODEL_BASE_CLASS[model_name] - - # Yield three source blocks per model in dependency order: - # 1. Individual CustomField subclasses (one per field key) - # 2. A CustomFields container model grouping all fields for this model - # 3. The extended model class that wires in the CustomFields container - yield "\n\n".join(custom_field_classes) - yield "\n".join( - [ - f"class {custom_fields_model_name}(CommonGrantsBaseModel):", - " model_config = ConfigDict(populate_by_name=True)", - *custom_fields_attrs, - ] - ) - yield "\n".join( - [ - f"class {model_name_with_extensions}({base_class}):", - " model_config = ConfigDict(populate_by_name=True)", - f" custom_fields: Optional[{custom_fields_model_name}] = Field( # type: ignore[assignment]", - " default=None,", - ' alias="customFields",', - " )", - ] - ) - - -def _render_schemas_py( - custom_fields: dict[str, dict[str, CustomFieldSpec]], - mappings_only_objs: set[str] | None = None, -) -> str: - """Render the full source of the generated ``schemas.py`` file. - - Produces a self-contained module containing typed ``CustomField`` subclasses, - a ``CustomFields`` container, and an extended model class for each entry in - ``custom_fields``. Also emits a ``_Schemas`` container object (attribute access - rather than dict lookup) and a module-level ``schemas`` instance. - - For objects that only have ``mappings`` (no ``custom_fields``), the ``_Schemas`` - object will expose the base SDK model class directly (e.g. ``schemas.Opportunity`` - will be ``OpportunityBase``). - - Args: - custom_fields: The flat custom fields mapping extracted from the plugin config. - mappings_only_objs: Set of object names that have mappings but no custom_fields. - These will be exposed on ``_Schemas`` as their base SDK class. - - Returns: - A string of valid Python source code ready to be written to disk. - """ - # _Schemas is a plain object (not a dict) so callers get attribute access: - # plugin.schemas.Opportunity rather than plugin.schemas["Opportunity"]. - # The dynamic __init__ assignment is necessary because model names aren't - # known until generation time, so a static class body can't be used. - model_names = list(custom_fields.keys()) - blocks = "\n\n\n".join(_model_blocks(custom_fields)) - mappings_only: set[str] = mappings_only_objs or set() - - # Build schema assignments: each attribute is a SchemaConfig instance so - # callers get a unified interface (plugin.schemas.Opportunity.common_schema for the - # model class, .to_common/.from_common for transforms). - # to_common/from_common default to None here; root __init__.py injects the - # real callables for any object that has transforms configured. - assignments: list[str] = [ - f" self.{name} = SchemaConfig(source_schema=dict, common_schema={name}, to_common=None, from_common=None)" - for name in model_names - ] - for obj in sorted(mappings_only): - if obj not in MODEL_BASE_CLASS: - raise ValueError( - f'Generator does not support model "{obj}". ' - f"Supported models: {sorted(MODEL_BASE_CLASS)}" - ) - base_class = MODEL_BASE_CLASS[obj] - assignments.append( - f" self.{obj} = SchemaConfig(source_schema=dict, common_schema={base_class}, to_common=None, from_common=None)" - ) - - schema_assignments = "\n".join(assignments or [" pass"]) - all_names = model_names + sorted(mappings_only) - all_exports = ", ".join([f'"{name}"' for name in all_names] + ['"schemas"']) - extra_imports = _collect_extra_imports(custom_fields) - extra_import_lines = ["", *extra_imports] if extra_imports else [] - - return "\n".join( - [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from __future__ import annotations", - "", - "from typing import Any, Optional", - "", - "from pydantic import ConfigDict, Field", - "", - "from common_grants_sdk.extensions.types import SchemaConfig", - "from common_grants_sdk.schemas.pydantic.base import CommonGrantsBaseModel", - "from common_grants_sdk.schemas.pydantic.fields import CustomField, CustomFieldType", - "from common_grants_sdk.schemas.pydantic.models import OpportunityBase", - *extra_import_lines, - *([blocks, ""] if blocks else []), - "class _Schemas:", - " def __init__(self) -> None:", - schema_assignments, - "", - "schemas = _Schemas()", - "", - f"__all__ = [{all_exports}]", - "", - ] - ) - - -def _render_generated_init_py() -> str: - """Render the source of the ``generated/__init__.py`` file. - - This file re-exports the ``schemas`` instance from ``schemas.py`` so that - the generated package can be imported as ``from .generated import schemas``. - - Returns: - A string of valid Python source code ready to be written to disk. - """ - return "\n".join( - [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from .schemas import schemas", - "", - '__all__ = ["schemas"]', - "", - ] - ) - - -def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) -> str: - """Render the source of the plugin directory's root __init__.py file. - - Emits a fully compiled Plugin instance. Transform callables are injected - into the _Schemas object (from generated/schemas.py) before Plugin is - constructed, so plugin.schemas.Opportunity.to_common etc. are populated. - """ - # Collect the sets of objects needing transform injection. - # Only count schemas that have explicit callable transforms, not those with custom_fields only. - explicit_objs: set[str] = ( - { - obj - for obj, s in config.schemas.items() - if s.to_common is not None or s.from_common is not None - } - if config.schemas - else set() - ) - mappings_objs: set[str] = ( - {obj for obj, s in config.schemas.items() if s.mappings is not None} - if config.schemas - else set() - ) - - needs_build_transforms = bool(mappings_objs - explicit_objs) - - # Build pre-plugin lines: inject transforms into the _Schemas object before - # constructing Plugin. schemas.py initialises each SchemaConfig with - # to_common=None/from_common=None; we mutate those attrs here. - inject_lines: list[str] = [] - - # Mappings-only objects: call build_transforms() then inject results. - for obj in sorted(mappings_objs - explicit_objs): - inject_lines += [ - f"_{obj}_to_common, _{obj}_from_common = build_transforms(", - f' to_common_mapping=config.schemas["{obj}"].mappings.to_common,', - f' from_common_mapping=config.schemas["{obj}"].mappings.from_common,', - f" common_schema=schemas.{obj}.common_schema,", - ")", - f"schemas.{obj}.to_common = _{obj}_to_common", - f"schemas.{obj}.from_common = _{obj}_from_common", - "", - ] - - # Explicit schemas: single inject_transforms() call handles all objects. - # Reassigning the return value preserves the concrete _Schemas type for mypy. - if explicit_objs: - inject_lines.append("schemas = inject_transforms(config, schemas)") - inject_lines.append("") - - imports = [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from __future__ import annotations", - "", - ] - - sdk_imports = {"Plugin"} - if explicit_objs: - sdk_imports.add("inject_transforms") - if needs_build_transforms: - sdk_imports.add("build_transforms") - imports.append( - f"from common_grants_sdk.extensions import {', '.join(sorted(sdk_imports))}" - ) - imports += [ - "from .cg_config import config", - "from .generated import schemas", - "", - ] - - plugin_lines = [ - f"{plugin_variable_name} = Plugin(", - " schemas=schemas,", - " meta=config.meta,", - ")", - ] - - pre_plugin = inject_lines # may be empty - - return "\n".join( - imports - + pre_plugin - + plugin_lines - + [ - "", - f'__all__ = ["{plugin_variable_name}", "schemas"]', - "", - ] - ) - - -def generate_plugin(plugin_dir: Path) -> Path: - """Run the full code generation pipeline for a single plugin directory. - - Loads ``cg_config.py``, creates the ``generated/`` subdirectory, and writes - three files: ``generated/schemas.py``, ``generated/__init__.py``, and the - root ``__init__.py`` that exports the ``Plugin`` instance. - - Args: - plugin_dir: Path to the plugin directory containing ``cg_config.py``. - - Returns: - The path to the ``generated/`` directory that was created. - - Raises: - FileNotFoundError: If ``cg_config.py`` does not exist in ``plugin_dir``. - RuntimeError: If the config file cannot be loaded or is invalid. - ValueError: If the config references an unsupported model name. - """ - plugin_dir = plugin_dir.resolve() - config_path = plugin_dir / "cg_config.py" - if not config_path.exists(): - raise FileNotFoundError(f"Could not find config file: {config_path}") - - config = _load_config(config_path) - custom_fields = _extract_custom_fields(config) - - # Determine objects that have mappings but no custom_fields — these need a - # pass-through entry in _Schemas pointing at the base SDK model class. - explicit_cf_objs: set[str] = set(custom_fields.keys()) - mappings_only_objs: set[str] = ( - { - obj - for obj, s in config.schemas.items() - if s.mappings is not None and obj not in explicit_cf_objs - } - if config.schemas - else set() - ) - # Third bucket: objects in config.schemas with explicit transforms but no - # custom_fields and no mappings entry. They also need a pass-through - # entry in _Schemas or schemas.<Obj> won't exist at import time. - transforms_only_objs: set[str] = ( - set(config.schemas.keys()) - explicit_cf_objs - mappings_only_objs - if config.schemas - else set() - ) - - # Validate that auto-generated transform objects have both mapping directions. - # Auto-generated objects: have mappings in schemas[obj] but no explicit callables. - if config.schemas: - explicit_schema_objs: set[str] = { - obj - for obj, s in config.schemas.items() - if s.to_common is not None or s.from_common is not None - } - for obj, schema in config.schemas.items(): - if schema.mappings is None or obj in explicit_schema_objs: - continue - if schema.mappings.to_common is None: - raise ValueError( - f'Plugin object "{obj}": mappings.to_common is required when ' - f"auto-generating transforms. Either provide a to_common mapping " - f"or pass an explicit to_common callable via schemas['{obj}']." - ) - if schema.mappings.from_common is None: - raise ValueError( - f'Plugin object "{obj}": mappings.from_common is required when ' - f"auto-generating transforms. Either provide a from_common mapping " - f"or pass an explicit from_common callable via schemas['{obj}']." - ) - - generated_dir = plugin_dir / "generated" - generated_dir.mkdir(parents=True, exist_ok=True) - - schemas_py = generated_dir / "schemas.py" - init_generated_py = generated_dir / "__init__.py" - root_init_py = plugin_dir / "__init__.py" - - schemas_py.write_text( - _render_schemas_py( - custom_fields, mappings_only_objs=mappings_only_objs | transforms_only_objs - ), - encoding="utf-8", - ) - init_generated_py.write_text(_render_generated_init_py(), encoding="utf-8") - - plugin_variable_name = _normalize_identifier(plugin_dir.name) - root_init_py.write_text( - _render_plugin_init_py(plugin_variable_name, config), encoding="utf-8" - ) - - return generated_dir - - -def main(argv: list[str] | None = None) -> int: - """CLI entry point for ``python -m common_grants_sdk.extensions.generate``. - - Parses command-line arguments and delegates to ``generate_plugin()``. - - Args: - argv: Argument list to parse. Defaults to ``sys.argv[1:]`` when ``None``. - - Returns: - Exit code (``0`` on success). - """ - parser = argparse.ArgumentParser( - prog="python -m common_grants_sdk.extensions.generate", - description="Generate typed plugin schemas from cg_config.py", - ) - parser.add_argument( - "--plugin", - nargs="+", - default=["."], - help="One or more plugin directories containing cg_config.py (default: current directory)", - ) - args = parser.parse_args(argv) - - for plugin_path in args.plugin: - generated_dir = generate_plugin(Path(plugin_path)) - print(f"Generated plugin schemas at {generated_dir}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lib/python-sdk/examples/plugin_custom_fields.py b/lib/python-sdk/examples/plugin_custom_fields.py deleted file mode 100644 index 738dba2de..000000000 --- a/lib/python-sdk/examples/plugin_custom_fields.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Example demonstrating how to use the plugin framework to validate and -access typed custom fields on an Opportunity. - -Before running this script, generate the typed models by running this -command from the plugin directory: - - lib/python-sdk - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions - -Then run this script from the lib/python-sdk directory: - - poetry run python examples/plugin_custom_fields.py -""" - -import sys -from pathlib import Path - -# Make the examples/ directory importable so that -# plugins.opportunity_extensions resolves correctly. -sys.path.insert(0, str(Path(__file__).parent)) - -from plugins.opportunity_extensions import opportunity_extensions # noqa: E402 -from plugins.opportunity_extensions.cg_config import config as opp_config # noqa: E402 - -# --------------------------------------------------------------------------- -# Sample API payload containing our four custom fields -# --------------------------------------------------------------------------- - -api_response = { - "id": "573525f2-8e15-4405-83fb-e6523511d893", - "title": "Community Health Innovation Grant", - "status": {"value": "open"}, - "description": "Funding for community-led health initiatives", - "createdAt": "2025-03-01T00:00:00Z", - "lastModifiedAt": "2025-03-15T00:00:00Z", - "customFields": { - "programArea": { - "fieldType": "string", - "value": "CFDA-93.243", - }, - "legacyGrantId": { - "fieldType": "integer", - "value": 98765, - }, - "eligibilityTypes": { - "fieldType": "array", - "value": ["nonprofit", "tribal", "city_government"], - }, - "awardCeiling": { - "fieldType": "number", - "value": 250000.00, - }, - }, -} - -# --------------------------------------------------------------------------- -# Use the model returned via opportunity_extensions -# --------------------------------------------------------------------------- - -opp = opportunity_extensions.schemas.Opportunity.common_schema.model_validate( - api_response -) - -assert opp.custom_fields is not None -assert opp.custom_fields.program_area is not None -assert opp.custom_fields.legacy_grant_id is not None -assert opp.custom_fields.eligibility_types is not None -assert opp.custom_fields.award_ceiling is not None - -print(f"Title: {opp.title}") -print(f"Status: {opp.status.value}") -print(f"program_area: {opp.custom_fields.program_area.value}") -print(f"legacy_grant_id: {opp.custom_fields.legacy_grant_id.value}") -print(f"eligibility_types:{opp.custom_fields.eligibility_types.value}") -print(f"award_ceiling: {opp.custom_fields.award_ceiling.value}") -print() - -# --------------------------------------------------------------------------- -# The plugin also exposes the original extension specs -# --------------------------------------------------------------------------- - -print("Registered custom fields:") -assert opp_config.schemas is not None -_opp_custom_fields = opp_config.schemas["Opportunity"].custom_fields -assert _opp_custom_fields is not None -for field_name, spec in _opp_custom_fields.items(): - print(f" {field_name}: {spec.field_type} — {spec.description}") diff --git a/lib/python-sdk/examples/plugins/__init__.py b/lib/python-sdk/examples/plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py deleted file mode 100644 index 4514e8f0e..000000000 --- a/lib/python-sdk/examples/plugins/grants_gov/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# This file is auto-generated. Do not edit it manually — it will be overwritten -# the next time `python -m common_grants_sdk.extensions.generate` is run. -from __future__ import annotations - -from common_grants_sdk.extensions import Plugin, inject_transforms -from .cg_config import config -from .generated import schemas - -schemas = inject_transforms(config, schemas) - -grants_gov = Plugin( - schemas=schemas, - meta=config.meta, -) - -__all__ = ["grants_gov", "schemas"] diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py deleted file mode 100644 index c155cfc00..000000000 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Grants.gov sample plugin — bidirectional transform PoC. - -Demonstrates the plugin framework shape using the grants.gov scenario. - -Usage (from lib/python-sdk/): - poetry run python examples/transforms.py - -Code generation (generates typed custom-field schemas): - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov -""" - -from typing import Any - -from common_grants_sdk.extensions import ( - CustomFieldSpec, - PluginExtensionsMeta, - SchemaInput, - build_transforms, - define_plugin, -) -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -from common_grants_sdk.utils.transformation import get_from_path - - -def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: - sep = spec.get("sep", " ") - parts = [get_from_path(data, path) for path in spec.get("fields", [])] - values = [str(p) for p in parts if p is not None] - return sep.join(values) if values else None - - -# --------------------------------------------------------------------------- -# Bidirectional transforms -# -# Both directions are author-provided — build_transforms() does not invert -# one into the other because many-to-one handlers like switch are not -# reversible. -# -# Convention: field extraction uses {"field": "dot.notation.path"} — bare -# string values are treated as literals by transform_from_mapping(), not -# as field paths. See Design Finding #2 in the spec for the open question -# about which convention is canonical. -# --------------------------------------------------------------------------- - -to_common, from_common = build_transforms( - handlers={"join": _join_fields}, - # to_common: grants.gov native → CommonGrants Opportunity - to_common_mapping={ - "title": {"field": "data.opportunity_title"}, - "status": { - "value": { - "match": { - "field": "data.opportunity_status", - "case": { - "forecasted": "forecasted", - "posted": "open", - "archived": "closed", - }, - "default": "custom", - } - }, - "description": { - "const": "The opportunity is currently accepting applications" - }, - }, - "funding": { - "minAwardAmount": { - "amount": {"field": "data.summary.award_floor"}, - "currency": {"const": "USD"}, - }, - "maxAwardAmount": { - "amount": {"field": "data.summary.award_ceiling"}, - "currency": {"const": "USD"}, - }, - }, - "keyDates": { - "appOpens": { - "name": {"const": "Open Date"}, - "date": {"field": "data.summary.forecasted_post_date"}, - "description": {"const": "Applications begin being accepted"}, - }, - "appDeadline": { - "name": {"const": "Application Deadline"}, - "date": {"field": "data.summary.forecasted_close_date"}, - "description": { - "const": "Final submission deadline for all grant applications" - }, - }, - }, - "customFields": { - "legacyIdStr": { - "value": {"numberToString": "data.opportunity_id"}, - }, - "priorityScore": { - "value": {"stringToNumber": "data.priority_score_str"}, - }, - "compositeLabel": { - "value": { - "join": { - "fields": ["data.opportunity_number", "data.opportunity_title"], - "sep": " — ", - } - }, - }, - }, - }, - # from_common: CommonGrants Opportunity → grants.gov native - from_common_mapping={ - "data": { - "opportunity_title": {"field": "title"}, - "opportunity_status": { - "match": { - "field": "status.value", - "case": { - "open": "posted", - "closed": "archived", - "forecasted": "forecasted", - }, - "default": "custom", - } - }, - "summary": { - "award_floor": {"field": "funding.minAwardAmount.amount"}, - "award_ceiling": {"field": "funding.maxAwardAmount.amount"}, - "forecasted_post_date": {"field": "keyDates.appOpens.date"}, - "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, - }, - "priority_score_str": { - "numberToString": "customFields.priorityScore.value" - }, - } - }, -) - -# --------------------------------------------------------------------------- -# Plugin config -# --------------------------------------------------------------------------- - -config = define_plugin( - meta=PluginExtensionsMeta( - name="grants-gov", - version="0.1.0", - sourceSystem="grants.gov", - capabilities=["customFields", "transforms"], - ), - schemas={ - "Opportunity": SchemaInput( - custom_fields={ - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - name="Legacy ID", - description="Unique identifier in legacy database", - ), - "legacyIdStr": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Legacy ID (string)", - description="Legacy ID coerced to a string via numberToString", - ), - "agencyName": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Agency", - description="Agency hosting the opportunity", - ), - "applicantTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - name="Applicant types", - description="Types of applicants eligible to apply", - ), - "priorityScore": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - name="Priority score", - description="Numeric priority score coerced from a string via stringToNumber", - ), - "compositeLabel": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Composite label", - description="Composite label '<opportunity_number> — <opportunity_title>' for round-trip recovery", - ), - }, - to_common=to_common, - from_common=from_common, - ) - }, -) diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py deleted file mode 100644 index c8f51a3ac..000000000 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# This file is auto-generated. Do not edit it manually — it will be overwritten -# the next time `python -m common_grants_sdk.extensions.generate` is run. -from __future__ import annotations - -from common_grants_sdk.extensions import Plugin -from .cg_config import config -from .generated import schemas - -opportunity_extensions = Plugin( - schemas=schemas, - meta=config.meta, -) - -__all__ = ["opportunity_extensions", "schemas"] diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py deleted file mode 100644 index 97a9d3765..000000000 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Plugin configuration for opportunity extensions. - -Defines custom field extensions for the Opportunity schema: -- HHS-specific fields (programArea, legacyGrantId) -- Local fields (eligibilityTypes, awardCeiling) -""" - -from typing import Any - -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, SchemaInput -from common_grants_sdk.extensions.plugin import PluginConfig -from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType - -config: PluginConfig[Any] = define_plugin( - schemas={ - "Opportunity": SchemaInput( - custom_fields={ - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - } - ) - } -) diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py deleted file mode 100644 index 637aa1ffb..000000000 --- a/lib/python-sdk/examples/transforms.py +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python3 -"""Bidirectional transform PoC — plugin transformation interface. - -Demonstrates source (grants.gov) → CommonGrants and CommonGrants → source -bidirectional transformations using the grants.gov sample plugin. - -Requires generated schemas (examples/plugins/grants_gov/generated/). -Generate them first (from lib/python-sdk/): - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov -Or run all plugins at once: - make plugins - -Then run (from lib/python-sdk/): - poetry run python examples/transforms.py -""" - -from __future__ import annotations - -import json -from typing import Any - -# When run as `poetry run python examples/transforms.py`, Python automatically -# adds the script's directory (examples/) to sys.path. Import from there using -# the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only -# works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. -from plugins.grants_gov import grants_gov as plugin -from plugins.grants_gov.generated.schemas import Opportunity - -from common_grants_sdk.extensions import build_transforms -from common_grants_sdk.utils.transformation import get_from_path - -# --------------------------------------------------------------------------- -# Sample grants.gov source data -# --------------------------------------------------------------------------- - -SOURCE_DATA: dict[str, Any] = { - "data": { - "agency_name": "Department of Examples", - "created_at": "2025-01-15T09:00:00Z", - "last_modified_at": "2025-04-01T12:30:00Z", - "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", - "opportunity_id": 12345, - "opportunity_number": "ABC-123-XYZ-001", - "opportunity_status": "posted", - "opportunity_title": "Research into conservation techniques", - "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "priority_score_str": "75", - "summary": { - "applicant_types": ["state_governments"], - "archive_date": "2025-05-01", - "award_ceiling": 100000, - "award_floor": 10000, - "forecasted_award_date": "2025-09-01", - "forecasted_close_date": "2025-07-15", - "forecasted_post_date": "2025-05-01", - }, - } -} - - -# --------------------------------------------------------------------------- -# Custom handlers: join_fields and split_field -# -# join_fields concatenates multiple source field values with a configurable -# separator. Mapping spec: {"join": {"fields": ["a.b", "c.d"], "sep": " — "}} -# -# split_field is the inverse: it splits a single field on a separator and -# returns the element at the given index. -# Mapping spec: {"split": {"field": "label", "sep": " — ", "index": 0}} -# --------------------------------------------------------------------------- - - -def join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: - """Custom handler that joins multiple field values with a separator.""" - sep = spec.get("sep", " ") - parts = [get_from_path(data, path) for path in spec.get("fields", [])] - values = [str(p) for p in parts if p is not None] - return sep.join(values) if values else None - - -def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: - """Custom handler that splits a field value and returns the element at index.""" - value = get_from_path(data, spec.get("field", "")) - if value is None: - return None - parts = str(value).split(spec.get("sep", " ")) - index = spec.get("index", 0) - return parts[index] if index < len(parts) else None - - -# Transform that uses the custom handlers and validates output against the generated -# Opportunity model. common_schema=Opportunity (from generated/schemas.py) ensures -# model_validate runs against the extended class with typed custom fields -# (legacyId, agencyName, applicantTypes), not just the base OpportunityBase. -to_common_with_custom, from_common_with_custom = build_transforms( - to_common_mapping={ - "id": {"field": "data.opportunity_uuid"}, - "title": {"field": "data.opportunity_title"}, - "description": {"field": "data.opportunity_description"}, - "createdAt": {"field": "data.created_at"}, - "lastModifiedAt": {"field": "data.last_modified_at"}, - "status": { - "value": { - "match": { - "field": "data.opportunity_status", - "case": { - "posted": "open", - "archived": "closed", - "forecasted": "forecasted", - }, - "default": "custom", - } - }, - }, - "customFields": { - "compositeLabel": { - "value": { - "join": { - "fields": ["data.opportunity_number", "data.opportunity_title"], - "sep": " — ", - } - }, - }, - "legacyId": { - "value": {"field": "data.opportunity_id"}, - }, - "agencyName": { - "value": {"field": "data.agency_name"}, - }, - "applicantTypes": { - "value": {"field": "data.summary.applicant_types"}, - }, - }, - }, - from_common_mapping={ - "data": { - # label is produced by the join handler above but gets dropped by - # model_validate (it is not a CG field), so from_common maps directly - # from the standard CG title field instead. - "opportunity_title": {"field": "title"}, - } - }, - handlers={"join": join_fields, "split": split_field}, - common_schema=Opportunity, -) - - -def _section(title: str) -> None: - print(f"\n{'=' * 60}") - print(title) - print("=" * 60) - - -def main() -> None: - opp = plugin.schemas.Opportunity - - _section("SOURCE DATA (grants.gov format)") - print(json.dumps(SOURCE_DATA, indent=2)) - - # --- to_common: grants.gov → CommonGrants --- - assert opp.to_common is not None - cg_result = opp.to_common(SOURCE_DATA) - - _section("to_common: grants.gov → CommonGrants") - if cg_result.errors: - print(f"ERRORS ({len(cg_result.errors)}):") - for err in cg_result.errors: - print(f" [path={err.path}] {err}") - else: - print("Errors: none") - print("\nResult:") - print(json.dumps(cg_result.result, indent=2)) - - # --- from_common: CommonGrants → grants.gov --- - assert opp.from_common is not None - native_result = opp.from_common(cg_result.result) - - _section("from_common: CommonGrants → grants.gov") - if native_result.errors: - print(f"ERRORS ({len(native_result.errors)}):") - for err in native_result.errors: - print(f" [path={err.path}] {err}") - else: - print("Errors: none") - print("\nResult:") - print(json.dumps(native_result.result, indent=2)) - - # --- Roundtrip comparison --- - # Note: SOURCE_DATA contains fields not covered by the mappings (agency_name, - # opportunity_id, etc.). Those fields are intentionally absent from the roundtrip - # output — the mapping layer is selective by design. - _section("ROUNDTRIP CHECK") - checks = [ - ( - "title", - SOURCE_DATA["data"]["opportunity_title"], - native_result.result.get("data", {}).get("opportunity_title"), - ), - ( - "status", - SOURCE_DATA["data"]["opportunity_status"], - native_result.result.get("data", {}).get("opportunity_status"), - ), - ( - "award_floor", - SOURCE_DATA["data"]["summary"]["award_floor"], - native_result.result.get("data", {}).get("summary", {}).get("award_floor"), - ), - ( - "award_ceiling", - SOURCE_DATA["data"]["summary"]["award_ceiling"], - native_result.result.get("data", {}) - .get("summary", {}) - .get("award_ceiling"), - ), - ( - "priority_score_str", - SOURCE_DATA["data"]["priority_score_str"], - native_result.result.get("data", {}).get("priority_score_str"), - ), - ] - all_pass = True - for field, original, roundtripped in checks: - ok = original == roundtripped - if not ok: - all_pass = False - status = "PASS" if ok else "FAIL" - print(f" [{status}] {field}: {original!r} -> {roundtripped!r}") - - print( - f"\nRoundtrip result ({len(checks)} mapped fields checked; unmapped fields dropped by design): {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}" - ) - - # --- Custom handler + model_validate demo --- - _section( - "CUSTOM HANDLER + MODEL VALIDATE DEMO (join / split / extended Opportunity)" - ) - print("Custom handlers: join, split") - print("common_model: generated Opportunity (with typed customFields)\n") - - custom_cg = to_common_with_custom(SOURCE_DATA) - - if custom_cg.errors: - print(f"ERRORS ({len(custom_cg.errors)}):") - for err in custom_cg.errors: - print(f" [path={err.path}] {err}") - else: - print("Validation: PASS — result is a typed Opportunity instance") - assert isinstance(custom_cg.result, Opportunity) - opp_instance = custom_cg.result - print(f"\n title: {opp_instance.title}") - print(f" id: {opp_instance.id}") - print(f" status: {opp_instance.status.value}") - if opp_instance.custom_fields: - cf = opp_instance.custom_fields - print("\n customFields (typed):") - if cf.legacy_id: - print( - f" legacyId.value: {cf.legacy_id.value!r} ({type(cf.legacy_id.value).__name__})" - ) - if cf.agency_name: - print( - f" agencyName.value: {cf.agency_name.value!r} ({type(cf.agency_name.value).__name__})" - ) - if cf.applicant_types: - print( - f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})" - ) - if cf.composite_label: - print( - f" compositeLabel.value: {cf.composite_label.value!r} ({type(cf.composite_label.value).__name__})" - ) - - custom_native = from_common_with_custom( - custom_cg.result if not custom_cg.errors else {} - ) - orig_title = SOURCE_DATA["data"]["opportunity_title"] - rt_title = custom_native.result.get("data", {}).get("opportunity_title") - print( - f"\n [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}" - ) - - # --- Plugin metadata --- - _section("PLUGIN METADATA") - assert plugin.meta is not None - print(f"name: {plugin.meta.name}") - print(f"version: {plugin.meta.version}") - print(f"sourceSystem: {plugin.meta.source_system}") - print(f"capabilities: {plugin.meta.capabilities}") - - -if __name__ == "__main__": - main() diff --git a/lib/python-sdk/tests/utils/test_plugin_generator.py b/lib/python-sdk/tests/utils/test_plugin_generator.py deleted file mode 100644 index 8b39a829d..000000000 --- a/lib/python-sdk/tests/utils/test_plugin_generator.py +++ /dev/null @@ -1,552 +0,0 @@ -from __future__ import annotations - -import importlib -import os -import shutil -import subprocess -import sys -from pathlib import Path -from typing import get_type_hints - -import pytest - -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec -from common_grants_sdk.extensions import PluginConfig -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType - - -def _sdk_root() -> Path: - return Path(__file__).resolve().parents[2] - - -def _env_with_sdk_pythonpath() -> dict[str, str]: - env = os.environ.copy() - sdk_root = str(_sdk_root()) - existing = env.get("PYTHONPATH") - env["PYTHONPATH"] = f"{sdk_root}{os.pathsep}{existing}" if existing else sdk_root - return env - - -def test_define_plugin_returns_config_with_schemas(): - from common_grants_sdk.extensions.types import SchemaInput - - schemas = { - "Opportunity": SchemaInput( - custom_fields={ - "program_area": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Grant category", - ) - } - ) - } - config = define_plugin(schemas=schemas) - - assert isinstance(config, PluginConfig) - assert config.schemas is schemas - assert config.schemas["Opportunity"].custom_fields is not None - assert "program_area" in config.schemas["Opportunity"].custom_fields - - -def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): - plugins_dir = tmp_path / "plugins" - plugins_dir.mkdir() - (plugins_dir / "__init__.py").write_text("", encoding="utf-8") - - plugin_dir = plugins_dir / "combined" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "program_area": CustomFieldSpec(', - ' field_type="string",', - ' description="Program area",', - " ),", - ' "eligibility_type": CustomFieldSpec(', - ' field_type="array",', - ' description="Types of eligible organizations",', - " ),", - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - cmd = [sys.executable, "-m", "common_grants_sdk.extensions.generate"] - run = subprocess.run( - cmd, - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - assert (plugin_dir / "generated" / "__init__.py").exists() - assert (plugin_dir / "generated" / "schemas.py").exists() - assert (plugin_dir / "__init__.py").exists() - - sys.path.insert(0, str(tmp_path)) - try: - combined_module = importlib.import_module("plugins.combined") - combined = getattr(combined_module, "combined") - opp_model = combined.schemas.Opportunity.common_schema - - type_hints = get_type_hints(opp_model, include_extras=False) - assert "custom_fields" in type_hints - - payload = { - "id": "573525f2-8e15-4405-83fb-e6523511d893", - "title": "Test Opportunity", - "status": {"value": "open"}, - "description": "Funding available for pilot projects", - "createdAt": "2026-01-01T00:00:00Z", - "lastModifiedAt": "2026-01-01T00:00:00Z", - "customFields": { - "program_area": { - "fieldType": "string", - "value": "Health", - }, - "eligibility_type": { - "fieldType": "array", - "value": ["nonprofit", "city_government"], - }, - }, - } - - opp = opp_model.model_validate(payload) - - assert opp.custom_fields is not None - assert opp.custom_fields.program_area is not None - assert opp.custom_fields.program_area.value == "Health" - assert opp.custom_fields.eligibility_type is not None - assert opp.custom_fields.eligibility_type.value == [ - "nonprofit", - "city_government", - ] - assert combined.schemas.Opportunity.common_schema is opp_model - finally: - sys.path.remove(str(tmp_path)) - - -def test_generate_emits_import_for_pydantic_model_in_cg_config(tmp_path: Path): - """spec.value set to a Pydantic model defined in cg_config.py should produce - a ``from ..cg_config import <Model>`` line in the generated schemas.py.""" - plugin_dir = tmp_path / "my_plugin" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from pydantic import BaseModel", - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "class AgentInfo(BaseModel):", - " name: str", - " email: str", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "point_of_contact": CustomFieldSpec(', - ' field_type="object",', - " value=AgentInfo,", - " ),", - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - run = subprocess.run( - [sys.executable, "-m", "common_grants_sdk.extensions.generate"], - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "from ..cg_config import AgentInfo" in schemas_src - assert "value: Optional[AgentInfo]" in schemas_src - - -def test_generate_emits_import_for_external_module_type(tmp_path: Path): - """spec.value set to a type from a real importable module should produce - a ``from <module> import <Type>`` line in the generated schemas.py.""" - plugin_dir = tmp_path / "my_plugin" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from datetime import datetime", - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "deadline": CustomFieldSpec(', - ' field_type="string",', - " value=datetime,", - " ),", - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - run = subprocess.run( - [sys.executable, "-m", "common_grants_sdk.extensions.generate"], - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "from datetime import datetime" in schemas_src - assert "value: Optional[datetime]" in schemas_src - - -def test_generate_auto_builds_transforms_from_mappings(tmp_path): - """When cg_config has schemas[obj].mappings but no explicit to_common/from_common, - the generated __init__.py calls build_transforms() automatically.""" - plugin_dir = tmp_path / "plugins" / "auto_transform" - plugin_dir.mkdir(parents=True) - (plugin_dir / "__init__.py").write_text("", encoding="utf-8") - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, SchemaMappings", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " mappings=SchemaMappings(", - ' to_common={"title": {"field": "data.title"}},', - " from_common={},", - " ),", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - from common_grants_sdk.extensions.generate import generate_plugin - - generate_plugin(plugin_dir) - - init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") - assert "build_transforms" in init_content - assert 'config.schemas["Opportunity"].mappings.to_common' in init_content - assert "_Opportunity_to_common" in init_content - assert "common_schema=schemas.Opportunity.common_schema" in init_content - - # Load the generated plugin and verify schemas are populated - import importlib - import sys - - # Remove any stale 'plugins' package from previous tests before inserting our path. - for key in list(sys.modules.keys()): - if key == "plugins" or key.startswith("plugins."): - del sys.modules[key] - - sys.path.insert(0, str(tmp_path)) - try: - mod = importlib.import_module("plugins.auto_transform") - plugin = getattr(mod, "auto_transform") - assert hasattr(plugin.schemas, "Opportunity") - assert plugin.schemas.Opportunity.to_common is not None - finally: - if str(tmp_path) in sys.path: - sys.path.remove(str(tmp_path)) - for key in list(sys.modules.keys()): - if key == "plugins" or key.startswith("plugins."): - del sys.modules[key] - - -@pytest.mark.skipif(shutil.which("pyright") is None, reason="pyright is not installed") -def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): - plugins_dir = tmp_path / "plugins" - plugins_dir.mkdir() - (plugins_dir / "__init__.py").write_text("", encoding="utf-8") - - plugin_dir = plugins_dir / "combined" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "eligibility_type": CustomFieldSpec(field_type="array"),', - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - run = subprocess.run( - [sys.executable, "-m", "common_grants_sdk.extensions.generate"], - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - typecheck_file = tmp_path / "typecheck.py" - typecheck_file.write_text( - "\n".join( - [ - "# pyright: strict", - "from plugins.combined import combined", - "", - "payload = {", - ' "id": "573525f2-8e15-4405-83fb-e6523511d893",', - ' "title": "Typed Opportunity",', - ' "status": {"value": "open"},', - ' "description": "Typed custom fields",', - ' "createdAt": "2026-01-01T00:00:00Z",', - ' "lastModifiedAt": "2026-01-01T00:00:00Z",', - ' "customFields": {"eligibility_type": {"fieldType": "array", "value": ["a"]}},', - "}", - "", - "opp = combined.schemas.Opportunity.common_schema.model_validate(payload)", - "if opp.custom_fields is not None and opp.custom_fields.eligibility_type is not None:", - " values = opp.custom_fields.eligibility_type.value", - " reveal_type(values)", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - env["PYTHONPATH"] = f"{tmp_path}{os.pathsep}{env['PYTHONPATH']}" - pyright = subprocess.run( - ["pyright", str(typecheck_file)], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert pyright.returncode == 0, pyright.stdout + "\n" + pyright.stderr - assert 'Type of "values" is "list[Any] | None"' in pyright.stdout - - -def test_generate_explicit_transforms(tmp_path): - """When cg_config has config.schemas with explicit to_common/from_common, - the generated __init__.py wires the supplied callables via inject_transforms.""" - from common_grants_sdk.extensions.generate import generate_plugin - - plugin_dir = tmp_path / "plugins" / "explicit_tf" - plugin_dir.mkdir(parents=True) - (plugin_dir / "__init__.py").write_text("", encoding="utf-8") - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, TransformResult", - "from common_grants_sdk.extensions import CustomFieldSpec", - "", - "def _to_common(native):", - " return TransformResult(result={'title': native.get('name', '')}, errors=[])", - "", - "def _from_common(common):", - " return TransformResult(result={'name': common.get('title', '')}, errors=[])", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - ' custom_fields={"legacyId": CustomFieldSpec(field_type="integer")},', - " to_common=_to_common,", - " from_common=_from_common,", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - generate_plugin(plugin_dir) - - init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") - # Explicit transforms use inject_transforms(), not per-object boilerplate - assert "build_transforms" not in init_content - assert "schemas = inject_transforms(config, schemas)" in init_content - # No per-object assignment lines - assert 'config.schemas["Opportunity"].to_common' not in init_content - assert 'config.schemas["Opportunity"].from_common' not in init_content - # ObjectSchemas is no longer constructed in __init__.py (only in schemas.py) - assert "ObjectSchemas" not in init_content - - # Load and verify the plugin works end-to-end - sys.path.insert(0, str(tmp_path)) - try: - mod = importlib.import_module("plugins.explicit_tf") - plugin = getattr(mod, "explicit_tf") - assert hasattr(plugin.schemas, "Opportunity") - opp_schemas = plugin.schemas.Opportunity - result = opp_schemas.to_common({"name": "Test Grant"}) - assert result.result == {"title": "Test Grant"} - assert result.errors == [] - finally: - if str(tmp_path) in sys.path: - sys.path.remove(str(tmp_path)) - for key in list(sys.modules.keys()): - if "explicit_tf" in key or key == "plugins": - del sys.modules[key] - - -def test_generate_transforms_only_no_custom_fields(tmp_path): - """Regression: config.schemas with only explicit transforms (no custom_fields, no extensions) - must produce a _Schemas entry for the object so inject_transforms() can access it at import. - """ - from common_grants_sdk.extensions.generate import generate_plugin - - plugin_dir = tmp_path / "plugins" / "transforms_only" - plugin_dir.mkdir(parents=True) - (plugin_dir / "__init__.py").write_text("", encoding="utf-8") - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, TransformResult", - "", - "def _to_common(native):", - " return TransformResult(result={'title': native.get('name', '')}, errors=[])", - "", - "def _from_common(common):", - " return TransformResult(result={'name': common.get('title', '')}, errors=[])", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " to_common=_to_common,", - " from_common=_from_common,", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - generate_plugin(plugin_dir) - - # schemas.py must assign self.Opportunity using the base SDK class - schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "self.Opportunity = SchemaConfig" in schemas_src - assert "OpportunityBase" in schemas_src - - # __init__.py must use inject_transforms (not build_transforms) - init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") - assert "inject_transforms" in init_content - assert "build_transforms" not in init_content - - # Importing must not raise AttributeError / ValueError - sys.path.insert(0, str(tmp_path)) - try: - mod = importlib.import_module("plugins.transforms_only") - plugin = getattr(mod, "transforms_only") - assert hasattr(plugin.schemas, "Opportunity") - result = plugin.schemas.Opportunity.to_common({"name": "Test Grant"}) - assert result.result == {"title": "Test Grant"} - assert result.errors == [] - finally: - if str(tmp_path) in sys.path: - sys.path.remove(str(tmp_path)) - for key in list(sys.modules.keys()): - if "transforms_only" in key or key == "plugins": - del sys.modules[key] - - -def test_generate_raises_on_missing_mapping_direction(tmp_path): - """generate_plugin raises ValueError when a mappings-only object is missing - one of its mapping directions (to_common or from_common is None).""" - from common_grants_sdk.extensions.generate import generate_plugin - - plugin_dir = tmp_path / "bad_plugin" - plugin_dir.mkdir(parents=True) - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, SchemaMappings", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " mappings=SchemaMappings(", - ' to_common={"title": {"field": "data.title"}},', - " # from_common intentionally omitted (None)", - " ),", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="from_common.*required"): - generate_plugin(plugin_dir) From 5db313fc128d7f62f6f5bf60fa8b8f932829a564 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 13:24:46 -0400 Subject: [PATCH 25/51] docs(py-sdk): rewrite extensions and examples READMEs Document the codegen-free design: the schema(...) factory, CustomField[V], PluginSchemas / define_plugin, declarative mappings vs hand-written functions, and consuming a plugin. Point the examples README at the single examples/plugins.py and drop the generate-step instructions. --- .../common_grants_sdk/extensions/README.md | 813 +++++------------- lib/python-sdk/examples/README.md | 154 +--- 2 files changed, 221 insertions(+), 746 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 0f0fa92ca..d7e3ab822 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -7,769 +7,340 @@ For background, see: - [Custom Fields catalog](https://commongrants.org/custom-fields/): The published set of recommended custom fields - [Extensions section of the CommonGrants specification](https://commongrants.org/protocol/specification/#extensions): How extensions fit into the protocol +The `common_grants_sdk.extensions` module contains the utilities for working with extensions: declaring custom fields on base schemas, bundling them into typed plugins, and transforming between a source system's format and the CommonGrants format. -The `common-grants/sdk/extensions` module contains the utilities for working with extensions: registering custom fields on base schemas, bundling them into reusable plugins and composing plugins together. +There is no build step. Plugins are plain Python: you declare custom fields as a `CustomFieldSet`, build each schema extension with the `schema(...)` factory, and assemble a `Plugin` with `define_plugin(...)`. Consumers import the plugin and get fully typed, non-optional access to every registered schema. ## Table of contents <!-- omit in toc --> -- [Key Concepts](#key-concepts) -- [Extending base models using Custom Fields](#extending-base-models-using-custom-fields) +- [Key concepts](#key-concepts) +- [Declaring custom fields](#declaring-custom-fields) - [Option 1: Ad hoc with `with_custom_fields()`](#option-1-ad-hoc-with-with_custom_fields) - - [Option 2: Build-time with plugins](#option-2-build-time-with-plugins) -- [Extracting Custom Field Values](#extracting-custom-field-values) + - [Option 2: A reusable plugin](#option-2-a-reusable-plugin) +- [Extracting custom field values](#extracting-custom-field-values) - [Plugins](#plugins) - - [What is a plugin?](#what-is-a-plugin) - - [Defining a plugin](#defining-a-plugin) + - [The `schema(...)` factory](#the-schema-factory) + - [Assembling a plugin](#assembling-a-plugin) + - [Consuming a plugin](#consuming-a-plugin) - [Publishing a plugin](#publishing-a-plugin) - - [Combining Plugins](#combining-plugins) -- [Bidirectional Transforms](#bidirectional-transforms) - - [Defining transforms](#defining-transforms) +- [Bidirectional transforms](#bidirectional-transforms) + - [Declarative mappings](#declarative-mappings) + - [Hand-written functions](#hand-written-functions) - [Mapping format](#mapping-format) - - [Using transforms](#using-transforms) - [Using plugins with the API client](#using-plugins-with-the-api-client) - [Best practices](#best-practices) - - [Field naming](#field-naming) - - [Keep plugins focused](#keep-plugins-focused) - - [Type safety](#type-safety) - - [Export value types alongside your plugin](#export-value-types-alongside-your-plugin) - - [Declare `common-grants-sdk` as a dependency](#declare-common-grants-sdk-as-a-dependency) - - [Avoid `"first_wins"` / `"last_wins"` in published plugins](#avoid-first_wins--last_wins-in-published-plugins) - -## Key Concepts - -Here are some key concepts that are used to define custom fields and plugins that extend base schemas from the CommonGrants protocol. +## Key concepts | Concept | Description | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | -| **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | -| **`SchemaExtensions`** | A legacy TypedDict mapping extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec`. Still accepted by `with_custom_fields()`. For plugins, declare custom fields inside `ObjectSchemasInput.custom_fields` instead. | -| **`Plugin`** | A dataclass assembled by the code generator. `.schemas` is a container object where each attribute (e.g. `.schemas.Opportunity`) is an `ObjectSchemas` instance providing the model class (`.common`), transform callables (`.to_common`, `.from_common`), and native type (`.native`). `.extensions` holds the serializable extension declarations. | -| **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | -| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict **or a Pydantic model instance** and returns a `TransformResult`. | -| **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | -| **`ObjectSchemasInput`**| Bundles `custom_fields`, `to_common`, and `from_common` for a single object type. Passed to `define_plugin()` via the `schemas` parameter. | - - - -## Extending base models using Custom Fields - -There are two ways to register custom fields on a base schema: at runtime (ad hoc) or at build-time (with plugins) +| --- | --- | +| **Custom field** | A key-value pair on a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | +| **`CustomField[V]`** | A Pydantic generic that is the single source of truth for a custom field. The static value type `V` anchors the typing; `fieldType` and the inspectable value type are *derived* from `V`, so they cannot drift. | +| **`CustomFieldSet`** | The base class an author subclasses to declare a schema's custom fields, each as `Optional[CustomField[V]] = Field(default=None, description=...)`. | +| **`Opportunity[CF]`** | The common Opportunity model, a Pydantic generic over its custom-fields container `CF`. `Opportunity[OpportunityFields]` is a fully concrete type; `Opportunity[NoCustomFields]` is the unextended form. | +| **`schema(...)`** | The factory that builds a schema extension. Overloads enforce, statically: mappings XOR hand-written functions XOR schema-only, and a `source` whenever transforms are present. Returns a `SchemaWithTransforms` or a `SchemaOnly`. | +| **`PluginSchemas`** | Maps your extensions to the registered extensible schemas, keyed by name (`PluginSchemas(Opportunity=...)`). Schemas you omit fall back to the base schema, never `None`. | +| **`define_plugin(...)`** | Assembles a `Plugin` from a `PluginSchemas` and a `PluginMeta`. | +| **`Plugin`** | The value consumers import. `plugin.schemas.Opportunity` is fully typed dot access. | +| **`build_transforms()`** | Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Used by `schema(..., mappings=...)` under the hood; also callable directly when you need custom handlers. | +| **`TransformResult`** | The return shape `(result, errors)` of every transform. Errors are non-fatal: a partial result is always returned alongside any errors. | +| **`PluginCustomFieldSpec`** | The resolved, inspection-only view of a custom field (`field_type`, `value`, `name`, `description`), exposed on `extension.custom_fields`. Derived from `CustomField[V]`; authors never construct it. | +| **`CustomFieldSpec`** | The runtime declaration consumed by `with_custom_fields()` (Option 1). `field_type` is a required input there. | + +## Declaring custom fields + +There are two ways to add custom fields to a base schema: ad hoc at runtime, or as a reusable plugin. ### Option 1: Ad hoc with `with_custom_fields()` -Use `with_custom_fields()` when you want to extend a single schema directly, without creating a reusable plugin. This is useful for one-off scripts, tests, or quick prototyping. + +Use `with_custom_fields()` to extend a single schema directly, without creating a plugin. Useful for one-off scripts, tests, or quick prototyping. ```python from pydantic import BaseModel -from datetime import datetime -from uuid import uuid4 -from common_grants_sdk.schemas.pydantic import ( - OpportunityBase, - CustomFieldType, - OppStatus, - OppStatusOptions, -) +from common_grants_sdk.schemas.pydantic import OpportunityBase, CustomFieldType from common_grants_sdk.extensions.specs import CustomFieldSpec -# Define a Pydantic schema for complex custom field values class LegacyIdValue(BaseModel): system: str id: int -# Add 2 custom fields to extend the base schema + fields = { "legacyId": CustomFieldSpec(field_type=CustomFieldType.OBJECT, value=LegacyIdValue), "groupName": CustomFieldSpec(field_type=CustomFieldType.STRING, value=str), } -Opportunity = OpportunityBase.with_custom_fields( - custom_fields=fields, model_name="Opportunity" -) +Opportunity = OpportunityBase.with_custom_fields(custom_fields=fields, model_name="Opportunity") -opp_data = { - "id": uuid4(), - "title": "Foo bar", - "status": OppStatus(value=OppStatusOptions.OPEN), - "description": "Example opportunity", - "createdAt": datetime.fromisoformat("2024-01-01T00:00:00+00:00"), - "lastModifiedAt": datetime.fromisoformat("2024-01-01T00:00:00+00:00"), - "customFields": { - "legacyId": { - "name": "legacyId", - "fieldType": "object", - "value": {"system": "legacy", "id": 12345}, - }, - "groupName": { - "name": "groupName", - "fieldType": "string", - "value": "TEST_GROUP", - }, - "ignoredForNow": {"type": "string", "value": "noop"}, - }, -} - -# Validate the fields to make them retrievable opp = Opportunity.model_validate(opp_data) print(opp.custom_fields["legacyId"]) ``` +### Option 2: A reusable plugin -### Option 2: Build-time with plugins - -Use `define_plugin()` when you want to create a **reusable, shareable** set of custom field definitions. Plugins are the recommended approach for any extensions that will be used across multiple files, projects, or teams. - -The following is an example `cg_config.py` file, which you pass to the build step. - +Declare custom fields as a `CustomFieldSet` and build a schema-only extension. `CustomField[V]` is the single source of truth: `fieldType` is derived from `V`, so the only per-field metadata is a description. ```python -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - } - ) - } -) -``` +from typing import Optional -After defining your plugins in `cg_config.py`, run the following command from the repo root (or the directory containing your plugin) to generate the plugin code: - -```bash -poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions -``` - -This generates the plugin and writes the output into a `generated/` subdirectory inside the named plugin directory. You can then import the plugin via its directory name. - - -```python -import sys -from pathlib import Path - -# Make the examples/ directory importable so that -# plugins.opportunity_extensions resolves correctly. -sys.path.insert(0, str(Path(__file__).parent)) - -from plugins.opportunity_extensions import opportunity_extensions # noqa: E402 - - -# --------------------------------------------------------------------------- -# Sample API payload containing our four custom fields -# --------------------------------------------------------------------------- - -api_response = { - "id": "573525f2-8e15-4405-83fb-e6523511d893", - "title": "Community Health Innovation Grant", - "status": {"value": "open"}, - "description": "Funding for community-led health initiatives", - "createdAt": "2025-03-01T00:00:00Z", - "lastModifiedAt": "2025-03-15T00:00:00Z", - "customFields": { - "programArea": { - "fieldType": "string", - "value": "CFDA-93.243", - }, - "legacyGrantId": { - "fieldType": "integer", - "value": 98765, - }, - "eligibilityTypes": { - "fieldType": "array", - "value": ["nonprofit", "tribal", "city_government"], - }, - "awardCeiling": { - "fieldType": "number", - "value": 250000.00, - }, - }, -} - -# --------------------------------------------------------------------------- -# Use the model returned via opportunity_extensions -# --------------------------------------------------------------------------- - -opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) +from pydantic import Field +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PluginMeta, + PluginSchemas, + define_plugin, + schema, +) +from common_grants_sdk.schemas.pydantic.models import Opportunity + + +class OpportunityFields(CustomFieldSet): + program_area: Optional[CustomField[str]] = Field( + default=None, description="HHS program area code (e.g. 'CFDA-93.243')" + ) + legacy_grant_id: Optional[CustomField[int]] = Field( + default=None, description="Numeric ID from the legacy grants management system" + ) + eligibility_types: Optional[CustomField[list[str]]] = Field( + default=None, description="Types of organizations eligible to apply" + ) + award_ceiling: Optional[CustomField[float]] = Field( + default=None, description="Maximum award amount in USD" + ) + + +opportunity_extensions = define_plugin( + PluginSchemas(Opportunity=schema(common=Opportunity[OpportunityFields])), + meta=PluginMeta(name="opportunity extensions", source_system="hhs"), +) ``` -## Extracting Custom Field Values - -Use `get_custom_field_value()` to safely retrieve typed values from `custom_fields`. It is useful in three main situations: - -1. **Unregistered fields** — you didn't use `with_custom_fields()` or a plugin to register a custom field and want to validate its value at runtime. -2. **Ad hoc fields** — you registered a field with `with_custom_fields()` for initial validation, but want type inference on the value after accessing it. -3. **Programmatic access** — you registered a field via a plugin (which provides type inference through dot-delimited access like `opp.custom_fields.foo.value`), but need to access fields by variable name, such as in a loop. +Custom-field keys are camelCase on the wire and snake_case in Python: a `CustomField[V]` named `legacy_grant_id` serializes to `legacyGrantId`. This is handled by an alias generator on the `CustomFieldSet` base, so authors and consumers use snake_case while JSON I/O stays camelCase. -The helper returns `None` if the key is absent (no `try/except` needed) and raises `ValueError` if the value is present but cannot be converted to the requested type. +## Extracting custom field values -The idiomatic form is the instance method on any `OpportunityBase` subclass: +Use `get_custom_field_value()` to safely retrieve typed values from `custom_fields`, e.g. for unregistered fields or programmatic (by-name) access. It returns `None` if the key is absent and raises `ValueError` if a present value cannot be converted. ```python from pydantic import BaseModel -from common_grants_sdk.schemas.pydantic import OpportunityBase, CustomFieldType +from common_grants_sdk.schemas.pydantic import OpportunityBase + class LegacyIdValue(BaseModel): system: str id: int -opp = OpportunityBase.model_validate(opp_data) - -# Returns Optional[LegacyIdValue], or None if key is absent -legacy = opp.get_custom_field_value("legacyId", LegacyIdValue) -if legacy is not None: - print(legacy.id) # typed as int - -# Returns Optional[str] -group = opp.get_custom_field_value("groupName", str) -# Returns None — no KeyError -missing = opp.get_custom_field_value("missing", str) +opp = OpportunityBase.model_validate(opp_data) +legacy = opp.get_custom_field_value("legacyId", LegacyIdValue) # Optional[LegacyIdValue] +group = opp.get_custom_field_value("groupName", str) # Optional[str] ``` -`get_custom_field_value()` works with both ad hoc (unregistered) and plugin-based (registered) custom fields. - - ## Plugins -### What is a plugin? +A plugin bundles custom fields and transforms for the CommonGrants schemas a source system extends. It is a `Plugin` value, built with no codegen. + +### The `schema(...)` factory -A plugin is a Python package that adds domain-specific custom fields to CommonGrants models. For example, a government agency might add a `legacy_grant_id` field to `Opportunity` to preserve backward compatibility with an existing system. +`schema(...)` builds one schema extension. Its overloads enforce the valid shapes statically and it validates registry membership, custom-field consistency, and mapping output keys at call (import) time: -Plugins are built from a `cg_config.py` spec file. The SDK's code generator reads that file and emits fully typed Pydantic models into a `generated/` subdirectory. Consumers import the generated schemas and get complete type safety without running the generator themselves. +- **Schema-only** — custom fields, no transforms: `schema(common=Opportunity[Fields])` returns a `SchemaOnly`. It has `parse(...)` but no `to_common`, so consumers cannot transform it. +- **Mappings** — declarative transforms: `schema(source=Src, common=Opportunity[Fields], mappings={...})` returns a `SchemaWithTransforms`. +- **Functions** — hand-written transforms: `schema(source=Src, common=Opportunity[Fields], to_common=fn, from_common=fn)` returns a `SchemaWithTransforms`. -### Defining a plugin +`common` must be a registered extensible schema (`Opportunity[...]`); an unregistered base raises `PluginDefinitionError`. -A plugin is a Python class that contains extension specs and generated schemas +### Assembling a plugin +`PluginSchemas` maps each extension to a registered schema name; `define_plugin` returns the `Plugin`. Schemas you omit fall back to the base schema (a `SchemaOnly` over `Opportunity[NoCustomFields]`), never `None`. ```python -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - } - ) - } +plugin = define_plugin( + PluginSchemas(Opportunity=schema(common=Opportunity[OpportunityFields])), + meta=PluginMeta(name="my-system", source_system="my-system.example.gov"), ) ``` -After running the build step the imported extension object will have 2 fields to use. -1. The `schemas` property used to access the properties on the model that was extended. -2. The `extensions` property to access information about the extensions added and their specific pydantic properties. - - -### Publishing a plugin - -#### Package structure - -A minimal plugin package has 2 user-defined files `cg_config.py` and `pyproject.toml` for export. - -For Poetry (and most `pyproject.toml`-based build systems), `pyproject.toml` lives at the **project root**, while `cg_config.py` and the generated files live inside the **Python package subdirectory**: - -``` -opportunity-extensions/ # Project root (outer folder — name it anything) -├── pyproject.toml # Package metadata (you write this) -└── opportunity_extensions/ # Python package (inner folder — must match packages config) - ├── __init__.py # Emitted by the generator — commit to the repo - ├── cg_config.py # Field specs (source of truth — you write this) - ├── py.typed # Marks the package as typed — you write this - └── generated/ # Emitted by the generator — commit to the repo - ├── __init__.py - └── schemas.py -``` - -The generator writes `generated/` and the package-level `__init__.py`. Only `cg_config.py` and `pyproject.toml` are hand-authored. - -> [!IMPORTANT] -> `cg_config.py` must be **inside** the Python package directory (alongside `__init__.py`), not at the project root next to `pyproject.toml`. Placing it at the project root will cause import errors when Poetry resolves the package. +### Consuming a plugin -Run the generator from the **project root**, pointing `--plugin` at the inner package directory: +Every registered schema is always present and fully typed: -```bash -poetry run python -m common_grants_sdk.extensions.generate --plugin opportunity_extensions -``` - -`pyproject.toml` (at the project root) defines the package metadata for export. - -```toml - [tool.poetry] - name = "opportunity-extensions" - version = "0.1.0" - description = "CommonGrants opportunity custom field extensions" - authors = ["Your Name <you@example.com>"] - packages = [{include = "opportunity_extensions"}] - - [tool.poetry.dependencies] - python = "^3.11" - common-grants-sdk = "^0.5.1" - - [tool.poetry.group.dev.dependencies] - pyright = "^1.1" - pytest = "^8.0" - - [build-system] - requires = ["poetry-core"] - build-backend = "poetry.core.masonry.api" +```python +opp = plugin.schemas.Opportunity.parse(api_response) +print(opp.custom_fields.program_area.value) # typed as str +print(opp.custom_fields.legacy_grant_id.value) # typed as int +# Inspect the resolved specs (field_type/value derived from CustomField[V]): +for name, spec in plugin.schemas.Opportunity.custom_fields.items(): + print(name, spec.field_type, spec.description) ``` -#### Shipping pre-built schemas - -Publishing steps: -1. Add an empty `py.typed` file so Pyright and mypy recognize your package as typed: - -``` -opportunity_extensions/ -└── py.typed -``` +### Publishing a plugin -2. Declare it in `pyproject.toml`: +A plugin is a normal Python module that exports the `Plugin` value, so publishing is standard packaging — no generated files to commit and no generate step. Add a `py.typed` marker so type checkers see your package as typed, declare `common-grants-sdk` as a dependency, and `poetry build` / `poetry publish`. ```toml [tool.poetry] +name = "opportunity-extensions" +version = "0.1.0" +packages = [{include = "opportunity_extensions"}] include = ["opportunity_extensions/py.typed"] -``` - -3. Build and verify the distribution: -```bash -poetry build -poetry publish +[tool.poetry.dependencies] +python = "^3.11" +common-grants-sdk = "^0.6.2" ``` -#### Consumer usage - -After installing the plugin (e.g. `poetry add opportunity-extensions`): - ```python from opportunity_extensions import opportunity_extensions -opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) -print(opp.custom_fields.program_area.value) # typed as str -print(opp.custom_fields.legacy_grant_id.value) # typed as int -``` - -#### Pre-publish checklist - -Before publishing a new version of your plugin: - -- [ ] `cg_config.py` field specs are up to date -- [ ] Generator has been re-run and the output is committed (`generated/schemas.py`, `generated/__init__.py`, `__init__.py`) -- [ ] All custom fields have the intended types (no unintended `Any` annotations in `generated/schemas.py`) -- [ ] `py.typed` marker is present and included in the package -- [ ] Package installs cleanly in a fresh virtual environment: `pip install dist/commongrants_my_plugin-*.whl` -- [ ] Imports resolve without errors: `from my_plugin import my_plugin` -- [ ] Custom field attributes are accessible and typed in the IDE after install -- [ ] A type checker passes with no errors: `pyright my_plugin` or `mypy my_plugin` -- [ ] A [changeset](../../lib/README.md) has been created with the correct revision type - -### Combining Plugins - -Custom fields from multiple logical sources are combined by declaring them all inside a single `ObjectSchemasInput.custom_fields` dict. Because the dict is plain Python, there is no special merge utility needed — just add the keys side by side: - -```python -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - # fields from a shared HHS package - "programArea": CustomFieldSpec(field_type=CustomFieldType.STRING), - "legacyGrantId": CustomFieldSpec(field_type=CustomFieldType.INTEGER), - # fields specific to this project - "eligibilityTypes": CustomFieldSpec(field_type=CustomFieldType.ARRAY), - "awardCeiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER), - } - ) - } -) +opp = opportunity_extensions.schemas.Opportunity.parse(api_response) +print(opp.custom_fields.program_area.value) # typed as str ``` -`merge_extensions()` is still available for merging `PluginExtensions` objects that carry declarative `mappings` (ADR-0017 transform configs). It no longer merges `custom_fields`. - -#### Verify type inference before publishing - -After building your package, import the plugin in a test file and confirm that `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. +## Bidirectional transforms -## Bidirectional Transforms +A `SchemaWithTransforms` maps between a source system's native format and the CommonGrants format. Both directions are always author-provided: `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. -Plugins can define bidirectional mappings between a source system's native data format and the CommonGrants format. These transforms are authored as plain Python dicts and compiled into callable functions by `build_transforms()`. +### Declarative mappings -### Defining transforms - -Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `schemas`: +Pass `mappings` to `schema(...)` and the SDK compiles them into typed, validated `to_common` / `from_common` callables: ```python -from common_grants_sdk.extensions import ( - CustomFieldSpec, - ObjectSchemasInput, - PluginExtensionsMeta, - build_transforms, - define_plugin, -) -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType - -to_common, from_common = build_transforms( - to_common_mapping={ - "title": {"field": "data.opportunity_title"}, - "status": { - "value": { - "match": { - "field": "data.opportunity_status", - "case": {"posted": "open", "archived": "closed", "forecasted": "forecasted"}, - "default": "custom", +from common_grants_sdk.extensions import PassthroughModel, schema +from common_grants_sdk.schemas.pydantic.models import Opportunity + +ext = schema( + source=PassthroughModel, + common=Opportunity[OpportunityFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } } }, - "description": {"const": "The opportunity is currently accepting applications"}, - }, - "funding": { - "minAwardAmount": { - "amount": {"field": "data.summary.award_floor"}, - "currency": {"const": "USD"}, + "customFields": { + "agencyCode": { + "value": {"field": "agency_code"}, + "name": {"const": "agencyCode"}, + "fieldType": {"const": "string"}, + } }, }, - }, - from_common_mapping={ - "data": { + "from_common": { + "opportunity_uuid": {"field": "id"}, "opportunity_title": {"field": "title"}, - "opportunity_status": { - "match": { - "field": "status.value", - "case": {"open": "posted", "closed": "archived", "forecasted": "forecasted"}, - "default": "custom", - } - }, - } - }, -) - -plugin = define_plugin( - meta=PluginExtensionsMeta( - name="my-system", - version="0.1.0", - source_system="my-system.example.gov", - capabilities=["customFields", "transforms"], - ), - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Unique identifier in legacy database", - ), - }, - to_common=to_common, - from_common=from_common, - ) + "agency_code": {"field": "customFields.agencyCode.value"}, + }, }, ) ``` -Both directions must be provided explicitly. `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. +`PassthroughModel` is a permissive source schema (`extra="allow"`) for when you do not want to model the source shape. Because it accepts arbitrary keys, output-path validation is skipped for it. -#### Hand-written callables - -`build_transforms()` is optional. You can supply any plain Python callable to `ObjectSchemasInput` as long as it matches the expected signature: - -```python -def to_common(native_data: dict) -> TransformResult: - ... - -def from_common(cg_data: dict) -> TransformResult: - ... - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - to_common=to_common, - from_common=from_common, - ) - }, -) -``` +### Hand-written functions -The key requirement when porting existing transform code is that **both callables must return `TransformResult`**. The type annotation enforces this, but it is easy to miss when wrapping a function that previously returned a plain dict. Wrap the return value like so: +When the transform is more than a mapping (e.g. it needs a custom handler or arbitrary Python), write the callables yourself and pass them via the functions overload. Use `validate_into` so the result is validated into the target model and any failure is routed to `TransformResult.errors`: ```python -from common_grants_sdk.extensions.types import TransformResult - -def to_common(native_data: dict) -> TransformResult: - result = my_existing_transform(native_data) # returns a plain dict - return TransformResult(result=result, errors=[]) -``` - -### Mapping format +from common_grants_sdk.extensions import TransformResult, schema, validate_into +from common_grants_sdk.schemas.pydantic.models import Opportunity -A mapping dict describes how to build an output object from a source dict. Each leaf node is either a literal value or a single-key dict that invokes a named handler. -| Handler | Syntax | Description | -|---|---|---| -| `const` | `{"const": "USD"}` | Returns a fixed literal value, ignoring source data | -| `field` | `{"field": "data.summary.award_floor"}` | Extracts a value using a dot-notation path | -| `match` | `{"match": {"field": "...", "case": {...}, "default": "..."}}` | Case-based lookup on a field value (canonical ADR name) | -| `switch` | `{"switch": {...}}` | Alias for `match`, kept for backward compatibility | -| `numberToString` | `{"numberToString": "data.summary.award_floor"}` | Extracts a numeric value and coerces it to a string | -| `stringToNumber` | `{"stringToNumber": "some.string.field"}` | Extracts a string and coerces it to `int` or `float` | +def to_common(source: GrantsGovOpportunity) -> TransformResult[Opportunity[OpportunityFields]]: + return validate_into(Opportunity[OpportunityFields], { + "id": source.opportunity_uuid, + "title": source.opportunity_title, + # ... + }) -Bare non-dict values (strings, numbers, booleans) in a mapping are treated as literals and passed through unchanged. Use `{"const": ...}` when you want a literal value inside a dict node that might otherwise be mistaken for a field name. -You can also register custom handlers by passing a `handlers` dict to `build_transforms()`: +def from_common(common: Opportunity[OpportunityFields]) -> TransformResult[GrantsGovOpportunity]: + # `common` is fully typed: common.custom_fields.agency_code.value -> str + return validate_into(GrantsGovOpportunity, {...}) -```python -def handle_upper(data, field_path): - val = get_from_path(data, field_path) - return val.upper() if isinstance(val, str) else val -to_common, from_common = build_transforms( - to_common_mapping={"title": {"upper": "data.opportunity_title"}}, - from_common_mapping={...}, - handlers={"upper": handle_upper}, +ext = schema( + source=GrantsGovOpportunity, + common=Opportunity[OpportunityFields], + to_common=to_common, + from_common=from_common, ) ``` -Custom handlers are merged with the defaults; they cannot override built-in handler names. +Hand-written functions own their own validation. In Python, returning a `TransformResult[Model]` means returning an actual Pydantic instance, which is validated on construction (and `validate_into` makes that ergonomic and routes errors). The SDK does not re-wrap them. -Handlers are scoped to the `build_transforms()` call they are registered on — they do not affect other calls: +When you need custom handlers, compile with `build_transforms()` directly and pass the callables to the functions overload: ```python -# Only the first pair of callables knows about "upper" -to_common_with_upper, _ = build_transforms( - to_common_mapping={"title": {"upper": "data.opportunity_title"}}, - from_common_mapping={}, - handlers={"upper": handle_upper}, -) +from common_grants_sdk.extensions import build_transforms -# This call has no knowledge of "upper" — using it in the mapping would fail at call time -to_common_plain, _ = build_transforms( - to_common_mapping={"title": {"field": "data.opportunity_title"}}, - from_common_mapping={}, +to_common, from_common = build_transforms( + handlers={"join": _join_fields}, + common_schema=Opportunity[OpportunityFields], + source_schema=PassthroughModel, + to_common_mapping={...}, + from_common_mapping={...}, ) ``` -### Using transforms - -The compiled callables are stored on the plugin's `schemas` object, accessible by attribute name. Each callable takes a data dict (or a Pydantic model instance) and returns a `TransformResult`: - -```python -opp_schemas = plugin.schemas.Opportunity - -# Source system → CommonGrants -result = opp_schemas.to_common(native_data) -if result.errors: - for err in result.errors: - print(f"[{err.path}] {err}") -else: - cg_data = result.result - -# CommonGrants → source system -result = opp_schemas.from_common(cg_data) -native_data = result.result -``` +### Mapping format -`TransformResult.errors` is always a list (empty on success). A non-empty errors list means the transform encountered a problem but still returned a partial result in `result`. +A mapping dict describes how to build an output object from a source object. Each leaf node is either a literal value or a single-key dict that invokes a named handler. -> [!IMPORTANT] -> When `common_model` is set on `build_transforms()`, `to_common` returns a validated Pydantic model instance in `result.result`. That instance can be passed directly to `from_common`. In that case, field paths in `from_common_mapping` must use the model's **camelCase alias names** (e.g. `"status.value"`, `"funding.minAwardAmount.amount"`), not Python snake_case attribute names. This matches the camelCase convention used throughout CommonGrants field paths. +| Handler | Syntax | Description | +| --- | --- | --- | +| `const` | `{"const": "USD"}` | A fixed literal, ignoring source data | +| `field` | `{"field": "data.summary.award_floor"}` | Extract a value via a dot-notation path | +| `match` | `{"match": {"field": "...", "case": {...}, "default": "..."}}` | Case-based lookup on a field value | +| `switch` | `{"switch": {...}}` | Alias for `match` | +| `numberToString` | `{"numberToString": "data.opportunity_id"}` | Extract a number and coerce to a string | +| `stringToNumber` | `{"stringToNumber": "data.priority_score_str"}` | Extract a string and coerce to `int`/`float` | -See `examples/transforms.py` for a complete working example with roundtrip verification. +Bare non-dict values are treated as literals. Register custom handlers by passing a `handlers` dict to `build_transforms()`; they are scoped to that call and cannot override built-in handler names. +When the source is a real (non-passthrough) model, mapping output keys may use either the model's field names or their camelCase aliases (e.g. `createdAt`). Field paths in `from_common` read from the common model, so they use its camelCase alias names (e.g. `status.value`, `customFields.agencyCode.value`). ## Using plugins with the API client -Pass a plugin's extended schema to the API client via the `schema` parameter. The client uses it to hydrate API responses into fully typed models. The `schema` parameter accepts any `Type[OpportunityBase]` subclass. +Pass an extension's `common_schema` to the client via the `schema` parameter to hydrate responses into fully typed models: ```python from common_grants_sdk import Client -from plugins.opportunity_extensions import opportunity_extensions client = Client(base_url="https://api.example.gov") +schema_cls = opportunity_extensions.schemas.Opportunity.common_schema -# Get a single opportunity with typed custom fields -opp = client.opportunities.get(opp_id, schema=opportunity_extensions.schemas.Opportunity.common) +opp = client.opportunities.get(opp_id, schema=schema_cls) print(opp.custom_fields.program_area.value) # typed as str -# List with the same schema -response = client.opportunities.list(schema=opportunity_extensions.schemas.Opportunity.common) -for opp in response.items: - print(opp.custom_fields.legacy_grant_id.value) # typed as int - -# Search with the same schema -results = client.opportunities.search( - search="health", - status=["open"], - schema=opportunity_extensions.schemas.Opportunity.common, -) +response = client.opportunities.list(schema=schema_cls) +results = client.opportunities.search(search="health", status=["open"], schema=schema_cls) ``` - ## Best practices ### Field naming -> [!IMPORTANT] -> The field key you define in `cg_config.py` **must match the key in the JSON data** you're parsing in order to validate your custom fields correctly. Because the CommonGrants standard uses `camelCase` for core attributes, you **should** use `camelCase` for custom field keys. - -The generator converts `camelCase` keys to `snake_case` Python attribute names automatically, using a Pydantic `alias` under the hood. For example, a field key of `"legacyGrantId"` becomes `opp.custom_fields.legacy_grant_id` in Python, while still mapping to `"legacyGrantId"` in JSON: - -```python -# cg_config.py -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "legacyGrantId": CustomFieldSpec( # camelCase key — matches the JSON - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - } - ) - } -) -``` - -```python -# JSON parsed correctly because the alias matches -api_response = { - "customFields": { - "legacyGrantId": {"fieldType": "integer", "value": 98765}, - }, -} - -opp = my_plugin.schemas.Opportunity.common.model_validate(api_response) -opp.custom_fields.legacy_grant_id.value # 98765, typed as int -``` - -Additional field naming guidelines: - -- Use descriptive, stable names. Renaming a published field is a breaking change. -- Prefix ambiguous names with your organization or system context (e.g. `hhsProgramArea` rather than `programArea`) when there is risk of collision with other plugins. +Custom-field keys are camelCase on the wire. Declare a `CustomField[V]` with a snake_case attribute name (`legacy_grant_id`); it serializes to and parses from `legacyGrantId` automatically. Use descriptive, stable names; renaming a published field is a breaking change. Prefix ambiguous names with your organization (e.g. `hhs_program_area`) when collisions with other plugins are likely. ### Keep plugins focused -A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, declare them all in one `ObjectSchemasInput.custom_fields` dict — Python dict literals compose cleanly without a special merge utility. +A plugin should represent a single logical concern (one agency's fields, one integration). Declare all of a schema's custom fields on one `CustomFieldSet`. ### Type safety -- Omitting `value` in `CustomFieldSpec` falls back to a sensible default (`str`, `int`, `float`, `bool`, `list[Any]`, or `dict[str, Any]`). Specify it explicitly when you need a more precise type. -- For complex object types, define a Pydantic `BaseModel` subclass and pass it as `value`: - - ```python - from pydantic import BaseModel - from common_grants_sdk import define_plugin - from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput - from common_grants_sdk.schemas.pydantic import CustomFieldType - - class LegacyRef(BaseModel): - system: str - id: int - - config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "legacyRef": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=LegacyRef, - description="Reference to the opportunity in the legacy system", - ), - } - ) - } - ) - ``` - -> [!NOTE] -> The generator only embeds the class name for builtins and SDK types. If a type lives in a third-party package **or your plugin's own module**, the generator falls back to `Any`. Define such types inside `common_grants_sdk` and use `from common_grants_sdk...` imports. - -### Export value types alongside your plugin - -When you define Pydantic models for complex `value` fields, export them as named exports from your package. Downstream consumers may need these types for use with `get_custom_field_value()`: - -```python -# cg_config.py of a plugin package -from pydantic import BaseModel -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -# Export value types so consumers can reference them directly -class ProgramAreaValue(BaseModel): - code: str - name: str - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "programArea": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=ProgramAreaValue, - description="The HHS program area for this opportunity", - ), - } - ) - } -) -``` - -This allows consumers to use `get_custom_field_value()` with the same type the plugin uses for validation: - -```python -from commongrants_hhs_plugin import hhs_plugin, ProgramAreaValue - -opp = hhs_plugin.schemas.Opportunity.common.model_validate(api_response) - -# Extract the value with full type safety using the exported type -area = opp.get_custom_field_value("programArea", ProgramAreaValue) -print(area.code) # typed as str -``` - -### Declare `common-grants-sdk` as a dependency - -Declare `common-grants-sdk` as a dependency in your plugin's `pyproject.toml` rather than expecting consumers to install it separately. This ensures the correct version of the SDK is always present when your plugin is installed. See [Publishing a plugin](#publishing-a-plugin) for a full `pyproject.toml` example. - -### Avoid `"first_wins"` / `"last_wins"` in published plugins - -When calling `merge_extensions()` with `"first_wins"` or `"last_wins"`, conflicts are resolved at runtime but the overridden field definitions are silently dropped. This is fine for local or ad hoc usage, but if you publish a package that uses one of these strategies internally, consumers have no indication that fields may have been overridden. - -Prefer the default `"error"` strategy in published plugins. If your extensions genuinely overlap with another plugin, resolve the conflicts explicitly before publishing rather than deferring to a lossy merge strategy. - +- The value type `V` on `CustomField[V]` is the single source of truth. `fieldType` and the inspectable value type are derived from it, so they cannot drift. +- For complex object values, use a Pydantic model as `V`: `Optional[CustomField[LegacyRef]]` derives `fieldType="object"` and gives consumers `value.system` / `value.id` typing. +- Run a type checker (`mypy` or `pyright`) over your plugin module; consumer typing is verified by `assert_type` lines in the SDK's tests. diff --git a/lib/python-sdk/examples/README.md b/lib/python-sdk/examples/README.md index 96b7bbc67..7ee6da6ab 100644 --- a/lib/python-sdk/examples/README.md +++ b/lib/python-sdk/examples/README.md @@ -119,142 +119,46 @@ None ``` -# Bidirectional transforms example +# Plugins example -This example demonstrates the plugin transform framework: mapping source system data (grants.gov format) to the CommonGrants format and back again, with a roundtrip consistency check. No API server is required — the script runs entirely offline using sample data defined in the file itself. - -**Step 1:** Generate the typed models for the grants.gov plugin (only needed once, or after changing `cg_config.py`): +`examples/plugins.py` is a single, self-contained file that both **defines** the +example plugins and **validates** them. It demonstrates every authoring scenario +(custom fields + mappings, custom fields + hand-written functions, mappings with no +custom fields, schema-only, and a realistic combined grants.gov plugin) plus +bidirectional round-trips. There is no build step — run it directly: ```bash cd lib/python-sdk -poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov -``` - -Or generate all example plugins at once with: - -```bash -make plugins -``` - -**Step 2:** Run the example: - -```bash -poetry run python examples/transforms.py +poetry run python examples/plugins.py ``` **Output Example:** -``` -============================================================ -SOURCE DATA (grants.gov format) -============================================================ -{ - "data": { - "agency_name": "Department of Examples", - "created_at": "2025-01-15T09:00:00Z", - "last_modified_at": "2025-04-01T12:30:00Z", - "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", - "opportunity_id": 12345, - "opportunity_number": "ABC-123-XYZ-001", - "opportunity_status": "posted", - "opportunity_title": "Research into conservation techniques", - "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "summary": { - "applicant_types": ["state_governments"], - "archive_date": "2025-05-01", - "award_ceiling": 100000, - "award_floor": 10000, - "forecasted_award_date": "2025-09-01", - "forecasted_close_date": "2025-07-15", - "forecasted_post_date": "2025-05-01" - } - } -} - -============================================================ -to_common: grants.gov → CommonGrants -============================================================ -Errors: none - -Result: -{ - "title": "Research into conservation techniques", - "status": { "value": "open", "description": "The opportunity is currently accepting applications" }, - "funding": { - "minAwardAmount": { "amount": 10000, "currency": "USD" }, - "maxAwardAmount": { "amount": 100000, "currency": "USD" } - }, - ... -} - -============================================================ -from_common: CommonGrants → grants.gov -============================================================ -Errors: none - -Result: -{ - "data": { - "opportunity_title": "Research into conservation techniques", - "opportunity_status": "posted", - "summary": { - "award_floor": 10000, - "award_ceiling": 100000, - "forecasted_post_date": "2025-05-01", - "forecasted_close_date": "2025-07-15" - } - } -} - -============================================================ -ROUNDTRIP CHECK -============================================================ - [PASS] title: 'Research into conservation techniques' -> 'Research into conservation techniques' - [PASS] status: 'posted' -> 'posted' - [PASS] award_floor: 10000 -> 10000 - [PASS] award_ceiling: 100000 -> 100000 - -Roundtrip result: ALL PASS -``` - -The transform mappings live in `examples/plugins/grants_gov/cg_config.py`. See the [extensions README](../common_grants_sdk/extensions/README.md#bidirectional-transforms) for a full explanation of the mapping format. - -# Plugin framework example - -This example uses the plugin framework to define four typed custom fields, generate static Pydantic models, and validate an API payload. - -**Step 1:** Generate the typed models (only needed once, or after changing `cg_config.py`): - -```bash -cd lib/python-sdk -poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions -cd ../../.. ``` - -**Step 2:** Run the example: - -```bash -poetry run python examples/plugin_custom_fields.py +Scenario 1 -- custom fields + mappings + [PASS] no transform errors + [PASS] title mapped + [PASS] agency_code.value typed str == 'HHS-123' + [PASS] inspect: agency_code field_type derived STRING + [PASS] round-trips (validated source instance) +Scenario 2 -- custom fields + hand-written functions + [PASS] title mapped + [PASS] agency_code.value == 'HHS-123' + [PASS] from_common -> typed source +Scenario 3 -- mappings, no custom fields + [PASS] title mapped +Scenario 4 -- custom fields only, no transforms + [PASS] schema-only legacy_grant_id.value typed int == 98765 + ... +grants.gov -- custom fields + transform with a custom handler + [PASS] no transform errors + [PASS] title mapped + [PASS] compositeLabel joined via custom handler + [PASS] from_common -> validated source instance ``` -**Output Example:** - -``` -Title: Community Health Innovation Grant -Status: open - -Custom fields: - program_area: CFDA-93.243 - legacy_grant_id: 98765 - eligibility_types:['nonprofit', 'tribal', 'city_government'] - award_ceiling: 250000.0 - -Registered extensions: - program_area: string — HHS program area code (e.g. 'CFDA-93.243') - legacy_grant_id: integer — Numeric ID from the legacy grants management system - eligibility_types: array — Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal') - award_ceiling: number — Maximum award amount in USD -``` +See the [extensions README](../common_grants_sdk/extensions/README.md) for the full +plugin and mapping-format documentation. ## Configuration From ca373ce647ba77d3db9751e2a5df01e0353c2b39 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:25:21 -0700 Subject: [PATCH 26/51] docs(sdk): fix stale PluginError references in transforms comments (#923) #916 renamed the transforms structured error from PluginError to TransformError but left 5 stale references in comments and one doc table. Update them to TransformError to match the type actually thrown. No functional change. --- lib/python-sdk/common_grants_sdk/extensions/README.md | 2 +- lib/ts-sdk/__tests__/extensions/transformation.spec.ts | 2 +- lib/ts-sdk/src/extensions/transforms.ts | 2 +- lib/ts-sdk/src/utils/transformation.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 0f0fa92ca..7ca03ea23 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -48,7 +48,7 @@ Here are some key concepts that are used to define custom fields and plugins tha | **`Plugin`** | A dataclass assembled by the code generator. `.schemas` is a container object where each attribute (e.g. `.schemas.Opportunity`) is an `ObjectSchemas` instance providing the model class (`.common`), transform callables (`.to_common`, `.from_common`), and native type (`.native`). `.extensions` holds the serializable extension declarations. | | **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | | **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict **or a Pydantic model instance** and returns a `TransformResult`. | -| **`TransformResult`** | A dataclass `(result: dict, errors: list[PluginError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | +| **`TransformResult`** | A dataclass `(result: dict, errors: list[TransformError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | | **`ObjectSchemasInput`**| Bundles `custom_fields`, `to_common`, and `from_common` for a single object type. Passed to `define_plugin()` via the `schemas` parameter. | diff --git a/lib/ts-sdk/__tests__/extensions/transformation.spec.ts b/lib/ts-sdk/__tests__/extensions/transformation.spec.ts index 2b2313bf0..d5da005d2 100644 --- a/lib/ts-sdk/__tests__/extensions/transformation.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/transformation.spec.ts @@ -109,7 +109,7 @@ describe("switchOnValue", () => { // which let a typo'd mapping like `{ match: "literal-where-spec-belongs" }` // produce missing fields with no error trail. New contract: throw, so the // walker wraps as `HandlerError` and the boundary materializes a - // `PluginError` with `handler: "match"`. + // `TransformError` with `handler: "match"`. expect(() => switchOnValue({ status: "posted" }, "garbage")).toThrow(/spec must be an object/); expect(() => switchOnValue({ status: "posted" }, null)).toThrow(/spec must be an object/); expect(() => switchOnValue({ status: "posted" }, 42)).toThrow(/spec must be an object/); diff --git a/lib/ts-sdk/src/extensions/transforms.ts b/lib/ts-sdk/src/extensions/transforms.ts index 205e5f8d8..92167574a 100644 --- a/lib/ts-sdk/src/extensions/transforms.ts +++ b/lib/ts-sdk/src/extensions/transforms.ts @@ -159,7 +159,7 @@ export interface BuiltTransforms<TSource, TCommon> { * * - Handler failures (a registered handler throws): the mapping walk * short-circuits on the first failure, so `errors` carries exactly one - * `PluginError` even when several fields would have failed. + * `TransformError` even when several fields would have failed. * - Zod-validation failures (`commonSchema` or `sourceSchema` provided): every * `ZodIssue` is flattened into a separate `TransformError`, so `errors` carries * the full set. diff --git a/lib/ts-sdk/src/utils/transformation.ts b/lib/ts-sdk/src/utils/transformation.ts index b76d152d0..6e22e3669 100644 --- a/lib/ts-sdk/src/utils/transformation.ts +++ b/lib/ts-sdk/src/utils/transformation.ts @@ -196,8 +196,8 @@ export function stringToNumber(data: unknown, fieldPath: unknown): number | null if (Number.isFinite(f)) return f; // Don't embed the source value in the error message — it could be PII when // transforming applicant data. The handler name and cause flow into - // `PluginError` separately for programmatic reasoning. Adopters who need - // the offending value can still read it from `PluginError.sourceValue` + // `TransformError` separately for programmatic reasoning. Adopters who need + // the offending value can still read it from `TransformError.sourceValue` // (which carries its own PII warning). throw new Error("stringToNumber: cannot convert source value to a number"); } From 67ae27c64c1cdf180c6f21d8e3f29d7840ff7491 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Tue, 16 Jun 2026 14:59:19 -0400 Subject: [PATCH 27/51] refactor(py-sdk): rename schema() params to common_schema / source_schema Align the schema(...) factory parameters with the resulting extension attributes (common_schema / source_schema) and the TS SDK's commonSchema / sourceSchema, so the authoring and consuming surfaces use the same names. --- .../common_grants_sdk/extensions/README.md | 18 +++---- .../common_grants_sdk/extensions/plugin.py | 2 +- .../common_grants_sdk/extensions/schema.py | 49 +++++++++++-------- lib/python-sdk/examples/plugins.py | 20 ++++---- .../tests/extensions/test_plugin.py | 6 +-- .../tests/extensions/test_schema.py | 24 ++++----- 6 files changed, 63 insertions(+), 56 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index d7e3ab822..d75477e00 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -112,7 +112,7 @@ class OpportunityFields(CustomFieldSet): opportunity_extensions = define_plugin( - PluginSchemas(Opportunity=schema(common=Opportunity[OpportunityFields])), + PluginSchemas(Opportunity=schema(common_schema=Opportunity[OpportunityFields])), meta=PluginMeta(name="opportunity extensions", source_system="hhs"), ) ``` @@ -146,9 +146,9 @@ A plugin bundles custom fields and transforms for the CommonGrants schemas a sou `schema(...)` builds one schema extension. Its overloads enforce the valid shapes statically and it validates registry membership, custom-field consistency, and mapping output keys at call (import) time: -- **Schema-only** — custom fields, no transforms: `schema(common=Opportunity[Fields])` returns a `SchemaOnly`. It has `parse(...)` but no `to_common`, so consumers cannot transform it. -- **Mappings** — declarative transforms: `schema(source=Src, common=Opportunity[Fields], mappings={...})` returns a `SchemaWithTransforms`. -- **Functions** — hand-written transforms: `schema(source=Src, common=Opportunity[Fields], to_common=fn, from_common=fn)` returns a `SchemaWithTransforms`. +- **Schema-only** — custom fields, no transforms: `schema(common_schema=Opportunity[Fields])` returns a `SchemaOnly`. It has `parse(...)` but no `to_common`, so consumers cannot transform it. +- **Mappings** — declarative transforms: `schema(source_schema=Src, common_schema=Opportunity[Fields], mappings={...})` returns a `SchemaWithTransforms`. +- **Functions** — hand-written transforms: `schema(source_schema=Src, common_schema=Opportunity[Fields], to_common=fn, from_common=fn)` returns a `SchemaWithTransforms`. `common` must be a registered extensible schema (`Opportunity[...]`); an unregistered base raises `PluginDefinitionError`. @@ -158,7 +158,7 @@ A plugin bundles custom fields and transforms for the CommonGrants schemas a sou ```python plugin = define_plugin( - PluginSchemas(Opportunity=schema(common=Opportunity[OpportunityFields])), + PluginSchemas(Opportunity=schema(common_schema=Opportunity[OpportunityFields])), meta=PluginMeta(name="my-system", source_system="my-system.example.gov"), ) ``` @@ -213,8 +213,8 @@ from common_grants_sdk.extensions import PassthroughModel, schema from common_grants_sdk.schemas.pydantic.models import Opportunity ext = schema( - source=PassthroughModel, - common=Opportunity[OpportunityFields], + source_schema=PassthroughModel, + common_schema=Opportunity[OpportunityFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -272,8 +272,8 @@ def from_common(common: Opportunity[OpportunityFields]) -> TransformResult[Grant ext = schema( - source=GrantsGovOpportunity, - common=Opportunity[OpportunityFields], + source_schema=GrantsGovOpportunity, + common_schema=Opportunity[OpportunityFields], to_common=to_common, from_common=from_common, ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index da4b01484..892cd9719 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -50,7 +50,7 @@ class PluginSchemas(Generic[_TOpportunity]): Opportunity: _TOpportunity = field( default_factory=lambda: cast( - _TOpportunity, schema(common=Opportunity[NoCustomFields]) + _TOpportunity, schema(common_schema=Opportunity[NoCustomFields]) ) ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py index 04e8033e4..4ccd0e8f0 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/schema.py +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -294,22 +294,22 @@ def parse(self, data: Any) -> TCommon: @overload def schema( - *, source: type[TSource], common: type[TCommon], mappings: Mappings + *, source_schema: type[TSource], common_schema: type[TCommon], mappings: Mappings ) -> SchemaWithTransforms[TSource, TCommon]: ... @overload def schema( *, - source: type[TSource], - common: type[TCommon], + source_schema: type[TSource], + common_schema: type[TCommon], to_common: Callable[[TSource], TransformResult[TCommon]], from_common: Callable[[TCommon], TransformResult[TSource]], ) -> SchemaWithTransforms[TSource, TCommon]: ... @overload -def schema(*, common: type[TCommon]) -> SchemaOnly[TCommon]: ... +def schema(*, common_schema: type[TCommon]) -> SchemaOnly[TCommon]: ... def schema( *, - source: Any = None, - common: Any, + source_schema: Any = None, + common_schema: Any, mappings: Optional[Mappings] = None, to_common: Any = None, from_common: Any = None, @@ -317,15 +317,15 @@ def schema( """Build a schema extension. The overloads enforce, statically: - ``mappings`` XOR hand-written ``to_common`` / ``from_common`` (both = no match), - - a ``source`` is required whenever transforms are present, - - a schema-only entry (``common`` only) returns a ``SchemaOnly`` whose + - a ``source_schema`` is required whenever transforms are present, + - a schema-only entry (``common_schema`` only) returns a ``SchemaOnly`` whose type has no ``to_common`` (so consumers cannot transform it). Registry membership, custom-field consistency, and mapping output keys are validated here at call (import) time, aggregated into one ``PluginDefinitionError``. """ errors: list[str] = [] - common_origin, custom_fields_model = _resolve_common(common) + common_origin, custom_fields_model = _resolve_common(common_schema) schema_name = _BASE_TO_NAME.get(common_origin) if schema_name is None: @@ -349,23 +349,25 @@ def schema( for direction in ("to_common", "from_common"): if direction not in mappings: errors.append(f"mappings: missing `{direction}` direction") - if "to_common" in mappings and _is_model_class(common): - unknown = set(mappings["to_common"]) - _output_field_names(common) + if "to_common" in mappings and _is_model_class(common_schema): + unknown = set(mappings["to_common"]) - _output_field_names(common_schema) if unknown: errors.append( f"mappings.to_common: unknown output field(s) {sorted(unknown)} " - f"for {common.__name__}" + f"for {common_schema.__name__}" ) if ( "from_common" in mappings - and _is_model_class(source) - and not _accepts_arbitrary_keys(source) + and _is_model_class(source_schema) + and not _accepts_arbitrary_keys(source_schema) ): - unknown_src = set(mappings["from_common"]) - _output_field_names(source) + unknown_src = set(mappings["from_common"]) - _output_field_names( + source_schema + ) if unknown_src: errors.append( f"mappings.from_common: unknown source field(s) " - f"{sorted(unknown_src)} for {source.__name__}" + f"{sorted(unknown_src)} for {source_schema.__name__}" ) if errors: @@ -380,14 +382,19 @@ def schema( to_fn, from_fn = build_transforms( mappings["to_common"], mappings["from_common"], - common_schema=common, - source_schema=source, + common_schema=common_schema, + source_schema=source_schema, ) return SchemaWithTransforms( - schema_name, common, custom_fields, source, to_fn, from_fn + schema_name, common_schema, custom_fields, source_schema, to_fn, from_fn ) if to_common is not None: return SchemaWithTransforms( - schema_name, common, custom_fields, source, to_common, from_common + schema_name, + common_schema, + custom_fields, + source_schema, + to_common, + from_common, ) - return SchemaOnly(schema_name, common, custom_fields) + return SchemaOnly(schema_name, common_schema, custom_fields) diff --git a/lib/python-sdk/examples/plugins.py b/lib/python-sdk/examples/plugins.py index ce2f3dce1..ca8602681 100644 --- a/lib/python-sdk/examples/plugins.py +++ b/lib/python-sdk/examples/plugins.py @@ -7,7 +7,7 @@ just the ``Plugin`` ``define_plugin`` returns, so each scenario is a self-contained plugin in this one file. -Run the consumer side with ``poetry run python examples/scenarios.py``. +Run the consumer side with ``poetry run python examples/plugins.py``. """ from __future__ import annotations @@ -89,8 +89,8 @@ class GrantsGovFields(CustomFieldSet): mappings_plugin = define_plugin( PluginSchemas( Opportunity=schema( - source=PassthroughModel, - common=Opportunity[AgencyFields], + source_schema=PassthroughModel, + common_schema=Opportunity[AgencyFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -192,8 +192,8 @@ def gadget_from_common( functions_plugin = define_plugin( PluginSchemas( Opportunity=schema( - source=GrantsGovOpportunity, - common=Opportunity[AgencyFields], + source_schema=GrantsGovOpportunity, + common_schema=Opportunity[AgencyFields], to_common=gadget_to_common, from_common=gadget_from_common, ) @@ -207,8 +207,8 @@ def gadget_from_common( base_plugin = define_plugin( PluginSchemas( Opportunity=schema( - source=PassthroughModel, - common=Opportunity[NoCustomFields], + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -241,7 +241,7 @@ def gadget_from_common( # --- Scenario 4: custom fields only, no transforms ---------------------------- opportunity_extensions = define_plugin( - PluginSchemas(Opportunity=schema(common=Opportunity[ExtensionFields])), + PluginSchemas(Opportunity=schema(common_schema=Opportunity[ExtensionFields])), meta=PluginMeta(name="opportunity extensions", source_system="hhs"), ) @@ -372,8 +372,8 @@ def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: grants_gov = define_plugin( PluginSchemas( Opportunity=schema( - source=PassthroughModel, - common=Opportunity[GrantsGovFields], + source_schema=PassthroughModel, + common_schema=Opportunity[GrantsGovFields], to_common=gg_to_common, from_common=gg_from_common, ) diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index 211dc26c5..f36caa7e4 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -38,7 +38,7 @@ def _meta() -> PluginMeta: def test_define_plugin_returns_plugin_with_schemas_and_meta(): - ext = schema(common=Opportunity[OpportunityFields]) + ext = schema(common_schema=Opportunity[OpportunityFields]) plugin = define_plugin(PluginSchemas(Opportunity=ext), meta=_meta()) assert isinstance(plugin, Plugin) assert plugin.schemas.Opportunity is ext @@ -59,8 +59,8 @@ def test_omitted_schema_falls_back_to_base_schema_only_extension(): def test_mappings_entry_is_a_transform_extension(): ext = schema( - source=PassthroughModel, - common=Opportunity[NoCustomFields], + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], mappings={ "to_common": {"title": {"field": "opportunity_title"}}, "from_common": {"opportunity_title": {"field": "title"}}, diff --git a/lib/python-sdk/tests/extensions/test_schema.py b/lib/python-sdk/tests/extensions/test_schema.py index 3ad684e15..1ef83c342 100644 --- a/lib/python-sdk/tests/extensions/test_schema.py +++ b/lib/python-sdk/tests/extensions/test_schema.py @@ -51,8 +51,8 @@ def _mappings_extension() -> ( SchemaWithTransforms[PassthroughModel, Opportunity[OpportunityFields]] ): return schema( - source=PassthroughModel, - common=Opportunity[OpportunityFields], + source_schema=PassthroughModel, + common_schema=Opportunity[OpportunityFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -124,7 +124,7 @@ def test_resolve_custom_field_specs_empty_for_no_container(): def test_schema_only_returns_schema_only_extension(): - ext = schema(common=Opportunity[OpportunityFields]) + ext = schema(common_schema=Opportunity[OpportunityFields]) assert isinstance(ext, SchemaOnly) assert ext.schema_name == "Opportunity" assert not hasattr(ext, "to_common") @@ -161,8 +161,8 @@ def from_common( return TransformResult(result=PassthroughModel(), errors=[]) ext = schema( - source=PassthroughModel, - common=Opportunity[NoCustomFields], + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], to_common=to_common, from_common=from_common, ) @@ -181,7 +181,7 @@ class NotRegistered(BaseModel): with pytest.raises( PluginDefinitionError, match="not a registered extensible schema" ): - schema(common=NotRegistered) + schema(common_schema=NotRegistered) def test_unextended_base_class_is_not_registered(): @@ -189,14 +189,14 @@ def test_unextended_base_class_is_not_registered(): with pytest.raises( PluginDefinitionError, match="not a registered extensible schema" ): - schema(common=OpportunityBase) + schema(common_schema=OpportunityBase) def test_unknown_to_common_output_field_raises(): with pytest.raises(PluginDefinitionError, match="unknown output field"): schema( - source=PassthroughModel, - common=Opportunity[NoCustomFields], + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], mappings={ "to_common": {"nope": {"const": 1}}, "from_common": {}, @@ -207,8 +207,8 @@ def test_unknown_to_common_output_field_raises(): def test_missing_mapping_direction_raises(): with pytest.raises(PluginDefinitionError, match="missing `from_common`"): schema( - source=PassthroughModel, - common=Opportunity[NoCustomFields], + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], mappings={"to_common": {"title": {"field": "x"}}}, # type: ignore[typeddict-item] ) @@ -235,7 +235,7 @@ def test_mappings_consumer_typed_and_round_trips() -> None: def test_schema_only_parse_typed() -> None: - ext = schema(common=Opportunity[OpportunityFields]) + ext = schema(common_schema=Opportunity[OpportunityFields]) parsed = ext.parse( { "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", From 15c32b792cb8bf8c754da27d6ca3b1d965003827 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Wed, 17 Jun 2026 07:48:32 -0700 Subject: [PATCH 28/51] fix(ts-sdk): guard date schemas against invalid Date input (#920) acceptDate ran toISOString() on any Date instance, so an invalid Date (e.g. new Date("garbage"), still instanceof Date) threw a RangeError from inside z.preprocess that escaped safeParse, breaking its never-throw contract. Guard on !isNaN(getTime()) so an invalid Date falls through to the string validator and returns a normal validation error instead. Adds regression tests for ISODate and OffsetDateTime. --- lib/ts-sdk/__tests__/schemas/zod/types.spec.ts | 13 +++++++++++++ lib/ts-sdk/src/schemas/zod/types.ts | 10 ++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts b/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts index 96a80ca1e..5e7cb792f 100644 --- a/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts +++ b/lib/ts-sdk/__tests__/schemas/zod/types.spec.ts @@ -170,6 +170,14 @@ describe("ISODate Schema", () => { expect(parsed.getUTCDate()).toBe(15); }); + it("returns a validation error (not a thrown RangeError) for an invalid Date", () => { + // Regression: an Invalid Date is still `instanceof Date`, so toISOString() + // throws a RangeError inside preprocess that escapes safeParse. The guard + // routes an invalid Date to the string validator so safeParse stays total. + expect(() => ISODateSchema.safeParse(new Date("not a date"))).not.toThrow(); + expect(ISODateSchema.safeParse(new Date("not a date")).success).toBe(false); + }); + it("should raise an error for an invalid ISODate", () => { // Must be in YYYY-MM-DD format expect(() => ISODateSchema.parse("01-01-2025")).toThrow(); @@ -327,6 +335,11 @@ describe("OffsetDateTime Schema", () => { expect(OffsetDateTimeSchema.parse(new Date("2025-01-01T00:00:00Z"))).toBeInstanceOf(Date); }); + it("returns a validation error (not a thrown RangeError) for an invalid Date", () => { + expect(() => OffsetDateTimeSchema.safeParse(new Date("not a date"))).not.toThrow(); + expect(OffsetDateTimeSchema.safeParse(new Date("not a date")).success).toBe(false); + }); + it("should raise an error for an invalid OffsetDateTime", () => { // Must be a valid ISO datetime string expect(() => OffsetDateTimeSchema.parse("not-a-datetime")).toThrow(); diff --git a/lib/ts-sdk/src/schemas/zod/types.ts b/lib/ts-sdk/src/schemas/zod/types.ts index 17689fc14..e74d4a610 100644 --- a/lib/ts-sdk/src/schemas/zod/types.ts +++ b/lib/ts-sdk/src/schemas/zod/types.ts @@ -45,14 +45,16 @@ const ensureUTC = (date: string) => { /** * Accept a `Date` as input by normalizing it to a string before the string * validators run, so a caller can pass either an ISO string or a `Date`. A - * string goes straight to the inner schema's strict validation; a `Date` is - * rendered by `toStr` into the string format the inner schema expects. Output - * is a `Date`. + * string goes straight to the inner schema's strict validation; a valid `Date` + * is rendered by `toStr` into the string format the inner schema expects. An + * invalid `Date` is passed through unchanged so the inner validator rejects it, + * rather than throwing a `RangeError` out of `toISOString()` and escaping + * `safeParse`. Output is a `Date`. */ const acceptDate = (toStr: (d: Date) => string) => (val: unknown): unknown => - val instanceof Date ? toStr(val) : val; + val instanceof Date && !isNaN(val.getTime()) ? toStr(val) : val; /** Schema for UTC datetime fields (accepts an ISO string or a `Date`; outputs a `Date`) */ export const UTCDateTimeSchema = z.preprocess( From 6db8a6ea0a19c7b0bf9d0b53c4f991bfe2b9d1d7 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 17 Jun 2026 12:03:58 -0400 Subject: [PATCH 29/51] test(py-sdk): cover the schema_name mismatch path in define_plugin Hand-build a SchemaOnly tagged with a different schema_name than its slot and assert define_plugin raises PluginDefinitionError. --- lib/python-sdk/tests/extensions/test_plugin.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index f36caa7e4..ce4fe5ec5 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -76,6 +76,21 @@ def test_define_plugin_rejects_non_extension_in_slot(): define_plugin(bad, meta=_meta()) +def test_define_plugin_rejects_schema_name_mismatch(): + """A slot holding an extension whose schema_name differs from the attribute name raises.""" + # Hand-build an extension tagged with a different schema name than its slot. + mismatched = SchemaOnly( + schema_name="Program", + common_schema=Opportunity[NoCustomFields], + custom_fields={}, + ) + bad = PluginSchemas(Opportunity=mismatched) # type: ignore[arg-type] + with pytest.raises( + PluginDefinitionError, match="attribute name must match the schema name" + ): + define_plugin(bad, meta=_meta()) + + # --------------------------------------------------------------------------- # Plugin container # --------------------------------------------------------------------------- From e2fff2c91ea18a20c133ff2551d07ff9d81b50f6 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 17 Jun 2026 12:16:46 -0400 Subject: [PATCH 30/51] fix(py-sdk): transform_from_mapping returns {} for an empty mapping node transform_node returned inside the key loop, so an empty dict never iterated and fell through to an implicit None. Return {} after the loop (reached only for an empty node), with a regression test for top-level and nested empty mappings. --- lib/python-sdk/common_grants_sdk/utils/transformation.py | 3 +++ lib/python-sdk/tests/utils/test_transformation.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index c36fed6eb..7662afcfd 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -261,5 +261,8 @@ def transform_node(node: Any, depth: int) -> Any: # ``` return {k: transform_node(v, depth + 1) for k, v in node.items()} + # An empty dict node maps to an empty dict (not None). + return {} + # Recursively walk the mapping until all nested transformations are applied return transform_node(mapping, depth) diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 797cd511a..20ff96245 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -111,6 +111,12 @@ def test_missing_field_returns_none(input_data): assert result == {"foo": None} +def test_empty_mapping_returns_empty_dict(input_data): + """An empty mapping (top-level or nested) yields an empty dict, not None.""" + assert transform_from_mapping(input_data, {}) == {} + assert transform_from_mapping(input_data, {"nested": {}}) == {"nested": {}} + + def test_literal_value(input_data): """ Test handling of literal values in the mapping. From 855f8766ea9dc8ac4bb8ab5e00b60665a95fd463 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 17 Jun 2026 12:18:15 -0400 Subject: [PATCH 31/51] fix(py-sdk): runtime guards for transform shape in schema() The overloads enforce the valid shapes for type-checkers, but the runtime body had no guards. Aggregate PluginDefinitionErrors for: mappings XOR explicit callables, both to_common/from_common required when either is given, and a source_schema required whenever transforms are declared. --- .../common_grants_sdk/extensions/schema.py | 19 +++++++ .../tests/extensions/test_schema.py | 49 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py index 4ccd0e8f0..65fd386c0 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/schema.py +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -345,6 +345,25 @@ def schema( ): errors.extend(_check_custom_fields(custom_fields_model)) + # Transform-shape guards. The overloads enforce these statically; these + # checks repeat them at runtime so dynamically-built or un-type-checked + # callers cannot slip through with a malformed extension. + has_callables = to_common is not None or from_common is not None + if mappings is not None and has_callables: + errors.append( + "cannot specify both `mappings` and `to_common`/`from_common`; " + "use mappings for declarative transforms or explicit callables, not both" + ) + if has_callables and not (to_common is not None and from_common is not None): + errors.append( + "both `to_common` and `from_common` are required when either is provided" + ) + if (mappings is not None or has_callables) and source_schema is None: + errors.append( + "a `source_schema` is required when transforms (mappings or " + "to_common/from_common) are declared" + ) + if mappings is not None: for direction in ("to_common", "from_common"): if direction not in mappings: diff --git a/lib/python-sdk/tests/extensions/test_schema.py b/lib/python-sdk/tests/extensions/test_schema.py index 1ef83c342..9afa17c91 100644 --- a/lib/python-sdk/tests/extensions/test_schema.py +++ b/lib/python-sdk/tests/extensions/test_schema.py @@ -213,6 +213,55 @@ def test_missing_mapping_direction_raises(): ) +def _noop(value): + return TransformResult(result=value, errors=[]) + + +def test_functions_without_source_schema_raises(): + with pytest.raises(PluginDefinitionError, match="source_schema` is required"): + schema( # type: ignore[call-overload] + common_schema=Opportunity[NoCustomFields], + to_common=_noop, + from_common=_noop, + ) + + +def test_mappings_without_source_schema_raises(): + with pytest.raises(PluginDefinitionError, match="source_schema` is required"): + schema( # type: ignore[call-overload] + common_schema=Opportunity[NoCustomFields], + mappings={ + "to_common": {"title": {"field": "x"}}, + "from_common": {}, + }, + ) + + +def test_one_sided_callable_raises(): + with pytest.raises( + PluginDefinitionError, match="both `to_common` and `from_common`" + ): + schema( # type: ignore[call-overload] + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], + to_common=_noop, + ) + + +def test_mappings_and_callables_together_raises(): + with pytest.raises(PluginDefinitionError, match="cannot specify both"): + schema( # type: ignore[call-overload] + source_schema=PassthroughModel, + common_schema=Opportunity[NoCustomFields], + mappings={ + "to_common": {"title": {"field": "x"}}, + "from_common": {}, + }, + to_common=_noop, + from_common=_noop, + ) + + # --------------------------------------------------------------------------- # Consumer typing + behavior (the make-or-break path) # --------------------------------------------------------------------------- From dd05c54fcc5a15fe36546644991f6ccc67ba4715 Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 17 Jun 2026 13:32:19 -0400 Subject: [PATCH 32/51] refactor(py-sdk): make CustomField generic, drop the duplicate Make the protocol CustomField generic over its value type V (default Any), so the bare CustomField keeps the untyped-value wire behavior while CustomField[T] gives authors/consumers a typed value. The extensions framework now imports this single CustomField instead of defining its own, matching the TS SDK's one-CustomField design. camelCase wire I/O is preserved via validation_alias/serialization_alias plus populate_by_name. --- .../common_grants_sdk/extensions/schema.py | 23 +++++--------- .../schemas/pydantic/fields/custom.py | 30 ++++++++++++++----- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py index 65fd386c0..9df275803 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/schema.py +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -36,7 +36,7 @@ ) from pydantic.alias_generators import to_camel -from ..schemas.pydantic.fields.custom import CustomFieldType +from ..schemas.pydantic.fields.custom import CustomField, CustomFieldType from ..schemas.pydantic.models import Opportunity from .specs import PluginCustomFieldSpec from .transforms import build_transforms @@ -55,7 +55,6 @@ "validate_into", ] -V = TypeVar("V") TSource = TypeVar("TSource", bound=BaseModel) TCommon = TypeVar("TCommon", bound=BaseModel) T = TypeVar("T", bound=BaseModel) @@ -78,23 +77,15 @@ class _CamelModel(BaseModel): ) -class CustomField(_CamelModel, Generic[V]): - """A single custom field: a typed ``value`` plus its metadata. +class CustomFieldSet(_CamelModel): + """Base class an author subclasses to declare a schema's custom fields. - ``CustomField[V]`` is the single source of truth -- ``field_type`` and the - inspectable value type are derived from ``V`` -- so authors declare a field as - ``Optional[CustomField[V]] = Field(default=None, description=...)`` and never - restate the type. + Each field is declared as ``Optional[CustomField[V]] = Field(default=None, + description=...)``. ``CustomField[V]`` is the single source of truth: its value + type ``V`` is the anchor from which ``field_type`` and the inspectable value type + are derived, so they cannot drift. """ - name: str - field_type: CustomFieldType - value: V - - -class CustomFieldSet(_CamelModel): - """Base class an author subclasses to declare a schema's custom fields.""" - class NoCustomFields(CustomFieldSet): """Marker for schema entries that declare no custom fields.""" diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py index 7606087bb..907a20edc 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py @@ -1,9 +1,10 @@ """Custom field types for the CommonGrants API.""" from enum import StrEnum -from typing import Any, Optional +from typing import Any, Generic, Optional -from pydantic import Field, HttpUrl +import typing_extensions as te +from pydantic import ConfigDict, Field, HttpUrl from ..base import CommonGrantsBaseModel @@ -20,8 +21,21 @@ class CustomFieldType(StrEnum): ARRAY = "array" -class CustomField(CommonGrantsBaseModel): - """Represents a custom field with type information and validation schema.""" +V = te.TypeVar("V", default=Any) + + +class CustomField(CommonGrantsBaseModel, Generic[V]): + """A custom field with type information and a typed value. + + Generic over its value type ``V`` (default ``Any``): the bare ``CustomField`` + keeps the protocol's untyped-value behavior, while ``CustomField[int]`` (or a + Pydantic model) gives plugin authors and consumers a concrete, inspectable + ``value`` type. ``populate_by_name`` plus ``validation_alias``/ + ``serialization_alias`` keep JSON I/O camelCase (``fieldType``) while + snake_case field-name construction (``CustomField(field_type=...)``) type-checks. + """ + + model_config = ConfigDict(populate_by_name=True) name: str = Field( ..., @@ -30,15 +44,17 @@ class CustomField(CommonGrantsBaseModel): ) field_type: CustomFieldType = Field( ..., - alias="fieldType", + validation_alias="fieldType", + serialization_alias="fieldType", description="The JSON schema type to use when de-serializing the `value` field", ) schema_url: Optional[HttpUrl] = Field( None, - alias="schema", + validation_alias="schema", + serialization_alias="schema", description="Link to the full JSON schema for this custom field", ) - value: Any = Field(..., description="Value of the custom field") + value: V = Field(..., description="Value of the custom field") description: Optional[str] = Field( None, description="Description of the custom field's purpose", From 250d8301643f0a55ba266fe21315c0ee00df79db Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 17 Jun 2026 14:02:04 -0400 Subject: [PATCH 33/51] refactor(py-sdk): collapse Opportunity into a generic OpportunityBase Make OpportunityBase itself generic over its custom-fields container, with a PEP-696 default of dict[str, CustomField], so the bare OpportunityBase keeps its concrete, non-generic behavior (model_validate, list[OpportunityBase], responses) while OpportunityBase[OpportunityFields] is the typed extended form. This removes the separate generic Opportunity model and points EXTENSIBLE_SCHEMA_MAP at OpportunityBase, matching the TS SDK's { Opportunity: OpportunityBaseSchema }. Also drop NoCustomFields: with the default in place, the unextended case is just bare OpportunityBase (an open dict[str, CustomField]), which matches the TS SDK's behavior of leaving the base schema's open customFields untouched. Verified the ca/pa example suites pass against this SDK (bare OpportunityBase usage), plus mypy + the full test suite. --- .../common_grants_sdk/extensions/README.md | 30 +++++------ .../common_grants_sdk/extensions/__init__.py | 2 - .../common_grants_sdk/extensions/plugin.py | 7 ++- .../common_grants_sdk/extensions/schema.py | 13 ++--- .../extensions/transforms.py | 2 +- .../schemas/pydantic/models/__init__.py | 2 - .../schemas/pydantic/models/opp_base.py | 19 +++++-- .../schemas/pydantic/models/opportunity.py | 36 ------------- lib/python-sdk/examples/plugins.py | 29 +++++------ .../tests/extensions/test_plugin.py | 11 ++-- .../tests/extensions/test_schema.py | 51 +++++++++---------- 11 files changed, 81 insertions(+), 121 deletions(-) delete mode 100644 lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index d75477e00..cbec8570c 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -37,8 +37,8 @@ There is no build step. Plugins are plain Python: you declare custom fields as a | **Custom field** | A key-value pair on a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | | **`CustomField[V]`** | A Pydantic generic that is the single source of truth for a custom field. The static value type `V` anchors the typing; `fieldType` and the inspectable value type are *derived* from `V`, so they cannot drift. | | **`CustomFieldSet`** | The base class an author subclasses to declare a schema's custom fields, each as `Optional[CustomField[V]] = Field(default=None, description=...)`. | -| **`Opportunity[CF]`** | The common Opportunity model, a Pydantic generic over its custom-fields container `CF`. `Opportunity[OpportunityFields]` is a fully concrete type; `Opportunity[NoCustomFields]` is the unextended form. | -| **`schema(...)`** | The factory that builds a schema extension. Overloads enforce, statically: mappings XOR hand-written functions XOR schema-only, and a `source` whenever transforms are present. Returns a `SchemaWithTransforms` or a `SchemaOnly`. | +| **`OpportunityBase[CF]`** | The common Opportunity model, a Pydantic generic over its custom-fields container `CF`. `OpportunityBase[OpportunityFields]` is a fully concrete type; `OpportunityBase` is the unextended form. | +| **`schema(...)`** | The factory that builds a schema extension. Overloads enforce, statically: mappings XOR hand-written functions XOR schema-only, and a `source_schema` whenever transforms are present. Returns a `SchemaWithTransforms` or a `SchemaOnly`. | | **`PluginSchemas`** | Maps your extensions to the registered extensible schemas, keyed by name (`PluginSchemas(Opportunity=...)`). Schemas you omit fall back to the base schema, never `None`. | | **`define_plugin(...)`** | Assembles a `Plugin` from a `PluginSchemas` and a `PluginMeta`. | | **`Plugin`** | The value consumers import. `plugin.schemas.Opportunity` is fully typed dot access. | @@ -112,7 +112,7 @@ class OpportunityFields(CustomFieldSet): opportunity_extensions = define_plugin( - PluginSchemas(Opportunity=schema(common_schema=Opportunity[OpportunityFields])), + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OpportunityFields])), meta=PluginMeta(name="opportunity extensions", source_system="hhs"), ) ``` @@ -146,19 +146,19 @@ A plugin bundles custom fields and transforms for the CommonGrants schemas a sou `schema(...)` builds one schema extension. Its overloads enforce the valid shapes statically and it validates registry membership, custom-field consistency, and mapping output keys at call (import) time: -- **Schema-only** — custom fields, no transforms: `schema(common_schema=Opportunity[Fields])` returns a `SchemaOnly`. It has `parse(...)` but no `to_common`, so consumers cannot transform it. -- **Mappings** — declarative transforms: `schema(source_schema=Src, common_schema=Opportunity[Fields], mappings={...})` returns a `SchemaWithTransforms`. -- **Functions** — hand-written transforms: `schema(source_schema=Src, common_schema=Opportunity[Fields], to_common=fn, from_common=fn)` returns a `SchemaWithTransforms`. +- **Schema-only** — custom fields, no transforms: `schema(common_schema=OpportunityBase[Fields])` returns a `SchemaOnly`. It has `parse(...)` but no `to_common`, so consumers cannot transform it. +- **Mappings** — declarative transforms: `schema(source_schema=Src, common_schema=OpportunityBase[Fields], mappings={...})` returns a `SchemaWithTransforms`. +- **Functions** — hand-written transforms: `schema(source_schema=Src, common_schema=OpportunityBase[Fields], to_common=fn, from_common=fn)` returns a `SchemaWithTransforms`. -`common` must be a registered extensible schema (`Opportunity[...]`); an unregistered base raises `PluginDefinitionError`. +`common_schema` must be a registered extensible schema (`OpportunityBase[...]`); an unregistered base raises `PluginDefinitionError`. ### Assembling a plugin -`PluginSchemas` maps each extension to a registered schema name; `define_plugin` returns the `Plugin`. Schemas you omit fall back to the base schema (a `SchemaOnly` over `Opportunity[NoCustomFields]`), never `None`. +`PluginSchemas` maps each extension to a registered schema name; `define_plugin` returns the `Plugin`. Schemas you omit fall back to the base schema (a `SchemaOnly` over `OpportunityBase`), never `None`. ```python plugin = define_plugin( - PluginSchemas(Opportunity=schema(common_schema=Opportunity[OpportunityFields])), + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OpportunityFields])), meta=PluginMeta(name="my-system", source_system="my-system.example.gov"), ) ``` @@ -214,7 +214,7 @@ from common_grants_sdk.schemas.pydantic.models import Opportunity ext = schema( source_schema=PassthroughModel, - common_schema=Opportunity[OpportunityFields], + common_schema=OpportunityBase[OpportunityFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -258,22 +258,22 @@ from common_grants_sdk.extensions import TransformResult, schema, validate_into from common_grants_sdk.schemas.pydantic.models import Opportunity -def to_common(source: GrantsGovOpportunity) -> TransformResult[Opportunity[OpportunityFields]]: - return validate_into(Opportunity[OpportunityFields], { +def to_common(source: GrantsGovOpportunity) -> TransformResult[OpportunityBase[OpportunityFields]]: + return validate_into(OpportunityBase[OpportunityFields], { "id": source.opportunity_uuid, "title": source.opportunity_title, # ... }) -def from_common(common: Opportunity[OpportunityFields]) -> TransformResult[GrantsGovOpportunity]: +def from_common(common: OpportunityBase[OpportunityFields]) -> TransformResult[GrantsGovOpportunity]: # `common` is fully typed: common.custom_fields.agency_code.value -> str return validate_into(GrantsGovOpportunity, {...}) ext = schema( source_schema=GrantsGovOpportunity, - common_schema=Opportunity[OpportunityFields], + common_schema=OpportunityBase[OpportunityFields], to_common=to_common, from_common=from_common, ) @@ -288,7 +288,7 @@ from common_grants_sdk.extensions import build_transforms to_common, from_common = build_transforms( handlers={"join": _join_fields}, - common_schema=Opportunity[OpportunityFields], + common_schema=OpportunityBase[OpportunityFields], source_schema=PassthroughModel, to_common_mapping={...}, from_common_mapping={...}, diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index c4d81f17a..464f396b4 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -10,7 +10,6 @@ EXTENSIBLE_SCHEMA_MAP, CustomField, CustomFieldSet, - NoCustomFields, PluginDefinitionError, SchemaOnly, SchemaWithTransforms, @@ -40,7 +39,6 @@ "CustomFieldSet", "CustomFieldSpec", "Handler", - "NoCustomFields", "PassthroughModel", "Plugin", "PluginCapability", diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 892cd9719..eb846f358 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -13,9 +13,8 @@ import typing_extensions as te -from ..schemas.pydantic.models import Opportunity +from ..schemas.pydantic.models import OpportunityBase from .schema import ( - NoCustomFields, PluginDefinitionError, SchemaOnly, SchemaWithTransforms, @@ -28,7 +27,7 @@ # The fallback for a schema a plugin does not extend: the base schema, no custom # fields, no transforms. A SchemaOnly type, so unextended slots have no # to_common either. -DefaultOpportunity = SchemaOnly[Opportunity[NoCustomFields]] +DefaultOpportunity = SchemaOnly[OpportunityBase] _TOpportunity = te.TypeVar("_TOpportunity", default=DefaultOpportunity) @@ -50,7 +49,7 @@ class PluginSchemas(Generic[_TOpportunity]): Opportunity: _TOpportunity = field( default_factory=lambda: cast( - _TOpportunity, schema(common_schema=Opportunity[NoCustomFields]) + _TOpportunity, schema(common_schema=OpportunityBase) ) ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py index 9df275803..c128a8868 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/schema.py +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -10,7 +10,7 @@ ``CustomField[V]`` is the single source of truth for a custom field: ``field_type`` and the inspectable value type are derived from ``V``, so they cannot drift from the typed declaration. The common models are generics over their custom-fields container -(``Opportunity[OpportunityFields]``), so consumers get concrete, non-optional types. +(``OpportunityBase[OpportunityFields]``), so consumers get concrete, non-optional types. """ from __future__ import annotations @@ -37,7 +37,7 @@ from pydantic.alias_generators import to_camel from ..schemas.pydantic.fields.custom import CustomField, CustomFieldType -from ..schemas.pydantic.models import Opportunity +from ..schemas.pydantic.models import OpportunityBase from .specs import PluginCustomFieldSpec from .transforms import build_transforms from .types import TransformError, TransformResult @@ -46,7 +46,6 @@ "EXTENSIBLE_SCHEMA_MAP", "CustomField", "CustomFieldSet", - "NoCustomFields", "PluginDefinitionError", "SchemaOnly", "SchemaWithTransforms", @@ -87,15 +86,11 @@ class CustomFieldSet(_CamelModel): """ -class NoCustomFields(CustomFieldSet): - """Marker for schema entries that declare no custom fields.""" - - # The extensible-schema registry: the closed set of models a plugin may extend, # mapped to their generic base model. Add an entry when a new model gains # custom-field support. EXTENSIBLE_SCHEMA_MAP: dict[str, type[BaseModel]] = { - "Opportunity": Opportunity, + "Opportunity": OpportunityBase, } _BASE_TO_NAME: dict[type[BaseModel], str] = { v: k for k, v in EXTENSIBLE_SCHEMA_MAP.items() @@ -237,7 +232,7 @@ def _accepts_arbitrary_keys(model: type[BaseModel]) -> bool: def _resolve_common(common: Any) -> tuple[Any, Any]: """Return ``(origin, custom_fields_model)`` for a common type. - A parameterized Pydantic generic (``Opportunity[OpportunityFields]``) is a + A parameterized Pydantic generic (``OpportunityBase[OpportunityFields]``) is a concrete subclass, so its origin/args live in ``__pydantic_generic_metadata__``. """ pyd_meta = getattr(common, "__pydantic_generic_metadata__", None) diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index c63151047..4f3c0dbe9 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -196,7 +196,7 @@ def handle_upper(data, path): ) common_schema: Optional Pydantic model class to validate the to_common output against. Pass the model that carries the plugin's custom fields (e.g. - ``Opportunity[OpportunityFields]``), not the base ``OpportunityBase``: + ``OpportunityBase[OpportunityFields]``), not the base ``OpportunityBase``: a base class only checks custom_fields against dict[str, CustomField] rather than the typed container. When provided, model_validate is called on the transform result and any ValidationErrors are appended to diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py index 4fac17cda..9c2ae819e 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/__init__.py @@ -4,10 +4,8 @@ from .opp_funding import OppFunding from .opp_status import OppStatus, OppStatusOptions from .opp_timeline import OppTimeline -from .opportunity import Opportunity __all__ = [ - "Opportunity", "OpportunityBase", "OppFunding", "OppStatus", diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py index 689357eb2..ed89ef020 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py @@ -2,10 +2,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar +from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar from uuid import UUID -from pydantic import Field, HttpUrl +import typing_extensions as te +from pydantic import ConfigDict, Field, HttpUrl from ..base import CommonGrantsBaseModel from ..fields import CustomField, SystemMetadata @@ -22,9 +23,17 @@ V = TypeVar("V") # Unbound to support both BaseModel subclasses and primitives +# The opportunity's custom-fields container. Defaults to the protocol's untyped +# representation (``dict[str, CustomField]``), so the bare ``OpportunityBase`` +# behaves exactly as a concrete model; plugin authors parameterize it with a typed +# ``CustomFieldSet`` (``OpportunityBase[OpportunityFields]``) for concrete access. +CF = te.TypeVar("CF", default="dict[str, CustomField]") -class OpportunityBase(SystemMetadata, CommonGrantsBaseModel): - """Base model for a funding opportunity with all core fields.""" + +class OpportunityBase(SystemMetadata, CommonGrantsBaseModel, Generic[CF]): + """Base model for a funding opportunity, generic over its custom-fields container.""" + + model_config = ConfigDict(populate_by_name=True) id: UUID = Field(..., description="Globally unique id for the opportunity") title: str = Field(..., description="Title or name of the funding opportunity") @@ -46,7 +55,7 @@ class OpportunityBase(SystemMetadata, CommonGrantsBaseModel): default=None, description="URL for the original source of the opportunity", ) - custom_fields: Optional[dict[str, CustomField]] = Field( + custom_fields: Optional[CF] = Field( default=None, alias="customFields", description="Additional custom fields specific to this opportunity", diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py deleted file mode 100644 index 7cc9dcc8c..000000000 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opportunity.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Generic Opportunity model parameterized by its custom-fields container.""" - -from __future__ import annotations - -from typing import Generic, Optional, TypeVar - -from pydantic import ConfigDict, Field - -from .opp_base import OpportunityBase - -CF = TypeVar("CF") - - -class Opportunity(OpportunityBase, Generic[CF]): - """The common Opportunity model, generic over its custom-fields container. - - ``OpportunityBase`` carries every core field; this subclass overrides the - ``custom_fields`` slot with the author's typed container ``CF`` so consumers - get concrete, non-optional dot access (e.g. ``opp.custom_fields.agency_code``). - Unextended plugins use ``Opportunity[NoCustomFields]``. - - ``populate_by_name=True`` lets callers construct with snake_case field names - (``created_at=...``) while JSON I/O stays camelCase via the inherited aliases. - """ - - model_config = ConfigDict( - from_attributes=True, - strict=False, - populate_by_name=True, - ) - - custom_fields: Optional[CF] = Field( # type: ignore[assignment] - default=None, - alias="customFields", - description="Additional custom fields specific to this opportunity", - ) diff --git a/lib/python-sdk/examples/plugins.py b/lib/python-sdk/examples/plugins.py index ca8602681..0e57605af 100644 --- a/lib/python-sdk/examples/plugins.py +++ b/lib/python-sdk/examples/plugins.py @@ -19,7 +19,6 @@ from common_grants_sdk.extensions import ( CustomField, CustomFieldSet, - NoCustomFields, PassthroughModel, PluginMeta, PluginSchemas, @@ -29,7 +28,7 @@ schema, validate_into, ) -from common_grants_sdk.schemas.pydantic.models import Opportunity +from common_grants_sdk.schemas.pydantic.models import OpportunityBase from common_grants_sdk.utils.transformation import get_from_path # --- Author-declared custom-field containers ---------------------------------- @@ -90,7 +89,7 @@ class GrantsGovFields(CustomFieldSet): PluginSchemas( Opportunity=schema( source_schema=PassthroughModel, - common_schema=Opportunity[AgencyFields], + common_schema=OpportunityBase[AgencyFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -145,9 +144,9 @@ class GrantsGovOpportunity(BaseModel): def gadget_to_common( source: GrantsGovOpportunity, -) -> TransformResult[Opportunity[AgencyFields]]: +) -> TransformResult[OpportunityBase[AgencyFields]]: return validate_into( - Opportunity[AgencyFields], + OpportunityBase[AgencyFields], { "id": source.opportunity_uuid, "title": source.opportunity_title, @@ -169,7 +168,7 @@ def gadget_to_common( def gadget_from_common( - common: Opportunity[AgencyFields], + common: OpportunityBase[AgencyFields], ) -> TransformResult[GrantsGovOpportunity]: # The make-or-break path: `common` is fully typed (agency_code.value -> str). agency = "" @@ -193,7 +192,7 @@ def gadget_from_common( PluginSchemas( Opportunity=schema( source_schema=GrantsGovOpportunity, - common_schema=Opportunity[AgencyFields], + common_schema=OpportunityBase[AgencyFields], to_common=gadget_to_common, from_common=gadget_from_common, ) @@ -208,7 +207,7 @@ def gadget_from_common( PluginSchemas( Opportunity=schema( source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -241,7 +240,7 @@ def gadget_from_common( # --- Scenario 4: custom fields only, no transforms ---------------------------- opportunity_extensions = define_plugin( - PluginSchemas(Opportunity=schema(common_schema=Opportunity[ExtensionFields])), + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[ExtensionFields])), meta=PluginMeta(name="opportunity extensions", source_system="hhs"), ) @@ -260,7 +259,7 @@ def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: # the other because many-to-one handlers like ``match`` are not reversible. _gg_to_common, _gg_from_common = build_transforms( handlers={"join": _join_fields}, - common_schema=Opportunity[GrantsGovFields], + common_schema=OpportunityBase[GrantsGovFields], source_schema=PassthroughModel, to_common_mapping={ "id": {"field": "data.opportunity_uuid"}, @@ -361,11 +360,11 @@ def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: # build_transforms returns loosely-typed callables (its output may be a dict on a # validation error). Restate the precise signatures for the typed consumer surface. gg_to_common = cast( - Callable[[PassthroughModel], TransformResult[Opportunity[GrantsGovFields]]], + Callable[[PassthroughModel], TransformResult[OpportunityBase[GrantsGovFields]]], _gg_to_common, ) gg_from_common = cast( - Callable[[Opportunity[GrantsGovFields]], TransformResult[PassthroughModel]], + Callable[[OpportunityBase[GrantsGovFields]], TransformResult[PassthroughModel]], _gg_from_common, ) @@ -373,7 +372,7 @@ def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: PluginSchemas( Opportunity=schema( source_schema=PassthroughModel, - common_schema=Opportunity[GrantsGovFields], + common_schema=OpportunityBase[GrantsGovFields], to_common=gg_to_common, from_common=gg_from_common, ) @@ -454,7 +453,7 @@ def main() -> None: res1 = mappings_plugin.schemas.Opportunity.to_common( PassthroughModel.model_validate(_FLAT_SOURCE) ) - assert_type(res1, TransformResult[Opportunity[AgencyFields]]) + assert_type(res1, TransformResult[OpportunityBase[AgencyFields]]) opp1 = res1.result _check("no transform errors", res1.errors == []) _check("title mapped", opp1.title == "Conservation research") @@ -501,7 +500,7 @@ def main() -> None: # Scenario 4 -- custom fields only, no transforms (schema-only) print("Scenario 4 -- custom fields only, no transforms") ext = opportunity_extensions.schemas.Opportunity - assert_type(ext, SchemaOnly[Opportunity[ExtensionFields]]) + assert_type(ext, SchemaOnly[OpportunityBase[ExtensionFields]]) parsed = ext.parse(_SCHEMA_ONLY_RECORD) if parsed.custom_fields and parsed.custom_fields.legacy_grant_id: assert_type(parsed.custom_fields.legacy_grant_id.value, int) diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index ce4fe5ec5..f73e5b6af 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -8,7 +8,6 @@ from common_grants_sdk.extensions import ( CustomField, CustomFieldSet, - NoCustomFields, PassthroughModel, Plugin, PluginMeta, @@ -19,7 +18,7 @@ schema, ) from common_grants_sdk.extensions.schema import PluginDefinitionError -from common_grants_sdk.schemas.pydantic.models import Opportunity +from common_grants_sdk.schemas.pydantic.models import OpportunityBase class OpportunityFields(CustomFieldSet): @@ -38,7 +37,7 @@ def _meta() -> PluginMeta: def test_define_plugin_returns_plugin_with_schemas_and_meta(): - ext = schema(common_schema=Opportunity[OpportunityFields]) + ext = schema(common_schema=OpportunityBase[OpportunityFields]) plugin = define_plugin(PluginSchemas(Opportunity=ext), meta=_meta()) assert isinstance(plugin, Plugin) assert plugin.schemas.Opportunity is ext @@ -54,13 +53,13 @@ def test_omitted_schema_falls_back_to_base_schema_only_extension(): assert entry.schema_name == "Opportunity" assert entry.custom_fields == {} # The base schema has no custom fields declared. - assert entry.common_schema is Opportunity[NoCustomFields] + assert entry.common_schema is OpportunityBase def test_mappings_entry_is_a_transform_extension(): ext = schema( source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, mappings={ "to_common": {"title": {"field": "opportunity_title"}}, "from_common": {"opportunity_title": {"field": "title"}}, @@ -81,7 +80,7 @@ def test_define_plugin_rejects_schema_name_mismatch(): # Hand-build an extension tagged with a different schema name than its slot. mismatched = SchemaOnly( schema_name="Program", - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, custom_fields={}, ) bad = PluginSchemas(Opportunity=mismatched) # type: ignore[arg-type] diff --git a/lib/python-sdk/tests/extensions/test_schema.py b/lib/python-sdk/tests/extensions/test_schema.py index 9afa17c91..cd06b4d9b 100644 --- a/lib/python-sdk/tests/extensions/test_schema.py +++ b/lib/python-sdk/tests/extensions/test_schema.py @@ -8,7 +8,6 @@ from common_grants_sdk.extensions import ( CustomField, CustomFieldSet, - NoCustomFields, PassthroughModel, SchemaOnly, SchemaWithTransforms, @@ -19,7 +18,7 @@ ) from common_grants_sdk.extensions.schema import PluginDefinitionError, _infer_field_type from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -from common_grants_sdk.schemas.pydantic.models import Opportunity, OpportunityBase +from common_grants_sdk.schemas.pydantic.models import OpportunityBase class LegacyRef(BaseModel): @@ -48,11 +47,11 @@ class OpportunityFields(CustomFieldSet): def _mappings_extension() -> ( - SchemaWithTransforms[PassthroughModel, Opportunity[OpportunityFields]] + SchemaWithTransforms[PassthroughModel, OpportunityBase[OpportunityFields]] ): return schema( source_schema=PassthroughModel, - common_schema=Opportunity[OpportunityFields], + common_schema=OpportunityBase[OpportunityFields], mappings={ "to_common": { "id": {"field": "opportunity_uuid"}, @@ -115,7 +114,7 @@ def test_resolve_custom_field_specs_derives_from_value_type(): def test_resolve_custom_field_specs_empty_for_no_container(): assert resolve_custom_field_specs(None) == {} - assert resolve_custom_field_specs(NoCustomFields) == {} + assert resolve_custom_field_specs(CustomFieldSet) == {} # --------------------------------------------------------------------------- @@ -124,7 +123,7 @@ def test_resolve_custom_field_specs_empty_for_no_container(): def test_schema_only_returns_schema_only_extension(): - ext = schema(common_schema=Opportunity[OpportunityFields]) + ext = schema(common_schema=OpportunityBase[OpportunityFields]) assert isinstance(ext, SchemaOnly) assert ext.schema_name == "Opportunity" assert not hasattr(ext, "to_common") @@ -140,9 +139,9 @@ def test_mappings_returns_transform_extension(): def test_functions_returns_transform_extension(): def to_common( _src: PassthroughModel, - ) -> TransformResult[Opportunity[NoCustomFields]]: + ) -> TransformResult[OpportunityBase]: return TransformResult( - result=Opportunity[NoCustomFields].model_validate( + result=OpportunityBase.model_validate( FLAT_SOURCE | { "id": FLAT_SOURCE["opportunity_uuid"], @@ -156,13 +155,13 @@ def to_common( ) def from_common( - _c: Opportunity[NoCustomFields], + _c: OpportunityBase, ) -> TransformResult[PassthroughModel]: return TransformResult(result=PassthroughModel(), errors=[]) ext = schema( source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, to_common=to_common, from_common=from_common, ) @@ -184,19 +183,19 @@ class NotRegistered(BaseModel): schema(common_schema=NotRegistered) -def test_unextended_base_class_is_not_registered(): - """Only the generic Opportunity is registered, not OpportunityBase.""" - with pytest.raises( - PluginDefinitionError, match="not a registered extensible schema" - ): - schema(common_schema=OpportunityBase) +def test_bare_opportunity_base_is_a_registered_schema_only(): + """Bare OpportunityBase (no custom fields) is the registered base schema.""" + ext = schema(common_schema=OpportunityBase) + assert isinstance(ext, SchemaOnly) + assert ext.schema_name == "Opportunity" + assert ext.custom_fields == {} def test_unknown_to_common_output_field_raises(): with pytest.raises(PluginDefinitionError, match="unknown output field"): schema( source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, mappings={ "to_common": {"nope": {"const": 1}}, "from_common": {}, @@ -208,7 +207,7 @@ def test_missing_mapping_direction_raises(): with pytest.raises(PluginDefinitionError, match="missing `from_common`"): schema( source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, mappings={"to_common": {"title": {"field": "x"}}}, # type: ignore[typeddict-item] ) @@ -220,7 +219,7 @@ def _noop(value): def test_functions_without_source_schema_raises(): with pytest.raises(PluginDefinitionError, match="source_schema` is required"): schema( # type: ignore[call-overload] - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, to_common=_noop, from_common=_noop, ) @@ -229,7 +228,7 @@ def test_functions_without_source_schema_raises(): def test_mappings_without_source_schema_raises(): with pytest.raises(PluginDefinitionError, match="source_schema` is required"): schema( # type: ignore[call-overload] - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, mappings={ "to_common": {"title": {"field": "x"}}, "from_common": {}, @@ -243,7 +242,7 @@ def test_one_sided_callable_raises(): ): schema( # type: ignore[call-overload] source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, to_common=_noop, ) @@ -252,7 +251,7 @@ def test_mappings_and_callables_together_raises(): with pytest.raises(PluginDefinitionError, match="cannot specify both"): schema( # type: ignore[call-overload] source_schema=PassthroughModel, - common_schema=Opportunity[NoCustomFields], + common_schema=OpportunityBase, mappings={ "to_common": {"title": {"field": "x"}}, "from_common": {}, @@ -270,7 +269,7 @@ def test_mappings_and_callables_together_raises(): def test_mappings_consumer_typed_and_round_trips() -> None: ext = _mappings_extension() res = ext.to_common(PassthroughModel.model_validate(FLAT_SOURCE)) - assert_type(res, TransformResult[Opportunity[OpportunityFields]]) + assert_type(res, TransformResult[OpportunityBase[OpportunityFields]]) assert res.errors == [] opp = res.result assert opp.title == "Conservation research" @@ -284,7 +283,7 @@ def test_mappings_consumer_typed_and_round_trips() -> None: def test_schema_only_parse_typed() -> None: - ext = schema(common_schema=Opportunity[OpportunityFields]) + ext = schema(common_schema=OpportunityBase[OpportunityFields]) parsed = ext.parse( { "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", @@ -302,7 +301,7 @@ def test_schema_only_parse_typed() -> None: }, } ) - assert_type(parsed, Opportunity[OpportunityFields]) + assert_type(parsed, OpportunityBase[OpportunityFields]) assert parsed.custom_fields is not None assert parsed.custom_fields.legacy_ref is not None assert_type(parsed.custom_fields.legacy_ref.value.id, int) @@ -330,7 +329,7 @@ def test_camel_case_round_trip(): } }, } - opp = Opportunity[OpportunityFields].model_validate(camel) + opp = OpportunityBase[OpportunityFields].model_validate(camel) # snake_case typed access assert opp.custom_fields is not None assert opp.custom_fields.agency_code is not None From 141864436e6219046dd1cd5157ec4b1baeb862ed Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 17 Jun 2026 17:30:44 -0400 Subject: [PATCH 34/51] fix(ts-sdk): annotate enum constants with local types for portable .d.ts Re-exporting SomeEnum.enum without an annotation made tsc synthesize a cross-package reference to zod when emitting constants.d.ts. Under a build-from-source git install, zod resolves to a non-portable temp path, so that synthesized reference triggered TS2742 and failed the build. Annotating each export with typeof SomeEnum.enum points the emitted type at the local ./schemas module, so the declaration no longer names zod's location and builds in any dependency layout. --- lib/ts-sdk/src/constants.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/ts-sdk/src/constants.ts b/lib/ts-sdk/src/constants.ts index 747cd2796..3293e8ddc 100644 --- a/lib/ts-sdk/src/constants.ts +++ b/lib/ts-sdk/src/constants.ts @@ -26,24 +26,26 @@ import { // Model enum constants // ############################################################################ -export const OppStatusOptions = OppStatusOptionsEnum.enum; -export const ApplicantTypeOptions = ApplicantTypeOptionsEnum.enum; -export const OppSortBy = OppSortByEnum.enum; +export const OppStatusOptions: typeof OppStatusOptionsEnum.enum = OppStatusOptionsEnum.enum; +export const ApplicantTypeOptions: typeof ApplicantTypeOptionsEnum.enum = + ApplicantTypeOptionsEnum.enum; +export const OppSortBy: typeof OppSortByEnum.enum = OppSortByEnum.enum; // ############################################################################ // Field enum constants // ############################################################################ -export const EventType = EventTypeEnum.enum; -export const CustomFieldType = CustomFieldTypeEnum.enum; +export const EventType: typeof EventTypeEnum.enum = EventTypeEnum.enum; +export const CustomFieldType: typeof CustomFieldTypeEnum.enum = CustomFieldTypeEnum.enum; // ############################################################################ // Filter operator enum constants // ############################################################################ -export const EquivalenceOperator = EquivalenceOperatorsEnum.enum; -export const ComparisonOperator = ComparisonOperatorsEnum.enum; -export const ArrayOperator = ArrayOperatorsEnum.enum; -export const StringOperator = StringOperatorsEnum.enum; -export const RangeOperator = RangeOperatorsEnum.enum; -export const AllOperators = AllOperatorsEnum.enum; +export const EquivalenceOperator: typeof EquivalenceOperatorsEnum.enum = + EquivalenceOperatorsEnum.enum; +export const ComparisonOperator: typeof ComparisonOperatorsEnum.enum = ComparisonOperatorsEnum.enum; +export const ArrayOperator: typeof ArrayOperatorsEnum.enum = ArrayOperatorsEnum.enum; +export const StringOperator: typeof StringOperatorsEnum.enum = StringOperatorsEnum.enum; +export const RangeOperator: typeof RangeOperatorsEnum.enum = RangeOperatorsEnum.enum; +export const AllOperators: typeof AllOperatorsEnum.enum = AllOperatorsEnum.enum; From 492fb00f96bfadfa28c0e870de97727b437a2ab9 Mon Sep 17 00:00:00 2001 From: Billy Daly <billy.daly@agile6.com> Date: Thu, 18 Jun 2026 15:13:18 -0400 Subject: [PATCH 35/51] refactor(py-sdk): adopt new plugin framework without codegen (#924) * feat(py-sdk): add codegen-free schema extension framework Add the building blocks for declaring plugins without a build step: - schema(...) factory returning discriminated SchemaWithTransforms / SchemaOnly extensions (mappings XOR functions XOR schema-only), with CustomField[V] as the single source of truth for custom fields (field_type and inspectable value derived from V), CustomFieldSet / NoCustomFields, and the EXTENSIBLE_SCHEMA_MAP registry. - Generic Opportunity[CF] common model over its custom-fields container. - PluginCustomFieldSpec: the resolved/inspection view of a custom field. * refactor(py-sdk): rewire the public surface onto the new framework - Rewrite plugin.py around PluginSchemas / Plugin / define_plugin / PluginMeta, replacing PluginConfig / inject_transforms. - Trim the codegen-only types (SchemaInput / SchemaConfig / SchemaMappings) and rename PluginExtensionsMeta to PluginMeta for parity with the TS SDK. - build_transforms validates from_common output into the source model and skips output-path checks for permissive (extra=allow) sources. - Update extensions and top-level package exports. - Break the schemas->extensions import cycle via TYPE_CHECKING guards. * refactor(py-sdk): port examples and tests to the new framework - Add examples/plugins.py: one file that defines every authoring scenario (mappings, hand-written functions, schema-only, and a combined grants.gov plugin) and validates them, with assert_type consumer lines. - Add test_schema.py covering the schema(...) factory, custom-field resolution, negative cases, and the camelCase round-trip. - Port test_plugin / test_types / test_plugin_registry to the new surface. * chore(py-sdk): remove the codegen build path Delete the codegen extensions pipeline now that plugins are plain Python: - extensions/generate.py and the Makefile generate target. - The per-plugin codegen example directories (cg_config.py + generated/) and the old transforms / plugin_custom_fields scripts. - tests/utils/test_plugin_generator.py (covered the deleted generator). * docs(py-sdk): rewrite extensions and examples READMEs Document the codegen-free design: the schema(...) factory, CustomField[V], PluginSchemas / define_plugin, declarative mappings vs hand-written functions, and consuming a plugin. Point the examples README at the single examples/plugins.py and drop the generate-step instructions. * refactor(py-sdk): rename schema() params to common_schema / source_schema Align the schema(...) factory parameters with the resulting extension attributes (common_schema / source_schema) and the TS SDK's commonSchema / sourceSchema, so the authoring and consuming surfaces use the same names. * test(py-sdk): cover the schema_name mismatch path in define_plugin Hand-build a SchemaOnly tagged with a different schema_name than its slot and assert define_plugin raises PluginDefinitionError. * fix(py-sdk): transform_from_mapping returns {} for an empty mapping node transform_node returned inside the key loop, so an empty dict never iterated and fell through to an implicit None. Return {} after the loop (reached only for an empty node), with a regression test for top-level and nested empty mappings. * fix(py-sdk): runtime guards for transform shape in schema() The overloads enforce the valid shapes for type-checkers, but the runtime body had no guards. Aggregate PluginDefinitionErrors for: mappings XOR explicit callables, both to_common/from_common required when either is given, and a source_schema required whenever transforms are declared. * refactor(py-sdk): make CustomField generic, drop the duplicate Make the protocol CustomField generic over its value type V (default Any), so the bare CustomField keeps the untyped-value wire behavior while CustomField[T] gives authors/consumers a typed value. The extensions framework now imports this single CustomField instead of defining its own, matching the TS SDK's one-CustomField design. camelCase wire I/O is preserved via validation_alias/serialization_alias plus populate_by_name. * refactor(py-sdk): collapse Opportunity into a generic OpportunityBase Make OpportunityBase itself generic over its custom-fields container, with a PEP-696 default of dict[str, CustomField], so the bare OpportunityBase keeps its concrete, non-generic behavior (model_validate, list[OpportunityBase], responses) while OpportunityBase[OpportunityFields] is the typed extended form. This removes the separate generic Opportunity model and points EXTENSIBLE_SCHEMA_MAP at OpportunityBase, matching the TS SDK's { Opportunity: OpportunityBaseSchema }. Also drop NoCustomFields: with the default in place, the unextended case is just bare OpportunityBase (an open dict[str, CustomField]), which matches the TS SDK's behavior of leaving the base schema's open customFields untouched. Verified the ca/pa example suites pass against this SDK (bare OpportunityBase usage), plus mypy + the full test suite. --- lib/python-sdk/Makefile | 5 +- lib/python-sdk/common_grants_sdk/__init__.py | 5 +- .../common_grants_sdk/extensions/README.md | 813 +++++------------- .../common_grants_sdk/extensions/__init__.py | 57 +- .../common_grants_sdk/extensions/generate.py | 659 -------------- .../common_grants_sdk/extensions/plugin.py | 193 ++--- .../common_grants_sdk/extensions/schema.py | 405 +++++++++ .../common_grants_sdk/extensions/specs.py | 31 +- .../extensions/transforms.py | 24 +- .../common_grants_sdk/extensions/types.py | 88 +- .../schemas/pydantic/fields/custom.py | 30 +- .../schemas/pydantic/models/opp_base.py | 25 +- .../common_grants_sdk/utils/custom_fields.py | 8 +- .../common_grants_sdk/utils/transformation.py | 3 + lib/python-sdk/examples/README.md | 154 +--- .../examples/plugin_custom_fields.py | 87 -- lib/python-sdk/examples/plugins.py | 539 ++++++++++++ lib/python-sdk/examples/plugins/__init__.py | 0 .../examples/plugins/grants_gov/__init__.py | 16 - .../examples/plugins/grants_gov/cg_config.py | 184 ---- .../opportunity_extensions/__init__.py | 14 - .../opportunity_extensions/cg_config.py | 39 - lib/python-sdk/examples/transforms.py | 293 ------- .../tests/extensions/test_plugin.py | 209 ++--- .../tests/extensions/test_schema.py | 347 ++++++++ lib/python-sdk/tests/extensions/test_types.py | 76 +- .../tests/schemas/test_plugin_registry.py | 75 +- .../tests/utils/test_plugin_generator.py | 552 ------------ .../tests/utils/test_transformation.py | 6 + 29 files changed, 1847 insertions(+), 3090 deletions(-) delete mode 100644 lib/python-sdk/common_grants_sdk/extensions/generate.py create mode 100644 lib/python-sdk/common_grants_sdk/extensions/schema.py delete mode 100644 lib/python-sdk/examples/plugin_custom_fields.py create mode 100644 lib/python-sdk/examples/plugins.py delete mode 100644 lib/python-sdk/examples/plugins/__init__.py delete mode 100644 lib/python-sdk/examples/plugins/grants_gov/__init__.py delete mode 100644 lib/python-sdk/examples/plugins/grants_gov/cg_config.py delete mode 100644 lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py delete mode 100644 lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py delete mode 100644 lib/python-sdk/examples/transforms.py create mode 100644 lib/python-sdk/tests/extensions/test_schema.py delete mode 100644 lib/python-sdk/tests/utils/test_plugin_generator.py diff --git a/lib/python-sdk/Makefile b/lib/python-sdk/Makefile index bd9d4614e..043a6fdbf 100644 --- a/lib/python-sdk/Makefile +++ b/lib/python-sdk/Makefile @@ -22,10 +22,7 @@ check-format: check-lint: $(RUNTIME_PREFIX) ruff check . -check-types: plugins +check-types: $(RUNTIME_PREFIX) mypy . checks: check-format check-lint check-types - -plugins: - $(RUNTIME_PREFIX) python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions examples/plugins/grants_gov diff --git a/lib/python-sdk/common_grants_sdk/__init__.py b/lib/python-sdk/common_grants_sdk/__init__.py index f70067807..9d2b9bad3 100644 --- a/lib/python-sdk/common_grants_sdk/__init__.py +++ b/lib/python-sdk/common_grants_sdk/__init__.py @@ -8,7 +8,7 @@ from . import schemas from .client import Auth, Client, Config -from .extensions import Plugin, PluginConfig, define_plugin +from .extensions import Plugin, PluginSchemas, define_plugin, schema __all__ = [ "schemas", @@ -16,6 +16,7 @@ "Auth", "Config", "Plugin", - "PluginConfig", + "PluginSchemas", "define_plugin", + "schema", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/README.md b/lib/python-sdk/common_grants_sdk/extensions/README.md index 7ca03ea23..cbec8570c 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/README.md +++ b/lib/python-sdk/common_grants_sdk/extensions/README.md @@ -7,769 +7,340 @@ For background, see: - [Custom Fields catalog](https://commongrants.org/custom-fields/): The published set of recommended custom fields - [Extensions section of the CommonGrants specification](https://commongrants.org/protocol/specification/#extensions): How extensions fit into the protocol +The `common_grants_sdk.extensions` module contains the utilities for working with extensions: declaring custom fields on base schemas, bundling them into typed plugins, and transforming between a source system's format and the CommonGrants format. -The `common-grants/sdk/extensions` module contains the utilities for working with extensions: registering custom fields on base schemas, bundling them into reusable plugins and composing plugins together. +There is no build step. Plugins are plain Python: you declare custom fields as a `CustomFieldSet`, build each schema extension with the `schema(...)` factory, and assemble a `Plugin` with `define_plugin(...)`. Consumers import the plugin and get fully typed, non-optional access to every registered schema. ## Table of contents <!-- omit in toc --> -- [Key Concepts](#key-concepts) -- [Extending base models using Custom Fields](#extending-base-models-using-custom-fields) +- [Key concepts](#key-concepts) +- [Declaring custom fields](#declaring-custom-fields) - [Option 1: Ad hoc with `with_custom_fields()`](#option-1-ad-hoc-with-with_custom_fields) - - [Option 2: Build-time with plugins](#option-2-build-time-with-plugins) -- [Extracting Custom Field Values](#extracting-custom-field-values) + - [Option 2: A reusable plugin](#option-2-a-reusable-plugin) +- [Extracting custom field values](#extracting-custom-field-values) - [Plugins](#plugins) - - [What is a plugin?](#what-is-a-plugin) - - [Defining a plugin](#defining-a-plugin) + - [The `schema(...)` factory](#the-schema-factory) + - [Assembling a plugin](#assembling-a-plugin) + - [Consuming a plugin](#consuming-a-plugin) - [Publishing a plugin](#publishing-a-plugin) - - [Combining Plugins](#combining-plugins) -- [Bidirectional Transforms](#bidirectional-transforms) - - [Defining transforms](#defining-transforms) +- [Bidirectional transforms](#bidirectional-transforms) + - [Declarative mappings](#declarative-mappings) + - [Hand-written functions](#hand-written-functions) - [Mapping format](#mapping-format) - - [Using transforms](#using-transforms) - [Using plugins with the API client](#using-plugins-with-the-api-client) - [Best practices](#best-practices) - - [Field naming](#field-naming) - - [Keep plugins focused](#keep-plugins-focused) - - [Type safety](#type-safety) - - [Export value types alongside your plugin](#export-value-types-alongside-your-plugin) - - [Declare `common-grants-sdk` as a dependency](#declare-common-grants-sdk-as-a-dependency) - - [Avoid `"first_wins"` / `"last_wins"` in published plugins](#avoid-first_wins--last_wins-in-published-plugins) - -## Key Concepts - -Here are some key concepts that are used to define custom fields and plugins that extend base schemas from the CommonGrants protocol. +## Key concepts | Concept | Description | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Custom field** | A key-value pair attached to a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | -| **`CustomFieldSpec`** | A Python dataclass that _describes_ a custom field: its `field_type`, optional `value` (a Python type for the `value` property), and optional `name` and `description`. | -| **`SchemaExtensions`** | A legacy TypedDict mapping extensible model names (e.g. `"Opportunity"`) to dicts of `CustomFieldSpec`. Still accepted by `with_custom_fields()`. For plugins, declare custom fields inside `ObjectSchemasInput.custom_fields` instead. | -| **`Plugin`** | A dataclass assembled by the code generator. `.schemas` is a container object where each attribute (e.g. `.schemas.Opportunity`) is an `ObjectSchemas` instance providing the model class (`.common`), transform callables (`.to_common`, `.from_common`), and native type (`.native`). `.extensions` holds the serializable extension declarations. | -| **`PluginExtensionsMeta`** | Optional metadata attached to a plugin: `name`, `version`, `source_system`, and `capabilities` (e.g. `["customFields", "transforms"]`). | -| **`build_transforms()`**| Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Each callable accepts a data dict **or a Pydantic model instance** and returns a `TransformResult`. | -| **`TransformResult`** | A dataclass `(result: dict, errors: list[TransformError])` returned by each transform callable. Errors are non-fatal — a partial result is always returned alongside any errors. | -| **`ObjectSchemasInput`**| Bundles `custom_fields`, `to_common`, and `from_common` for a single object type. Passed to `define_plugin()` via the `schemas` parameter. | - - - -## Extending base models using Custom Fields - -There are two ways to register custom fields on a base schema: at runtime (ad hoc) or at build-time (with plugins) +| --- | --- | +| **Custom field** | A key-value pair on a resource's `customFields` property. Each field has a `name`, `fieldType`, `value`, and optional `description`. | +| **`CustomField[V]`** | A Pydantic generic that is the single source of truth for a custom field. The static value type `V` anchors the typing; `fieldType` and the inspectable value type are *derived* from `V`, so they cannot drift. | +| **`CustomFieldSet`** | The base class an author subclasses to declare a schema's custom fields, each as `Optional[CustomField[V]] = Field(default=None, description=...)`. | +| **`OpportunityBase[CF]`** | The common Opportunity model, a Pydantic generic over its custom-fields container `CF`. `OpportunityBase[OpportunityFields]` is a fully concrete type; `OpportunityBase` is the unextended form. | +| **`schema(...)`** | The factory that builds a schema extension. Overloads enforce, statically: mappings XOR hand-written functions XOR schema-only, and a `source_schema` whenever transforms are present. Returns a `SchemaWithTransforms` or a `SchemaOnly`. | +| **`PluginSchemas`** | Maps your extensions to the registered extensible schemas, keyed by name (`PluginSchemas(Opportunity=...)`). Schemas you omit fall back to the base schema, never `None`. | +| **`define_plugin(...)`** | Assembles a `Plugin` from a `PluginSchemas` and a `PluginMeta`. | +| **`Plugin`** | The value consumers import. `plugin.schemas.Opportunity` is fully typed dot access. | +| **`build_transforms()`** | Compiles a pair of mapping dicts into `(to_common, from_common)` callables. Used by `schema(..., mappings=...)` under the hood; also callable directly when you need custom handlers. | +| **`TransformResult`** | The return shape `(result, errors)` of every transform. Errors are non-fatal: a partial result is always returned alongside any errors. | +| **`PluginCustomFieldSpec`** | The resolved, inspection-only view of a custom field (`field_type`, `value`, `name`, `description`), exposed on `extension.custom_fields`. Derived from `CustomField[V]`; authors never construct it. | +| **`CustomFieldSpec`** | The runtime declaration consumed by `with_custom_fields()` (Option 1). `field_type` is a required input there. | + +## Declaring custom fields + +There are two ways to add custom fields to a base schema: ad hoc at runtime, or as a reusable plugin. ### Option 1: Ad hoc with `with_custom_fields()` -Use `with_custom_fields()` when you want to extend a single schema directly, without creating a reusable plugin. This is useful for one-off scripts, tests, or quick prototyping. + +Use `with_custom_fields()` to extend a single schema directly, without creating a plugin. Useful for one-off scripts, tests, or quick prototyping. ```python from pydantic import BaseModel -from datetime import datetime -from uuid import uuid4 -from common_grants_sdk.schemas.pydantic import ( - OpportunityBase, - CustomFieldType, - OppStatus, - OppStatusOptions, -) +from common_grants_sdk.schemas.pydantic import OpportunityBase, CustomFieldType from common_grants_sdk.extensions.specs import CustomFieldSpec -# Define a Pydantic schema for complex custom field values class LegacyIdValue(BaseModel): system: str id: int -# Add 2 custom fields to extend the base schema + fields = { "legacyId": CustomFieldSpec(field_type=CustomFieldType.OBJECT, value=LegacyIdValue), "groupName": CustomFieldSpec(field_type=CustomFieldType.STRING, value=str), } -Opportunity = OpportunityBase.with_custom_fields( - custom_fields=fields, model_name="Opportunity" -) +Opportunity = OpportunityBase.with_custom_fields(custom_fields=fields, model_name="Opportunity") -opp_data = { - "id": uuid4(), - "title": "Foo bar", - "status": OppStatus(value=OppStatusOptions.OPEN), - "description": "Example opportunity", - "createdAt": datetime.fromisoformat("2024-01-01T00:00:00+00:00"), - "lastModifiedAt": datetime.fromisoformat("2024-01-01T00:00:00+00:00"), - "customFields": { - "legacyId": { - "name": "legacyId", - "fieldType": "object", - "value": {"system": "legacy", "id": 12345}, - }, - "groupName": { - "name": "groupName", - "fieldType": "string", - "value": "TEST_GROUP", - }, - "ignoredForNow": {"type": "string", "value": "noop"}, - }, -} - -# Validate the fields to make them retrievable opp = Opportunity.model_validate(opp_data) print(opp.custom_fields["legacyId"]) ``` +### Option 2: A reusable plugin -### Option 2: Build-time with plugins - -Use `define_plugin()` when you want to create a **reusable, shareable** set of custom field definitions. Plugins are the recommended approach for any extensions that will be used across multiple files, projects, or teams. - -The following is an example `cg_config.py` file, which you pass to the build step. - +Declare custom fields as a `CustomFieldSet` and build a schema-only extension. `CustomField[V]` is the single source of truth: `fieldType` is derived from `V`, so the only per-field metadata is a description. ```python -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - } - ) - } -) -``` +from typing import Optional -After defining your plugins in `cg_config.py`, run the following command from the repo root (or the directory containing your plugin) to generate the plugin code: - -```bash -poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions -``` - -This generates the plugin and writes the output into a `generated/` subdirectory inside the named plugin directory. You can then import the plugin via its directory name. - - -```python -import sys -from pathlib import Path - -# Make the examples/ directory importable so that -# plugins.opportunity_extensions resolves correctly. -sys.path.insert(0, str(Path(__file__).parent)) - -from plugins.opportunity_extensions import opportunity_extensions # noqa: E402 - - -# --------------------------------------------------------------------------- -# Sample API payload containing our four custom fields -# --------------------------------------------------------------------------- - -api_response = { - "id": "573525f2-8e15-4405-83fb-e6523511d893", - "title": "Community Health Innovation Grant", - "status": {"value": "open"}, - "description": "Funding for community-led health initiatives", - "createdAt": "2025-03-01T00:00:00Z", - "lastModifiedAt": "2025-03-15T00:00:00Z", - "customFields": { - "programArea": { - "fieldType": "string", - "value": "CFDA-93.243", - }, - "legacyGrantId": { - "fieldType": "integer", - "value": 98765, - }, - "eligibilityTypes": { - "fieldType": "array", - "value": ["nonprofit", "tribal", "city_government"], - }, - "awardCeiling": { - "fieldType": "number", - "value": 250000.00, - }, - }, -} - -# --------------------------------------------------------------------------- -# Use the model returned via opportunity_extensions -# --------------------------------------------------------------------------- - -opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) +from pydantic import Field +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PluginMeta, + PluginSchemas, + define_plugin, + schema, +) +from common_grants_sdk.schemas.pydantic.models import Opportunity + + +class OpportunityFields(CustomFieldSet): + program_area: Optional[CustomField[str]] = Field( + default=None, description="HHS program area code (e.g. 'CFDA-93.243')" + ) + legacy_grant_id: Optional[CustomField[int]] = Field( + default=None, description="Numeric ID from the legacy grants management system" + ) + eligibility_types: Optional[CustomField[list[str]]] = Field( + default=None, description="Types of organizations eligible to apply" + ) + award_ceiling: Optional[CustomField[float]] = Field( + default=None, description="Maximum award amount in USD" + ) + + +opportunity_extensions = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OpportunityFields])), + meta=PluginMeta(name="opportunity extensions", source_system="hhs"), +) ``` -## Extracting Custom Field Values - -Use `get_custom_field_value()` to safely retrieve typed values from `custom_fields`. It is useful in three main situations: - -1. **Unregistered fields** — you didn't use `with_custom_fields()` or a plugin to register a custom field and want to validate its value at runtime. -2. **Ad hoc fields** — you registered a field with `with_custom_fields()` for initial validation, but want type inference on the value after accessing it. -3. **Programmatic access** — you registered a field via a plugin (which provides type inference through dot-delimited access like `opp.custom_fields.foo.value`), but need to access fields by variable name, such as in a loop. +Custom-field keys are camelCase on the wire and snake_case in Python: a `CustomField[V]` named `legacy_grant_id` serializes to `legacyGrantId`. This is handled by an alias generator on the `CustomFieldSet` base, so authors and consumers use snake_case while JSON I/O stays camelCase. -The helper returns `None` if the key is absent (no `try/except` needed) and raises `ValueError` if the value is present but cannot be converted to the requested type. +## Extracting custom field values -The idiomatic form is the instance method on any `OpportunityBase` subclass: +Use `get_custom_field_value()` to safely retrieve typed values from `custom_fields`, e.g. for unregistered fields or programmatic (by-name) access. It returns `None` if the key is absent and raises `ValueError` if a present value cannot be converted. ```python from pydantic import BaseModel -from common_grants_sdk.schemas.pydantic import OpportunityBase, CustomFieldType +from common_grants_sdk.schemas.pydantic import OpportunityBase + class LegacyIdValue(BaseModel): system: str id: int -opp = OpportunityBase.model_validate(opp_data) - -# Returns Optional[LegacyIdValue], or None if key is absent -legacy = opp.get_custom_field_value("legacyId", LegacyIdValue) -if legacy is not None: - print(legacy.id) # typed as int - -# Returns Optional[str] -group = opp.get_custom_field_value("groupName", str) -# Returns None — no KeyError -missing = opp.get_custom_field_value("missing", str) +opp = OpportunityBase.model_validate(opp_data) +legacy = opp.get_custom_field_value("legacyId", LegacyIdValue) # Optional[LegacyIdValue] +group = opp.get_custom_field_value("groupName", str) # Optional[str] ``` -`get_custom_field_value()` works with both ad hoc (unregistered) and plugin-based (registered) custom fields. - - ## Plugins -### What is a plugin? +A plugin bundles custom fields and transforms for the CommonGrants schemas a source system extends. It is a `Plugin` value, built with no codegen. + +### The `schema(...)` factory -A plugin is a Python package that adds domain-specific custom fields to CommonGrants models. For example, a government agency might add a `legacy_grant_id` field to `Opportunity` to preserve backward compatibility with an existing system. +`schema(...)` builds one schema extension. Its overloads enforce the valid shapes statically and it validates registry membership, custom-field consistency, and mapping output keys at call (import) time: -Plugins are built from a `cg_config.py` spec file. The SDK's code generator reads that file and emits fully typed Pydantic models into a `generated/` subdirectory. Consumers import the generated schemas and get complete type safety without running the generator themselves. +- **Schema-only** — custom fields, no transforms: `schema(common_schema=OpportunityBase[Fields])` returns a `SchemaOnly`. It has `parse(...)` but no `to_common`, so consumers cannot transform it. +- **Mappings** — declarative transforms: `schema(source_schema=Src, common_schema=OpportunityBase[Fields], mappings={...})` returns a `SchemaWithTransforms`. +- **Functions** — hand-written transforms: `schema(source_schema=Src, common_schema=OpportunityBase[Fields], to_common=fn, from_common=fn)` returns a `SchemaWithTransforms`. -### Defining a plugin +`common_schema` must be a registered extensible schema (`OpportunityBase[...]`); an unregistered base raises `PluginDefinitionError`. -A plugin is a Python class that contains extension specs and generated schemas +### Assembling a plugin +`PluginSchemas` maps each extension to a registered schema name; `define_plugin` returns the `Plugin`. Schemas you omit fall back to the base schema (a `SchemaOnly` over `OpportunityBase`), never `None`. ```python -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal')", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - } - ) - } +plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OpportunityFields])), + meta=PluginMeta(name="my-system", source_system="my-system.example.gov"), ) ``` -After running the build step the imported extension object will have 2 fields to use. -1. The `schemas` property used to access the properties on the model that was extended. -2. The `extensions` property to access information about the extensions added and their specific pydantic properties. - - -### Publishing a plugin - -#### Package structure - -A minimal plugin package has 2 user-defined files `cg_config.py` and `pyproject.toml` for export. - -For Poetry (and most `pyproject.toml`-based build systems), `pyproject.toml` lives at the **project root**, while `cg_config.py` and the generated files live inside the **Python package subdirectory**: - -``` -opportunity-extensions/ # Project root (outer folder — name it anything) -├── pyproject.toml # Package metadata (you write this) -└── opportunity_extensions/ # Python package (inner folder — must match packages config) - ├── __init__.py # Emitted by the generator — commit to the repo - ├── cg_config.py # Field specs (source of truth — you write this) - ├── py.typed # Marks the package as typed — you write this - └── generated/ # Emitted by the generator — commit to the repo - ├── __init__.py - └── schemas.py -``` - -The generator writes `generated/` and the package-level `__init__.py`. Only `cg_config.py` and `pyproject.toml` are hand-authored. - -> [!IMPORTANT] -> `cg_config.py` must be **inside** the Python package directory (alongside `__init__.py`), not at the project root next to `pyproject.toml`. Placing it at the project root will cause import errors when Poetry resolves the package. +### Consuming a plugin -Run the generator from the **project root**, pointing `--plugin` at the inner package directory: +Every registered schema is always present and fully typed: -```bash -poetry run python -m common_grants_sdk.extensions.generate --plugin opportunity_extensions -``` - -`pyproject.toml` (at the project root) defines the package metadata for export. - -```toml - [tool.poetry] - name = "opportunity-extensions" - version = "0.1.0" - description = "CommonGrants opportunity custom field extensions" - authors = ["Your Name <you@example.com>"] - packages = [{include = "opportunity_extensions"}] - - [tool.poetry.dependencies] - python = "^3.11" - common-grants-sdk = "^0.5.1" - - [tool.poetry.group.dev.dependencies] - pyright = "^1.1" - pytest = "^8.0" - - [build-system] - requires = ["poetry-core"] - build-backend = "poetry.core.masonry.api" +```python +opp = plugin.schemas.Opportunity.parse(api_response) +print(opp.custom_fields.program_area.value) # typed as str +print(opp.custom_fields.legacy_grant_id.value) # typed as int +# Inspect the resolved specs (field_type/value derived from CustomField[V]): +for name, spec in plugin.schemas.Opportunity.custom_fields.items(): + print(name, spec.field_type, spec.description) ``` -#### Shipping pre-built schemas - -Publishing steps: -1. Add an empty `py.typed` file so Pyright and mypy recognize your package as typed: - -``` -opportunity_extensions/ -└── py.typed -``` +### Publishing a plugin -2. Declare it in `pyproject.toml`: +A plugin is a normal Python module that exports the `Plugin` value, so publishing is standard packaging — no generated files to commit and no generate step. Add a `py.typed` marker so type checkers see your package as typed, declare `common-grants-sdk` as a dependency, and `poetry build` / `poetry publish`. ```toml [tool.poetry] +name = "opportunity-extensions" +version = "0.1.0" +packages = [{include = "opportunity_extensions"}] include = ["opportunity_extensions/py.typed"] -``` - -3. Build and verify the distribution: -```bash -poetry build -poetry publish +[tool.poetry.dependencies] +python = "^3.11" +common-grants-sdk = "^0.6.2" ``` -#### Consumer usage - -After installing the plugin (e.g. `poetry add opportunity-extensions`): - ```python from opportunity_extensions import opportunity_extensions -opp = opportunity_extensions.schemas.Opportunity.common.model_validate(api_response) -print(opp.custom_fields.program_area.value) # typed as str -print(opp.custom_fields.legacy_grant_id.value) # typed as int -``` - -#### Pre-publish checklist - -Before publishing a new version of your plugin: - -- [ ] `cg_config.py` field specs are up to date -- [ ] Generator has been re-run and the output is committed (`generated/schemas.py`, `generated/__init__.py`, `__init__.py`) -- [ ] All custom fields have the intended types (no unintended `Any` annotations in `generated/schemas.py`) -- [ ] `py.typed` marker is present and included in the package -- [ ] Package installs cleanly in a fresh virtual environment: `pip install dist/commongrants_my_plugin-*.whl` -- [ ] Imports resolve without errors: `from my_plugin import my_plugin` -- [ ] Custom field attributes are accessible and typed in the IDE after install -- [ ] A type checker passes with no errors: `pyright my_plugin` or `mypy my_plugin` -- [ ] A [changeset](../../lib/README.md) has been created with the correct revision type - -### Combining Plugins - -Custom fields from multiple logical sources are combined by declaring them all inside a single `ObjectSchemasInput.custom_fields` dict. Because the dict is plain Python, there is no special merge utility needed — just add the keys side by side: - -```python -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - # fields from a shared HHS package - "programArea": CustomFieldSpec(field_type=CustomFieldType.STRING), - "legacyGrantId": CustomFieldSpec(field_type=CustomFieldType.INTEGER), - # fields specific to this project - "eligibilityTypes": CustomFieldSpec(field_type=CustomFieldType.ARRAY), - "awardCeiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER), - } - ) - } -) +opp = opportunity_extensions.schemas.Opportunity.parse(api_response) +print(opp.custom_fields.program_area.value) # typed as str ``` -`merge_extensions()` is still available for merging `PluginExtensions` objects that carry declarative `mappings` (ADR-0017 transform configs). It no longer merges `custom_fields`. - -#### Verify type inference before publishing - -After building your package, import the plugin in a test file and confirm that `.schemas` parse types resolve correctly. Hover over the types in your editor to confirm they are not `any`. +## Bidirectional transforms -## Bidirectional Transforms +A `SchemaWithTransforms` maps between a source system's native format and the CommonGrants format. Both directions are always author-provided: `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. -Plugins can define bidirectional mappings between a source system's native data format and the CommonGrants format. These transforms are authored as plain Python dicts and compiled into callable functions by `build_transforms()`. +### Declarative mappings -### Defining transforms - -Use `build_transforms()` to compile a pair of mapping dicts into `(to_common, from_common)` callables, then pass them to `define_plugin()` via `schemas`: +Pass `mappings` to `schema(...)` and the SDK compiles them into typed, validated `to_common` / `from_common` callables: ```python -from common_grants_sdk.extensions import ( - CustomFieldSpec, - ObjectSchemasInput, - PluginExtensionsMeta, - build_transforms, - define_plugin, -) -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType - -to_common, from_common = build_transforms( - to_common_mapping={ - "title": {"field": "data.opportunity_title"}, - "status": { - "value": { - "match": { - "field": "data.opportunity_status", - "case": {"posted": "open", "archived": "closed", "forecasted": "forecasted"}, - "default": "custom", +from common_grants_sdk.extensions import PassthroughModel, schema +from common_grants_sdk.schemas.pydantic.models import Opportunity + +ext = schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase[OpportunityFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } } }, - "description": {"const": "The opportunity is currently accepting applications"}, - }, - "funding": { - "minAwardAmount": { - "amount": {"field": "data.summary.award_floor"}, - "currency": {"const": "USD"}, + "customFields": { + "agencyCode": { + "value": {"field": "agency_code"}, + "name": {"const": "agencyCode"}, + "fieldType": {"const": "string"}, + } }, }, - }, - from_common_mapping={ - "data": { + "from_common": { + "opportunity_uuid": {"field": "id"}, "opportunity_title": {"field": "title"}, - "opportunity_status": { - "match": { - "field": "status.value", - "case": {"open": "posted", "closed": "archived", "forecasted": "forecasted"}, - "default": "custom", - } - }, - } - }, -) - -plugin = define_plugin( - meta=PluginExtensionsMeta( - name="my-system", - version="0.1.0", - source_system="my-system.example.gov", - capabilities=["customFields", "transforms"], - ), - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Unique identifier in legacy database", - ), - }, - to_common=to_common, - from_common=from_common, - ) + "agency_code": {"field": "customFields.agencyCode.value"}, + }, }, ) ``` -Both directions must be provided explicitly. `build_transforms()` does not invert one mapping from the other, because many-to-one handlers like `match` are not reversible. +`PassthroughModel` is a permissive source schema (`extra="allow"`) for when you do not want to model the source shape. Because it accepts arbitrary keys, output-path validation is skipped for it. -#### Hand-written callables - -`build_transforms()` is optional. You can supply any plain Python callable to `ObjectSchemasInput` as long as it matches the expected signature: - -```python -def to_common(native_data: dict) -> TransformResult: - ... - -def from_common(cg_data: dict) -> TransformResult: - ... - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - to_common=to_common, - from_common=from_common, - ) - }, -) -``` +### Hand-written functions -The key requirement when porting existing transform code is that **both callables must return `TransformResult`**. The type annotation enforces this, but it is easy to miss when wrapping a function that previously returned a plain dict. Wrap the return value like so: +When the transform is more than a mapping (e.g. it needs a custom handler or arbitrary Python), write the callables yourself and pass them via the functions overload. Use `validate_into` so the result is validated into the target model and any failure is routed to `TransformResult.errors`: ```python -from common_grants_sdk.extensions.types import TransformResult - -def to_common(native_data: dict) -> TransformResult: - result = my_existing_transform(native_data) # returns a plain dict - return TransformResult(result=result, errors=[]) -``` - -### Mapping format +from common_grants_sdk.extensions import TransformResult, schema, validate_into +from common_grants_sdk.schemas.pydantic.models import Opportunity -A mapping dict describes how to build an output object from a source dict. Each leaf node is either a literal value or a single-key dict that invokes a named handler. -| Handler | Syntax | Description | -|---|---|---| -| `const` | `{"const": "USD"}` | Returns a fixed literal value, ignoring source data | -| `field` | `{"field": "data.summary.award_floor"}` | Extracts a value using a dot-notation path | -| `match` | `{"match": {"field": "...", "case": {...}, "default": "..."}}` | Case-based lookup on a field value (canonical ADR name) | -| `switch` | `{"switch": {...}}` | Alias for `match`, kept for backward compatibility | -| `numberToString` | `{"numberToString": "data.summary.award_floor"}` | Extracts a numeric value and coerces it to a string | -| `stringToNumber` | `{"stringToNumber": "some.string.field"}` | Extracts a string and coerces it to `int` or `float` | +def to_common(source: GrantsGovOpportunity) -> TransformResult[OpportunityBase[OpportunityFields]]: + return validate_into(OpportunityBase[OpportunityFields], { + "id": source.opportunity_uuid, + "title": source.opportunity_title, + # ... + }) -Bare non-dict values (strings, numbers, booleans) in a mapping are treated as literals and passed through unchanged. Use `{"const": ...}` when you want a literal value inside a dict node that might otherwise be mistaken for a field name. -You can also register custom handlers by passing a `handlers` dict to `build_transforms()`: +def from_common(common: OpportunityBase[OpportunityFields]) -> TransformResult[GrantsGovOpportunity]: + # `common` is fully typed: common.custom_fields.agency_code.value -> str + return validate_into(GrantsGovOpportunity, {...}) -```python -def handle_upper(data, field_path): - val = get_from_path(data, field_path) - return val.upper() if isinstance(val, str) else val -to_common, from_common = build_transforms( - to_common_mapping={"title": {"upper": "data.opportunity_title"}}, - from_common_mapping={...}, - handlers={"upper": handle_upper}, +ext = schema( + source_schema=GrantsGovOpportunity, + common_schema=OpportunityBase[OpportunityFields], + to_common=to_common, + from_common=from_common, ) ``` -Custom handlers are merged with the defaults; they cannot override built-in handler names. +Hand-written functions own their own validation. In Python, returning a `TransformResult[Model]` means returning an actual Pydantic instance, which is validated on construction (and `validate_into` makes that ergonomic and routes errors). The SDK does not re-wrap them. -Handlers are scoped to the `build_transforms()` call they are registered on — they do not affect other calls: +When you need custom handlers, compile with `build_transforms()` directly and pass the callables to the functions overload: ```python -# Only the first pair of callables knows about "upper" -to_common_with_upper, _ = build_transforms( - to_common_mapping={"title": {"upper": "data.opportunity_title"}}, - from_common_mapping={}, - handlers={"upper": handle_upper}, -) +from common_grants_sdk.extensions import build_transforms -# This call has no knowledge of "upper" — using it in the mapping would fail at call time -to_common_plain, _ = build_transforms( - to_common_mapping={"title": {"field": "data.opportunity_title"}}, - from_common_mapping={}, +to_common, from_common = build_transforms( + handlers={"join": _join_fields}, + common_schema=OpportunityBase[OpportunityFields], + source_schema=PassthroughModel, + to_common_mapping={...}, + from_common_mapping={...}, ) ``` -### Using transforms - -The compiled callables are stored on the plugin's `schemas` object, accessible by attribute name. Each callable takes a data dict (or a Pydantic model instance) and returns a `TransformResult`: - -```python -opp_schemas = plugin.schemas.Opportunity - -# Source system → CommonGrants -result = opp_schemas.to_common(native_data) -if result.errors: - for err in result.errors: - print(f"[{err.path}] {err}") -else: - cg_data = result.result - -# CommonGrants → source system -result = opp_schemas.from_common(cg_data) -native_data = result.result -``` +### Mapping format -`TransformResult.errors` is always a list (empty on success). A non-empty errors list means the transform encountered a problem but still returned a partial result in `result`. +A mapping dict describes how to build an output object from a source object. Each leaf node is either a literal value or a single-key dict that invokes a named handler. -> [!IMPORTANT] -> When `common_model` is set on `build_transforms()`, `to_common` returns a validated Pydantic model instance in `result.result`. That instance can be passed directly to `from_common`. In that case, field paths in `from_common_mapping` must use the model's **camelCase alias names** (e.g. `"status.value"`, `"funding.minAwardAmount.amount"`), not Python snake_case attribute names. This matches the camelCase convention used throughout CommonGrants field paths. +| Handler | Syntax | Description | +| --- | --- | --- | +| `const` | `{"const": "USD"}` | A fixed literal, ignoring source data | +| `field` | `{"field": "data.summary.award_floor"}` | Extract a value via a dot-notation path | +| `match` | `{"match": {"field": "...", "case": {...}, "default": "..."}}` | Case-based lookup on a field value | +| `switch` | `{"switch": {...}}` | Alias for `match` | +| `numberToString` | `{"numberToString": "data.opportunity_id"}` | Extract a number and coerce to a string | +| `stringToNumber` | `{"stringToNumber": "data.priority_score_str"}` | Extract a string and coerce to `int`/`float` | -See `examples/transforms.py` for a complete working example with roundtrip verification. +Bare non-dict values are treated as literals. Register custom handlers by passing a `handlers` dict to `build_transforms()`; they are scoped to that call and cannot override built-in handler names. +When the source is a real (non-passthrough) model, mapping output keys may use either the model's field names or their camelCase aliases (e.g. `createdAt`). Field paths in `from_common` read from the common model, so they use its camelCase alias names (e.g. `status.value`, `customFields.agencyCode.value`). ## Using plugins with the API client -Pass a plugin's extended schema to the API client via the `schema` parameter. The client uses it to hydrate API responses into fully typed models. The `schema` parameter accepts any `Type[OpportunityBase]` subclass. +Pass an extension's `common_schema` to the client via the `schema` parameter to hydrate responses into fully typed models: ```python from common_grants_sdk import Client -from plugins.opportunity_extensions import opportunity_extensions client = Client(base_url="https://api.example.gov") +schema_cls = opportunity_extensions.schemas.Opportunity.common_schema -# Get a single opportunity with typed custom fields -opp = client.opportunities.get(opp_id, schema=opportunity_extensions.schemas.Opportunity.common) +opp = client.opportunities.get(opp_id, schema=schema_cls) print(opp.custom_fields.program_area.value) # typed as str -# List with the same schema -response = client.opportunities.list(schema=opportunity_extensions.schemas.Opportunity.common) -for opp in response.items: - print(opp.custom_fields.legacy_grant_id.value) # typed as int - -# Search with the same schema -results = client.opportunities.search( - search="health", - status=["open"], - schema=opportunity_extensions.schemas.Opportunity.common, -) +response = client.opportunities.list(schema=schema_cls) +results = client.opportunities.search(search="health", status=["open"], schema=schema_cls) ``` - ## Best practices ### Field naming -> [!IMPORTANT] -> The field key you define in `cg_config.py` **must match the key in the JSON data** you're parsing in order to validate your custom fields correctly. Because the CommonGrants standard uses `camelCase` for core attributes, you **should** use `camelCase` for custom field keys. - -The generator converts `camelCase` keys to `snake_case` Python attribute names automatically, using a Pydantic `alias` under the hood. For example, a field key of `"legacyGrantId"` becomes `opp.custom_fields.legacy_grant_id` in Python, while still mapping to `"legacyGrantId"` in JSON: - -```python -# cg_config.py -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "legacyGrantId": CustomFieldSpec( # camelCase key — matches the JSON - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - } - ) - } -) -``` - -```python -# JSON parsed correctly because the alias matches -api_response = { - "customFields": { - "legacyGrantId": {"fieldType": "integer", "value": 98765}, - }, -} - -opp = my_plugin.schemas.Opportunity.common.model_validate(api_response) -opp.custom_fields.legacy_grant_id.value # 98765, typed as int -``` - -Additional field naming guidelines: - -- Use descriptive, stable names. Renaming a published field is a breaking change. -- Prefix ambiguous names with your organization or system context (e.g. `hhsProgramArea` rather than `programArea`) when there is risk of collision with other plugins. +Custom-field keys are camelCase on the wire. Declare a `CustomField[V]` with a snake_case attribute name (`legacy_grant_id`); it serializes to and parses from `legacyGrantId` automatically. Use descriptive, stable names; renaming a published field is a breaking change. Prefix ambiguous names with your organization (e.g. `hhs_program_area`) when collisions with other plugins are likely. ### Keep plugins focused -A plugin should represent a single logical concern (one agency's fields, one integration's needs, or one domain concept). If you need fields from multiple concerns, declare them all in one `ObjectSchemasInput.custom_fields` dict — Python dict literals compose cleanly without a special merge utility. +A plugin should represent a single logical concern (one agency's fields, one integration). Declare all of a schema's custom fields on one `CustomFieldSet`. ### Type safety -- Omitting `value` in `CustomFieldSpec` falls back to a sensible default (`str`, `int`, `float`, `bool`, `list[Any]`, or `dict[str, Any]`). Specify it explicitly when you need a more precise type. -- For complex object types, define a Pydantic `BaseModel` subclass and pass it as `value`: - - ```python - from pydantic import BaseModel - from common_grants_sdk import define_plugin - from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput - from common_grants_sdk.schemas.pydantic import CustomFieldType - - class LegacyRef(BaseModel): - system: str - id: int - - config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "legacyRef": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=LegacyRef, - description="Reference to the opportunity in the legacy system", - ), - } - ) - } - ) - ``` - -> [!NOTE] -> The generator only embeds the class name for builtins and SDK types. If a type lives in a third-party package **or your plugin's own module**, the generator falls back to `Any`. Define such types inside `common_grants_sdk` and use `from common_grants_sdk...` imports. - -### Export value types alongside your plugin - -When you define Pydantic models for complex `value` fields, export them as named exports from your package. Downstream consumers may need these types for use with `get_custom_field_value()`: - -```python -# cg_config.py of a plugin package -from pydantic import BaseModel -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, ObjectSchemasInput -from common_grants_sdk.schemas.pydantic import CustomFieldType - -# Export value types so consumers can reference them directly -class ProgramAreaValue(BaseModel): - code: str - name: str - -config = define_plugin( - schemas={ - "Opportunity": ObjectSchemasInput( - custom_fields={ - "programArea": CustomFieldSpec( - field_type=CustomFieldType.OBJECT, - value=ProgramAreaValue, - description="The HHS program area for this opportunity", - ), - } - ) - } -) -``` - -This allows consumers to use `get_custom_field_value()` with the same type the plugin uses for validation: - -```python -from commongrants_hhs_plugin import hhs_plugin, ProgramAreaValue - -opp = hhs_plugin.schemas.Opportunity.common.model_validate(api_response) - -# Extract the value with full type safety using the exported type -area = opp.get_custom_field_value("programArea", ProgramAreaValue) -print(area.code) # typed as str -``` - -### Declare `common-grants-sdk` as a dependency - -Declare `common-grants-sdk` as a dependency in your plugin's `pyproject.toml` rather than expecting consumers to install it separately. This ensures the correct version of the SDK is always present when your plugin is installed. See [Publishing a plugin](#publishing-a-plugin) for a full `pyproject.toml` example. - -### Avoid `"first_wins"` / `"last_wins"` in published plugins - -When calling `merge_extensions()` with `"first_wins"` or `"last_wins"`, conflicts are resolved at runtime but the overridden field definitions are silently dropped. This is fine for local or ad hoc usage, but if you publish a package that uses one of these strategies internally, consumers have no indication that fields may have been overridden. - -Prefer the default `"error"` strategy in published plugins. If your extensions genuinely overlap with another plugin, resolve the conflicts explicitly before publishing rather than deferring to a lossy merge strategy. - +- The value type `V` on `CustomField[V]` is the single source of truth. `fieldType` and the inspectable value type are derived from it, so they cannot drift. +- For complex object values, use a Pydantic model as `V`: `Optional[CustomField[LegacyRef]]` derives `fieldType="object"` and gives consumers `value.system` / `value.id` typing. +- Run a type checker (`mypy` or `pyright`) over your plugin module; consumer typing is verified by `assert_type` lines in the SDK's tests. diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 0fb2c8148..464f396b4 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,34 +1,59 @@ """Public extension APIs for the CommonGrants Python SDK.""" -from .plugin import Plugin, PluginConfig, define_plugin, inject_transforms -from .specs import ConflictStrategy, CustomFieldSpec, SchemaExtensions +from .plugin import ( + Plugin, + PluginMeta, + PluginSchemas, + define_plugin, +) +from .schema import ( + EXTENSIBLE_SCHEMA_MAP, + CustomField, + CustomFieldSet, + PluginDefinitionError, + SchemaOnly, + SchemaWithTransforms, + resolve_custom_field_specs, + schema, + validate_into, +) +from .specs import ( + ConflictStrategy, + CustomFieldSpec, + PluginCustomFieldSpec, + SchemaExtensions, +) from .transforms import build_transforms from .types import ( Handler, + PassthroughModel, PluginCapability, - PluginExtensionsMeta, - SchemaConfig, - SchemaInput, - SchemaMappings, TransformError, TransformResult, ) __all__ = [ + "EXTENSIBLE_SCHEMA_MAP", "ConflictStrategy", + "CustomField", + "CustomFieldSet", "CustomFieldSpec", - "Plugin", - "PluginConfig", - "SchemaExtensions", - "define_plugin", - "inject_transforms", - "build_transforms", "Handler", + "PassthroughModel", + "Plugin", "PluginCapability", - "PluginExtensionsMeta", - "SchemaConfig", - "SchemaInput", - "SchemaMappings", + "PluginCustomFieldSpec", + "PluginDefinitionError", + "PluginMeta", + "PluginSchemas", + "SchemaExtensions", + "SchemaOnly", "TransformError", + "SchemaWithTransforms", "TransformResult", + "build_transforms", + "define_plugin", + "resolve_custom_field_specs", + "schema", + "validate_into", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/generate.py b/lib/python-sdk/common_grants_sdk/extensions/generate.py deleted file mode 100644 index 5b6b222ba..000000000 --- a/lib/python-sdk/common_grants_sdk/extensions/generate.py +++ /dev/null @@ -1,659 +0,0 @@ -"""Generate typed plugin schema models from a plugin config.""" - -from __future__ import annotations - -import argparse -import importlib.util -import keyword -import re -from pathlib import Path -from typing import Any, Iterable - -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -from common_grants_sdk.utils.json import snake -from .plugin import PluginConfig -from .specs import CustomFieldSpec - -# Maps extensible model names to the SDK base class they extend in generated code. -# Add an entry here when a new model gains customFields support. -MODEL_BASE_CLASS: dict[str, str] = { - "Opportunity": "OpportunityBase", -} - -# Default Python type annotation strings written into generated source for each field type. -# ARRAY and OBJECT use parameterized forms (list[Any], dict[str, Any]) as practical defaults; -# callers can override these by setting spec.value to a more specific type. -FIELD_TYPE_DEFAULT_ANNOTATION: dict[CustomFieldType, str] = { - CustomFieldType.STRING: "str", - CustomFieldType.NUMBER: "float", - CustomFieldType.INTEGER: "int", - CustomFieldType.BOOLEAN: "bool", - CustomFieldType.OBJECT: "dict[str, Any]", - CustomFieldType.ARRAY: "list[Any]", -} - - -def _load_config(config_path: Path) -> PluginConfig[Any]: - """Load and validate a plugin config file, returning the PluginConfig object. - - Uses importlib to load cg_config.py as an isolated module so it doesn't - pollute sys.modules and can be loaded from any directory at runtime. - - Args: - config_path: Absolute path to the cg_config.py file. - - Returns: - The ``PluginConfig`` bound to the module-level ``config`` variable. - - Raises: - RuntimeError: If the file cannot be loaded or does not expose a valid - ``config`` variable created by ``define_plugin()``. - """ - spec = importlib.util.spec_from_file_location( - f"cg_plugin_config_{config_path.parent.name}", config_path - ) - if spec is None or spec.loader is None: - raise RuntimeError(f"Unable to load plugin config from {config_path}") - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - config = getattr(module, "config", None) - if not isinstance(config, PluginConfig): - raise RuntimeError( - 'Plugin config must expose a "config" variable created by define_plugin()' - ) - return config - - -def _extract_custom_fields( - config: PluginConfig[Any], -) -> dict[str, dict[str, CustomFieldSpec]]: - """Extract custom field specs from config.schemas into the flat shape used by generators. - - Returns an empty dict if config.schemas is None or has no schemas with custom_fields. - """ - if config.schemas is None: - return {} - return { - obj: schema.custom_fields - for obj, schema in config.schemas.items() - if schema.custom_fields is not None - } - - -def _normalize_identifier(name: str) -> str: - """Convert an arbitrary string into a valid Python identifier. - - Args: - name: The raw string to normalise (e.g. a directory name or field key). - - Returns: - A non-empty string that is a legal Python identifier and not a keyword. - - Example:: - - _normalize_identifier("my-plugin") # "my_plugin" - _normalize_identifier("123abc") # "plugin_123abc" - _normalize_identifier("class") # "class_plugin" - """ - # Replace any non-word characters with underscores to produce a valid Python identifier. - ident = re.sub(r"\W+", "_", name.strip()) - if not ident: - ident = "plugin" # empty string edge case - if ident[0].isdigit(): - ident = f"plugin_{ident}" # identifiers can't start with a digit - if keyword.iskeyword(ident): - ident = f"{ident}_plugin" # e.g. "class" -> "class_plugin" - return ident - - -def _to_pascal(value: str) -> str: - """Convert a snake_case or kebab-case string to PascalCase. - - Args: - value: The string to convert (e.g. ``"eligibility_type"``). - - Returns: - PascalCase string (e.g. ``"EligibilityType"``). Falls back to - ``"Field"`` if the input contains no alphanumeric characters. - """ - parts = re.split(r"[^A-Za-z0-9]+", value) - return "".join(p[:1].upper() + p[1:] for p in parts if p) or "Field" - - -def _resolve_field_type(field_type: CustomFieldType | str) -> CustomFieldType: - """Normalise a field type value to a ``CustomFieldType`` enum member. - - Accepts either an already-resolved enum member or a plain string (e.g. - ``"string"``, ``"array"``) as used in ``cg_config.py`` shorthand. - - Args: - field_type: A ``CustomFieldType`` member or its string value. - - Returns: - The corresponding ``CustomFieldType`` enum member. - - Raises: - ValueError: If the string does not match any ``CustomFieldType`` value. - """ - if isinstance(field_type, CustomFieldType): - return field_type - return CustomFieldType(field_type) - - -def _annotation_for_spec(spec: CustomFieldSpec, resolved_type: CustomFieldType) -> str: - """Determine the Python type annotation string for a custom field's ``value`` property. - - Handles three cases based on what ``spec.value`` contains: - - - ``None``: looks up a default annotation from ``FIELD_TYPE_DEFAULT_ANNOTATION`` - (e.g. ``field_type="array"`` → ``"list[Any]"``). - - A plain ``type`` object (e.g. ``int``, ``MyModel``): uses ``__name__``. Builtin - and SDK types are already importable in the generated file; external types (e.g. - Pydantic models from ``cg_config.py`` or third-party packages) will have their - import emitted by :func:`_collect_extra_imports`. - - A generic alias or complex type (e.g. ``list[str]``, ``Optional[int]``): converts - to string, strips the ``typing.`` prefix added by older Python versions, and unwraps - ``<class 'str'>``-style repr strings produced by ``str()`` in some contexts. - - Args: - spec: The ``CustomFieldSpec`` for the field being generated. - resolved_type: The normalised ``CustomFieldType`` for the field. - - Returns: - A string suitable for use as a type annotation in generated Python source, - e.g. ``"str"``, ``"list[str]"``, ``"dict[str, Any]"``, or ``"MyModel"``. - """ - if spec.value is None: - return FIELD_TYPE_DEFAULT_ANNOTATION[resolved_type] - - value = spec.value - if isinstance(value, type): - # Always use the bare name; _collect_extra_imports handles the import - # statement for any type that isn't already available in the generated file. - return value.__name__ - - # spec.value is a generic alias or other non-type (e.g. list[str], Optional[int]). - # Strip the "typing." prefix so the annotation is valid in generated source that - # uses `from __future__ import annotations`. - rendered = str(value).replace("typing.", "") - # str(type_obj) for builtins renders as "<class 'str'>" rather than "str" — unwrap it. - if rendered in { - "<class 'str'>", - "<class 'int'>", - "<class 'float'>", - "<class 'bool'>", - }: - return rendered.split("'")[1] - return rendered or "Any" - - -def _collect_extra_imports( - custom_fields: dict[str, dict[str, CustomFieldSpec]], -) -> list[str]: - """Collect import lines needed for external types used as ``spec.value``. - - Walks all specs and returns one ``import`` line per distinct external type. - Types from ``builtins`` or ``common_grants_sdk`` are skipped because they - are already available in the generated file without an explicit import. - - Types loaded from a ``cg_config.py`` module (identified by the synthetic - ``cg_plugin_config_*`` module name that :func:`_load_config` assigns) are - imported via a relative ``from ..cg_config import`` statement. All other - types are imported using their ``__module__`` path directly. - - Args: - custom_fields: The flat custom fields mapping extracted from the plugin config. - - Returns: - A deduplicated list of import-statement strings in the order they were - first encountered, ready to be inserted into the generated source file. - """ - seen: set[tuple[str, str]] = set() - imports: list[str] = [] - - for fields in custom_fields.values(): - for spec in fields.values(): - if not isinstance(spec.value, type): - continue - module = spec.value.__module__ - name = spec.value.__name__ - if module == "builtins" or module.startswith("common_grants_sdk"): - continue - key = (module, name) - if key in seen: - continue - seen.add(key) - if module.startswith("cg_plugin_config_"): - # Type was defined in cg_config.py, loaded via importlib with a - # synthetic module name — import it relative to the generated/ dir. - imports.append(f"from ..cg_config import {name}") - else: - imports.append(f"from {module} import {name}") - - return imports - - -def _model_blocks( - custom_fields: dict[str, dict[str, CustomFieldSpec]], -) -> Iterable[str]: - """Yield source-code blocks for every model defined in the extensions mapping. - - For each model, yields three blocks in dependency order: - - 1. One ``CustomField`` subclass per field key (typed ``value`` property). - 2. A ``CustomFields`` container model grouping all fields for the model. - 3. The extended model class that adds a ``custom_fields`` attribute typed to - the container. - - Args: - custom_fields: The flat custom fields mapping extracted from the plugin config. - - Yields: - Source-code strings to be joined and written into ``schemas.py``. - - Raises: - ValueError: If a model name is not present in ``MODEL_BASE_CLASS``. - """ - for model_name, fields in custom_fields.items(): - if model_name not in MODEL_BASE_CLASS: - raise ValueError( - f'Generator does not support model "{model_name}". ' - f"Supported models: {sorted(MODEL_BASE_CLASS)}" - ) - - custom_field_classes: list[str] = [] - custom_fields_attrs: list[str] = [] - - for field_key, spec in fields.items(): - resolved_type = _resolve_field_type(spec.field_type) - # Derive generated class names and the snake_case attribute name for this field. - field_cls_name = f"{model_name}{_to_pascal(field_key)}CustomField" - attr_name = _normalize_identifier(snake(field_key)) - value_annotation = _annotation_for_spec( - spec=spec, resolved_type=resolved_type - ) - # Use spec.name as the runtime display name if provided, otherwise fall back - # to the field key (the dict key in SchemaInput.custom_fields). - field_name_default = spec.name or field_key - # repr() produces a quoted string literal safe to embed directly in source code. - description_default = repr(spec.description) if spec.description else "None" - - custom_field_classes.append( - "\n".join( - [ - f"class {field_cls_name}(CustomField):", - " model_config = ConfigDict(populate_by_name=True)", - " field_type: CustomFieldType = Field(", - f" default=CustomFieldType.{resolved_type.name},", - ' alias="fieldType",', - " )", - f" name: str = Field(default={field_name_default!r})", - f" description: Optional[str] = Field(default={description_default})", - f" value: Optional[{value_annotation}] = None", - ] - ) - ) - custom_fields_attrs.append( - "\n".join( - [ - f" {attr_name}: Optional[{field_cls_name}] = Field(", - " default=None,", - f" alias={field_key!r},", - " )", - ] - ) - ) - - custom_fields_model_name = f"{model_name}CustomFields" - model_name_with_extensions = model_name - base_class = MODEL_BASE_CLASS[model_name] - - # Yield three source blocks per model in dependency order: - # 1. Individual CustomField subclasses (one per field key) - # 2. A CustomFields container model grouping all fields for this model - # 3. The extended model class that wires in the CustomFields container - yield "\n\n".join(custom_field_classes) - yield "\n".join( - [ - f"class {custom_fields_model_name}(CommonGrantsBaseModel):", - " model_config = ConfigDict(populate_by_name=True)", - *custom_fields_attrs, - ] - ) - yield "\n".join( - [ - f"class {model_name_with_extensions}({base_class}):", - " model_config = ConfigDict(populate_by_name=True)", - f" custom_fields: Optional[{custom_fields_model_name}] = Field( # type: ignore[assignment]", - " default=None,", - ' alias="customFields",', - " )", - ] - ) - - -def _render_schemas_py( - custom_fields: dict[str, dict[str, CustomFieldSpec]], - mappings_only_objs: set[str] | None = None, -) -> str: - """Render the full source of the generated ``schemas.py`` file. - - Produces a self-contained module containing typed ``CustomField`` subclasses, - a ``CustomFields`` container, and an extended model class for each entry in - ``custom_fields``. Also emits a ``_Schemas`` container object (attribute access - rather than dict lookup) and a module-level ``schemas`` instance. - - For objects that only have ``mappings`` (no ``custom_fields``), the ``_Schemas`` - object will expose the base SDK model class directly (e.g. ``schemas.Opportunity`` - will be ``OpportunityBase``). - - Args: - custom_fields: The flat custom fields mapping extracted from the plugin config. - mappings_only_objs: Set of object names that have mappings but no custom_fields. - These will be exposed on ``_Schemas`` as their base SDK class. - - Returns: - A string of valid Python source code ready to be written to disk. - """ - # _Schemas is a plain object (not a dict) so callers get attribute access: - # plugin.schemas.Opportunity rather than plugin.schemas["Opportunity"]. - # The dynamic __init__ assignment is necessary because model names aren't - # known until generation time, so a static class body can't be used. - model_names = list(custom_fields.keys()) - blocks = "\n\n\n".join(_model_blocks(custom_fields)) - mappings_only: set[str] = mappings_only_objs or set() - - # Build schema assignments: each attribute is a SchemaConfig instance so - # callers get a unified interface (plugin.schemas.Opportunity.common_schema for the - # model class, .to_common/.from_common for transforms). - # to_common/from_common default to None here; root __init__.py injects the - # real callables for any object that has transforms configured. - assignments: list[str] = [ - f" self.{name} = SchemaConfig(source_schema=dict, common_schema={name}, to_common=None, from_common=None)" - for name in model_names - ] - for obj in sorted(mappings_only): - if obj not in MODEL_BASE_CLASS: - raise ValueError( - f'Generator does not support model "{obj}". ' - f"Supported models: {sorted(MODEL_BASE_CLASS)}" - ) - base_class = MODEL_BASE_CLASS[obj] - assignments.append( - f" self.{obj} = SchemaConfig(source_schema=dict, common_schema={base_class}, to_common=None, from_common=None)" - ) - - schema_assignments = "\n".join(assignments or [" pass"]) - all_names = model_names + sorted(mappings_only) - all_exports = ", ".join([f'"{name}"' for name in all_names] + ['"schemas"']) - extra_imports = _collect_extra_imports(custom_fields) - extra_import_lines = ["", *extra_imports] if extra_imports else [] - - return "\n".join( - [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from __future__ import annotations", - "", - "from typing import Any, Optional", - "", - "from pydantic import ConfigDict, Field", - "", - "from common_grants_sdk.extensions.types import SchemaConfig", - "from common_grants_sdk.schemas.pydantic.base import CommonGrantsBaseModel", - "from common_grants_sdk.schemas.pydantic.fields import CustomField, CustomFieldType", - "from common_grants_sdk.schemas.pydantic.models import OpportunityBase", - *extra_import_lines, - *([blocks, ""] if blocks else []), - "class _Schemas:", - " def __init__(self) -> None:", - schema_assignments, - "", - "schemas = _Schemas()", - "", - f"__all__ = [{all_exports}]", - "", - ] - ) - - -def _render_generated_init_py() -> str: - """Render the source of the ``generated/__init__.py`` file. - - This file re-exports the ``schemas`` instance from ``schemas.py`` so that - the generated package can be imported as ``from .generated import schemas``. - - Returns: - A string of valid Python source code ready to be written to disk. - """ - return "\n".join( - [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from .schemas import schemas", - "", - '__all__ = ["schemas"]', - "", - ] - ) - - -def _render_plugin_init_py(plugin_variable_name: str, config: PluginConfig[Any]) -> str: - """Render the source of the plugin directory's root __init__.py file. - - Emits a fully compiled Plugin instance. Transform callables are injected - into the _Schemas object (from generated/schemas.py) before Plugin is - constructed, so plugin.schemas.Opportunity.to_common etc. are populated. - """ - # Collect the sets of objects needing transform injection. - # Only count schemas that have explicit callable transforms, not those with custom_fields only. - explicit_objs: set[str] = ( - { - obj - for obj, s in config.schemas.items() - if s.to_common is not None or s.from_common is not None - } - if config.schemas - else set() - ) - mappings_objs: set[str] = ( - {obj for obj, s in config.schemas.items() if s.mappings is not None} - if config.schemas - else set() - ) - - needs_build_transforms = bool(mappings_objs - explicit_objs) - - # Build pre-plugin lines: inject transforms into the _Schemas object before - # constructing Plugin. schemas.py initialises each SchemaConfig with - # to_common=None/from_common=None; we mutate those attrs here. - inject_lines: list[str] = [] - - # Mappings-only objects: call build_transforms() then inject results. - for obj in sorted(mappings_objs - explicit_objs): - inject_lines += [ - f"_{obj}_to_common, _{obj}_from_common = build_transforms(", - f' to_common_mapping=config.schemas["{obj}"].mappings.to_common,', - f' from_common_mapping=config.schemas["{obj}"].mappings.from_common,', - f" common_schema=schemas.{obj}.common_schema,", - ")", - f"schemas.{obj}.to_common = _{obj}_to_common", - f"schemas.{obj}.from_common = _{obj}_from_common", - "", - ] - - # Explicit schemas: single inject_transforms() call handles all objects. - # Reassigning the return value preserves the concrete _Schemas type for mypy. - if explicit_objs: - inject_lines.append("schemas = inject_transforms(config, schemas)") - inject_lines.append("") - - imports = [ - "# This file is auto-generated. Do not edit it manually — it will be overwritten", - "# the next time `python -m common_grants_sdk.extensions.generate` is run.", - "from __future__ import annotations", - "", - ] - - sdk_imports = {"Plugin"} - if explicit_objs: - sdk_imports.add("inject_transforms") - if needs_build_transforms: - sdk_imports.add("build_transforms") - imports.append( - f"from common_grants_sdk.extensions import {', '.join(sorted(sdk_imports))}" - ) - imports += [ - "from .cg_config import config", - "from .generated import schemas", - "", - ] - - plugin_lines = [ - f"{plugin_variable_name} = Plugin(", - " schemas=schemas,", - " meta=config.meta,", - ")", - ] - - pre_plugin = inject_lines # may be empty - - return "\n".join( - imports - + pre_plugin - + plugin_lines - + [ - "", - f'__all__ = ["{plugin_variable_name}", "schemas"]', - "", - ] - ) - - -def generate_plugin(plugin_dir: Path) -> Path: - """Run the full code generation pipeline for a single plugin directory. - - Loads ``cg_config.py``, creates the ``generated/`` subdirectory, and writes - three files: ``generated/schemas.py``, ``generated/__init__.py``, and the - root ``__init__.py`` that exports the ``Plugin`` instance. - - Args: - plugin_dir: Path to the plugin directory containing ``cg_config.py``. - - Returns: - The path to the ``generated/`` directory that was created. - - Raises: - FileNotFoundError: If ``cg_config.py`` does not exist in ``plugin_dir``. - RuntimeError: If the config file cannot be loaded or is invalid. - ValueError: If the config references an unsupported model name. - """ - plugin_dir = plugin_dir.resolve() - config_path = plugin_dir / "cg_config.py" - if not config_path.exists(): - raise FileNotFoundError(f"Could not find config file: {config_path}") - - config = _load_config(config_path) - custom_fields = _extract_custom_fields(config) - - # Determine objects that have mappings but no custom_fields — these need a - # pass-through entry in _Schemas pointing at the base SDK model class. - explicit_cf_objs: set[str] = set(custom_fields.keys()) - mappings_only_objs: set[str] = ( - { - obj - for obj, s in config.schemas.items() - if s.mappings is not None and obj not in explicit_cf_objs - } - if config.schemas - else set() - ) - # Third bucket: objects in config.schemas with explicit transforms but no - # custom_fields and no mappings entry. They also need a pass-through - # entry in _Schemas or schemas.<Obj> won't exist at import time. - transforms_only_objs: set[str] = ( - set(config.schemas.keys()) - explicit_cf_objs - mappings_only_objs - if config.schemas - else set() - ) - - # Validate that auto-generated transform objects have both mapping directions. - # Auto-generated objects: have mappings in schemas[obj] but no explicit callables. - if config.schemas: - explicit_schema_objs: set[str] = { - obj - for obj, s in config.schemas.items() - if s.to_common is not None or s.from_common is not None - } - for obj, schema in config.schemas.items(): - if schema.mappings is None or obj in explicit_schema_objs: - continue - if schema.mappings.to_common is None: - raise ValueError( - f'Plugin object "{obj}": mappings.to_common is required when ' - f"auto-generating transforms. Either provide a to_common mapping " - f"or pass an explicit to_common callable via schemas['{obj}']." - ) - if schema.mappings.from_common is None: - raise ValueError( - f'Plugin object "{obj}": mappings.from_common is required when ' - f"auto-generating transforms. Either provide a from_common mapping " - f"or pass an explicit from_common callable via schemas['{obj}']." - ) - - generated_dir = plugin_dir / "generated" - generated_dir.mkdir(parents=True, exist_ok=True) - - schemas_py = generated_dir / "schemas.py" - init_generated_py = generated_dir / "__init__.py" - root_init_py = plugin_dir / "__init__.py" - - schemas_py.write_text( - _render_schemas_py( - custom_fields, mappings_only_objs=mappings_only_objs | transforms_only_objs - ), - encoding="utf-8", - ) - init_generated_py.write_text(_render_generated_init_py(), encoding="utf-8") - - plugin_variable_name = _normalize_identifier(plugin_dir.name) - root_init_py.write_text( - _render_plugin_init_py(plugin_variable_name, config), encoding="utf-8" - ) - - return generated_dir - - -def main(argv: list[str] | None = None) -> int: - """CLI entry point for ``python -m common_grants_sdk.extensions.generate``. - - Parses command-line arguments and delegates to ``generate_plugin()``. - - Args: - argv: Argument list to parse. Defaults to ``sys.argv[1:]`` when ``None``. - - Returns: - Exit code (``0`` on success). - """ - parser = argparse.ArgumentParser( - prog="python -m common_grants_sdk.extensions.generate", - description="Generate typed plugin schemas from cg_config.py", - ) - parser.add_argument( - "--plugin", - nargs="+", - default=["."], - help="One or more plugin directories containing cg_config.py (default: current directory)", - ) - args = parser.parse_args(argv) - - for plugin_path in args.plugin: - generated_dir = generate_plugin(Path(plugin_path)) - print(f"Generated plugin schemas at {generated_dir}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 49cd62d26..eb846f358 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -1,156 +1,91 @@ -"""Plugin configuration and composition APIs.""" +"""Plugin assembly: ``PluginSchemas``, ``Plugin``, and ``define_plugin``. -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Generic, TypeVar, overload - -from .types import PluginExtensionsMeta +A plugin maps the schema extensions an author builds with ``schema(...)`` onto the +registered extensible schemas, keyed by registry name. Schemas a plugin does not +extend fall back to the base schema (a ``SchemaOnly``), never ``None``, so +consumers get fully-typed, non-optional dot access: ``plugin.schemas.Opportunity``. +""" -T = TypeVar("T") -TSchemas = TypeVar("TSchemas") -_TSchemasContainer = TypeVar("_TSchemasContainer") +from __future__ import annotations +from dataclasses import dataclass, field, fields +from typing import Any, Generic, TypeVar, cast -@dataclass(frozen=True) -class PluginConfig(Generic[TSchemas]): - """Build-time plugin config produced by define_plugin() and consumed by generate.py. +import typing_extensions as te - Generic on TSchemas so the precise type of the schemas dict is preserved — e.g. - PluginConfig[dict[str, SchemaInput[MyNative, MyCg]]] — rather than being - widened to SchemaInput[Any, Any] at the storage boundary. +from ..schemas.pydantic.models import OpportunityBase +from .schema import ( + PluginDefinitionError, + SchemaOnly, + SchemaWithTransforms, + schema, +) +from .types import PluginMeta - Stores inputs as-is — no compilation occurs at define_plugin() call time. - generate.py compiles this into a fully resolved Plugin by injecting the generated - Pydantic model class as the common schema for each SchemaInput entry, and - auto-generating build_transforms() calls for any object that has - schemas[obj].mappings but no explicit schemas[obj].to_common / from_common. +SchemasT = TypeVar("SchemasT") - All fields are optional so adopters can start with only what they need. - """ +# The fallback for a schema a plugin does not extend: the base schema, no custom +# fields, no transforms. A SchemaOnly type, so unextended slots have no +# to_common either. +DefaultOpportunity = SchemaOnly[OpportunityBase] - meta: PluginExtensionsMeta | None = None - schemas: TSchemas | None = None +_TOpportunity = te.TypeVar("_TOpportunity", default=DefaultOpportunity) @dataclass -class Plugin(Generic[T]): - """Runtime plugin container assembled by generate.py after code generation. - - schemas: the _Schemas object from generated/schemas.py. Each attribute is a - SchemaConfig instance providing unified access to the model class and - transforms for that object: - plugin.schemas.Opportunity.common_schema → the Pydantic model class (includes - any custom fields declared by the plugin) - plugin.schemas.Opportunity.to_common → transform callable (or None) - plugin.schemas.Opportunity.from_common → transform callable (or None) - plugin.schemas.Opportunity.source_schema → the source system's type (or dict) - """ - - schemas: T - meta: PluginExtensionsMeta | None = None - - -@overload -def define_plugin( - meta: PluginExtensionsMeta | None = ..., - schemas: None = ..., -) -> PluginConfig[None]: ... - +class PluginSchemas(Generic[_TOpportunity]): + """Maps your extensions to the extensible schemas. Construct it directly. -@overload -def define_plugin( - meta: PluginExtensionsMeta | None = ..., - schemas: TSchemas = ..., -) -> PluginConfig[TSchemas]: ... + Pass one extension per schema you extend, keyed by the registered schema name. + Schemas you omit fall back to the base schema (a ``SchemaOnly``), never + ``None``. Unknown schema names are a *static* error, and each slot's type is + inferred concretely, so consumers get non-optional dot access:: + plugin = define_plugin(PluginSchemas(Opportunity=opp_ext), meta=...) + plugin.schemas.Opportunity # the extension you passed -def define_plugin( - meta: PluginExtensionsMeta | None = None, - schemas: Any = None, -) -> PluginConfig[Any]: - """Create a PluginConfig consumed by the code generator. - - No compilation occurs here — inputs are stored as-is. The code generator - (generate.py) compiles SchemaInput → SchemaConfig by injecting - the common model from the generated schemas, and auto-wires build_transforms() - for any object that has schemas[obj].mappings but no explicit callables. - - The return type is generic on the schemas argument: passing a typed dict - (e.g. {"Opportunity": SchemaInput[MyNative, MyCg](...) }) preserves - those per-object generics on the returned PluginConfig rather than widening - them to Any. - - Raises: - ValueError: If any schema entry specifies both mappings and explicit - to_common/from_common callables (XOR constraint). + There is one field per registered extensible schema. """ - if schemas: - for obj_name, schema_input in schemas.items(): - has_mappings = schema_input.mappings is not None - has_callables = ( - schema_input.to_common is not None - or schema_input.from_common is not None - ) - if has_mappings and has_callables: - raise ValueError( - f"define_plugin: {obj_name} cannot specify both mappings and explicit " - f"to_common/from_common. " - f"Use mappings for declarative transforms or provide explicit callables, not both." - ) - return PluginConfig( - meta=meta, - schemas=schemas, + + Opportunity: _TOpportunity = field( + default_factory=lambda: cast( + _TOpportunity, schema(common_schema=OpportunityBase) + ) ) -def inject_transforms( - config: PluginConfig[Any], schemas: _TSchemasContainer -) -> _TSchemasContainer: - """Wire transform callables from plugin config into the generated schemas container. +@dataclass(frozen=True) +class Plugin(Generic[SchemasT]): + """The plugin singleton consumers import. - Called by the generated plugin __init__.py to inject to_common/from_common - callables (and the native type) from cg_config into the SchemaConfig instances - produced by the code generator. + ``schemas`` is a typed frozen dataclass, so ``plugin.schemas.Opportunity`` is + fully typed (dot access). + """ - Iterates over all entries in config.schemas that have at least one callable, - validates that both directions are present, then sets the attributes on the - matching schemas container attribute (e.g. schemas.Opportunity). + schemas: SchemasT + meta: PluginMeta - Returns the same schemas container (mutated in place) so callers can write - ``schemas = inject_transforms(config, schemas)`` and retain the concrete - generated type rather than widening to Any. - Args: - config: The PluginConfig produced by define_plugin(). - schemas: The generated _Schemas container from generated/schemas.py. +def define_plugin(schemas: SchemasT, *, meta: PluginMeta) -> Plugin[SchemasT]: + """Assemble the plugin from a ``PluginSchemas`` instance and metadata. - Returns: - The same schemas container, with transform callables injected. + Each attribute name must equal the entry's ``schema_name``, so + ``schemas.Opportunity`` really holds the Opportunity extensible schema. Raises: - ValueError: If a schema with any callable is missing its counterpart, - or if the object name is not found in the schemas container. + PluginDefinitionError: If any slot does not hold a schema extension, or holds + one whose ``schema_name`` does not match its attribute name. """ - if not config.schemas: - return schemas - for obj_name, schema_input in config.schemas.items(): - if schema_input.to_common is None and schema_input.from_common is None: - continue - obj_schemas = getattr(schemas, obj_name, None) - if obj_schemas is None: - raise ValueError( - f"Plugin object {obj_name!r}: not found in generated schemas" - ) - if schema_input.to_common is None: - raise ValueError( - f"Plugin object {obj_name!r}: to_common callable is required" - ) - if schema_input.from_common is None: - raise ValueError( - f"Plugin object {obj_name!r}: from_common callable is required" + errors: list[str] = [] + for fld in fields(cast(Any, schemas)): + entry = getattr(schemas, fld.name) + if not isinstance(entry, (SchemaWithTransforms, SchemaOnly)): + errors.append(f"schemas.{fld.name}: not a schema extension") + elif entry.schema_name != fld.name: + errors.append( + f"schemas.{fld.name}: holds the {entry.schema_name!r} extensible " + f"schema; the attribute name must match the schema name" ) - obj_schemas.source_schema = schema_input.source_schema or dict - obj_schemas.to_common = schema_input.to_common - obj_schemas.from_common = schema_input.from_common - return schemas + if errors: + raise PluginDefinitionError("plugin", errors) + return Plugin(schemas=schemas, meta=meta) diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py new file mode 100644 index 000000000..c128a8868 --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -0,0 +1,405 @@ +"""Schema extensions: the ``schema(...)`` factory and its building blocks. + +Authors build each schema extension with the overloaded ``schema(...)`` factory, +which returns a discriminated ``SchemaWithTransforms`` or ``SchemaOnly``. The +overloads enforce, statically: mappings XOR hand-written transforms, a source when +transforms are present, and no ``to_common`` on schema-only entries. Registry +membership, custom-field consistency, and mapping output keys are validated when the +extension is built, aggregated into one error. + +``CustomField[V]`` is the single source of truth for a custom field: ``field_type`` +and the inspectable value type are derived from ``V``, so they cannot drift from the +typed declaration. The common models are generics over their custom-fields container +(``OpportunityBase[OpportunityFields]``), so consumers get concrete, non-optional types. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + Any, + Callable, + Generic, + Optional, + TypeGuard, + TypeVar, + get_args, + get_origin, + overload, +) + +from pydantic import ( + AliasGenerator, + BaseModel, + ConfigDict, + ValidationError, +) +from pydantic.alias_generators import to_camel + +from ..schemas.pydantic.fields.custom import CustomField, CustomFieldType +from ..schemas.pydantic.models import OpportunityBase +from .specs import PluginCustomFieldSpec +from .transforms import build_transforms +from .types import TransformError, TransformResult + +__all__ = [ + "EXTENSIBLE_SCHEMA_MAP", + "CustomField", + "CustomFieldSet", + "PluginDefinitionError", + "SchemaOnly", + "SchemaWithTransforms", + "resolve_custom_field_specs", + "schema", + "validate_into", +] + +TSource = TypeVar("TSource", bound=BaseModel) +TCommon = TypeVar("TCommon", bound=BaseModel) +T = TypeVar("T", bound=BaseModel) + + +class _CamelModel(BaseModel): + """Base model that is camelCase on the wire and snake_case in code. + + Authors and consumers use snake_case field names (``field_type``); JSON I/O uses + camelCase (``fieldType``). Construction by field name type-checks and validation + accepts either casing. + """ + + model_config = ConfigDict( + alias_generator=AliasGenerator( + validation_alias=to_camel, + serialization_alias=to_camel, + ), + populate_by_name=True, + ) + + +class CustomFieldSet(_CamelModel): + """Base class an author subclasses to declare a schema's custom fields. + + Each field is declared as ``Optional[CustomField[V]] = Field(default=None, + description=...)``. ``CustomField[V]`` is the single source of truth: its value + type ``V`` is the anchor from which ``field_type`` and the inspectable value type + are derived, so they cannot drift. + """ + + +# The extensible-schema registry: the closed set of models a plugin may extend, +# mapped to their generic base model. Add an entry when a new model gains +# custom-field support. +EXTENSIBLE_SCHEMA_MAP: dict[str, type[BaseModel]] = { + "Opportunity": OpportunityBase, +} +_BASE_TO_NAME: dict[type[BaseModel], str] = { + v: k for k, v in EXTENSIBLE_SCHEMA_MAP.items() +} + + +class PluginDefinitionError(Exception): + """Raised at definition (import) time, listing every problem at once.""" + + def __init__(self, cls_name: str, errors: list[str]) -> None: + self.errors = errors + body = "\n".join(f" - {e}" for e in errors) + super().__init__(f"{cls_name} is not a valid plugin definition:\n{body}") + + +def _is_model_class(obj: Any) -> TypeGuard[type[BaseModel]]: + return isinstance(obj, type) and issubclass(obj, BaseModel) + + +def _infer_field_type(value_type: Any) -> Optional[CustomFieldType]: + """Derive the JSON-schema ``field_type`` tag from a ``CustomField[V]`` value type. + + Returns ``None`` for value types that do not map to a ``CustomFieldType``. + """ + origin = get_origin(value_type) + if value_type is bool: # before int: bool is a subclass of int + return CustomFieldType.BOOLEAN + if value_type is int: + return CustomFieldType.INTEGER + if value_type is float: + return CustomFieldType.NUMBER + if value_type is str: + return CustomFieldType.STRING + if isinstance(value_type, type) and issubclass(value_type, BaseModel): + return CustomFieldType.OBJECT + if value_type is dict or origin is dict: + return CustomFieldType.OBJECT + if value_type is list or origin is list: + return CustomFieldType.ARRAY + return None + + +def _value_type(annotation: Any) -> Any: + """Pull ``V`` out of an ``Optional[CustomField[V]]`` annotation. + + ``CustomField[V]`` is a parameterized Pydantic generic (a concrete subclass), so + its argument lives in ``__pydantic_generic_metadata__`` rather than via + ``get_origin``/``get_args``. + """ + candidates = get_args(annotation) or (annotation,) + for cand in candidates: + meta = getattr(cand, "__pydantic_generic_metadata__", None) + if meta and meta.get("origin") is CustomField: + args = meta.get("args", ()) + return args[0] if args else None + if get_origin(cand) is CustomField: # non-concrete fallback + args = get_args(cand) + return args[0] if args else None + return None + + +def _check_custom_fields(container: type[BaseModel]) -> list[str]: + """Ensure each custom field's value type ``V`` maps to a known ``field_type``. + + Nothing is reconciled: ``field_type`` and the inspectable value type are derived + from ``CustomField[V]``, so the only failure is a ``V`` we cannot map. + """ + errors: list[str] = [] + for name, info in container.model_fields.items(): + value_type = _value_type(info.annotation) + if _infer_field_type(value_type) is None: + label = getattr(value_type, "__name__", value_type) + errors.append( + f"custom field {name!r}: cannot derive a field_type from value type " + f"{label}; use a supported CustomField[V]" + ) + return errors + + +def resolve_custom_field_specs( + container: Optional[type[BaseModel]], +) -> dict[str, PluginCustomFieldSpec]: + """Build the resolved spec for each custom field, for consumer inspection. + + Everything is derived from the single source of truth, ``CustomField[V]``: + ``field_type`` and the inspectable value type come from ``V``, ``name`` is the + attribute name, and ``description`` is the Pydantic field description. This is + what ``extension.custom_fields`` exposes; because nothing is authored twice, it + cannot drift. + """ + if not (isinstance(container, type) and issubclass(container, CustomFieldSet)): + return {} + out: dict[str, PluginCustomFieldSpec] = {} + for name, info in container.model_fields.items(): + value_type = _value_type(info.annotation) + out[name] = PluginCustomFieldSpec( + field_type=_infer_field_type(value_type), + value=value_type, + name=name, + description=info.description or "", + ) + return out + + +def validate_into(model: type[T], data: Any) -> TransformResult[T]: + """Validate ``data`` into ``model``, routing failures to ``errors``. + + The helper hand-written transform authors use so their ``to_common`` / + ``from_common`` return the validated model on success, or the raw data alongside + structured ``TransformError`` entries on failure. + """ + try: + return TransformResult(result=model.model_validate(data), errors=[]) + except ValidationError as exc: + errors = [ + TransformError(e["msg"], path=".".join(str(loc) for loc in e["loc"])) + for e in exc.errors() + ] + return TransformResult(result=data, errors=errors) + + +def _output_field_names(model: type[BaseModel]) -> set[str]: + """Valid top-level output keys for a model: field names plus their aliases.""" + names: set[str] = set(model.model_fields.keys()) + for info in model.model_fields.values(): + if info.alias: + names.add(info.alias) + return names + + +def _accepts_arbitrary_keys(model: type[BaseModel]) -> bool: + """True for permissive sources (PassthroughModel) where path checks are moot.""" + return model.model_config.get("extra") == "allow" + + +Mappings = dict[str, Any] + + +def _resolve_common(common: Any) -> tuple[Any, Any]: + """Return ``(origin, custom_fields_model)`` for a common type. + + A parameterized Pydantic generic (``OpportunityBase[OpportunityFields]``) is a + concrete subclass, so its origin/args live in ``__pydantic_generic_metadata__``. + """ + pyd_meta = getattr(common, "__pydantic_generic_metadata__", None) + if pyd_meta and pyd_meta.get("origin"): + args = pyd_meta.get("args", ()) + return pyd_meta["origin"], (args[0] if args else None) + return (get_origin(common) or common), (get_args(common)[:1] or (None,))[0] + + +@dataclass +class SchemaWithTransforms(Generic[TSource, TCommon]): + """A schema extension with transforms. Built by ``schema(...)``, never by hand. + + ``custom_fields`` exposes the resolved specs (field_type, value, name, + description) so consumers can inspect each declared custom field. + """ + + schema_name: str + common_schema: type[TCommon] + custom_fields: dict[str, PluginCustomFieldSpec] + source_schema: type[TSource] + to_common: Callable[[TSource], TransformResult[TCommon]] + from_common: Callable[[TCommon], TransformResult[TSource]] + + def parse(self, data: Any) -> TCommon: + return self.common_schema.model_validate(data) + + +@dataclass +class SchemaOnly(Generic[TCommon]): + """A schema extension with custom fields but no transforms. Built by ``schema(...)``. + + It deliberately has no ``to_common`` / ``from_common``, so a consumer cannot call + a transform on a schema-only entry -- that is a static error. ``custom_fields`` + exposes the resolved specs for consumer inspection. + """ + + schema_name: str + common_schema: type[TCommon] + custom_fields: dict[str, PluginCustomFieldSpec] + + def parse(self, data: Any) -> TCommon: + return self.common_schema.model_validate(data) + + +@overload +def schema( + *, source_schema: type[TSource], common_schema: type[TCommon], mappings: Mappings +) -> SchemaWithTransforms[TSource, TCommon]: ... +@overload +def schema( + *, + source_schema: type[TSource], + common_schema: type[TCommon], + to_common: Callable[[TSource], TransformResult[TCommon]], + from_common: Callable[[TCommon], TransformResult[TSource]], +) -> SchemaWithTransforms[TSource, TCommon]: ... +@overload +def schema(*, common_schema: type[TCommon]) -> SchemaOnly[TCommon]: ... +def schema( + *, + source_schema: Any = None, + common_schema: Any, + mappings: Optional[Mappings] = None, + to_common: Any = None, + from_common: Any = None, +) -> Any: + """Build a schema extension. The overloads enforce, statically: + + - ``mappings`` XOR hand-written ``to_common`` / ``from_common`` (both = no match), + - a ``source_schema`` is required whenever transforms are present, + - a schema-only entry (``common_schema`` only) returns a ``SchemaOnly`` whose + type has no ``to_common`` (so consumers cannot transform it). + + Registry membership, custom-field consistency, and mapping output keys are + validated here at call (import) time, aggregated into one ``PluginDefinitionError``. + """ + errors: list[str] = [] + common_origin, custom_fields_model = _resolve_common(common_schema) + + schema_name = _BASE_TO_NAME.get(common_origin) + if schema_name is None: + base_label = getattr(common_origin, "__name__", repr(common_origin)) + raise PluginDefinitionError( + "schema", + [ + f"common base {base_label} is not a registered extensible schema " + f"(expected one of {sorted(EXTENSIBLE_SCHEMA_MAP)})" + ], + ) + + if ( + custom_fields_model is not None + and isinstance(custom_fields_model, type) + and issubclass(custom_fields_model, CustomFieldSet) + ): + errors.extend(_check_custom_fields(custom_fields_model)) + + # Transform-shape guards. The overloads enforce these statically; these + # checks repeat them at runtime so dynamically-built or un-type-checked + # callers cannot slip through with a malformed extension. + has_callables = to_common is not None or from_common is not None + if mappings is not None and has_callables: + errors.append( + "cannot specify both `mappings` and `to_common`/`from_common`; " + "use mappings for declarative transforms or explicit callables, not both" + ) + if has_callables and not (to_common is not None and from_common is not None): + errors.append( + "both `to_common` and `from_common` are required when either is provided" + ) + if (mappings is not None or has_callables) and source_schema is None: + errors.append( + "a `source_schema` is required when transforms (mappings or " + "to_common/from_common) are declared" + ) + + if mappings is not None: + for direction in ("to_common", "from_common"): + if direction not in mappings: + errors.append(f"mappings: missing `{direction}` direction") + if "to_common" in mappings and _is_model_class(common_schema): + unknown = set(mappings["to_common"]) - _output_field_names(common_schema) + if unknown: + errors.append( + f"mappings.to_common: unknown output field(s) {sorted(unknown)} " + f"for {common_schema.__name__}" + ) + if ( + "from_common" in mappings + and _is_model_class(source_schema) + and not _accepts_arbitrary_keys(source_schema) + ): + unknown_src = set(mappings["from_common"]) - _output_field_names( + source_schema + ) + if unknown_src: + errors.append( + f"mappings.from_common: unknown source field(s) " + f"{sorted(unknown_src)} for {source_schema.__name__}" + ) + + if errors: + raise PluginDefinitionError(schema_name, errors) + + custom_fields = resolve_custom_field_specs(custom_fields_model) + + if mappings is not None: + # Passing both schemas makes the compiled callables validate their output: + # to_common into the common model, from_common into the source model. So + # both directions return a validated instance (not a raw dict) on success. + to_fn, from_fn = build_transforms( + mappings["to_common"], + mappings["from_common"], + common_schema=common_schema, + source_schema=source_schema, + ) + return SchemaWithTransforms( + schema_name, common_schema, custom_fields, source_schema, to_fn, from_fn + ) + if to_common is not None: + return SchemaWithTransforms( + schema_name, + common_schema, + custom_fields, + source_schema, + to_common, + from_common, + ) + return SchemaOnly(schema_name, common_schema, custom_fields) diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index 4b4124b57..a91dd925c 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -12,7 +12,13 @@ @dataclass class CustomFieldSpec: - """Custom Field spec class to support adding custom fields""" + """Runtime custom-field declaration consumed by the registration path. + + Used by ``utils.custom_fields.add_custom_fields`` (and + ``OpportunityBase.with_custom_fields``) to build a typed custom-fields container + at runtime. ``field_type`` is a required input here: it selects the value type + when ``value`` is not given, and is pinned on the resulting ``CustomField``. + """ field_type: CustomFieldType value: Optional[Any] = None @@ -20,6 +26,29 @@ class CustomFieldSpec: description: str = "" +@dataclass +class PluginCustomFieldSpec: + """Resolved, inspection-only view of a single custom field. + + Mirrors the field shape of :class:`CustomFieldSpec`, but authors never construct + it. The ``schema(...)`` factory produces it via ``resolve_custom_field_specs`` + from a ``CustomField[V]`` declaration on a ``CustomFieldSet``, and exposes it to + consumers through ``extension.custom_fields`` so they can introspect each field + without it ever drifting from the typed declaration: + + - ``field_type`` -- the JSON-schema tag derived from ``V`` (``str -> string``, + a Pydantic model -> ``object``, ...); ``None`` when ``V`` cannot be mapped. + - ``value`` -- the static value type ``V`` itself, for runtime inspection. + - ``name`` -- the attribute name on the container. + - ``description`` -- the Pydantic field description. + """ + + field_type: Optional[CustomFieldType] = None + value: Optional[Any] = None + name: str = "" + description: str = "" + + class SchemaExtensions(TypedDict, total=False): """Maps extensible model names to custom field specifications.""" diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index b4099a1ad..4f3c0dbe9 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -34,12 +34,18 @@ def _validate_output_paths( ) -> None: """Validate that top-level output keys in mapping are valid fields on model. - Called when common_schema is supplied to build_transforms(). Custom fields - declared by the plugin appear as regular model fields on the generated common + Called when common_schema or source_schema is supplied to build_transforms(). + Custom fields declared by the plugin appear as regular model fields on the common model and are therefore treated as valid output paths automatically. + Models configured with ``extra="allow"`` (e.g. ``PassthroughModel``) accept + arbitrary keys, so every output key is valid and the check is skipped. + Raises ValueError if any top-level key is not a field name or alias on model. """ + if model.model_config.get("extra") == "allow": + return + valid_names: set[str] = set(model.model_fields.keys()) for field_info in model.model_fields.values(): if field_info.alias: @@ -53,7 +59,7 @@ def _validate_output_paths( raise ValueError( f"build_transforms ({direction}_mapping): unknown output {noun} " f"{sorted(invalid)!r} for model {model.__name__}. " - f"Declare them as custom_fields in SchemaInput or check the field name." + f"Declare them on the schema's CustomFieldSet or check the field name." ) @@ -189,13 +195,11 @@ def handle_upper(data, path): handlers={"upper": handle_upper}, ) common_schema: Optional Pydantic model class to validate the to_common output - against. Must be the fully extended generated model class (e.g. the - generated Opportunity from generated/schemas.py), NOT the base class - (e.g. OpportunityBase). Passing a base class will silently weaken - validation — custom_fields will only be checked against - dict[str, CustomField] rather than the typed container produced by the - plugin's custom field declarations. When provided, model_validate is - called on the transform result and any ValidationErrors are appended to + against. Pass the model that carries the plugin's custom fields (e.g. + ``OpportunityBase[OpportunityFields]``), not the base ``OpportunityBase``: + a base class only checks custom_fields against dict[str, CustomField] + rather than the typed container. When provided, model_validate is called + on the transform result and any ValidationErrors are appended to TransformResult.errors rather than raised. Note on result shape: when common_schema is set, TransformResult.result diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 493bba1eb..3d88d6f47 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -7,10 +7,6 @@ from pydantic import BaseModel, ConfigDict, Field -from .specs import CustomFieldSpec - -TSource = TypeVar("TSource") -TCommon = TypeVar("TCommon") T = TypeVar("T") # Capability enum — Literal rather than StrEnum to stay JSON-safe. @@ -20,6 +16,19 @@ Handler = Callable[[Any, Any], Any] +class PassthroughModel(BaseModel): + """Permissive source schema, which preserves the transformation result dict as is. + + Validates only that the input is a mapping and preserves arbitrary keys + (``extra="allow"``) without constraining any field. Use it as the + ``source_schema`` on a transform entry to satisfy the source-schema + requirement without modeling the source-system shape (e.g. in tests or + early development). + """ + + model_config = ConfigDict(extra="allow") + + class TransformError(Exception): """Structured transformation error per ADR-0022 Decision #9. @@ -63,20 +72,7 @@ class TransformResult(Generic[T]): errors: list[TransformError] -class SchemaMappings(BaseModel): - """ADR-0017 declarative mapping dicts for a single object. - - Each direction is author-provided — build_transforms() does not invert one into - the other because many-to-one handlers like switch are not reversible (Decision #6). - """ - - model_config = ConfigDict(populate_by_name=True) - - to_common: dict[str, Any] | None = Field(default=None, alias="toCommon") - from_common: dict[str, Any] | None = Field(default=None, alias="fromCommon") - - -class PluginExtensionsMeta(BaseModel): +class PluginMeta(BaseModel): """Plugin identity and capability declaration. name and source_system are required so that plugin registries and @@ -88,58 +84,8 @@ class PluginExtensionsMeta(BaseModel): model_config = ConfigDict(populate_by_name=True) name: str - source_system: str = Field(alias="sourceSystem") + source_system: str = Field( + validation_alias="sourceSystem", serialization_alias="sourceSystem" + ) version: str | None = None capabilities: list[PluginCapability] | None = None - - -@dataclass -class SchemaInput(Generic[TSource, TCommon]): - """Input type provided by plugin authors inside define_plugin(schemas=...). - - This is the single surface for all per-object declarations. Plugin authors supply - to_common and from_common as plain callables — either hand-written or generated - via build_transforms(). source_schema defaults to dict[str, Any] if omitted. - - custom_fields declares any extra fields this object exposes beyond the base - CommonGrants schema. The code generator reads these and emits typed subclasses. - - mappings holds optional declarative mappings. When present and no - explicit to_common / from_common is supplied, the code generator auto-invokes - build_transforms() on these. Explicit callables take priority and disable - auto-wiring for that object. - - common_schema is intentionally absent here. It is injected by define_plugin() during - compilation from SchemaInput → SchemaConfig, resolved from the generated - model classes produced by the code generator. Plugin authors never set it directly — - cg_config.py cannot import from generated/ (it is the input to generation). - """ - - source_schema: type[TSource] | None = None - custom_fields: dict[str, CustomFieldSpec] | None = None - mappings: SchemaMappings | None = None - to_common: Callable[[TSource], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TSource]] | None = None - - -@dataclass -class SchemaConfig(Generic[TSource, TCommon]): - """Runtime compiled schema container for a single object (ADR-0022). - - Bundles the type information and transform callables for one schema object - (e.g. Opportunity). Accessed via attribute lookup on the plugin's schemas - container: plugin.schemas.Opportunity. - - source_schema: The source system's Python type (defaults to dict when not specified). - common_schema: The CommonGrants-format Pydantic model class produced by the generator. - If the plugin declares custom_fields, this is a generated subclass of - the base CG model (e.g. OpportunityBase) with those fields already - baked in as typed attributes. - to_common: Transforms source_data → TransformResult[common_schema] (None if not configured). - from_common: Transforms common_data → TransformResult[source_schema] (None if not configured). - """ - - source_schema: type[TSource] - common_schema: type[TCommon] - to_common: Callable[[TSource], TransformResult[TCommon]] | None = None - from_common: Callable[[TCommon], TransformResult[TSource]] | None = None diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py index 7606087bb..907a20edc 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py @@ -1,9 +1,10 @@ """Custom field types for the CommonGrants API.""" from enum import StrEnum -from typing import Any, Optional +from typing import Any, Generic, Optional -from pydantic import Field, HttpUrl +import typing_extensions as te +from pydantic import ConfigDict, Field, HttpUrl from ..base import CommonGrantsBaseModel @@ -20,8 +21,21 @@ class CustomFieldType(StrEnum): ARRAY = "array" -class CustomField(CommonGrantsBaseModel): - """Represents a custom field with type information and validation schema.""" +V = te.TypeVar("V", default=Any) + + +class CustomField(CommonGrantsBaseModel, Generic[V]): + """A custom field with type information and a typed value. + + Generic over its value type ``V`` (default ``Any``): the bare ``CustomField`` + keeps the protocol's untyped-value behavior, while ``CustomField[int]`` (or a + Pydantic model) gives plugin authors and consumers a concrete, inspectable + ``value`` type. ``populate_by_name`` plus ``validation_alias``/ + ``serialization_alias`` keep JSON I/O camelCase (``fieldType``) while + snake_case field-name construction (``CustomField(field_type=...)``) type-checks. + """ + + model_config = ConfigDict(populate_by_name=True) name: str = Field( ..., @@ -30,15 +44,17 @@ class CustomField(CommonGrantsBaseModel): ) field_type: CustomFieldType = Field( ..., - alias="fieldType", + validation_alias="fieldType", + serialization_alias="fieldType", description="The JSON schema type to use when de-serializing the `value` field", ) schema_url: Optional[HttpUrl] = Field( None, - alias="schema", + validation_alias="schema", + serialization_alias="schema", description="Link to the full JSON schema for this custom field", ) - value: Any = Field(..., description="Value of the custom field") + value: V = Field(..., description="Value of the custom field") description: Optional[str] = Field( None, description="Description of the custom field's purpose", diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py index eac3bc183..ed89ef020 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py @@ -1,9 +1,12 @@ """Base model for funding opportunities.""" -from typing import Any, Optional, Type, TypeVar +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar from uuid import UUID -from pydantic import Field, HttpUrl +import typing_extensions as te +from pydantic import ConfigDict, Field, HttpUrl from ..base import CommonGrantsBaseModel from ..fields import CustomField, SystemMetadata @@ -14,13 +17,23 @@ add_custom_fields, get_custom_field_value, ) -from common_grants_sdk.extensions.specs import CustomFieldSpec + +if TYPE_CHECKING: + from common_grants_sdk.extensions.specs import CustomFieldSpec V = TypeVar("V") # Unbound to support both BaseModel subclasses and primitives +# The opportunity's custom-fields container. Defaults to the protocol's untyped +# representation (``dict[str, CustomField]``), so the bare ``OpportunityBase`` +# behaves exactly as a concrete model; plugin authors parameterize it with a typed +# ``CustomFieldSet`` (``OpportunityBase[OpportunityFields]``) for concrete access. +CF = te.TypeVar("CF", default="dict[str, CustomField]") + + +class OpportunityBase(SystemMetadata, CommonGrantsBaseModel, Generic[CF]): + """Base model for a funding opportunity, generic over its custom-fields container.""" -class OpportunityBase(SystemMetadata, CommonGrantsBaseModel): - """Base model for a funding opportunity with all core fields.""" + model_config = ConfigDict(populate_by_name=True) id: UUID = Field(..., description="Globally unique id for the opportunity") title: str = Field(..., description="Title or name of the funding opportunity") @@ -42,7 +55,7 @@ class OpportunityBase(SystemMetadata, CommonGrantsBaseModel): default=None, description="URL for the original source of the opportunity", ) - custom_fields: Optional[dict[str, CustomField]] = Field( + custom_fields: Optional[CF] = Field( default=None, alias="customFields", description="Additional custom fields specific to this opportunity", diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index 05e5ba6c6..c98db17ef 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -1,9 +1,13 @@ -from typing import Optional, Any, Type, TypeVar +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional, Any, Type, TypeVar from pydantic import BaseModel, Field, create_model, ConfigDict, model_validator from ..schemas.pydantic.fields import CustomField, CustomFieldType from ..schemas.pydantic.base import CommonGrantsBaseModel from common_grants_sdk.utils.json import snake -from common_grants_sdk.extensions.specs import CustomFieldSpec + +if TYPE_CHECKING: + from common_grants_sdk.extensions.specs import CustomFieldSpec T = TypeVar("T", bound=BaseModel) # For add_custom_fields V = TypeVar("V") # For get_custom_field_value (unbound to support primitives) diff --git a/lib/python-sdk/common_grants_sdk/utils/transformation.py b/lib/python-sdk/common_grants_sdk/utils/transformation.py index c36fed6eb..7662afcfd 100644 --- a/lib/python-sdk/common_grants_sdk/utils/transformation.py +++ b/lib/python-sdk/common_grants_sdk/utils/transformation.py @@ -261,5 +261,8 @@ def transform_node(node: Any, depth: int) -> Any: # ``` return {k: transform_node(v, depth + 1) for k, v in node.items()} + # An empty dict node maps to an empty dict (not None). + return {} + # Recursively walk the mapping until all nested transformations are applied return transform_node(mapping, depth) diff --git a/lib/python-sdk/examples/README.md b/lib/python-sdk/examples/README.md index 96b7bbc67..7ee6da6ab 100644 --- a/lib/python-sdk/examples/README.md +++ b/lib/python-sdk/examples/README.md @@ -119,142 +119,46 @@ None ``` -# Bidirectional transforms example +# Plugins example -This example demonstrates the plugin transform framework: mapping source system data (grants.gov format) to the CommonGrants format and back again, with a roundtrip consistency check. No API server is required — the script runs entirely offline using sample data defined in the file itself. - -**Step 1:** Generate the typed models for the grants.gov plugin (only needed once, or after changing `cg_config.py`): +`examples/plugins.py` is a single, self-contained file that both **defines** the +example plugins and **validates** them. It demonstrates every authoring scenario +(custom fields + mappings, custom fields + hand-written functions, mappings with no +custom fields, schema-only, and a realistic combined grants.gov plugin) plus +bidirectional round-trips. There is no build step — run it directly: ```bash cd lib/python-sdk -poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov -``` - -Or generate all example plugins at once with: - -```bash -make plugins -``` - -**Step 2:** Run the example: - -```bash -poetry run python examples/transforms.py +poetry run python examples/plugins.py ``` **Output Example:** -``` -============================================================ -SOURCE DATA (grants.gov format) -============================================================ -{ - "data": { - "agency_name": "Department of Examples", - "created_at": "2025-01-15T09:00:00Z", - "last_modified_at": "2025-04-01T12:30:00Z", - "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", - "opportunity_id": 12345, - "opportunity_number": "ABC-123-XYZ-001", - "opportunity_status": "posted", - "opportunity_title": "Research into conservation techniques", - "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "summary": { - "applicant_types": ["state_governments"], - "archive_date": "2025-05-01", - "award_ceiling": 100000, - "award_floor": 10000, - "forecasted_award_date": "2025-09-01", - "forecasted_close_date": "2025-07-15", - "forecasted_post_date": "2025-05-01" - } - } -} - -============================================================ -to_common: grants.gov → CommonGrants -============================================================ -Errors: none - -Result: -{ - "title": "Research into conservation techniques", - "status": { "value": "open", "description": "The opportunity is currently accepting applications" }, - "funding": { - "minAwardAmount": { "amount": 10000, "currency": "USD" }, - "maxAwardAmount": { "amount": 100000, "currency": "USD" } - }, - ... -} - -============================================================ -from_common: CommonGrants → grants.gov -============================================================ -Errors: none - -Result: -{ - "data": { - "opportunity_title": "Research into conservation techniques", - "opportunity_status": "posted", - "summary": { - "award_floor": 10000, - "award_ceiling": 100000, - "forecasted_post_date": "2025-05-01", - "forecasted_close_date": "2025-07-15" - } - } -} - -============================================================ -ROUNDTRIP CHECK -============================================================ - [PASS] title: 'Research into conservation techniques' -> 'Research into conservation techniques' - [PASS] status: 'posted' -> 'posted' - [PASS] award_floor: 10000 -> 10000 - [PASS] award_ceiling: 100000 -> 100000 - -Roundtrip result: ALL PASS -``` - -The transform mappings live in `examples/plugins/grants_gov/cg_config.py`. See the [extensions README](../common_grants_sdk/extensions/README.md#bidirectional-transforms) for a full explanation of the mapping format. - -# Plugin framework example - -This example uses the plugin framework to define four typed custom fields, generate static Pydantic models, and validate an API payload. - -**Step 1:** Generate the typed models (only needed once, or after changing `cg_config.py`): - -```bash -cd lib/python-sdk -poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions -cd ../../.. ``` - -**Step 2:** Run the example: - -```bash -poetry run python examples/plugin_custom_fields.py +Scenario 1 -- custom fields + mappings + [PASS] no transform errors + [PASS] title mapped + [PASS] agency_code.value typed str == 'HHS-123' + [PASS] inspect: agency_code field_type derived STRING + [PASS] round-trips (validated source instance) +Scenario 2 -- custom fields + hand-written functions + [PASS] title mapped + [PASS] agency_code.value == 'HHS-123' + [PASS] from_common -> typed source +Scenario 3 -- mappings, no custom fields + [PASS] title mapped +Scenario 4 -- custom fields only, no transforms + [PASS] schema-only legacy_grant_id.value typed int == 98765 + ... +grants.gov -- custom fields + transform with a custom handler + [PASS] no transform errors + [PASS] title mapped + [PASS] compositeLabel joined via custom handler + [PASS] from_common -> validated source instance ``` -**Output Example:** - -``` -Title: Community Health Innovation Grant -Status: open - -Custom fields: - program_area: CFDA-93.243 - legacy_grant_id: 98765 - eligibility_types:['nonprofit', 'tribal', 'city_government'] - award_ceiling: 250000.0 - -Registered extensions: - program_area: string — HHS program area code (e.g. 'CFDA-93.243') - legacy_grant_id: integer — Numeric ID from the legacy grants management system - eligibility_types: array — Types of organizations eligible to apply (e.g. 'nonprofit', 'tribal') - award_ceiling: number — Maximum award amount in USD -``` +See the [extensions README](../common_grants_sdk/extensions/README.md) for the full +plugin and mapping-format documentation. ## Configuration diff --git a/lib/python-sdk/examples/plugin_custom_fields.py b/lib/python-sdk/examples/plugin_custom_fields.py deleted file mode 100644 index 738dba2de..000000000 --- a/lib/python-sdk/examples/plugin_custom_fields.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Example demonstrating how to use the plugin framework to validate and -access typed custom fields on an Opportunity. - -Before running this script, generate the typed models by running this -command from the plugin directory: - - lib/python-sdk - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/opportunity_extensions - -Then run this script from the lib/python-sdk directory: - - poetry run python examples/plugin_custom_fields.py -""" - -import sys -from pathlib import Path - -# Make the examples/ directory importable so that -# plugins.opportunity_extensions resolves correctly. -sys.path.insert(0, str(Path(__file__).parent)) - -from plugins.opportunity_extensions import opportunity_extensions # noqa: E402 -from plugins.opportunity_extensions.cg_config import config as opp_config # noqa: E402 - -# --------------------------------------------------------------------------- -# Sample API payload containing our four custom fields -# --------------------------------------------------------------------------- - -api_response = { - "id": "573525f2-8e15-4405-83fb-e6523511d893", - "title": "Community Health Innovation Grant", - "status": {"value": "open"}, - "description": "Funding for community-led health initiatives", - "createdAt": "2025-03-01T00:00:00Z", - "lastModifiedAt": "2025-03-15T00:00:00Z", - "customFields": { - "programArea": { - "fieldType": "string", - "value": "CFDA-93.243", - }, - "legacyGrantId": { - "fieldType": "integer", - "value": 98765, - }, - "eligibilityTypes": { - "fieldType": "array", - "value": ["nonprofit", "tribal", "city_government"], - }, - "awardCeiling": { - "fieldType": "number", - "value": 250000.00, - }, - }, -} - -# --------------------------------------------------------------------------- -# Use the model returned via opportunity_extensions -# --------------------------------------------------------------------------- - -opp = opportunity_extensions.schemas.Opportunity.common_schema.model_validate( - api_response -) - -assert opp.custom_fields is not None -assert opp.custom_fields.program_area is not None -assert opp.custom_fields.legacy_grant_id is not None -assert opp.custom_fields.eligibility_types is not None -assert opp.custom_fields.award_ceiling is not None - -print(f"Title: {opp.title}") -print(f"Status: {opp.status.value}") -print(f"program_area: {opp.custom_fields.program_area.value}") -print(f"legacy_grant_id: {opp.custom_fields.legacy_grant_id.value}") -print(f"eligibility_types:{opp.custom_fields.eligibility_types.value}") -print(f"award_ceiling: {opp.custom_fields.award_ceiling.value}") -print() - -# --------------------------------------------------------------------------- -# The plugin also exposes the original extension specs -# --------------------------------------------------------------------------- - -print("Registered custom fields:") -assert opp_config.schemas is not None -_opp_custom_fields = opp_config.schemas["Opportunity"].custom_fields -assert _opp_custom_fields is not None -for field_name, spec in _opp_custom_fields.items(): - print(f" {field_name}: {spec.field_type} — {spec.description}") diff --git a/lib/python-sdk/examples/plugins.py b/lib/python-sdk/examples/plugins.py new file mode 100644 index 000000000..0e57605af --- /dev/null +++ b/lib/python-sdk/examples/plugins.py @@ -0,0 +1,539 @@ +"""Plugin authoring examples -- every scenario in one module. + +Each scenario builds a schema extension with the overloaded ``schema(...)`` factory +(mappings XOR hand-written functions XOR schema-only, enforced statically), then +assembles a ``Plugin`` with ``define_plugin``. ``CustomField[V]`` is the single +source of truth for custom fields: ``field_type`` is derived from ``V``. A plugin is +just the ``Plugin`` ``define_plugin`` returns, so each scenario is a self-contained +plugin in this one file. + +Run the consumer side with ``poetry run python examples/plugins.py``. +""" + +from __future__ import annotations + +from typing import Any, Callable, Optional, cast + +from pydantic import BaseModel, Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PassthroughModel, + PluginMeta, + PluginSchemas, + TransformResult, + build_transforms, + define_plugin, + schema, + validate_into, +) +from common_grants_sdk.schemas.pydantic.models import OpportunityBase +from common_grants_sdk.utils.transformation import get_from_path + +# --- Author-declared custom-field containers ---------------------------------- + + +class AgencyFields(CustomFieldSet): + """The single custom field shared by the mappings and functions scenarios.""" + + agency_code: Optional[CustomField[str]] = Field( + default=None, description="Agency code carried from grants.gov." + ) + + +class ExtensionFields(CustomFieldSet): + """HHS- and locality-specific custom fields (schema-only scenario).""" + + program_area: Optional[CustomField[str]] = Field( + default=None, description="HHS program area code (e.g. 'CFDA-93.243')" + ) + legacy_grant_id: Optional[CustomField[int]] = Field( + default=None, description="Numeric ID from the legacy grants management system" + ) + eligibility_types: Optional[CustomField[list[str]]] = Field( + default=None, description="Types of organizations eligible to apply" + ) + award_ceiling: Optional[CustomField[float]] = Field( + default=None, description="Maximum award amount in USD" + ) + + +class GrantsGovFields(CustomFieldSet): + """Custom fields grants.gov carries on the Opportunity schema.""" + + legacy_id: Optional[CustomField[int]] = Field( + default=None, description="Unique identifier in legacy database" + ) + legacy_id_str: Optional[CustomField[str]] = Field( + default=None, description="Legacy ID coerced to a string via numberToString" + ) + agency_name: Optional[CustomField[str]] = Field( + default=None, description="Agency hosting the opportunity" + ) + applicant_types: Optional[CustomField[list[str]]] = Field( + default=None, description="Types of applicants eligible to apply" + ) + priority_score: Optional[CustomField[float]] = Field( + default=None, description="Numeric priority score coerced from a string" + ) + composite_label: Optional[CustomField[str]] = Field( + default=None, + description="Composite '<opportunity_number> -- <opportunity_title>' label", + ) + + +# --- Scenario 1: custom fields + declarative mappings ------------------------- + +mappings_plugin = define_plugin( + PluginSchemas( + Opportunity=schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase[AgencyFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "description": {"field": "opportunity_description"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } + } + }, + "customFields": { + "agencyCode": { + "value": {"field": "agency_code"}, + "name": {"const": "agencyCode"}, + "fieldType": {"const": "string"}, + } + }, + }, + "from_common": { + "opportunity_uuid": {"field": "id"}, + "opportunity_title": {"field": "title"}, + "opportunity_status": {"const": "posted"}, + "agency_code": {"field": "customFields.agencyCode.value"}, + }, + }, + ) + ), + meta=PluginMeta(name="grants.gov (mappings)", source_system="grants.gov"), +) + + +# --- Scenario 2: custom fields + hand-written functions ----------------------- + + +class GrantsGovOpportunity(BaseModel): + """The grants.gov source shape (validated on the from_common output).""" + + opportunity_uuid: str + opportunity_title: str + opportunity_description: str + opportunity_status: str + created_at: str + last_modified_at: str + agency_code: str + + +def gadget_to_common( + source: GrantsGovOpportunity, +) -> TransformResult[OpportunityBase[AgencyFields]]: + return validate_into( + OpportunityBase[AgencyFields], + { + "id": source.opportunity_uuid, + "title": source.opportunity_title, + "description": source.opportunity_description, + "status": { + "value": "open" if source.opportunity_status == "posted" else "custom" + }, + "createdAt": source.created_at, + "lastModifiedAt": source.last_modified_at, + "customFields": { + "agencyCode": { + "name": "agencyCode", + "fieldType": "string", + "value": source.agency_code, + } + }, + }, + ) + + +def gadget_from_common( + common: OpportunityBase[AgencyFields], +) -> TransformResult[GrantsGovOpportunity]: + # The make-or-break path: `common` is fully typed (agency_code.value -> str). + agency = "" + if common.custom_fields and common.custom_fields.agency_code: + agency = common.custom_fields.agency_code.value + return validate_into( + GrantsGovOpportunity, + { + "opportunity_uuid": str(common.id), + "opportunity_title": common.title, + "opportunity_description": common.description, + "opportunity_status": "posted", + "created_at": common.created_at.isoformat(), + "last_modified_at": common.last_modified_at.isoformat(), + "agency_code": agency, + }, + ) + + +functions_plugin = define_plugin( + PluginSchemas( + Opportunity=schema( + source_schema=GrantsGovOpportunity, + common_schema=OpportunityBase[AgencyFields], + to_common=gadget_to_common, + from_common=gadget_from_common, + ) + ), + meta=PluginMeta(name="grants.gov (functions)", source_system="grants.gov"), +) + + +# --- Scenario 3: declarative mappings, no custom fields ----------------------- + +base_plugin = define_plugin( + PluginSchemas( + Opportunity=schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase, + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "description": {"field": "opportunity_description"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open", "archived": "closed"}, + "default": "custom", + } + } + }, + }, + "from_common": { + "opportunity_uuid": {"field": "id"}, + "opportunity_title": {"field": "title"}, + "opportunity_status": {"const": "posted"}, + }, + }, + ) + ), + meta=PluginMeta(name="grants.gov (no custom fields)", source_system="grants.gov"), +) + + +# --- Scenario 4: custom fields only, no transforms ---------------------------- + +opportunity_extensions = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[ExtensionFields])), + meta=PluginMeta(name="opportunity extensions", source_system="hhs"), +) + + +# --- Realistic combined plugin: grants.gov ------------------------------------ + + +def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: + sep = spec.get("sep", " ") + parts = [get_from_path(data, path) for path in spec.get("fields", [])] + values = [str(p) for p in parts if p is not None] + return sep.join(values) if values else None + + +# Both directions are author-provided -- build_transforms() does not invert one into +# the other because many-to-one handlers like ``match`` are not reversible. +_gg_to_common, _gg_from_common = build_transforms( + handlers={"join": _join_fields}, + common_schema=OpportunityBase[GrantsGovFields], + source_schema=PassthroughModel, + to_common_mapping={ + "id": {"field": "data.opportunity_uuid"}, + "title": {"field": "data.opportunity_title"}, + "description": {"field": "data.opportunity_description"}, + "createdAt": {"field": "data.created_at"}, + "lastModifiedAt": {"field": "data.last_modified_at"}, + "status": { + "value": { + "match": { + "field": "data.opportunity_status", + "case": { + "forecasted": "forecasted", + "posted": "open", + "archived": "closed", + }, + "default": "custom", + } + }, + "description": { + "const": "The opportunity is currently accepting applications" + }, + }, + "funding": { + "minAwardAmount": { + "amount": {"numberToString": "data.summary.award_floor"}, + "currency": {"const": "USD"}, + }, + "maxAwardAmount": { + "amount": {"numberToString": "data.summary.award_ceiling"}, + "currency": {"const": "USD"}, + }, + }, + "keyDates": { + "appOpens": { + "name": {"const": "Open Date"}, + "date": {"field": "data.summary.forecasted_post_date"}, + "description": {"const": "Applications begin being accepted"}, + }, + "appDeadline": { + "name": {"const": "Application Deadline"}, + "date": {"field": "data.summary.forecasted_close_date"}, + "description": { + "const": "Final submission deadline for all grant applications" + }, + }, + }, + "customFields": { + "legacyIdStr": { + "value": {"numberToString": "data.opportunity_id"}, + "name": {"const": "legacyIdStr"}, + "fieldType": {"const": "string"}, + }, + "priorityScore": { + "value": {"stringToNumber": "data.priority_score_str"}, + "name": {"const": "priorityScore"}, + "fieldType": {"const": "number"}, + }, + "compositeLabel": { + "value": { + "join": { + "fields": ["data.opportunity_number", "data.opportunity_title"], + "sep": " -- ", + } + }, + "name": {"const": "compositeLabel"}, + "fieldType": {"const": "string"}, + }, + }, + }, + from_common_mapping={ + "data": { + "opportunity_title": {"field": "title"}, + "opportunity_status": { + "match": { + "field": "status.value", + "case": { + "open": "posted", + "closed": "archived", + "forecasted": "forecasted", + }, + "default": "custom", + } + }, + "summary": { + "award_floor": {"stringToNumber": "funding.minAwardAmount.amount"}, + "award_ceiling": {"stringToNumber": "funding.maxAwardAmount.amount"}, + "forecasted_post_date": {"field": "keyDates.appOpens.date"}, + "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, + }, + "priority_score_str": { + "numberToString": "customFields.priorityScore.value" + }, + } + }, +) + +# build_transforms returns loosely-typed callables (its output may be a dict on a +# validation error). Restate the precise signatures for the typed consumer surface. +gg_to_common = cast( + Callable[[PassthroughModel], TransformResult[OpportunityBase[GrantsGovFields]]], + _gg_to_common, +) +gg_from_common = cast( + Callable[[OpportunityBase[GrantsGovFields]], TransformResult[PassthroughModel]], + _gg_from_common, +) + +grants_gov = define_plugin( + PluginSchemas( + Opportunity=schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase[GrantsGovFields], + to_common=gg_to_common, + from_common=gg_from_common, + ) + ), + meta=PluginMeta( + name="grants-gov", + version="0.1.0", + source_system="grants.gov", + capabilities=["customFields", "transforms"], + ), +) + + +# --- Consumer side: validate each plugin in this same file -------------------- +# +# Consumers use non-optional dot access on ``plugin.schemas.Opportunity``. The +# ``assert_type`` lines document the concrete static types each plugin yields. +# Run with: poetry run python examples/plugins.py + +from typing import assert_type # noqa: E402 + +from common_grants_sdk.extensions import SchemaOnly # noqa: E402 + +_FLAT_SOURCE = { + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "opportunity_title": "Conservation research", + "opportunity_description": "Funding for conservation.", + "opportunity_status": "posted", + "created_at": "2025-01-01T00:00:00Z", + "last_modified_at": "2025-01-01T00:00:00Z", + "agency_code": "HHS-123", +} + +_GRANTS_GOV_SOURCE = { + "data": { + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "opportunity_id": 12345, + "opportunity_number": "ABC-123-XYZ-001", + "opportunity_title": "Research into conservation techniques", + "opportunity_description": "Funding to advance conservation research.", + "opportunity_status": "posted", + "created_at": "2025-01-15T09:00:00Z", + "last_modified_at": "2025-04-01T12:30:00Z", + "priority_score_str": "75", + "summary": { + "award_floor": 10000, + "award_ceiling": 100000, + "forecasted_post_date": "2025-05-01", + "forecasted_close_date": "2025-07-15", + }, + } +} + +_SCHEMA_ONLY_RECORD = { + "id": "573525f2-8e15-4405-83fb-e6523511d893", + "title": "Community Health Innovation Grant", + "status": {"value": "open"}, + "description": "Funding for community-led health initiatives", + "createdAt": "2025-03-01T00:00:00Z", + "lastModifiedAt": "2025-03-15T00:00:00Z", + "customFields": { + "legacyGrantId": { + "name": "legacyGrantId", + "fieldType": "integer", + "value": 98765, + } + }, +} + + +def _check(label: str, ok: bool) -> None: + print(f" [{'PASS' if ok else 'FAIL'}] {label}") + + +def main() -> None: + # Scenario 1 -- custom fields + declarative mappings + print("Scenario 1 -- custom fields + mappings") + res1 = mappings_plugin.schemas.Opportunity.to_common( + PassthroughModel.model_validate(_FLAT_SOURCE) + ) + assert_type(res1, TransformResult[OpportunityBase[AgencyFields]]) + opp1 = res1.result + _check("no transform errors", res1.errors == []) + _check("title mapped", opp1.title == "Conservation research") + cf1 = opp1.custom_fields + if cf1 and cf1.agency_code: + assert_type(cf1.agency_code.value, str) + _check( + "agency_code.value typed str == 'HHS-123'", + cf1.agency_code.value == "HHS-123", + ) + specs = mappings_plugin.schemas.Opportunity.custom_fields + _check( + "inspect: agency_code field_type derived STRING", + specs["agency_code"].field_type is not None + and specs["agency_code"].field_type.value == "string", + ) + back1 = mappings_plugin.schemas.Opportunity.from_common(opp1) + _check("round-trips (validated source instance)", not back1.errors) + + # Scenario 2 -- custom fields + hand-written functions + print("Scenario 2 -- custom fields + hand-written functions") + res2 = functions_plugin.schemas.Opportunity.to_common( + GrantsGovOpportunity(**_FLAT_SOURCE) + ) + g = res2.result + _check("title mapped", g.title == "Conservation research") + if g.custom_fields and g.custom_fields.agency_code: + assert_type(g.custom_fields.agency_code.value, str) + _check( + "agency_code.value == 'HHS-123'", + g.custom_fields.agency_code.value == "HHS-123", + ) + back2 = functions_plugin.schemas.Opportunity.from_common(g) + assert_type(back2, TransformResult[GrantsGovOpportunity]) + _check("from_common -> typed source", back2.result.agency_code == "HHS-123") + + # Scenario 3 -- declarative mappings, no custom fields + print("Scenario 3 -- mappings, no custom fields") + res3 = base_plugin.schemas.Opportunity.to_common( + PassthroughModel.model_validate(_FLAT_SOURCE) + ) + _check("title mapped", res3.result.title == "Conservation research") + + # Scenario 4 -- custom fields only, no transforms (schema-only) + print("Scenario 4 -- custom fields only, no transforms") + ext = opportunity_extensions.schemas.Opportunity + assert_type(ext, SchemaOnly[OpportunityBase[ExtensionFields]]) + parsed = ext.parse(_SCHEMA_ONLY_RECORD) + if parsed.custom_fields and parsed.custom_fields.legacy_grant_id: + assert_type(parsed.custom_fields.legacy_grant_id.value, int) + _check( + "schema-only legacy_grant_id.value typed int == 98765", + parsed.custom_fields.legacy_grant_id.value == 98765, + ) + print(" inspect: resolved specs") + for name, spec in ext.custom_fields.items(): + print(f" {name}: {spec.field_type} -- {spec.description}") + + # Realistic combined plugin -- grants.gov (custom handler + round-trip) + print("grants.gov -- custom fields + transform with a custom handler") + res5 = grants_gov.schemas.Opportunity.to_common( + PassthroughModel.model_validate(_GRANTS_GOV_SOURCE) + ) + _check("no transform errors", not res5.errors) + gg = res5.result + _check("title mapped", gg.title == "Research into conservation techniques") + if gg.custom_fields and gg.custom_fields.composite_label: + _check( + "compositeLabel joined via custom handler", + gg.custom_fields.composite_label.value + == "ABC-123-XYZ-001 -- Research into conservation techniques", + ) + back5 = grants_gov.schemas.Opportunity.from_common(gg) + assert_type(back5, TransformResult[PassthroughModel]) + native = back5.result.model_dump(by_alias=True) + _check( + "from_common -> validated source instance", + native["data"]["opportunity_title"] == "Research into conservation techniques", + ) + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/examples/plugins/__init__.py b/lib/python-sdk/examples/plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/python-sdk/examples/plugins/grants_gov/__init__.py b/lib/python-sdk/examples/plugins/grants_gov/__init__.py deleted file mode 100644 index 4514e8f0e..000000000 --- a/lib/python-sdk/examples/plugins/grants_gov/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# This file is auto-generated. Do not edit it manually — it will be overwritten -# the next time `python -m common_grants_sdk.extensions.generate` is run. -from __future__ import annotations - -from common_grants_sdk.extensions import Plugin, inject_transforms -from .cg_config import config -from .generated import schemas - -schemas = inject_transforms(config, schemas) - -grants_gov = Plugin( - schemas=schemas, - meta=config.meta, -) - -__all__ = ["grants_gov", "schemas"] diff --git a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py b/lib/python-sdk/examples/plugins/grants_gov/cg_config.py deleted file mode 100644 index c155cfc00..000000000 --- a/lib/python-sdk/examples/plugins/grants_gov/cg_config.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Grants.gov sample plugin — bidirectional transform PoC. - -Demonstrates the plugin framework shape using the grants.gov scenario. - -Usage (from lib/python-sdk/): - poetry run python examples/transforms.py - -Code generation (generates typed custom-field schemas): - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov -""" - -from typing import Any - -from common_grants_sdk.extensions import ( - CustomFieldSpec, - PluginExtensionsMeta, - SchemaInput, - build_transforms, - define_plugin, -) -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -from common_grants_sdk.utils.transformation import get_from_path - - -def _join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: - sep = spec.get("sep", " ") - parts = [get_from_path(data, path) for path in spec.get("fields", [])] - values = [str(p) for p in parts if p is not None] - return sep.join(values) if values else None - - -# --------------------------------------------------------------------------- -# Bidirectional transforms -# -# Both directions are author-provided — build_transforms() does not invert -# one into the other because many-to-one handlers like switch are not -# reversible. -# -# Convention: field extraction uses {"field": "dot.notation.path"} — bare -# string values are treated as literals by transform_from_mapping(), not -# as field paths. See Design Finding #2 in the spec for the open question -# about which convention is canonical. -# --------------------------------------------------------------------------- - -to_common, from_common = build_transforms( - handlers={"join": _join_fields}, - # to_common: grants.gov native → CommonGrants Opportunity - to_common_mapping={ - "title": {"field": "data.opportunity_title"}, - "status": { - "value": { - "match": { - "field": "data.opportunity_status", - "case": { - "forecasted": "forecasted", - "posted": "open", - "archived": "closed", - }, - "default": "custom", - } - }, - "description": { - "const": "The opportunity is currently accepting applications" - }, - }, - "funding": { - "minAwardAmount": { - "amount": {"field": "data.summary.award_floor"}, - "currency": {"const": "USD"}, - }, - "maxAwardAmount": { - "amount": {"field": "data.summary.award_ceiling"}, - "currency": {"const": "USD"}, - }, - }, - "keyDates": { - "appOpens": { - "name": {"const": "Open Date"}, - "date": {"field": "data.summary.forecasted_post_date"}, - "description": {"const": "Applications begin being accepted"}, - }, - "appDeadline": { - "name": {"const": "Application Deadline"}, - "date": {"field": "data.summary.forecasted_close_date"}, - "description": { - "const": "Final submission deadline for all grant applications" - }, - }, - }, - "customFields": { - "legacyIdStr": { - "value": {"numberToString": "data.opportunity_id"}, - }, - "priorityScore": { - "value": {"stringToNumber": "data.priority_score_str"}, - }, - "compositeLabel": { - "value": { - "join": { - "fields": ["data.opportunity_number", "data.opportunity_title"], - "sep": " — ", - } - }, - }, - }, - }, - # from_common: CommonGrants Opportunity → grants.gov native - from_common_mapping={ - "data": { - "opportunity_title": {"field": "title"}, - "opportunity_status": { - "match": { - "field": "status.value", - "case": { - "open": "posted", - "closed": "archived", - "forecasted": "forecasted", - }, - "default": "custom", - } - }, - "summary": { - "award_floor": {"field": "funding.minAwardAmount.amount"}, - "award_ceiling": {"field": "funding.maxAwardAmount.amount"}, - "forecasted_post_date": {"field": "keyDates.appOpens.date"}, - "forecasted_close_date": {"field": "keyDates.appDeadline.date"}, - }, - "priority_score_str": { - "numberToString": "customFields.priorityScore.value" - }, - } - }, -) - -# --------------------------------------------------------------------------- -# Plugin config -# --------------------------------------------------------------------------- - -config = define_plugin( - meta=PluginExtensionsMeta( - name="grants-gov", - version="0.1.0", - sourceSystem="grants.gov", - capabilities=["customFields", "transforms"], - ), - schemas={ - "Opportunity": SchemaInput( - custom_fields={ - "legacyId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - name="Legacy ID", - description="Unique identifier in legacy database", - ), - "legacyIdStr": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Legacy ID (string)", - description="Legacy ID coerced to a string via numberToString", - ), - "agencyName": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Agency", - description="Agency hosting the opportunity", - ), - "applicantTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - name="Applicant types", - description="Types of applicants eligible to apply", - ), - "priorityScore": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - name="Priority score", - description="Numeric priority score coerced from a string via stringToNumber", - ), - "compositeLabel": CustomFieldSpec( - field_type=CustomFieldType.STRING, - name="Composite label", - description="Composite label '<opportunity_number> — <opportunity_title>' for round-trip recovery", - ), - }, - to_common=to_common, - from_common=from_common, - ) - }, -) diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py b/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py deleted file mode 100644 index c8f51a3ac..000000000 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# This file is auto-generated. Do not edit it manually — it will be overwritten -# the next time `python -m common_grants_sdk.extensions.generate` is run. -from __future__ import annotations - -from common_grants_sdk.extensions import Plugin -from .cg_config import config -from .generated import schemas - -opportunity_extensions = Plugin( - schemas=schemas, - meta=config.meta, -) - -__all__ = ["opportunity_extensions", "schemas"] diff --git a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py b/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py deleted file mode 100644 index 97a9d3765..000000000 --- a/lib/python-sdk/examples/plugins/opportunity_extensions/cg_config.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Plugin configuration for opportunity extensions. - -Defines custom field extensions for the Opportunity schema: -- HHS-specific fields (programArea, legacyGrantId) -- Local fields (eligibilityTypes, awardCeiling) -""" - -from typing import Any - -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec, SchemaInput -from common_grants_sdk.extensions.plugin import PluginConfig -from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType - -config: PluginConfig[Any] = define_plugin( - schemas={ - "Opportunity": SchemaInput( - custom_fields={ - "programArea": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="HHS program area code (e.g. 'CFDA-93.243')", - ), - "legacyGrantId": CustomFieldSpec( - field_type=CustomFieldType.INTEGER, - description="Numeric ID from the legacy grants management system", - ), - "eligibilityTypes": CustomFieldSpec( - field_type=CustomFieldType.ARRAY, - description="Types of organizations eligible to apply", - ), - "awardCeiling": CustomFieldSpec( - field_type=CustomFieldType.NUMBER, - description="Maximum award amount in USD", - ), - } - ) - } -) diff --git a/lib/python-sdk/examples/transforms.py b/lib/python-sdk/examples/transforms.py deleted file mode 100644 index 637aa1ffb..000000000 --- a/lib/python-sdk/examples/transforms.py +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python3 -"""Bidirectional transform PoC — plugin transformation interface. - -Demonstrates source (grants.gov) → CommonGrants and CommonGrants → source -bidirectional transformations using the grants.gov sample plugin. - -Requires generated schemas (examples/plugins/grants_gov/generated/). -Generate them first (from lib/python-sdk/): - poetry run python -m common_grants_sdk.extensions.generate --plugin examples/plugins/grants_gov -Or run all plugins at once: - make plugins - -Then run (from lib/python-sdk/): - poetry run python examples/transforms.py -""" - -from __future__ import annotations - -import json -from typing import Any - -# When run as `poetry run python examples/transforms.py`, Python automatically -# adds the script's directory (examples/) to sys.path. Import from there using -# the `plugins.` prefix (not `examples.plugins.`) — the `examples.` prefix only -# works in -c or interactive contexts where lib/python-sdk/ is sys.path[0]. -from plugins.grants_gov import grants_gov as plugin -from plugins.grants_gov.generated.schemas import Opportunity - -from common_grants_sdk.extensions import build_transforms -from common_grants_sdk.utils.transformation import get_from_path - -# --------------------------------------------------------------------------- -# Sample grants.gov source data -# --------------------------------------------------------------------------- - -SOURCE_DATA: dict[str, Any] = { - "data": { - "agency_name": "Department of Examples", - "created_at": "2025-01-15T09:00:00Z", - "last_modified_at": "2025-04-01T12:30:00Z", - "opportunity_description": "Funding to advance research into conservation techniques for endangered ecosystems.", - "opportunity_id": 12345, - "opportunity_number": "ABC-123-XYZ-001", - "opportunity_status": "posted", - "opportunity_title": "Research into conservation techniques", - "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "priority_score_str": "75", - "summary": { - "applicant_types": ["state_governments"], - "archive_date": "2025-05-01", - "award_ceiling": 100000, - "award_floor": 10000, - "forecasted_award_date": "2025-09-01", - "forecasted_close_date": "2025-07-15", - "forecasted_post_date": "2025-05-01", - }, - } -} - - -# --------------------------------------------------------------------------- -# Custom handlers: join_fields and split_field -# -# join_fields concatenates multiple source field values with a configurable -# separator. Mapping spec: {"join": {"fields": ["a.b", "c.d"], "sep": " — "}} -# -# split_field is the inverse: it splits a single field on a separator and -# returns the element at the given index. -# Mapping spec: {"split": {"field": "label", "sep": " — ", "index": 0}} -# --------------------------------------------------------------------------- - - -def join_fields(data: dict[str, Any], spec: dict[str, Any]) -> str | None: - """Custom handler that joins multiple field values with a separator.""" - sep = spec.get("sep", " ") - parts = [get_from_path(data, path) for path in spec.get("fields", [])] - values = [str(p) for p in parts if p is not None] - return sep.join(values) if values else None - - -def split_field(data: dict[str, Any], spec: dict[str, Any]) -> str | None: - """Custom handler that splits a field value and returns the element at index.""" - value = get_from_path(data, spec.get("field", "")) - if value is None: - return None - parts = str(value).split(spec.get("sep", " ")) - index = spec.get("index", 0) - return parts[index] if index < len(parts) else None - - -# Transform that uses the custom handlers and validates output against the generated -# Opportunity model. common_schema=Opportunity (from generated/schemas.py) ensures -# model_validate runs against the extended class with typed custom fields -# (legacyId, agencyName, applicantTypes), not just the base OpportunityBase. -to_common_with_custom, from_common_with_custom = build_transforms( - to_common_mapping={ - "id": {"field": "data.opportunity_uuid"}, - "title": {"field": "data.opportunity_title"}, - "description": {"field": "data.opportunity_description"}, - "createdAt": {"field": "data.created_at"}, - "lastModifiedAt": {"field": "data.last_modified_at"}, - "status": { - "value": { - "match": { - "field": "data.opportunity_status", - "case": { - "posted": "open", - "archived": "closed", - "forecasted": "forecasted", - }, - "default": "custom", - } - }, - }, - "customFields": { - "compositeLabel": { - "value": { - "join": { - "fields": ["data.opportunity_number", "data.opportunity_title"], - "sep": " — ", - } - }, - }, - "legacyId": { - "value": {"field": "data.opportunity_id"}, - }, - "agencyName": { - "value": {"field": "data.agency_name"}, - }, - "applicantTypes": { - "value": {"field": "data.summary.applicant_types"}, - }, - }, - }, - from_common_mapping={ - "data": { - # label is produced by the join handler above but gets dropped by - # model_validate (it is not a CG field), so from_common maps directly - # from the standard CG title field instead. - "opportunity_title": {"field": "title"}, - } - }, - handlers={"join": join_fields, "split": split_field}, - common_schema=Opportunity, -) - - -def _section(title: str) -> None: - print(f"\n{'=' * 60}") - print(title) - print("=" * 60) - - -def main() -> None: - opp = plugin.schemas.Opportunity - - _section("SOURCE DATA (grants.gov format)") - print(json.dumps(SOURCE_DATA, indent=2)) - - # --- to_common: grants.gov → CommonGrants --- - assert opp.to_common is not None - cg_result = opp.to_common(SOURCE_DATA) - - _section("to_common: grants.gov → CommonGrants") - if cg_result.errors: - print(f"ERRORS ({len(cg_result.errors)}):") - for err in cg_result.errors: - print(f" [path={err.path}] {err}") - else: - print("Errors: none") - print("\nResult:") - print(json.dumps(cg_result.result, indent=2)) - - # --- from_common: CommonGrants → grants.gov --- - assert opp.from_common is not None - native_result = opp.from_common(cg_result.result) - - _section("from_common: CommonGrants → grants.gov") - if native_result.errors: - print(f"ERRORS ({len(native_result.errors)}):") - for err in native_result.errors: - print(f" [path={err.path}] {err}") - else: - print("Errors: none") - print("\nResult:") - print(json.dumps(native_result.result, indent=2)) - - # --- Roundtrip comparison --- - # Note: SOURCE_DATA contains fields not covered by the mappings (agency_name, - # opportunity_id, etc.). Those fields are intentionally absent from the roundtrip - # output — the mapping layer is selective by design. - _section("ROUNDTRIP CHECK") - checks = [ - ( - "title", - SOURCE_DATA["data"]["opportunity_title"], - native_result.result.get("data", {}).get("opportunity_title"), - ), - ( - "status", - SOURCE_DATA["data"]["opportunity_status"], - native_result.result.get("data", {}).get("opportunity_status"), - ), - ( - "award_floor", - SOURCE_DATA["data"]["summary"]["award_floor"], - native_result.result.get("data", {}).get("summary", {}).get("award_floor"), - ), - ( - "award_ceiling", - SOURCE_DATA["data"]["summary"]["award_ceiling"], - native_result.result.get("data", {}) - .get("summary", {}) - .get("award_ceiling"), - ), - ( - "priority_score_str", - SOURCE_DATA["data"]["priority_score_str"], - native_result.result.get("data", {}).get("priority_score_str"), - ), - ] - all_pass = True - for field, original, roundtripped in checks: - ok = original == roundtripped - if not ok: - all_pass = False - status = "PASS" if ok else "FAIL" - print(f" [{status}] {field}: {original!r} -> {roundtripped!r}") - - print( - f"\nRoundtrip result ({len(checks)} mapped fields checked; unmapped fields dropped by design): {'ALL PASS' if all_pass else 'SOME FIELDS DIFFER'}" - ) - - # --- Custom handler + model_validate demo --- - _section( - "CUSTOM HANDLER + MODEL VALIDATE DEMO (join / split / extended Opportunity)" - ) - print("Custom handlers: join, split") - print("common_model: generated Opportunity (with typed customFields)\n") - - custom_cg = to_common_with_custom(SOURCE_DATA) - - if custom_cg.errors: - print(f"ERRORS ({len(custom_cg.errors)}):") - for err in custom_cg.errors: - print(f" [path={err.path}] {err}") - else: - print("Validation: PASS — result is a typed Opportunity instance") - assert isinstance(custom_cg.result, Opportunity) - opp_instance = custom_cg.result - print(f"\n title: {opp_instance.title}") - print(f" id: {opp_instance.id}") - print(f" status: {opp_instance.status.value}") - if opp_instance.custom_fields: - cf = opp_instance.custom_fields - print("\n customFields (typed):") - if cf.legacy_id: - print( - f" legacyId.value: {cf.legacy_id.value!r} ({type(cf.legacy_id.value).__name__})" - ) - if cf.agency_name: - print( - f" agencyName.value: {cf.agency_name.value!r} ({type(cf.agency_name.value).__name__})" - ) - if cf.applicant_types: - print( - f" applicantTypes.value: {cf.applicant_types.value!r} ({type(cf.applicant_types.value).__name__})" - ) - if cf.composite_label: - print( - f" compositeLabel.value: {cf.composite_label.value!r} ({type(cf.composite_label.value).__name__})" - ) - - custom_native = from_common_with_custom( - custom_cg.result if not custom_cg.errors else {} - ) - orig_title = SOURCE_DATA["data"]["opportunity_title"] - rt_title = custom_native.result.get("data", {}).get("opportunity_title") - print( - f"\n [{'PASS' if orig_title == rt_title else 'FAIL'}] opportunity_title: {orig_title!r} -> {rt_title!r}" - ) - - # --- Plugin metadata --- - _section("PLUGIN METADATA") - assert plugin.meta is not None - print(f"name: {plugin.meta.name}") - print(f"version: {plugin.meta.version}") - print(f"sourceSystem: {plugin.meta.source_system}") - print(f"capabilities: {plugin.meta.capabilities}") - - -if __name__ == "__main__": - main() diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index 072ee1306..f73e5b6af 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -1,154 +1,101 @@ -"""Tests for plugin.py — Plugin and PluginConfig API.""" +"""Tests for plugin.py -- PluginSchemas / Plugin / define_plugin assembly.""" -import pytest +from typing import Optional -from common_grants_sdk.extensions.plugin import Plugin, PluginConfig, define_plugin -from common_grants_sdk.extensions.types import ( - SchemaInput, - PluginExtensionsMeta, - TransformResult, +import pytest +from pydantic import Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PassthroughModel, + Plugin, + PluginMeta, + PluginSchemas, + SchemaOnly, + SchemaWithTransforms, + define_plugin, + schema, ) +from common_grants_sdk.extensions.schema import PluginDefinitionError +from common_grants_sdk.schemas.pydantic.models import OpportunityBase -def test_define_plugin_no_args(): - """define_plugin() with no args returns PluginConfig with all fields None.""" - config = define_plugin() - assert isinstance(config, PluginConfig) - assert config.meta is None - assert config.schemas is None - - -def test_define_plugin_with_meta_and_schemas(): - meta = PluginExtensionsMeta(name="test", source_system="test-system") - - def passthrough(x): - return TransformResult(result=x, errors=[]) - - schemas = { - "Opportunity": SchemaInput(to_common=passthrough, from_common=passthrough) - } - config = define_plugin(meta=meta, schemas=schemas) - assert config.meta is meta - assert config.meta.name == "test" - assert config.schemas is schemas - - -def test_define_plugin_schemas_callable_roundtrip(): - """config.schemas["Opportunity"].to_common(data) works.""" - - def always_transformed(_x): - return TransformResult(result={"transformed": True}, errors=[]) - - config = define_plugin( - schemas={ - "Opportunity": SchemaInput( - to_common=always_transformed, from_common=always_transformed - ) - }, +class OpportunityFields(CustomFieldSet): + agency_code: Optional[CustomField[str]] = Field( + default=None, description="Agency code" ) - result = config.schemas["Opportunity"].to_common({"raw": "data"}) - assert result.result == {"transformed": True} - assert result.errors == [] - - -def test_plugin_fields_default_to_none(): - """Plugin.schemas holds the container; meta defaults to None.""" - base = Plugin(schemas=object()) - assert base.meta is None -def test_plugin_fields_populated(): - meta = PluginExtensionsMeta(name="p", source_system="s") - full = Plugin(schemas=object(), meta=meta) - assert full.meta is meta - - -def test_plugin_schemas_is_attribute_container(): - """Plugin.schemas holds the _Schemas object (no generated_schemas field).""" - s = object() - p = Plugin(schemas=s) - assert p.schemas is s - assert not hasattr(p, "generated_schemas") +def _meta() -> PluginMeta: + return PluginMeta(name="test", source_system="test-system") # --------------------------------------------------------------------------- -# XOR constraint tests +# define_plugin assembly # --------------------------------------------------------------------------- -def test_define_plugin_xor_mappings_and_both_callables_raises(): - """Providing mappings AND both explicit callables raises ValueError.""" - from common_grants_sdk.extensions.types import SchemaMappings +def test_define_plugin_returns_plugin_with_schemas_and_meta(): + ext = schema(common_schema=OpportunityBase[OpportunityFields]) + plugin = define_plugin(PluginSchemas(Opportunity=ext), meta=_meta()) + assert isinstance(plugin, Plugin) + assert plugin.schemas.Opportunity is ext + assert plugin.meta.name == "test" + assert plugin.meta.source_system == "test-system" + + +def test_omitted_schema_falls_back_to_base_schema_only_extension(): + """An unextended PluginSchemas slot is a SchemaOnly over the base, never None.""" + plugin = define_plugin(PluginSchemas(), meta=_meta()) + entry = plugin.schemas.Opportunity + assert isinstance(entry, SchemaOnly) + assert entry.schema_name == "Opportunity" + assert entry.custom_fields == {} + # The base schema has no custom fields declared. + assert entry.common_schema is OpportunityBase + + +def test_mappings_entry_is_a_transform_extension(): + ext = schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase, + mappings={ + "to_common": {"title": {"field": "opportunity_title"}}, + "from_common": {"opportunity_title": {"field": "title"}}, + }, + ) + plugin = define_plugin(PluginSchemas(Opportunity=ext), meta=_meta()) + assert isinstance(plugin.schemas.Opportunity, SchemaWithTransforms) - def noop(x): - return TransformResult(result=x, errors=[]) - with pytest.raises( - ValueError, - match="cannot specify both mappings and explicit to_common/from_common", - ): - define_plugin( - schemas={ - "Opportunity": SchemaInput( - mappings=SchemaMappings( - to_common={"title": {"field": "native_title"}}, - from_common={"native_title": {"field": "title"}}, - ), - to_common=noop, - from_common=noop, - ) - } - ) - - -def test_define_plugin_xor_mappings_and_one_callable_raises(): - """Providing mappings AND a single explicit callable raises ValueError.""" - from common_grants_sdk.extensions.types import SchemaMappings - - def noop(x): - return TransformResult(result=x, errors=[]) +def test_define_plugin_rejects_non_extension_in_slot(): + bad = PluginSchemas(Opportunity="not an extension") # type: ignore[arg-type] + with pytest.raises(PluginDefinitionError, match="not a schema extension"): + define_plugin(bad, meta=_meta()) + +def test_define_plugin_rejects_schema_name_mismatch(): + """A slot holding an extension whose schema_name differs from the attribute name raises.""" + # Hand-build an extension tagged with a different schema name than its slot. + mismatched = SchemaOnly( + schema_name="Program", + common_schema=OpportunityBase, + custom_fields={}, + ) + bad = PluginSchemas(Opportunity=mismatched) # type: ignore[arg-type] with pytest.raises( - ValueError, - match="cannot specify both mappings and explicit to_common/from_common", + PluginDefinitionError, match="attribute name must match the schema name" ): - define_plugin( - schemas={ - "Opportunity": SchemaInput( - mappings=SchemaMappings( - to_common={"title": {"field": "native_title"}}, - from_common={"native_title": {"field": "title"}}, - ), - to_common=noop, - ) - } - ) - - -def test_define_plugin_mappings_without_callables_is_valid(): - """Providing mappings without explicit callables does not raise.""" - from common_grants_sdk.extensions.types import SchemaMappings - - config = define_plugin( - schemas={ - "Opportunity": SchemaInput( - mappings=SchemaMappings( - to_common={"title": {"field": "native_title"}}, - from_common={"native_title": {"field": "title"}}, - ) - ) - } - ) - assert config.schemas is not None + define_plugin(bad, meta=_meta()) -def test_define_plugin_callables_without_mappings_is_valid(): - """Providing explicit callables without mappings does not raise.""" +# --------------------------------------------------------------------------- +# Plugin container +# --------------------------------------------------------------------------- - def noop(x): - return TransformResult(result=x, errors=[]) - config = define_plugin( - schemas={"Opportunity": SchemaInput(to_common=noop, from_common=noop)} - ) - assert config.schemas is not None +def test_plugin_is_frozen(): + plugin = define_plugin(PluginSchemas(), meta=_meta()) + with pytest.raises((AttributeError, TypeError)): + plugin.meta = _meta() # type: ignore[misc] diff --git a/lib/python-sdk/tests/extensions/test_schema.py b/lib/python-sdk/tests/extensions/test_schema.py new file mode 100644 index 000000000..cd06b4d9b --- /dev/null +++ b/lib/python-sdk/tests/extensions/test_schema.py @@ -0,0 +1,347 @@ +"""Tests for the codegen-free schema(...) factory and its building blocks.""" + +from typing import Optional, assert_type + +import pytest +from pydantic import BaseModel, Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PassthroughModel, + SchemaOnly, + SchemaWithTransforms, + TransformResult, + resolve_custom_field_specs, + schema, + validate_into, +) +from common_grants_sdk.extensions.schema import PluginDefinitionError, _infer_field_type +from common_grants_sdk.schemas.pydantic.fields import CustomFieldType +from common_grants_sdk.schemas.pydantic.models import OpportunityBase + + +class LegacyRef(BaseModel): + system: str + id: int + + +class OpportunityFields(CustomFieldSet): + agency_code: Optional[CustomField[str]] = Field( + default=None, description="Agency code" + ) + legacy_id: Optional[CustomField[int]] = Field(default=None) + legacy_ref: Optional[CustomField[LegacyRef]] = Field(default=None) + tags: Optional[CustomField[list[str]]] = Field(default=None) + + +FLAT_SOURCE = { + "opportunity_uuid": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "opportunity_title": "Conservation research", + "opportunity_description": "Funding.", + "opportunity_status": "posted", + "created_at": "2025-01-01T00:00:00Z", + "last_modified_at": "2025-01-01T00:00:00Z", + "agency_code": "HHS-123", +} + + +def _mappings_extension() -> ( + SchemaWithTransforms[PassthroughModel, OpportunityBase[OpportunityFields]] +): + return schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase[OpportunityFields], + mappings={ + "to_common": { + "id": {"field": "opportunity_uuid"}, + "title": {"field": "opportunity_title"}, + "description": {"field": "opportunity_description"}, + "createdAt": {"field": "created_at"}, + "lastModifiedAt": {"field": "last_modified_at"}, + "status": { + "value": { + "match": { + "field": "opportunity_status", + "case": {"posted": "open"}, + "default": "custom", + } + } + }, + "customFields": { + "agencyCode": { + "value": {"field": "agency_code"}, + "name": {"const": "agencyCode"}, + "fieldType": {"const": "string"}, + } + }, + }, + "from_common": { + "opportunity_uuid": {"field": "id"}, + "opportunity_title": {"field": "title"}, + "agency_code": {"field": "customFields.agencyCode.value"}, + }, + }, + ) + + +# --------------------------------------------------------------------------- +# _infer_field_type / resolve_custom_field_specs +# --------------------------------------------------------------------------- + + +def test_infer_field_type_mapping(): + assert _infer_field_type(str) == CustomFieldType.STRING + assert _infer_field_type(int) == CustomFieldType.INTEGER + assert _infer_field_type(float) == CustomFieldType.NUMBER + assert _infer_field_type(bool) == CustomFieldType.BOOLEAN + assert _infer_field_type(LegacyRef) == CustomFieldType.OBJECT + assert _infer_field_type(list[str]) == CustomFieldType.ARRAY + assert _infer_field_type(dict[str, int]) == CustomFieldType.OBJECT + + +def test_resolve_custom_field_specs_derives_from_value_type(): + specs = resolve_custom_field_specs(OpportunityFields) + assert specs["agency_code"].field_type == CustomFieldType.STRING + assert specs["agency_code"].value is str + assert specs["agency_code"].name == "agency_code" + assert specs["agency_code"].description == "Agency code" + assert specs["legacy_id"].field_type == CustomFieldType.INTEGER + assert specs["legacy_ref"].field_type == CustomFieldType.OBJECT + assert specs["legacy_ref"].value is LegacyRef + assert specs["tags"].field_type == CustomFieldType.ARRAY + + +def test_resolve_custom_field_specs_empty_for_no_container(): + assert resolve_custom_field_specs(None) == {} + assert resolve_custom_field_specs(CustomFieldSet) == {} + + +# --------------------------------------------------------------------------- +# schema(...) discrimination +# --------------------------------------------------------------------------- + + +def test_schema_only_returns_schema_only_extension(): + ext = schema(common_schema=OpportunityBase[OpportunityFields]) + assert isinstance(ext, SchemaOnly) + assert ext.schema_name == "Opportunity" + assert not hasattr(ext, "to_common") + + +def test_mappings_returns_transform_extension(): + ext = _mappings_extension() + assert isinstance(ext, SchemaWithTransforms) + assert ext.source_schema is PassthroughModel + assert ext.custom_fields["agency_code"].field_type == CustomFieldType.STRING + + +def test_functions_returns_transform_extension(): + def to_common( + _src: PassthroughModel, + ) -> TransformResult[OpportunityBase]: + return TransformResult( + result=OpportunityBase.model_validate( + FLAT_SOURCE + | { + "id": FLAT_SOURCE["opportunity_uuid"], + "title": FLAT_SOURCE["opportunity_title"], + "status": {"value": "open"}, + "createdAt": FLAT_SOURCE["created_at"], + "lastModifiedAt": FLAT_SOURCE["last_modified_at"], + } + ), + errors=[], + ) + + def from_common( + _c: OpportunityBase, + ) -> TransformResult[PassthroughModel]: + return TransformResult(result=PassthroughModel(), errors=[]) + + ext = schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase, + to_common=to_common, + from_common=from_common, + ) + assert isinstance(ext, SchemaWithTransforms) + + +# --------------------------------------------------------------------------- +# Negative cases (validated at schema(...) call time) +# --------------------------------------------------------------------------- + + +def test_unregistered_base_raises(): + class NotRegistered(BaseModel): + x: int + + with pytest.raises( + PluginDefinitionError, match="not a registered extensible schema" + ): + schema(common_schema=NotRegistered) + + +def test_bare_opportunity_base_is_a_registered_schema_only(): + """Bare OpportunityBase (no custom fields) is the registered base schema.""" + ext = schema(common_schema=OpportunityBase) + assert isinstance(ext, SchemaOnly) + assert ext.schema_name == "Opportunity" + assert ext.custom_fields == {} + + +def test_unknown_to_common_output_field_raises(): + with pytest.raises(PluginDefinitionError, match="unknown output field"): + schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase, + mappings={ + "to_common": {"nope": {"const": 1}}, + "from_common": {}, + }, + ) + + +def test_missing_mapping_direction_raises(): + with pytest.raises(PluginDefinitionError, match="missing `from_common`"): + schema( + source_schema=PassthroughModel, + common_schema=OpportunityBase, + mappings={"to_common": {"title": {"field": "x"}}}, # type: ignore[typeddict-item] + ) + + +def _noop(value): + return TransformResult(result=value, errors=[]) + + +def test_functions_without_source_schema_raises(): + with pytest.raises(PluginDefinitionError, match="source_schema` is required"): + schema( # type: ignore[call-overload] + common_schema=OpportunityBase, + to_common=_noop, + from_common=_noop, + ) + + +def test_mappings_without_source_schema_raises(): + with pytest.raises(PluginDefinitionError, match="source_schema` is required"): + schema( # type: ignore[call-overload] + common_schema=OpportunityBase, + mappings={ + "to_common": {"title": {"field": "x"}}, + "from_common": {}, + }, + ) + + +def test_one_sided_callable_raises(): + with pytest.raises( + PluginDefinitionError, match="both `to_common` and `from_common`" + ): + schema( # type: ignore[call-overload] + source_schema=PassthroughModel, + common_schema=OpportunityBase, + to_common=_noop, + ) + + +def test_mappings_and_callables_together_raises(): + with pytest.raises(PluginDefinitionError, match="cannot specify both"): + schema( # type: ignore[call-overload] + source_schema=PassthroughModel, + common_schema=OpportunityBase, + mappings={ + "to_common": {"title": {"field": "x"}}, + "from_common": {}, + }, + to_common=_noop, + from_common=_noop, + ) + + +# --------------------------------------------------------------------------- +# Consumer typing + behavior (the make-or-break path) +# --------------------------------------------------------------------------- + + +def test_mappings_consumer_typed_and_round_trips() -> None: + ext = _mappings_extension() + res = ext.to_common(PassthroughModel.model_validate(FLAT_SOURCE)) + assert_type(res, TransformResult[OpportunityBase[OpportunityFields]]) + assert res.errors == [] + opp = res.result + assert opp.title == "Conservation research" + assert opp.custom_fields is not None + assert opp.custom_fields.agency_code is not None + assert_type(opp.custom_fields.agency_code.value, str) + assert opp.custom_fields.agency_code.value == "HHS-123" + back = ext.from_common(opp) + assert_type(back, TransformResult[PassthroughModel]) + assert back.errors == [] + + +def test_schema_only_parse_typed() -> None: + ext = schema(common_schema=OpportunityBase[OpportunityFields]) + parsed = ext.parse( + { + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "title": "T", + "status": {"value": "open"}, + "description": "d", + "createdAt": "2025-01-01T00:00:00Z", + "lastModifiedAt": "2025-01-01T00:00:00Z", + "customFields": { + "legacyRef": { + "name": "legacyRef", + "fieldType": "object", + "value": {"system": "legacy", "id": 7}, + } + }, + } + ) + assert_type(parsed, OpportunityBase[OpportunityFields]) + assert parsed.custom_fields is not None + assert parsed.custom_fields.legacy_ref is not None + assert_type(parsed.custom_fields.legacy_ref.value.id, int) + assert parsed.custom_fields.legacy_ref.value.id == 7 + + +# --------------------------------------------------------------------------- +# camelCase round-trip +# --------------------------------------------------------------------------- + + +def test_camel_case_round_trip(): + camel = { + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "title": "T", + "status": {"value": "open"}, + "description": "d", + "createdAt": "2025-01-01T00:00:00Z", + "lastModifiedAt": "2025-01-01T00:00:00Z", + "customFields": { + "agencyCode": { + "name": "agencyCode", + "fieldType": "string", + "value": "HHS-123", + } + }, + } + opp = OpportunityBase[OpportunityFields].model_validate(camel) + # snake_case typed access + assert opp.custom_fields is not None + assert opp.custom_fields.agency_code is not None + assert opp.custom_fields.agency_code.value == "HHS-123" + # serializes back to camelCase + dumped = opp.model_dump(by_alias=True, mode="json") + assert "customFields" in dumped + assert dumped["customFields"]["agencyCode"]["fieldType"] == "string" + assert "createdAt" in dumped + + +def test_validate_into_routes_errors(): + res = validate_into(LegacyRef, {"system": "x"}) # missing id + assert res.errors + assert any("id" in (e.path or "") for e in res.errors) diff --git a/lib/python-sdk/tests/extensions/test_types.py b/lib/python-sdk/tests/extensions/test_types.py index 85e76ad71..7b18132b1 100644 --- a/lib/python-sdk/tests/extensions/test_types.py +++ b/lib/python-sdk/tests/extensions/test_types.py @@ -1,15 +1,11 @@ -"""Tests for ADR-0022 types defined in common_grants_sdk.extensions.types.""" +"""Tests for the plugin framework types in common_grants_sdk.extensions.types.""" -from common_grants_sdk.extensions.specs import CustomFieldSpec from common_grants_sdk.extensions.types import ( - PluginExtensionsMeta, - SchemaConfig, - SchemaInput, - SchemaMappings, + PassthroughModel, + PluginMeta, TransformError, TransformResult, ) -from common_grants_sdk.schemas.pydantic.fields.custom import CustomFieldType # --- TransformError --- @@ -40,7 +36,7 @@ def test_transform_error_structured_fields(): def test_transform_error_source_value_excluded_from_str_and_repr(): - """source_value must not appear in str(err) or repr(err) — PII defence per ADR-0022 Decision #9.""" + """source_value must not appear in str(err) or repr(err) -- PII defence.""" sensitive = {"ssn": "123-45-6789"} err = TransformError("transform failed", source_value=sensitive) assert "123-45-6789" not in str(err) @@ -61,69 +57,25 @@ def test_transform_result(): assert partial.errors[0] is err -# --- SchemaMappings --- +# --- PluginMeta --- -def test_schema_mappings(): - assert SchemaMappings().to_common is None - assert SchemaMappings().from_common is None - m = SchemaMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) - assert m.to_common == {"title": "x"} - assert m.from_common == {"x": "title"} - - -# --- PluginExtensionsMeta --- - - -def test_plugin_extensions_meta(): - m = PluginExtensionsMeta(name="grants.gov", sourceSystem="grants.gov") +def test_plugin_extensions_meta_camel_alias(): + m = PluginMeta(name="grants.gov", sourceSystem="grants.gov") assert m.name == "grants.gov" assert m.source_system == "grants.gov" assert m.version is None assert m.capabilities is None -# --- SchemaInput --- - - -def test_schema_input(): - assert SchemaInput().source_schema is None - assert SchemaInput().custom_fields is None - assert SchemaInput().mappings is None - assert SchemaInput().to_common is None - - spec = CustomFieldSpec(field_type=CustomFieldType.INTEGER) - inp = SchemaInput(custom_fields={"legacyId": spec}) - assert inp.custom_fields == {"legacyId": spec} - - m = SchemaMappings(toCommon={"title": "x"}, fromCommon={"x": "title"}) - inp_with_mappings = SchemaInput(mappings=m) - assert inp_with_mappings.mappings.to_common == {"title": "x"} - - def passthrough(x): - return TransformResult(result=x, errors=[]) - - inp2 = SchemaInput(to_common=passthrough, from_common=passthrough) - assert inp2.to_common is passthrough - - -# --- SchemaConfig --- +def test_plugin_extensions_meta_populate_by_name(): + m = PluginMeta(name="grants.gov", source_system="grants.gov") + assert m.source_system == "grants.gov" -def test_schema_config(): - def passthrough(x): - return TransformResult(result=x, errors=[]) +# --- PassthroughModel --- - schemas = SchemaConfig( - source_schema=dict, - common_schema=dict, - to_common=passthrough, - from_common=passthrough, - ) - assert schemas.source_schema is dict - assert schemas.common_schema is dict - # to_common and from_common are optional — omitting them is valid - minimal = SchemaConfig(source_schema=dict, common_schema=dict) - assert minimal.to_common is None - assert minimal.from_common is None +def test_passthrough_model_accepts_arbitrary_keys(): + instance = PassthroughModel.model_validate({"data": {"any": "shape"}, "n": 1}) + assert instance.model_dump() == {"data": {"any": "shape"}, "n": 1} diff --git a/lib/python-sdk/tests/schemas/test_plugin_registry.py b/lib/python-sdk/tests/schemas/test_plugin_registry.py index 0d0524be4..5c3fdb9e8 100644 --- a/lib/python-sdk/tests/schemas/test_plugin_registry.py +++ b/lib/python-sdk/tests/schemas/test_plugin_registry.py @@ -1,12 +1,8 @@ -"""Tests for OpportunityBase.with_custom_fields().""" - -from typing import Any +"""Tests for OpportunityBase.with_custom_fields() (runtime custom-field registration).""" import pytest from common_grants_sdk.extensions import CustomFieldSpec -from common_grants_sdk.extensions import Plugin -from common_grants_sdk.extensions.types import SchemaConfig from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.schemas.pydantic.models.opp_base import OpportunityBase @@ -15,28 +11,12 @@ # --------------------------------------------------------------------------- -class _Schemas: - """Minimal schemas container that mirrors the generated _Schemas class.""" - - Opportunity: SchemaConfig[Any, Any] - - -def _make_plugin( - field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity" -) -> "Plugin[_Schemas]": - """Build a Plugin whose schemas.Opportunity.common_schema is produced by with_custom_fields().""" - extended = OpportunityBase.with_custom_fields( +def _extended(field_specs: dict[str, CustomFieldSpec], model_name: str = "Opportunity"): + """Return an OpportunityBase subclass produced by with_custom_fields().""" + return OpportunityBase.with_custom_fields( custom_fields=field_specs, model_name=model_name, ) - s = _Schemas() - s.Opportunity = SchemaConfig(source_schema=dict, common_schema=extended) - return Plugin(schemas=s) - - -def _make_plugin_without_opportunity() -> "Plugin[_Schemas]": - """Build a Plugin that has no Opportunity schema.""" - return Plugin(schemas=_Schemas()) # --------------------------------------------------------------------------- @@ -45,8 +25,8 @@ def _make_plugin_without_opportunity() -> "Plugin[_Schemas]": @pytest.fixture -def simple_plugin() -> "Plugin[_Schemas]": - return _make_plugin( +def simple_opportunity(): + return _extended( { "program_area": CustomFieldSpec( field_type=CustomFieldType.STRING, @@ -80,32 +60,20 @@ def sample_payload() -> dict: # --------------------------------------------------------------------------- -# Plugin schema tests +# Schema construction tests # --------------------------------------------------------------------------- -def test_plugin_schema_is_subclass_of_opportunity_base(simple_plugin): - Opportunity = simple_plugin.schemas.Opportunity.common_schema - - assert Opportunity is not OpportunityBase - assert issubclass(Opportunity, OpportunityBase) - - -def test_plugin_without_opportunity_has_no_opportunity_schema(): - plugin = _make_plugin_without_opportunity() - - assert not hasattr(plugin.schemas, "Opportunity") +def test_with_custom_fields_is_subclass_of_opportunity_base(simple_opportunity): + assert simple_opportunity is not OpportunityBase + assert issubclass(simple_opportunity, OpportunityBase) -def test_two_plugins_produce_distinct_schemas(simple_plugin): - second_plugin = _make_plugin( +def test_two_calls_produce_distinct_schemas(simple_opportunity): + second = _extended( {"award_ceiling": CustomFieldSpec(field_type=CustomFieldType.NUMBER)} ) - - assert ( - simple_plugin.schemas.Opportunity.common_schema - is not second_plugin.schemas.Opportunity.common_schema - ) + assert simple_opportunity is not second # --------------------------------------------------------------------------- @@ -113,23 +81,12 @@ def test_two_plugins_produce_distinct_schemas(simple_plugin): # --------------------------------------------------------------------------- -def test_plugin_schema_validates_payload_and_exposes_typed_custom_fields( - simple_plugin, sample_payload +def test_validates_payload_and_exposes_typed_custom_fields( + simple_opportunity, sample_payload ): - Opportunity = simple_plugin.schemas.Opportunity.common_schema - - opp = Opportunity.model_validate(sample_payload) + opp = simple_opportunity.model_validate(sample_payload) assert opp.title == "Community Health Grant" assert opp.custom_fields is not None assert opp.custom_fields.program_area.value == "Health" assert opp.custom_fields.eligibility_types.value == ["nonprofit", "tribal"] - - -def test_plugin_schema_validates_custom_fields(simple_plugin, sample_payload): - Opportunity = simple_plugin.schemas.Opportunity.common_schema - - opp = Opportunity.model_validate(sample_payload) - - assert opp.custom_fields.program_area.value == "Health" - assert opp.custom_fields.eligibility_types.value == ["nonprofit", "tribal"] diff --git a/lib/python-sdk/tests/utils/test_plugin_generator.py b/lib/python-sdk/tests/utils/test_plugin_generator.py deleted file mode 100644 index 8b39a829d..000000000 --- a/lib/python-sdk/tests/utils/test_plugin_generator.py +++ /dev/null @@ -1,552 +0,0 @@ -from __future__ import annotations - -import importlib -import os -import shutil -import subprocess -import sys -from pathlib import Path -from typing import get_type_hints - -import pytest - -from common_grants_sdk import define_plugin -from common_grants_sdk.extensions import CustomFieldSpec -from common_grants_sdk.extensions import PluginConfig -from common_grants_sdk.schemas.pydantic.fields import CustomFieldType - - -def _sdk_root() -> Path: - return Path(__file__).resolve().parents[2] - - -def _env_with_sdk_pythonpath() -> dict[str, str]: - env = os.environ.copy() - sdk_root = str(_sdk_root()) - existing = env.get("PYTHONPATH") - env["PYTHONPATH"] = f"{sdk_root}{os.pathsep}{existing}" if existing else sdk_root - return env - - -def test_define_plugin_returns_config_with_schemas(): - from common_grants_sdk.extensions.types import SchemaInput - - schemas = { - "Opportunity": SchemaInput( - custom_fields={ - "program_area": CustomFieldSpec( - field_type=CustomFieldType.STRING, - description="Grant category", - ) - } - ) - } - config = define_plugin(schemas=schemas) - - assert isinstance(config, PluginConfig) - assert config.schemas is schemas - assert config.schemas["Opportunity"].custom_fields is not None - assert "program_area" in config.schemas["Opportunity"].custom_fields - - -def test_generate_cli_emits_plugin_and_typed_models(tmp_path: Path): - plugins_dir = tmp_path / "plugins" - plugins_dir.mkdir() - (plugins_dir / "__init__.py").write_text("", encoding="utf-8") - - plugin_dir = plugins_dir / "combined" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "program_area": CustomFieldSpec(', - ' field_type="string",', - ' description="Program area",', - " ),", - ' "eligibility_type": CustomFieldSpec(', - ' field_type="array",', - ' description="Types of eligible organizations",', - " ),", - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - cmd = [sys.executable, "-m", "common_grants_sdk.extensions.generate"] - run = subprocess.run( - cmd, - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - assert (plugin_dir / "generated" / "__init__.py").exists() - assert (plugin_dir / "generated" / "schemas.py").exists() - assert (plugin_dir / "__init__.py").exists() - - sys.path.insert(0, str(tmp_path)) - try: - combined_module = importlib.import_module("plugins.combined") - combined = getattr(combined_module, "combined") - opp_model = combined.schemas.Opportunity.common_schema - - type_hints = get_type_hints(opp_model, include_extras=False) - assert "custom_fields" in type_hints - - payload = { - "id": "573525f2-8e15-4405-83fb-e6523511d893", - "title": "Test Opportunity", - "status": {"value": "open"}, - "description": "Funding available for pilot projects", - "createdAt": "2026-01-01T00:00:00Z", - "lastModifiedAt": "2026-01-01T00:00:00Z", - "customFields": { - "program_area": { - "fieldType": "string", - "value": "Health", - }, - "eligibility_type": { - "fieldType": "array", - "value": ["nonprofit", "city_government"], - }, - }, - } - - opp = opp_model.model_validate(payload) - - assert opp.custom_fields is not None - assert opp.custom_fields.program_area is not None - assert opp.custom_fields.program_area.value == "Health" - assert opp.custom_fields.eligibility_type is not None - assert opp.custom_fields.eligibility_type.value == [ - "nonprofit", - "city_government", - ] - assert combined.schemas.Opportunity.common_schema is opp_model - finally: - sys.path.remove(str(tmp_path)) - - -def test_generate_emits_import_for_pydantic_model_in_cg_config(tmp_path: Path): - """spec.value set to a Pydantic model defined in cg_config.py should produce - a ``from ..cg_config import <Model>`` line in the generated schemas.py.""" - plugin_dir = tmp_path / "my_plugin" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from pydantic import BaseModel", - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "class AgentInfo(BaseModel):", - " name: str", - " email: str", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "point_of_contact": CustomFieldSpec(', - ' field_type="object",', - " value=AgentInfo,", - " ),", - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - run = subprocess.run( - [sys.executable, "-m", "common_grants_sdk.extensions.generate"], - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "from ..cg_config import AgentInfo" in schemas_src - assert "value: Optional[AgentInfo]" in schemas_src - - -def test_generate_emits_import_for_external_module_type(tmp_path: Path): - """spec.value set to a type from a real importable module should produce - a ``from <module> import <Type>`` line in the generated schemas.py.""" - plugin_dir = tmp_path / "my_plugin" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from datetime import datetime", - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "deadline": CustomFieldSpec(', - ' field_type="string",', - " value=datetime,", - " ),", - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - run = subprocess.run( - [sys.executable, "-m", "common_grants_sdk.extensions.generate"], - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "from datetime import datetime" in schemas_src - assert "value: Optional[datetime]" in schemas_src - - -def test_generate_auto_builds_transforms_from_mappings(tmp_path): - """When cg_config has schemas[obj].mappings but no explicit to_common/from_common, - the generated __init__.py calls build_transforms() automatically.""" - plugin_dir = tmp_path / "plugins" / "auto_transform" - plugin_dir.mkdir(parents=True) - (plugin_dir / "__init__.py").write_text("", encoding="utf-8") - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, SchemaMappings", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " mappings=SchemaMappings(", - ' to_common={"title": {"field": "data.title"}},', - " from_common={},", - " ),", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - from common_grants_sdk.extensions.generate import generate_plugin - - generate_plugin(plugin_dir) - - init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") - assert "build_transforms" in init_content - assert 'config.schemas["Opportunity"].mappings.to_common' in init_content - assert "_Opportunity_to_common" in init_content - assert "common_schema=schemas.Opportunity.common_schema" in init_content - - # Load the generated plugin and verify schemas are populated - import importlib - import sys - - # Remove any stale 'plugins' package from previous tests before inserting our path. - for key in list(sys.modules.keys()): - if key == "plugins" or key.startswith("plugins."): - del sys.modules[key] - - sys.path.insert(0, str(tmp_path)) - try: - mod = importlib.import_module("plugins.auto_transform") - plugin = getattr(mod, "auto_transform") - assert hasattr(plugin.schemas, "Opportunity") - assert plugin.schemas.Opportunity.to_common is not None - finally: - if str(tmp_path) in sys.path: - sys.path.remove(str(tmp_path)) - for key in list(sys.modules.keys()): - if key == "plugins" or key.startswith("plugins."): - del sys.modules[key] - - -@pytest.mark.skipif(shutil.which("pyright") is None, reason="pyright is not installed") -def test_generate_models_typecheck_with_pyright_strict(tmp_path: Path): - plugins_dir = tmp_path / "plugins" - plugins_dir.mkdir() - (plugins_dir / "__init__.py").write_text("", encoding="utf-8") - - plugin_dir = plugins_dir / "combined" - plugin_dir.mkdir() - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions import CustomFieldSpec", - "from common_grants_sdk.extensions.types import SchemaInput", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " custom_fields={", - ' "eligibility_type": CustomFieldSpec(field_type="array"),', - " },", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - run = subprocess.run( - [sys.executable, "-m", "common_grants_sdk.extensions.generate"], - cwd=plugin_dir, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert run.returncode == 0, run.stderr - - typecheck_file = tmp_path / "typecheck.py" - typecheck_file.write_text( - "\n".join( - [ - "# pyright: strict", - "from plugins.combined import combined", - "", - "payload = {", - ' "id": "573525f2-8e15-4405-83fb-e6523511d893",', - ' "title": "Typed Opportunity",', - ' "status": {"value": "open"},', - ' "description": "Typed custom fields",', - ' "createdAt": "2026-01-01T00:00:00Z",', - ' "lastModifiedAt": "2026-01-01T00:00:00Z",', - ' "customFields": {"eligibility_type": {"fieldType": "array", "value": ["a"]}},', - "}", - "", - "opp = combined.schemas.Opportunity.common_schema.model_validate(payload)", - "if opp.custom_fields is not None and opp.custom_fields.eligibility_type is not None:", - " values = opp.custom_fields.eligibility_type.value", - " reveal_type(values)", - "", - ] - ), - encoding="utf-8", - ) - - env = _env_with_sdk_pythonpath() - env["PYTHONPATH"] = f"{tmp_path}{os.pathsep}{env['PYTHONPATH']}" - pyright = subprocess.run( - ["pyright", str(typecheck_file)], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - check=False, - ) - assert pyright.returncode == 0, pyright.stdout + "\n" + pyright.stderr - assert 'Type of "values" is "list[Any] | None"' in pyright.stdout - - -def test_generate_explicit_transforms(tmp_path): - """When cg_config has config.schemas with explicit to_common/from_common, - the generated __init__.py wires the supplied callables via inject_transforms.""" - from common_grants_sdk.extensions.generate import generate_plugin - - plugin_dir = tmp_path / "plugins" / "explicit_tf" - plugin_dir.mkdir(parents=True) - (plugin_dir / "__init__.py").write_text("", encoding="utf-8") - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, TransformResult", - "from common_grants_sdk.extensions import CustomFieldSpec", - "", - "def _to_common(native):", - " return TransformResult(result={'title': native.get('name', '')}, errors=[])", - "", - "def _from_common(common):", - " return TransformResult(result={'name': common.get('title', '')}, errors=[])", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - ' custom_fields={"legacyId": CustomFieldSpec(field_type="integer")},', - " to_common=_to_common,", - " from_common=_from_common,", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - generate_plugin(plugin_dir) - - init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") - # Explicit transforms use inject_transforms(), not per-object boilerplate - assert "build_transforms" not in init_content - assert "schemas = inject_transforms(config, schemas)" in init_content - # No per-object assignment lines - assert 'config.schemas["Opportunity"].to_common' not in init_content - assert 'config.schemas["Opportunity"].from_common' not in init_content - # ObjectSchemas is no longer constructed in __init__.py (only in schemas.py) - assert "ObjectSchemas" not in init_content - - # Load and verify the plugin works end-to-end - sys.path.insert(0, str(tmp_path)) - try: - mod = importlib.import_module("plugins.explicit_tf") - plugin = getattr(mod, "explicit_tf") - assert hasattr(plugin.schemas, "Opportunity") - opp_schemas = plugin.schemas.Opportunity - result = opp_schemas.to_common({"name": "Test Grant"}) - assert result.result == {"title": "Test Grant"} - assert result.errors == [] - finally: - if str(tmp_path) in sys.path: - sys.path.remove(str(tmp_path)) - for key in list(sys.modules.keys()): - if "explicit_tf" in key or key == "plugins": - del sys.modules[key] - - -def test_generate_transforms_only_no_custom_fields(tmp_path): - """Regression: config.schemas with only explicit transforms (no custom_fields, no extensions) - must produce a _Schemas entry for the object so inject_transforms() can access it at import. - """ - from common_grants_sdk.extensions.generate import generate_plugin - - plugin_dir = tmp_path / "plugins" / "transforms_only" - plugin_dir.mkdir(parents=True) - (plugin_dir / "__init__.py").write_text("", encoding="utf-8") - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, TransformResult", - "", - "def _to_common(native):", - " return TransformResult(result={'title': native.get('name', '')}, errors=[])", - "", - "def _from_common(common):", - " return TransformResult(result={'name': common.get('title', '')}, errors=[])", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " to_common=_to_common,", - " from_common=_from_common,", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - generate_plugin(plugin_dir) - - # schemas.py must assign self.Opportunity using the base SDK class - schemas_src = (plugin_dir / "generated" / "schemas.py").read_text(encoding="utf-8") - assert "self.Opportunity = SchemaConfig" in schemas_src - assert "OpportunityBase" in schemas_src - - # __init__.py must use inject_transforms (not build_transforms) - init_content = (plugin_dir / "__init__.py").read_text(encoding="utf-8") - assert "inject_transforms" in init_content - assert "build_transforms" not in init_content - - # Importing must not raise AttributeError / ValueError - sys.path.insert(0, str(tmp_path)) - try: - mod = importlib.import_module("plugins.transforms_only") - plugin = getattr(mod, "transforms_only") - assert hasattr(plugin.schemas, "Opportunity") - result = plugin.schemas.Opportunity.to_common({"name": "Test Grant"}) - assert result.result == {"title": "Test Grant"} - assert result.errors == [] - finally: - if str(tmp_path) in sys.path: - sys.path.remove(str(tmp_path)) - for key in list(sys.modules.keys()): - if "transforms_only" in key or key == "plugins": - del sys.modules[key] - - -def test_generate_raises_on_missing_mapping_direction(tmp_path): - """generate_plugin raises ValueError when a mappings-only object is missing - one of its mapping directions (to_common or from_common is None).""" - from common_grants_sdk.extensions.generate import generate_plugin - - plugin_dir = tmp_path / "bad_plugin" - plugin_dir.mkdir(parents=True) - - (plugin_dir / "cg_config.py").write_text( - "\n".join( - [ - "from common_grants_sdk import define_plugin", - "from common_grants_sdk.extensions.types import SchemaInput, SchemaMappings", - "", - "config = define_plugin(", - " schemas={", - ' "Opportunity": SchemaInput(', - " mappings=SchemaMappings(", - ' to_common={"title": {"field": "data.title"}},', - " # from_common intentionally omitted (None)", - " ),", - " )", - " },", - ")", - "", - ] - ), - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="from_common.*required"): - generate_plugin(plugin_dir) diff --git a/lib/python-sdk/tests/utils/test_transformation.py b/lib/python-sdk/tests/utils/test_transformation.py index 797cd511a..20ff96245 100644 --- a/lib/python-sdk/tests/utils/test_transformation.py +++ b/lib/python-sdk/tests/utils/test_transformation.py @@ -111,6 +111,12 @@ def test_missing_field_returns_none(input_data): assert result == {"foo": None} +def test_empty_mapping_returns_empty_dict(input_data): + """An empty mapping (top-level or nested) yields an empty dict, not None.""" + assert transform_from_mapping(input_data, {}) == {} + assert transform_from_mapping(input_data, {"nested": {}}) == {"nested": {}} + + def test_literal_value(input_data): """ Test handling of literal values in the mapping. From 54bff1cef0b6102c48a4c199d8f62f77aed750db Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:23:24 -0700 Subject: [PATCH 36/51] feat(py-sdk): codegen-free typed custom-filter authoring DX (#934) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(py-sdk): typed-declaration custom-filter authoring DX Add a static-typing layer over the throw-based custom-filter runtime so a plugin author registers a route's filters by declaring a TypedDict and a consumer gets per-key narrowing on the registered filters with no call-site annotations. - extensions/routes.py (new): RouteFilters[TF] phantom carrier; frozen, covariant ResourceRoutes / Routes carriers (PEP 696 defaults). Static-typing only; the runtime classifier does not consume them. - filters/opportunity.py: OpportunityFilters TypedDict bag keyed to the standard opportunity filters; clean value-model aliases (StringArray, NumberComparison, ...) for author declarations. - extensions/plugin.py: define_plugin gains routes= (default Routes()); Plugin carries it and exposes search_filters_type(), a constrained-self projection that recovers the registered filter TypedDict. Transport is out-of-scope (ADR-0022), so this returns the typed filter slot, not a client. - examples/typed_filters.py: author + consumer DX with assert_type narrowing. - test_plugin.py: route threading, default carrier, frozen carriers. Narrows cleanly under both mypy and pyright. The covariant dataclass fields need type: ignore[misc] for mypy (covariant-as-parameter) but not pyright; the constrained-self projection narrows identically under both. make checks + pytest green (369 passed). * docs(py-sdk): end-to-end consumer example searching with custom filters Add examples/consumer_search_with_filters.py: the full downstream flow an adopter writes — author registers opportunities.search custom filters (typed OppSearchFilters declaration + runtime PluginRoutes), a stubbed client runs classify_filters to build the three-bucket OppFilters request body and returns typed OpportunityBase rows, and the consumer builds filters with f.* and iterates the results. Transport is stubbed per ADR-0022 (the SDK client is a typed slot; real transport is downstream). The production client/opportunities.py:search() is NOT modified — it currently takes only search+status; the filters= parameter shown is the proposed signature being finalized (PR #934 open question 1), kept local to the example. Runnable: poetry run python examples/consumer_search_with_filters.py make checks + pytest green (369 passed); mypy + pyright clean. * docs(py-sdk): drop redundant filters annotation in consumer example * feat(py-sdk): overload f.* builders to precise filter models Overload each f.* builder so its static return type is the precise filter model the value implies — f.eq("x") -> StringComparisonFilter, f.eq(5) -> NumberComparisonFilter, f.eq(True) -> BooleanComparisonFilter, f.in_([str]) -> StringArrayFilter, f.gte(3) -> NumberComparisonFilter, f.between(0,1) -> NumberRangeFilter. Value types that don't map to a precise model (Money dicts, ISODate strings, date/Money ranges) fall through to a DefaultFilter overload, so ad-hoc and Money/date usage still works. This closes the cross-SDK parity gap flagged on the consumer example: the runtime-ergonomic f.* call site now composes with the typed OppSearchFilters bag with no annotation and full per-key narrowing. Runtime dispatches on value type (bool before int/float, since bool is an int subclass) to construct the matching model; model_dump() wire shape stays {operator, value}, identical to before — classify_filters and the wire body are unchanged. Fallout fixed: - classify_filters consumer_filters param widened from Mapping[str, DefaultFilter | dict] to Mapping[str, BaseModel | dict] (the precise models are siblings of DefaultFilter, not subclasses). - consumer_search_with_filters.py: search(filters=) typed to OppSearchFilters; the consumer call composes annotation-free with assert_type proof; "doesn't compose" caveat removed. - test_f_helper_wire_values asserts the JSON wire dump (the real contract) rather than .value, which is now a sub-model for range filters. Type-checker note: pyright flags the precise/Any overload overlap (and the bool/int overlap) which mypy does not; suppressed per-builder on the first overload. Both checkers clean on the changed files. make checks + pyright + pytest green (369 passed); all three examples run. * chore(py-sdk): tidy typed-filter authoring DX - test_plugin fixture: declare agency with the StringArray alias (the canonical authoring shape) rather than list[str] - plugin: import the TF TypeVar from routes instead of redeclaring it - plugin: note why the shared module-level routes default is safe to reuse - consumer example: use list[]/type[] builtins to match SDK style * test(py-sdk): pin numberRange round-trip; tidy filter comment and example import - add a NUMBER_RANGE registered-filter round-trip test: f.between(int, int) validates and model_dump recurses the NumberRange sub-model to {min, max} (the moneyRange analog was covered; the numeric path was not) - drop the inline bool-before-int comment in _equivalence; the block comment above the eq/neq overloads already explains the bool ⊂ int ordering - drop the redundant StringArrayFilter import in typed_filters.py and use the StringArray alias in the runtime guards (StringArray is StringArrayFilter), matching the NumberComparison alias usage alongside it * fix(py-sdk): type range fields with explicit *RangeFilter to remove a name collision DateRange / MoneyRange / NumberRange are the range value sub-models exported from the filters package root, so reusing those names as filter aliases in .opportunity made one public name resolve to two different types by import path: a consumer extending OpportunityFilters who imported a *Range name from the package root got the value sub-model, not the filter — silently breaking per-key narrowing for any custom range filter. - drop the three colliding *Range filter aliases (the *Comparison / *Array aliases have no clash and stay) - type the standard range fields (closeDateRange, *AwardAmountRange) with the explicit *RangeFilter names, which are unambiguous everywhere - example imports the *RangeFilter names from the package root * docs(py-sdk): drop PR-ref and point-in-time framing from consumer example * docs(py-sdk): move ADR-0022 rationale out of comments; fix runtime-unchanged note Comments carry brief intent only; the transport-scope rationale lives in the PR description. Also corrects the typed_filters docstring, which wrongly listed f.* among the unchanged runtime (f.* now returns precise filter models). * docs(py-sdk): trim wire-body narration from filter docstrings Keep docstrings to quirks and intent. Remove the runtime/wire-shape paragraph from _FHelpers (the bool-ordering quirk it restated is already documented on the bool overload and the runtime guard), and drop the request-body / wire-shape clauses from the test_f_helper_wire_values docstring, leaving the assertion intent. * address review: drop search_filters_type carriers, wire client custom filters - Remove search_filters_type() + Routes/ResourceRoutes/RouteFilters carriers; reshape Plugin.routes to PluginRoutes (ADR-0022 Decision #10) - Opportunities.search() takes filters=/routes=, building the body via classify_filters (mode="json" so date filters serialize to ISO strings) - Rewrite consumer example to call the real Client + plugin.routes - Rewrite OpportunityFilters docstring; drop the redundant typed_filters example --- .changeset/python-typed-custom-filters.md | 9 + .../common_grants_sdk/client/opportunities.py | 41 ++- .../common_grants_sdk/extensions/filters.py | 239 +++++++++++++----- .../common_grants_sdk/extensions/plugin.py | 29 ++- .../schemas/pydantic/filters/__init__.py | 19 ++ .../schemas/pydantic/filters/opportunity.py | 48 +++- .../examples/consumer_search_with_filters.py | 125 +++++++++ .../tests/client/test_opportunities.py | 69 ++++- .../tests/extensions/test_filters.py | 24 +- .../tests/extensions/test_plugin.py | 39 +++ 10 files changed, 556 insertions(+), 86 deletions(-) create mode 100644 .changeset/python-typed-custom-filters.md create mode 100644 lib/python-sdk/examples/consumer_search_with_filters.py diff --git a/.changeset/python-typed-custom-filters.md b/.changeset/python-typed-custom-filters.md new file mode 100644 index 000000000..517d7d740 --- /dev/null +++ b/.changeset/python-typed-custom-filters.md @@ -0,0 +1,9 @@ +--- +"common-grants-sdk": minor +--- + +Add codegen-free typed custom-filter authoring and client-side filtered search to the Python SDK. + +- `OpportunityFilters` (a `TypedDict`) is the typed authoring surface for opportunity-search filters: a consumer annotates their filter dict with it — or a `total=False` subclass — to get per-key autocomplete and value-type narrowing, and to give each custom filter its own typed key. Clean value aliases (`StringArray`, `NumberComparison`, `DateComparison`, …) read at the call site. +- `define_plugin(routes=...)` accepts a route-keyed `PluginRoutes` map of `CustomFilterSpec` (`routes[resource][method].filters[name]`, per ADR-0022 Decision #10) and threads it onto `plugin.routes`. +- `Opportunities.search()` gains optional `filters` and `routes` parameters: the consumer's filter dict is classified via `classify_filters` into the three-bucket search body (default named fields + a `customFilters` record) and POSTed, returning typed `OpportunityBase` rows. `status` is now an optional shorthand merged into the filter set. diff --git a/lib/python-sdk/common_grants_sdk/client/opportunities.py b/lib/python-sdk/common_grants_sdk/client/opportunities.py index 03dbd4246..fd05f49ce 100644 --- a/lib/python-sdk/common_grants_sdk/client/opportunities.py +++ b/lib/python-sdk/common_grants_sdk/client/opportunities.py @@ -13,8 +13,10 @@ Paginated, ) from ..schemas.pydantic.models.opp_status import OppStatusOptions +from ..extensions.filters import classify_filters +from ..extensions.types import PluginRoutes from .types import ItemsT -from typing import List +from typing import Any, List, Mapping if TYPE_CHECKING: from .client import Client @@ -108,34 +110,47 @@ def get( def search( self, search: str, - status: List[OppStatusOptions], + status: List[OppStatusOptions] | None = None, + filters: Mapping[str, Any] | None = None, + routes: PluginRoutes | None = None, page: int | None = None, page_size: int | None = None, schema: Type[OpportunityBase] | None = OpportunityBase, ) -> OpportunitiesSearchResponse: - """Search for opportunties by a query string + """Search for opportunities by a query string, with optional custom filters. Args: search: The string to search for. - status: List of statuses to search on. + status: Status shorthand; merged in as the ``status`` filter when given + (a ``status`` key already in ``filters`` takes precedence). + filters: Consumer filter dict (standard + custom keys), classified into + the request body via ``classify_filters``. Registered custom filters + land in ``customFilters``; register them via ``define_plugin(routes=...)``. + routes: Route-keyed custom-filter registration (e.g. ``plugin.routes``) + used to classify which keys are registered custom filters. page: Page number (1-indexed). If None, method will fetch all items across all pages and aggregate them into a single response. page_size: Number of items per page. If None, uses the default from client config. schema: OpportunityBase to support custom fields added by the caller. - Returns: OpportunitiesSearchResponse with items and pagination info - Raises: - APIError: if the API request fails + Raises: + APIError: if the API request fails + FilterError: if a filter is called with an operator its type does not allow """ + consumer_filters: dict[str, Any] = dict(filters or {}) + if status is not None and "status" not in consumer_filters: + consumer_filters["status"] = {"operator": "in", "value": status} + + opp_filters = classify_filters( + routes or {}, "opportunities", "search", consumer_filters + ) request = { - "filters": { - "status": {"operator": "in", "value": status}, - }, + "filters": opp_filters.model_dump(by_alias=True, exclude_none=True), "pagination": {"page": 1, "pageSize": 10}, "search": search, "sorting": {"sortBy": "lastModifiedAt", "sortOrder": "desc"}, @@ -143,10 +158,12 @@ def search( request_data = OpportunitySearchRequest.model_validate(request) - # Call client method to get paginated response + # Call client method to get paginated response. mode="json" so date filter + # values serialize to ISO strings — httpx encodes json= with stdlib json, + # which rejects the datetime.date objects pydantic parses dates into. paginated_response: Paginated[ItemsT] = self.client.search( # type: ignore[valid-type] f"{self.path}/search", - request_data.model_dump(by_alias=True, exclude_unset=True), + request_data.model_dump(by_alias=True, exclude_unset=True, mode="json"), page=page, page_size=page_size, ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index 4aa93da1c..b736a958e 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -17,7 +17,7 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Any, Optional +from typing import Any, Optional, Union, overload from pydantic import BaseModel, ValidationError @@ -41,6 +41,7 @@ from common_grants_sdk.schemas.pydantic.filters.numeric import ( NumberArrayFilter, NumberComparisonFilter, + NumberRange, NumberRangeFilter, ) from common_grants_sdk.schemas.pydantic.filters.opportunity import ( @@ -61,70 +62,190 @@ class _FHelpers: - """Helper namespace for building ``DefaultFilter`` instances. + """Helper namespace for building filter values, typed to the precise model. Import as ``from common_grants_sdk.extensions import f`` and use as ``f.eq("open")``, ``f.in_([...])``, etc. - ``in_`` and ``not_in`` avoid Python reserved words; the wire operator - values they emit are still ``"in"`` / ``"notIn"``. + Each builder is overloaded so the *static* return type is the precise filter + model the value implies — ``f.eq("x")`` is a ``StringComparisonFilter``, + ``f.eq(5)`` a ``NumberComparisonFilter``, ``f.eq(True)`` a + ``BooleanComparisonFilter``, ``f.in_([...])`` a ``StringArrayFilter`` / + ``NumberArrayFilter`` — so a typed filter bag (``OppSearchFilters``) composes + with ``f.*`` at the call site with no annotation. Value types that don't map + to a precise model (a Money dict, an ``ISODate``) fall through the final + overload to ``DefaultFilter``, so ad-hoc and Money/date usage still works. + + ``in_`` and ``not_in`` avoid Python reserved words; the wire operator values + they emit are still ``"in"`` / ``"notIn"``. """ - def eq(self, value: Any) -> DefaultFilter: - """Return DefaultFilter(operator="eq", value=value).""" - return DefaultFilter(operator=EquivalenceOperator.EQUAL, value=value) - - def neq(self, value: Any) -> DefaultFilter: - """Return DefaultFilter(operator="neq", value=value).""" - return DefaultFilter(operator=EquivalenceOperator.NOT_EQUAL, value=value) - - def gt(self, value: Any) -> DefaultFilter: - """Return DefaultFilter(operator="gt", value=value).""" - return DefaultFilter(operator=ComparisonOperator.GREATER_THAN, value=value) - - def gte(self, value: Any) -> DefaultFilter: - """Return DefaultFilter(operator="gte", value=value).""" - return DefaultFilter( - operator=ComparisonOperator.GREATER_THAN_OR_EQUAL, value=value - ) - - def lt(self, value: Any) -> DefaultFilter: - """Return DefaultFilter(operator="lt", value=value).""" - return DefaultFilter(operator=ComparisonOperator.LESS_THAN, value=value) - - def lte(self, value: Any) -> DefaultFilter: - """Return DefaultFilter(operator="lte", value=value).""" - return DefaultFilter( - operator=ComparisonOperator.LESS_THAN_OR_EQUAL, value=value - ) - - def in_(self, value: list[Any]) -> DefaultFilter: - """Return DefaultFilter with wire operator "in" (Python keyword workaround: f.in_).""" - return DefaultFilter(operator=ArrayOperator.IN, value=value) - - def not_in(self, value: list[Any]) -> DefaultFilter: - """Return DefaultFilter with wire operator "notIn".""" - return DefaultFilter(operator=ArrayOperator.NOT_IN, value=value) - - def like(self, value: str) -> DefaultFilter: - """Return DefaultFilter(operator="like", value=value).""" - return DefaultFilter(operator=StringOperator.LIKE, value=value) - - def not_like(self, value: str) -> DefaultFilter: - """Return DefaultFilter(operator="notLike", value=value).""" - return DefaultFilter(operator=StringOperator.NOT_LIKE, value=value) - - def between(self, min: Any, max: Any) -> DefaultFilter: - """Return DefaultFilter(operator="between", value={"min": min, "max": max}).""" - return DefaultFilter( - operator=RangeOperator.BETWEEN, value={"min": min, "max": max} - ) - - def outside(self, min: Any, max: Any) -> DefaultFilter: - """Return DefaultFilter(operator="outside", value={"min": min, "max": max}).""" - return DefaultFilter( - operator=RangeOperator.OUTSIDE, value={"min": min, "max": max} + # --- eq / neq: bool -> Boolean, str -> String, number -> Number, else Default --- + # bool overload first: bool ⊂ int, so a literal True would otherwise match the + # number arm. The intentional overlap is suppressed on the bool overload. + @overload + def eq(self, value: bool) -> BooleanComparisonFilter: ... # type: ignore[overload-overlap] # pyright: ignore[reportOverlappingOverload] + @overload + def eq(self, value: str) -> StringComparisonFilter: ... + @overload + def eq(self, value: Union[int, float]) -> NumberComparisonFilter: ... + @overload + def eq(self, value: Any) -> DefaultFilter: ... + def eq(self, value: Any) -> BaseModel: + """Build an ``eq`` filter, typed to the value's precise model.""" + return self._equivalence(EquivalenceOperator.EQUAL, value) + + @overload + def neq(self, value: bool) -> BooleanComparisonFilter: ... # type: ignore[overload-overlap] # pyright: ignore[reportOverlappingOverload] + @overload + def neq(self, value: str) -> StringComparisonFilter: ... + @overload + def neq(self, value: Union[int, float]) -> NumberComparisonFilter: ... + @overload + def neq(self, value: Any) -> DefaultFilter: ... + def neq(self, value: Any) -> BaseModel: + """Build a ``neq`` filter, typed to the value's precise model.""" + return self._equivalence(EquivalenceOperator.NOT_EQUAL, value) + + def _equivalence(self, operator: EquivalenceOperator, value: Any) -> BaseModel: + if isinstance(value, bool): # bool ⊂ int — must precede the number arm + return BooleanComparisonFilter(operator=operator, value=value) + if isinstance(value, str): + return StringComparisonFilter(operator=operator, value=value) + if isinstance(value, (int, float)): + return NumberComparisonFilter(operator=operator, value=value) + return DefaultFilter(operator=operator, value=value) + + # --- gt / gte / lt / lte: number -> Number, else Default (Money dict, date) --- + # Precise (number) overload first; its intentional overlap with the Any fallback + # is suppressed on the first overload of each builder. + @overload + def gt( # pyright: ignore[reportOverlappingOverload] + self, value: Union[int, float] + ) -> NumberComparisonFilter: ... + @overload + def gt(self, value: Any) -> DefaultFilter: ... + def gt(self, value: Any) -> BaseModel: + """Build a ``gt`` filter, typed to the value's precise model.""" + return self._comparison(ComparisonOperator.GREATER_THAN, value) + + @overload + def gte( # pyright: ignore[reportOverlappingOverload] + self, value: Union[int, float] + ) -> NumberComparisonFilter: ... + @overload + def gte(self, value: Any) -> DefaultFilter: ... + def gte(self, value: Any) -> BaseModel: + """Build a ``gte`` filter, typed to the value's precise model.""" + return self._comparison(ComparisonOperator.GREATER_THAN_OR_EQUAL, value) + + @overload + def lt( # pyright: ignore[reportOverlappingOverload] + self, value: Union[int, float] + ) -> NumberComparisonFilter: ... + @overload + def lt(self, value: Any) -> DefaultFilter: ... + def lt(self, value: Any) -> BaseModel: + """Build a ``lt`` filter, typed to the value's precise model.""" + return self._comparison(ComparisonOperator.LESS_THAN, value) + + @overload + def lte( # pyright: ignore[reportOverlappingOverload] + self, value: Union[int, float] + ) -> NumberComparisonFilter: ... + @overload + def lte(self, value: Any) -> DefaultFilter: ... + def lte(self, value: Any) -> BaseModel: + """Build a ``lte`` filter, typed to the value's precise model.""" + return self._comparison(ComparisonOperator.LESS_THAN_OR_EQUAL, value) + + def _comparison(self, operator: ComparisonOperator, value: Any) -> BaseModel: + # Numbers map to NumberComparisonFilter; bool is excluded (it is not a + # comparable number here and NumberComparisonFilter rejects it). Money + # dicts and ISODate strings fall through to DefaultFilter, where the + # registered filter_type model validates them at call time. + if not isinstance(value, bool) and isinstance(value, (int, float)): + return NumberComparisonFilter(operator=operator, value=value) + return DefaultFilter(operator=operator, value=value) + + # --- in_ / not_in: list[str] -> StringArray, list[number] -> NumberArray --- + @overload + def in_( # pyright: ignore[reportOverlappingOverload] + self, value: list[str] + ) -> StringArrayFilter: ... + @overload + def in_(self, value: list[Union[int, float]]) -> NumberArrayFilter: ... + @overload + def in_(self, value: list[Any]) -> DefaultFilter: ... + def in_(self, value: list[Any]) -> BaseModel: + """Build an ``in`` filter (Python keyword workaround: ``f.in_``).""" + return self._array(ArrayOperator.IN, value) + + @overload + def not_in( # pyright: ignore[reportOverlappingOverload] + self, value: list[str] + ) -> StringArrayFilter: ... + @overload + def not_in(self, value: list[Union[int, float]]) -> NumberArrayFilter: ... + @overload + def not_in(self, value: list[Any]) -> DefaultFilter: ... + def not_in(self, value: list[Any]) -> BaseModel: + """Build a ``notIn`` filter.""" + return self._array(ArrayOperator.NOT_IN, value) + + def _array(self, operator: ArrayOperator, value: list[Any]) -> BaseModel: + # An all-string list is a StringArrayFilter; an all-number (non-bool) list + # is a NumberArrayFilter. Mixed / Money-dict lists fall through to + # DefaultFilter. An empty list is treated as a string array (the common + # case; the registered filter_type re-validates either way). + if all(isinstance(v, str) for v in value): + return StringArrayFilter(operator=operator, value=value) + if all(not isinstance(v, bool) and isinstance(v, (int, float)) for v in value): + return NumberArrayFilter(operator=operator, value=value) + return DefaultFilter(operator=operator, value=value) + + # --- like / not_like: always String --- + def like(self, value: str) -> StringComparisonFilter: + """Build a ``like`` string filter.""" + return StringComparisonFilter(operator=StringOperator.LIKE, value=value) + + def not_like(self, value: str) -> StringComparisonFilter: + """Build a ``notLike`` string filter.""" + return StringComparisonFilter(operator=StringOperator.NOT_LIKE, value=value) + + # --- between / outside: number range -> NumberRange, else Default --- + @overload + def between( # pyright: ignore[reportOverlappingOverload] + self, min: Union[int, float], max: Union[int, float] + ) -> NumberRangeFilter: ... + @overload + def between(self, min: Any, max: Any) -> DefaultFilter: ... + def between(self, min: Any, max: Any) -> BaseModel: + """Build a ``between`` range filter over ``{min, max}``.""" + return self._range(RangeOperator.BETWEEN, min, max) + + @overload + def outside( # pyright: ignore[reportOverlappingOverload] + self, min: Union[int, float], max: Union[int, float] + ) -> NumberRangeFilter: ... + @overload + def outside(self, min: Any, max: Any) -> DefaultFilter: ... + def outside(self, min: Any, max: Any) -> BaseModel: + """Build an ``outside`` range filter over ``{min, max}``.""" + return self._range(RangeOperator.OUTSIDE, min, max) + + def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: + # A numeric min+max is a NumberRangeFilter; date / Money ranges (whose + # value sub-models differ) fall through to DefaultFilter, where the + # registered filter_type (DATE_RANGE / MONEY_RANGE) validates the shape. + numeric = all( + not isinstance(v, bool) and isinstance(v, (int, float)) for v in (min, max) ) + if numeric: + return NumberRangeFilter( + operator=operator, value=NumberRange(min=min, max=max) + ) + return DefaultFilter(operator=operator, value={"min": min, "max": max}) #: Module-level singleton — use as ``f.eq(...)``, ``f.in_([...])``, etc. @@ -343,7 +464,7 @@ def classify_filters( routes: PluginRoutes, resource: str, method: str, - consumer_filters: Mapping[str, DefaultFilter | dict[str, Any]], + consumer_filters: Mapping[str, Union[BaseModel, dict[str, Any]]], ) -> OppFilters: """Classify consumer filter dict into the ``OppFilters`` search request body. diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index eb846f358..17f0b50a7 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -1,8 +1,9 @@ """Plugin assembly: ``PluginSchemas``, ``Plugin``, and ``define_plugin``. A plugin maps the schema extensions an author builds with ``schema(...)`` onto the -registered extensible schemas, keyed by registry name. Schemas a plugin does not -extend fall back to the base schema (a ``SchemaOnly``), never ``None``, so +registered extensible schemas, keyed by registry name, and (optionally) registers +per-route custom filters via ``routes`` (a ``PluginRoutes`` map). Schemas a plugin +does not extend fall back to the base schema (a ``SchemaOnly``), never ``None``, so consumers get fully-typed, non-optional dot access: ``plugin.schemas.Opportunity``. """ @@ -20,7 +21,7 @@ SchemaWithTransforms, schema, ) -from .types import PluginMeta +from .types import PluginMeta, PluginRoutes SchemasT = TypeVar("SchemasT") @@ -59,19 +60,31 @@ class Plugin(Generic[SchemasT]): """The plugin singleton consumers import. ``schemas`` is a typed frozen dataclass, so ``plugin.schemas.Opportunity`` is - fully typed (dot access). + fully typed (dot access). ``routes`` is the route-keyed custom-filter + registration (``PluginRoutes``) a consumer passes to ``classify_filters``. """ schemas: SchemasT + routes: PluginRoutes meta: PluginMeta -def define_plugin(schemas: SchemasT, *, meta: PluginMeta) -> Plugin[SchemasT]: - """Assemble the plugin from a ``PluginSchemas`` instance and metadata. +def define_plugin( + schemas: SchemasT, + *, + routes: PluginRoutes | None = None, + meta: PluginMeta, +) -> Plugin[SchemasT]: + """Assemble the plugin from schema extensions, optional route registrations, + and metadata. - Each attribute name must equal the entry's ``schema_name``, so + Each schema attribute name must equal the entry's ``schema_name``, so ``schemas.Opportunity`` really holds the Opportunity extensible schema. + ``routes`` is the route-keyed custom-filter registration (``PluginRoutes``), + threaded onto the returned plugin so a consumer can pass ``plugin.routes`` to + ``classify_filters``. Omitted, it defaults to an empty map. + Raises: PluginDefinitionError: If any slot does not hold a schema extension, or holds one whose ``schema_name`` does not match its attribute name. @@ -88,4 +101,4 @@ def define_plugin(schemas: SchemasT, *, meta: PluginMeta) -> Plugin[SchemasT]: ) if errors: raise PluginDefinitionError("plugin", errors) - return Plugin(schemas=schemas, meta=meta) + return Plugin(schemas=schemas, routes=routes or {}, meta=meta) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py index 9ea43cf06..2f535ec77 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py @@ -28,9 +28,20 @@ # Opportunity Filters "OppDefaultFilters", "OppFilters", + "OpportunityFilters", # String Filters "StringArrayFilter", "StringComparisonFilter", + # Typed authoring-DX aliases. Range filters have no clean alias: ``DateRange`` + # / ``MoneyRange`` / ``NumberRange`` (above) are the range *value* sub-models, + # so range fields are typed with the explicit ``*RangeFilter`` names instead. + "BooleanComparison", + "DateComparison", + "MoneyComparison", + "NumberArray", + "NumberComparison", + "StringArray", + "StringComparison", ] from .base import ( @@ -60,8 +71,16 @@ NumberRangeFilter, ) from .opportunity import ( + BooleanComparison, + DateComparison, + MoneyComparison, + NumberArray, + NumberComparison, OppDefaultFilters, OppFilters, + OpportunityFilters, + StringArray, + StringComparison, ) from .string import ( StringArrayFilter, diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py index 197a21145..bb6cab981 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py @@ -1,13 +1,30 @@ """Schemas for the CommonGrants API.""" -from typing import Optional +from typing import Optional, TypedDict from .base import DefaultFilter -from .date import DateRangeFilter -from .money import MoneyRangeFilter -from .string import StringArrayFilter +from .boolean import BooleanComparisonFilter +from .date import DateComparisonFilter, DateRangeFilter +from .money import MoneyComparisonFilter, MoneyRangeFilter +from .numeric import NumberArrayFilter, NumberComparisonFilter +from .string import StringArrayFilter, StringComparisonFilter from pydantic import BaseModel, Field +# Clean aliases for the typed authoring DX. A plugin author declaring custom +# filters reads ``agency: StringArray`` rather than ``agency: StringArrayFilter``; +# the alias *is* the Pydantic value model, so call-site values still validate. +StringArray = StringArrayFilter +StringComparison = StringComparisonFilter +NumberArray = NumberArrayFilter +NumberComparison = NumberComparisonFilter +DateComparison = DateComparisonFilter +MoneyComparison = MoneyComparisonFilter +BooleanComparison = BooleanComparisonFilter + +# Range filters have no clean alias: ``DateRange`` / ``MoneyRange`` / ``NumberRange`` +# are already the range *value* sub-models (a range filter's ``.value``), so range +# fields are typed with the explicit ``*RangeFilter`` names to avoid the collision. + class OppDefaultFilters(BaseModel): """Standard filters available for searching opportunities.""" @@ -46,3 +63,26 @@ class OppFilters(OppDefaultFilters): description="Additional custom filters to apply to the search", alias="customFilters", ) + + +class OpportunityFilters(TypedDict, total=False): + """Typed authoring surface for opportunity filters — the dict consumers + annotate, extended to give each custom filter its own typed key. + + Keys are the wire names a consumer passes to ``classify_filters`` (the + camelCase aliases of :class:`OppDefaultFilters`), each typed to its value + model. A plugin author extends this to register custom filters for the route:: + + class OppSearchFilters(OpportunityFilters, total=False): + agency: StringArray + + ``total=False`` so every standard key is optional. This is the static + authoring surface only; the throw-based ``classify_filters`` runtime is + unchanged and does not consume it. + """ + + status: StringArray + closeDateRange: DateRangeFilter + totalFundingAvailableRange: MoneyRangeFilter + minAwardAmountRange: MoneyRangeFilter + maxAwardAmountRange: MoneyRangeFilter diff --git a/lib/python-sdk/examples/consumer_search_with_filters.py b/lib/python-sdk/examples/consumer_search_with_filters.py new file mode 100644 index 000000000..c80c8506f --- /dev/null +++ b/lib/python-sdk/examples/consumer_search_with_filters.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""End-to-end consumer: search opportunities WITH custom filters, via the SDK client. + +The full downstream flow an adopter writes against the SDK: + + AUTHOR registers the custom filters a route accepts on the plugin + (``define_plugin(routes=...)`` → ``plugin.routes``, a ``PluginRoutes`` map + of ``CustomFilterSpec``), and extends the ``OpportunityFilters`` TypedDict + so the call-site filter dict narrows per key. + CONSUMER builds a filter dict with the ``f.*`` builders and calls + ``client.opportunities.search(search=..., filters=..., routes=plugin.routes)``. + The client runs ``classify_filters`` to build the three-bucket request body + (default named fields + ``customFilters`` record), POSTs it, and returns + typed ``OpportunityBase`` rows. + +The ``f.*`` builders return the precise filter model per value type +(``f.in_([...])`` -> ``StringArrayFilter``, ``f.gte(3)`` -> ``NumberComparisonFilter``), +so a dict built with ``f.*`` composes directly with the typed ``OppSearchFilters`` bag +the route registered — each key narrows to its value model (the ``assert_type`` lines +prove it). + +Run against a CommonGrants endpoint on http://localhost:8000 (see examples/README.md). +The Pennsylvania reference API (``examples/pa-opportunity-example``) accepts the +``customFilters`` body and echoes it back in ``filterInfo`` (it narrows results on the +standard ``status`` filter; how a deployment applies custom filters is its own concern): + + poetry run python examples/consumer_search_with_filters.py +""" + +from __future__ import annotations + +from typing_extensions import assert_type + +from common_grants_sdk.client import Client +from common_grants_sdk.client.config import Config +from common_grants_sdk.extensions import ( + CustomFilterSpec, + CustomFilterType, + PluginMeta, + PluginSchemas, + define_plugin, + f, + schema, +) +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + NumberComparison, + OpportunityFilters, + StringArray, +) +from common_grants_sdk.schemas.pydantic.models import OpportunityBase + +# --------------------------------------------------------------------------- +# Author: register the opportunities-search custom filters on the plugin +# --------------------------------------------------------------------------- +# Typed authoring surface: extend OpportunityFilters so the call-site filter dict +# narrows per key (agency -> StringArray, awardCount -> NumberComparison). + + +class OppSearchFilters(OpportunityFilters, total=False): + """The opportunities-search filters this plugin accepts: standard + custom.""" + + agency: StringArray + awardCount: NumberComparison + + +# Route registration the client's classifier consumes (see define_plugin(routes=...)). +# Each custom filter's value shape is enforced at call time by its filter_type model. +plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), + routes={ + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ), + "awardCount": CustomFilterSpec( + filter_type=CustomFilterType.NUMBER_COMPARISON, + description="Number of awards expected", + ), + } + } + } + }, + meta=PluginMeta(name="grants.gov", source_system="grants.gov"), +) + + +# --------------------------------------------------------------------------- +# Consumer: build filters, search, iterate the typed results +# --------------------------------------------------------------------------- +def demo() -> None: + print("Consumer: search opportunities with custom filters\n") + + config = Config( + base_url="http://localhost:8000", api_key="two_orgs_user_key", timeout=5.0 + ) + client = Client(config) + + # A filter dict built with the f.* helpers. The builders return the precise + # per-key models, so each value narrows to exactly the type OppSearchFilters + # declares. These assert_type lines are the composition contract. + assert_type(f.in_(["open", "forecasted"]), StringArray) # status / agency value + assert_type(f.gte(3), NumberComparison) # awardCount value + + filters: OppSearchFilters = { + "status": f.in_(["open", "forecasted"]), # default -> top level + "agency": f.in_(["NSF", "NIH"]), # registered custom -> customFilters + "awardCount": f.gte(3), # registered custom -> customFilters + } + + response = client.opportunities.search( + search="conservation", + filters=filters, + routes=plugin.routes, + ) + + print(f" -> {len(response.items)} opportunities returned (typed OpportunityBase):") + for opp in response.items: + # opp is a typed OpportunityBase: dot access on concrete fields. + print(f" - {opp.title} [{opp.status.value}] ({opp.id})") + + +if __name__ == "__main__": + demo() diff --git a/lib/python-sdk/tests/client/test_opportunities.py b/lib/python-sdk/tests/client/test_opportunities.py index a22a01389..5d75ae3ea 100644 --- a/lib/python-sdk/tests/client/test_opportunities.py +++ b/lib/python-sdk/tests/client/test_opportunities.py @@ -19,7 +19,11 @@ OpportunitiesSearchResponse, ) from common_grants_sdk.schemas.pydantic.models.opp_status import OppStatusOptions -from common_grants_sdk.extensions.specs import CustomFieldSpec +from common_grants_sdk.extensions.specs import ( + CustomFieldSpec, + CustomFilterSpec, + CustomFilterType, +) @pytest.fixture @@ -1173,3 +1177,66 @@ def mock_post(*args, **kwargs): search="local", status=[OppStatusOptions.OPEN], page=None ) assert exc_info.value.error.status == 500 + + def test_search_routes_custom_filters_into_request_body( + self, client, mock_httpx_client, sample_search_response + ): + """search(filters=, routes=) lands registered custom filters in the customFilters bucket.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + routes = { + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + client.opportunities.search( + search="local", + filters={"agency": {"operator": "in", "value": ["NSF", "NIH"]}}, + routes=routes, + page=1, + ) + + body = mock_httpx_client.post.call_args[1]["json"] + assert body["filters"]["customFilters"]["agency"]["value"] == ["NSF", "NIH"] + + def test_search_date_filter_body_is_json_serializable( + self, client, mock_httpx_client, sample_search_response + ): + """A date-valued filter must reach the wire as ISO strings, not datetime.date. + + Regression: httpx encodes ``json=`` with the stdlib ``json.dumps``, which + raises on ``datetime.date``; the body dump must use ``mode="json"``. + """ + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + client.opportunities.search( + search="local", + filters={ + "closeDateRange": { + "operator": "between", + "value": {"min": "2026-01-01", "max": "2026-12-31"}, + } + }, + page=1, + ) + + body = mock_httpx_client.post.call_args[1]["json"] + # httpx encodes json= with the stdlib encoder, which rejects datetime.date. + json.dumps(body) + assert body["filters"]["closeDateRange"]["value"]["min"] == "2026-01-01" diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py index ac410f13e..a332b06f2 100644 --- a/lib/python-sdk/tests/extensions/test_filters.py +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -67,12 +67,16 @@ def test_f_helper_wire_values(helper, args, operator, value): """Every f helper produces its documented wire operator and value shape. + Asserts the JSON wire dump, not the in-memory ``.value``: the builders return + precise filter models whose ``.value`` may be a sub-model (e.g. ``NumberRange``). + Includes the reserved-word workarounds: Python f.in_ / f.not_in produce wire operators "in" / "notIn". """ flt = getattr(f, helper)(*args) - assert flt.operator == operator - assert flt.value == value + wire = flt.model_dump(by_alias=True, mode="json") + assert wire["operator"] == operator + assert wire["value"] == value # --------------------------------------------------------------------------- @@ -364,6 +368,22 @@ def test_validate_filter_call_money_range_rejects_comparison_operator(): ) +def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): + """A registered numberRange filter round-trips its NumberRange sub-model to wire dict. + + f.between(int, int) returns a NumberRangeFilter whose ``.value`` is a NumberRange + sub-model (not a plain dict); validation must accept it and model_dump must recurse + the sub-model to ``{min, max}``. A regression that left ``.value`` as an un-dumped + NumberRange object — or that dropped the int payload — would ship a non-JSON body. + The moneyRange analog above is covered; this pins the numeric path. + """ + spec = CustomFilterSpec(filter_type=CustomFilterType.NUMBER_RANGE) + validated = validate_filter_call(spec, "awardCount", f.between(0, 1000)) + wire = validated.model_dump(by_alias=True, exclude_none=True, mode="json") + assert wire["operator"] == "between" + assert wire["value"] == {"min": 0, "max": 1000} + + def test_classify_default_wrong_shape_raises_plugin_error(): """A wrong-shaped DEFAULT filter raises FilterError, not a raw pydantic ValidationError. diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index f73e5b6af..c5db350f7 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -8,9 +8,12 @@ from common_grants_sdk.extensions import ( CustomField, CustomFieldSet, + CustomFilterSpec, + CustomFilterType, PassthroughModel, Plugin, PluginMeta, + PluginRoutes, PluginSchemas, SchemaOnly, SchemaWithTransforms, @@ -99,3 +102,39 @@ def test_plugin_is_frozen(): plugin = define_plugin(PluginSchemas(), meta=_meta()) with pytest.raises((AttributeError, TypeError)): plugin.meta = _meta() # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# Route registration (PluginRoutes threaded onto the plugin) +# --------------------------------------------------------------------------- + + +def test_define_plugin_threads_routes_onto_plugin(): + """The PluginRoutes map passed to define_plugin is threaded onto the plugin. + + Regression: if define_plugin dropped ``routes=``, plugin.routes would be empty + and the registered custom filters would be lost. + """ + registered: PluginRoutes = { + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), + routes=registered, + meta=_meta(), + ) + assert plugin.routes is registered + + +def test_define_plugin_defaults_routes_to_empty_map(): + """Omitting ``routes`` yields a concrete, non-optional empty map (never None).""" + plugin = define_plugin(PluginSchemas(), meta=_meta()) + assert plugin.routes == {} From 89e2b00aa2f2a7959c0609896c8ae1489daf3e90 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:27:58 -0700 Subject: [PATCH 37/51] docs(adr): amend ADR-0012 + ADR-0022 to match the implemented custom-filters design (#941) * docs(adr): #898 use operator (not operation) in ADR-0012 filter schema * docs(adr): #898 record route-keyed custom-filter declaration in ADR-0022 * docs(adr): #898 align ADR-0012 value-field wording with TypeSpec --- .../docs/governance/adr/0012-filtering.md | 20 ++++++------- .../governance/adr/0022-plugin-framework.mdx | 29 ++++++++++++++----- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/website/src/content/docs/governance/adr/0012-filtering.md b/website/src/content/docs/governance/adr/0012-filtering.md index aaa7ae143..72aa5b350 100644 --- a/website/src/content/docs/governance/adr/0012-filtering.md +++ b/website/src/content/docs/governance/adr/0012-filtering.md @@ -28,14 +28,14 @@ API endpoints that support filtering should be POST operations that accept a `fi - The `filters` parameter MUST be included at the root of the request body. - Each filter MUST conform to the `Filter` schema, which contains: -| Property | Type | Required | Description | -| ----------- | ---- | -------- | ---------------------------------------- | -| `operation` | enum | Yes | The operation to perform on the value | -| `value` | any | Yes | The data to use for the filter operation | +| Property | Type | Required | Description | +| ---------- | ---- | -------- | ----------------------------------------- | +| `operator` | enum | Yes | The operator to apply to the filter value | +| `value` | any | Yes | The value to use for the filter operation | -- Supported operations: +- Supported operators: -| Operation | Description | Supported `value` types | +| Operator | Description | Supported `value` types | | --------- | ------------------------ | -------------------------------------------- | | `eq` | Equal to | string, number, boolean, date | | `neq` | Not equal to | string, number, boolean, date | @@ -63,14 +63,14 @@ An example of a request body that includes a _standard_ filter: "filters": { "title": { "value": "example", - "operation": "like" + "operator": "like" }, "closedDateRange": { "value": { "min": "2024-01-01", "max": "2024-01-31" }, - "operation": "between" + "operator": "between" } } } @@ -83,12 +83,12 @@ An example of a request body that includes both _standard_ and _custom_ filters: "filters": { "title": { "value": "example", - "operation": "like" + "operator": "like" }, "customFilters": { "agency": { "value": ["Department of Transportation"], - "operation": "in" + "operator": "in" } } } diff --git a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx index f72528391..d004f047d 100644 --- a/website/src/content/docs/governance/adr/0022-plugin-framework.mdx +++ b/website/src/content/docs/governance/adr/0022-plugin-framework.mdx @@ -31,7 +31,7 @@ We decided to: 1. **Keep "plugin" as the unified term** for both the published npm/PyPI packages in the website catalog and the runtime SDK object. No change to `PluginSourceEntry` or `src/content/plugins/index.json`. The existing `definePlugin()` function is expanded to accept the full set of top-level fields described below. -2. **Use functional grouping at the top level** with two keys — `meta` and `schemas` — rather than grouping by object name at the root. Client configuration, auth, and custom filters are deferred to future capabilities and are not part of the current Plugin shape. +2. **Use functional grouping at the top level** with keys for `meta`, `schemas`, and `routes` — rather than grouping by object name at the root. Client configuration and auth are deferred to future capabilities and are not part of the current Plugin shape. 3. **Use per-object grouping inside `schemas`** where it reflects real coupling: each object's source schema, CommonGrants schema, and bidirectional transforms are tightly coupled and change together. @@ -47,6 +47,8 @@ We decided to: 9. **Transformation errors carry structured context.** SDK-emitted transformation errors extend a single `PluginError` base carrying field path, handler name, source value, and underlying cause, enabling programmatic reasoning without parsing error text. The source value may contain PII when transforming applicant data; adopters are responsible for redacting it before logging or re-raising, and the SDK does not redact by default. +10. **Declare custom filters per route, not per schema.** Custom filters are declared in a route-keyed `routes` map — `routes[resource][method].filters[name]` carries a `CustomFilterSpec`. Filters attach to resource methods rather than schemas because filter availability varies by method, and classification resolves a filter by resource + method — a schema-keyed map cannot express this. A filter's `filterType` determines its allowed operators (derived, not authored). The SDKs provide the route declaration in language-idiomatic typed forms. + The resulting Plugin shape: ``` @@ -127,8 +129,14 @@ interface SchemaMappings { fromCommon?: Record<string, unknown>; // ADR-0017 mapping: CommonGrants → source } -// Scalar types only — filters are query parameters, not schema fields -type CustomFilterType = "string" | "number" | "integer" | "boolean"; +// filterType names a semantic filter type (e.g. "stringArray", "dateRange", +// "moneyComparison"); each maps 1:1 to a base filter with auto-derived operators. +// See the SDK for the enumerated set. +type CustomFilterType = + | "stringComparison" + | "stringArray" + | "dateRange" + | "moneyComparison"; interface CustomFilterSpec { filterType: CustomFilterType; @@ -140,6 +148,9 @@ interface Plugin { schemas: Partial< Record<ExtensibleSchemaName, SchemaConfig<unknown, unknown>> >; + // Route-keyed custom filter declarations (Decision #10): + // resource → method → { filters: { name: CustomFilterSpec } }. Typed as PluginRoutes in the SDK. + routes?: PluginRoutes; } // Input object for definePlugin(). Using a named-options object makes it easy to add @@ -149,6 +160,8 @@ interface DefinePluginOptions { // Plugin authors provide input schemas and transforms; definePlugin() compiles them // into the full SchemaConfig runtime type, extending the base schema with any customFields. schemas?: Partial<Record<ExtensibleSchemaName, SchemaInput>>; + // Route-keyed custom filter declarations, passed through to Plugin.routes unchanged. + routes?: PluginRoutes; } // Factory: all options are optional so adopters can start with only what they need @@ -271,8 +284,10 @@ class ObjectSchemasInput(Generic[TSource, TCommon]): to_common: Callable[[TSource], TransformResult[TCommon]] | None = None from_common: Callable[[TCommon], TransformResult[TSource]] | None = None -# Scalar types only — filters are query parameters, not schema fields -CustomFilterType = Literal['string', 'number', 'integer', 'boolean'] +# filter_type names a semantic filter type (e.g. 'stringArray', 'dateRange', +# 'moneyComparison'); each maps 1:1 to a base filter with auto-derived operators. +# See the SDK for the enumerated set. +CustomFilterType = Literal['stringComparison', 'stringArray', 'dateRange', 'moneyComparison'] @dataclass class CustomFilterSpec: @@ -558,7 +573,7 @@ print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms"] ### Consequences - **Positive consequences** - - Top-level surface (`meta`, `schemas`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only + - Top-level surface (`meta`, `schemas`, `routes`) is short, closed, and stable — adding protocol objects adds a key under `schemas` only - Per-object grouping inside `schemas` preserves the real coupling between source schema, CommonGrants schema, and bidirectional transforms — they share type signatures and change together - All per-object declarations (custom fields, source type, declarative mappings, and explicit callables) are co-located under `schemas.<Object>` — no split across multiple top-level keys - `toCommon`/`fromCommon` can be plain hand-written functions, generated via `buildTransforms()` and passed in `schemas`, or auto-generated by `definePlugin()` from mappings declared in `schemas.<Object>.mappings` — plugin authors are not required to use a declarative mapping format @@ -568,7 +583,7 @@ print(grants_gov_plugin.meta.capabilities) # ["customFields", "transforms"] - `customFields` is optional — the `customFields`-only config structure remains valid; existing plugin packages require only minimal code changes to adopt `definePlugin()` - All top-level Plugin fields are optional — adopters can start with only what they need and expand incrementally - **Negative consequences** - - Client configuration, auth, and custom filters are deferred to future capabilities — current Plugin shape does not support them + - Client configuration and auth are deferred to future capabilities — current Plugin shape does not support them ### Criteria From 4b9b00ac19ab75e40d56698cff735c723bc98a68 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Fri, 26 Jun 2026 12:32:59 -0700 Subject: [PATCH 38/51] spike(sdk): classify custom filters behind opportunities.search() in both SDKs (#928) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * spike(ts-sdk): classify custom filters behind opportunities.search() Move custom-filter classification off the consumer and into the API client: `search({ filters, routes })` classifies a flat bag via `classifyFilters` inside `buildSearchBody` — matching the existing `statuses` shorthand — so consumers no longer call `classifyFilters` by hand. `routes` arrives per-call, mirroring the `schema` option. Also exports `FILTER_TYPE_SCHEMAS` read-only in both SDKs for consumer introspection. Spike / exploratory — see SPIKE-FINDINGS.md. * style(ts-sdk): apply prettier formatting for check:format * chore(spike): drop SPIKE-FINDINGS.md; findings live in the PR description * spike(python-sdk): classify custom filters behind opportunities.search() Mirror the TS change in the Python client: search() gains `filters` (flat bag) + `routes`, classifies via `classify_filters` to build the OppFilters body — the same role the `status` shorthand plays — and `status` becomes an optional shorthand. Brings the search-endpoint integration to full TS/Python parity (the catalog export was already in both SDKs). * fix(sdk): validate default filters against their real type; drop speculative export classifyFilters now validates default filters (status, closeDateRange, ...) against their real field schema in OppDefaultFiltersSchema and raises FilterError, matching the Python SDK. Default fields previously got a shape-only check and were silently deferred to the server, so a malformed default filter shipped without complaint. Also removes the FILTER_TYPE_SCHEMAS public export from both SDKs: it has no in-repo consumer and the TS copy was a plain mutable const despite the read-only claim. It stays internal until a real consumer need lands (a public export is a one-way door). * fix(sdk): raise on status shorthand/filters-bag collision search() now raises FilterError when status is supplied via both the statuses/status shorthand and a status key in the filters bag, instead of silently letting the shorthand win (last-write). Symmetric across both SDKs. Adds error-path tests in both SDKs that a malformed registered filter surfaces FilterError out of search(), and clarifies the CustomFilterBag typing doc (envelope check at the type level; per-filterType value validation is runtime). * docs(ts-sdk): fix filter-narrowing wording and @ts-expect-error count The compile-time assertions file claimed two @ts-expect-error directives but ships one; corrected. Describes the static guarantee as filter-envelope checking ({ operator, value } shape), with per-filterType value validation noted as runtime (classifyFilters). * refactor(sdk): client-bound routes and surfaced filter feedback - Bind `routes` at client construction instead of passing it per call to search() in both SDKs. routes is fixed plugin config, not per-call data; in TS the client is generic over the routes type so the filter-name narrowing on search({ filters }) still resolves from the bound routes. - Python: surface the server's filterInfo/sortInfo (including filterInfo.errors) instead of fabricating empty placeholders. client.search validates into Filtered and the pagination layer preserves the first page's response envelope, matching the TS SDK's behavior. - Add a Python classifier-level test for MoneyRangeFilter default-field validation, the one default filter type not previously covered. * fix(sdk): omit empty filters from search wire body; keep R generic in buildSearchBody - Python search() omits the `filters` key when neither a filters bag nor the status shorthand is given, matching the TS SDK (which only sets the key when present) — identical wire body on the no-filter path. - TS buildSearchBody takes SearchOptions<OppSchema, R> so the private helper preserves the routes generic instead of widening to the default and casting. * feat(sdk): fail-soft custom-filter validation in search() (collect, don't throw) Custom-filter validation inside opportunities.search() now collects errors instead of throwing on the first bad filter, mirroring the ADR-0022 transform { result, errors } contract. Both SDKs, identical wire/response behavior. - classifyFilters / classify_filters return { result, errors }: a key that fails validation is dropped from the request body and its FilterError collected, rather than aborting the whole call. - search() merges the collected errors into the response's filterInfo.errors (flattened "path: message", client errors ahead of server) and never throws on a call-time filter problem. - status collision (statuses shorthand + filters.status) resolves filters-wins with a collected warning, instead of throwing. - validateRoutes / validate_routes (registration-time) still throw — a malformed plugin declaration has no result to return. * chore(python-sdk): drop ADR-0012 refs from filter comments Strip the ADR citation from the Filtered-validation comment and the filter/sort-info test docstring. Keeps the intent and the Filtered IS-A Paginated quirk; ADR rationale belongs in the PR description, not inline. * fix(sdk): null-safe filterInfo merge + validate routes at construction - TS: guard mergeFilterErrors against a missing filterInfo on the auto-paginate path so a dropped filter stays fail-soft (never throws) - Both SDKs: validate routes at client construction (validateRoutes was exported but never wired in); README updated to match - Strip stale comment framing (a "today", a review "FINDING" label) and a tooling prefix from published source - Tests: cut a wording-coupled assertion, rename a mislabeled test, add regression tests for the two behavior changes * test(ts-sdk): pin client-before-server order of filterInfo.errors Mirror the Python ordering assertion: a search whose dropped-filter error must precede the server-provided error. Closes the cross-SDK coverage gap where TS asserted only error membership, not order. * style(ts-sdk): prettier-format the routes-validation test * feat(sdk): reject custom filters on unsupported routes; address review feedback Route-support validation: validate_routes / validateRoutes now raise FilterError when custom filters are declared on a route that does not support them (only opportunities.search is filter-capable), in both SDKs, each with a regression test. Also from review: - ClassifyResult is generic (Generic[T] / ClassifyResult<T>) so the result container can extend past opportunities; classify returns ClassifyResult[OppFilters]. - validateFilterCall / validate_filter_call are now internal — dropped from the public extensions surface; classifyFilters runs call-time validation. Docs updated. - Rename SearchFilterNames -> CustomFilterNames (TS) for vocabulary consistency. - Client.search: note that request_data already carries the assembled filters. - De-emphasize a shouted code comment. --- .../common_grants_sdk/client/client.py | 22 +- .../common_grants_sdk/client/opportunities.py | 86 ++-- .../common_grants_sdk/client/pagination.py | 33 +- .../common_grants_sdk/extensions/__init__.py | 5 +- .../common_grants_sdk/extensions/filters.py | 214 +++++--- .../common_grants_sdk/extensions/types.py | 31 +- .../examples/consumer_search_with_filters.py | 9 +- lib/python-sdk/examples/custom_filters.py | 46 +- lib/python-sdk/tests/client/test_client.py | 76 ++- .../tests/client/test_opportunities.py | 253 +++++++-- .../tests/extensions/test_filters.py | 479 ++++++++++++------ .../__tests__/client/opportunities.spec.ts | 218 +++++++- .../__tests__/client/search-filters-types.ts | 72 +++ .../extensions/custom-filters.spec.ts | 171 +++++-- lib/ts-sdk/examples/custom-filters.ts | 8 +- lib/ts-sdk/src/client/client.ts | 16 +- lib/ts-sdk/src/client/opportunities.ts | 143 +++++- lib/ts-sdk/src/extensions/README.md | 30 +- lib/ts-sdk/src/extensions/custom-filters.ts | 93 +++- lib/ts-sdk/src/extensions/index.ts | 3 +- 20 files changed, 1572 insertions(+), 436 deletions(-) create mode 100644 lib/ts-sdk/__tests__/client/search-filters-types.ts diff --git a/lib/python-sdk/common_grants_sdk/client/client.py b/lib/python-sdk/common_grants_sdk/client/client.py index bbcd1fbc6..ce50f7936 100644 --- a/lib/python-sdk/common_grants_sdk/client/client.py +++ b/lib/python-sdk/common_grants_sdk/client/client.py @@ -11,7 +11,9 @@ from .opportunities import Opportunities from .pagination import pagination from .types import ItemsT -from ..schemas.pydantic.responses import Paginated +from ..extensions.filters import validate_routes +from ..extensions.types import PluginRoutes +from ..schemas.pydantic.responses import Filtered, Paginated class Client: @@ -21,6 +23,7 @@ def __init__( self, config: Optional[Config] = None, auth: Optional[Auth] = None, + routes: Optional[PluginRoutes] = None, ): """Initialize the CommonGrants client. @@ -28,10 +31,21 @@ def __init__( config: Optional Config instance. If None, a default Config is created. auth: Optional Auth instance. If None, API key authentication is used with the key from config. + routes: Optional plugin ``routes`` declaration (fixed plugin config). + Bound and validated once here; used to classify registered custom + filters in ``opportunities.search``. + + Raises: + FilterError: If ``routes`` declares an unknown ``filter_type``, a + custom filter name that collides with a default filter name, or + custom filters on a route that does not support them (e.g. + ``opportunities.list``). """ self.config = config or Config() self.auth = auth or Auth.api_key(self.config.api_key) self.http = httpx.Client(timeout=self.config.timeout) + self.routes = routes or {} + validate_routes(self.routes) self.opportunities = Opportunities(client=self) def post(self, path: str, **kwargs) -> httpx.Response: @@ -178,11 +192,15 @@ def search( page_size = self.config.page_size try: + # request_data already includes any filters assembled by the resource method. api_response = self.post( path, json=request_data, params={"page": page, "pageSize": page_size} ) api_response.raise_for_status() - result_dict = Paginated[dict].model_validate(api_response.json()) + # Validate into Filtered so the server's sortInfo/filterInfo (incl. + # filterInfo.errors) survive instead of being dropped. + # Filtered IS-A Paginated, so the existing cast still holds. + result_dict = Filtered[dict, dict].model_validate(api_response.json()) result = cast(Paginated[ItemsT], result_dict) except httpx.HTTPError as e: diff --git a/lib/python-sdk/common_grants_sdk/client/opportunities.py b/lib/python-sdk/common_grants_sdk/client/opportunities.py index fd05f49ce..334a0afb1 100644 --- a/lib/python-sdk/common_grants_sdk/client/opportunities.py +++ b/lib/python-sdk/common_grants_sdk/client/opportunities.py @@ -14,7 +14,7 @@ ) from ..schemas.pydantic.models.opp_status import OppStatusOptions from ..extensions.filters import classify_filters -from ..extensions.types import PluginRoutes +from ..extensions.types import FilterError from .types import ItemsT from typing import Any, List, Mapping @@ -111,50 +111,70 @@ def search( self, search: str, status: List[OppStatusOptions] | None = None, - filters: Mapping[str, Any] | None = None, - routes: PluginRoutes | None = None, page: int | None = None, page_size: int | None = None, schema: Type[OpportunityBase] | None = OpportunityBase, + filters: Mapping[str, Any] | None = None, ) -> OpportunitiesSearchResponse: - """Search for opportunities by a query string, with optional custom filters. + """Search for opportunties by a query string Args: search: The string to search for. - status: Status shorthand; merged in as the ``status`` filter when given - (a ``status`` key already in ``filters`` takes precedence). - filters: Consumer filter dict (standard + custom keys), classified into - the request body via ``classify_filters``. Registered custom filters - land in ``customFilters``; register them via ``define_plugin(routes=...)``. - routes: Route-keyed custom-filter registration (e.g. ``plugin.routes``) - used to classify which keys are registered custom filters. + status: List of statuses to search on (shorthand for the ``status`` + default filter). page: Page number (1-indexed). If None, method will fetch all items across all pages and aggregate them into a single response. page_size: Number of items per page. If None, uses the default from client config. schema: OpportunityBase to support custom fields added by the caller. + filters: Flat custom-filter dict (``{name: {"operator", "value"}}``, + e.g. built with the ``f`` helper), classified via + ``classify_filters``. Registered custom filters validate against + the specs declared in the client's ``routes`` (bound at construction). Returns: OpportunitiesSearchResponse with items and pagination info - Raises: - APIError: if the API request fails - FilterError: if a filter is called with an operator its type does not allow + Raises: + APIError: if the API request fails """ - consumer_filters: dict[str, Any] = dict(filters or {}) - if status is not None and "status" not in consumer_filters: - consumer_filters["status"] = {"operator": "in", "value": status} - - opp_filters = classify_filters( - routes or {}, "opportunities", "search", consumer_filters - ) - request = { - "filters": opp_filters.model_dump(by_alias=True, exclude_none=True), + # Classify the custom-filter dict. Fail-soft: invalid filters are dropped + # and their errors collected here rather than raised. + filters_body: dict = {} + filter_errors: list[FilterError] = [] + if filters: + classified = classify_filters( + self.client.routes or {}, "opportunities", "search", filters + ) + filter_errors.extend(classified.errors) + filters_body = classified.result.model_dump( + by_alias=True, exclude_none=True, mode="json" + ) + + if status: + if "status" in filters_body: + # ``status`` given via both the shorthand and ``filters``: + # ``filters`` wins, the shorthand is ignored, a warning collected. + filter_errors.append( + FilterError( + "specified via both the status shorthand and the filters " + "argument; used the filters value", + path="filters.status", + source_value=status, + ) + ) + else: + filters_body["status"] = {"operator": "in", "value": status} + + request: dict = { "pagination": {"page": 1, "pageSize": 10}, "search": search, "sorting": {"sortBy": "lastModifiedAt", "sortOrder": "desc"}, } + # Only set the filters key when non-empty. + if filters_body: + request["filters"] = filters_body request_data = OpportunitySearchRequest.model_validate(request) @@ -174,18 +194,18 @@ def search( for item in paginated_response.items ] - # Convert paginated_response to dict and replace items with hydrated models + # Replace items with the hydrated models. response_data = paginated_response.model_dump(by_alias=True) response_data["items"] = items - response_data["sortInfo"] = { - "sortBy": "", - "sortOrder": "", - "customSortBy": None, - "errros": [], - } - - response_data["filterInfo"] = {"filters": {}, "errors": []} + # Merge collected client-side errors into filterInfo.errors, flattened to + # "{path}: {message}" and ordered before any existing entries. + if filter_errors: + filter_info = response_data.setdefault("filterInfo", {}) + server_errors = filter_info.get("errors") or [] + filter_info["errors"] = [ + f"{e.path}: {e}" for e in filter_errors + ] + server_errors - # Hydrate OpportunitiesListResponse from response data + # Hydrate OpportunitiesSearchResponse from response data return OpportunitiesSearchResponse.model_validate(response_data) diff --git a/lib/python-sdk/common_grants_sdk/client/pagination.py b/lib/python-sdk/common_grants_sdk/client/pagination.py index b44b11ec1..7ebdc2567 100644 --- a/lib/python-sdk/common_grants_sdk/client/pagination.py +++ b/lib/python-sdk/common_grants_sdk/client/pagination.py @@ -67,6 +67,7 @@ def wrapper( # Otherwise, fetch all pages items: list[dict] = [] + first_response: Paginated[ItemsT] | None = None latest_response: Paginated[ItemsT] | None = None current_page = 1 @@ -85,6 +86,8 @@ def wrapper( *bound.args, **bound.kwargs ) items.extend(cast(list[dict], page_response.items)) + if first_response is None: + first_response = page_response latest_response = page_response # Break if max items is reached or no more pages @@ -102,17 +105,33 @@ def wrapper( # Trim items array to not exceed max items limit items = items[: self.config.list_items_limit] - # Build aggregated response + aggregated_pagination_info = PaginatedResultsInfo( + page=1, + pageSize=len(items) or page_size, + totalItems=len(items), + totalPages=1, + ) + + # Build aggregated response. Copy the first page's response so any extra + # envelope fields (search -> Filtered's sortInfo/filterInfo) are preserved; + # this is a no-op for plain Paginated (list), which has no such fields. + if first_response is not None: + return cast( + Paginated[ItemsT], + first_response.model_copy( + update={ + "items": cast(list[ItemsT], items), + "pagination_info": aggregated_pagination_info, + } + ), + ) + + # No response was fetched (e.g. empty result set with no pages). return Paginated[ItemsT]( status=latest_response.status if latest_response else 200, message=latest_response.message if latest_response else "Success", items=cast(list[ItemsT], items), - paginationInfo=PaginatedResultsInfo( - page=1, - pageSize=len(items) or page_size, - totalItems=len(items), - totalPages=1, - ), + paginationInfo=aggregated_pagination_info, ) return wrapper diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index ac57c8bc2..d51efc5e7 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -1,6 +1,6 @@ """Public extension APIs for the CommonGrants Python SDK.""" -from .filters import classify_filters, f, validate_filter_call, validate_routes +from .filters import classify_filters, f, validate_routes from .plugin import ( Plugin, PluginMeta, @@ -28,6 +28,7 @@ ) from .transforms import build_transforms from .types import ( + ClassifyResult, FilterError, Handler, PassthroughModel, @@ -62,12 +63,12 @@ "schema", "validate_into", # Custom filters + "ClassifyResult", "FilterError", "PluginRoutes", "classify_filters", "CustomFilterSpec", "CustomFilterType", "f", - "validate_filter_call", "validate_routes", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index b736a958e..789ce77ad 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -5,10 +5,11 @@ - ``FILTER_TYPE_SCHEMAS`` — map from CustomFilterType to the Pydantic validation model. - ``DEFAULT_FILTER_NAMES`` — frozenset of all core default-filter field names (snake + alias). - ``validate_routes(routes)`` — registration-time validator; raises FilterError. -- ``validate_filter_call(spec, filter_name, value)`` — call-time validator; returns the - wire-ready DefaultFilter; raises FilterError. -- ``classify_filters(routes, resource, method, consumer_filters)`` — classifier producing - the ``OppFilters`` search request body (default named fields + ``customFilters``). +- ``validate_filter_call(spec, filter_name, value)`` — call-time validator; fail-soft, + returns ``(wire-ready DefaultFilter, None)`` on success or ``(None, FilterError)``. +- ``classify_filters(routes, resource, method, consumer_filters)`` — fail-soft classifier + producing a ``ClassifyResult`` (the valid-only ``OppFilters`` search request body plus + the collected ``FilterError``s; never raises on a bad call-time filter value). No generate.py / codegen dependency. Correctness is enforced at runtime by Pydantic v2. """ @@ -54,7 +55,7 @@ ) from .specs import CustomFilterSpec, CustomFilterType -from .types import FilterError, PluginRoutes +from .types import ClassifyResult, FilterError, PluginRoutes # --------------------------------------------------------------------------- # f.* helpers @@ -321,6 +322,12 @@ def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: # validate_routes — registration-time validator # --------------------------------------------------------------------------- +# (resource, method) pairs whose custom filters this client classifies. A route +# is filter-capable when its core operation declares a ``filters`` parameter +# (lib/core routes); this set hardcodes that subset. As more routes gain filter +# support, derive it from the contract rather than extending this literal by hand. +SUPPORTED_CUSTOM_FILTER_ROUTES: set[tuple[str, str]] = {("opportunities", "search")} + def validate_routes(routes: PluginRoutes) -> None: """Registration-time validator for a plugin's route filter declarations. @@ -330,6 +337,9 @@ def validate_routes(routes: PluginRoutes) -> None: 2. Filter name that collides with a core default-filter name in ``DEFAULT_FILTER_NAMES`` (the escape-hatch collision check; a namespaced key such as ``gov.<system>@<filterName>`` passes through as ad-hoc instead). + 3. Filters declared on a route that does not support custom filters, i.e. a + ``(resource, method)`` not in ``SUPPORTED_CUSTOM_FILTER_ROUTES`` (e.g. + ``opportunities.list``, whose core operation declares no ``filters``). Duplicate custom-filter names need no check: ``routes`` is dict-keyed, so a duplicate name cannot reach this validator (a duplicated literal key collapses @@ -349,6 +359,16 @@ def validate_routes(routes: PluginRoutes) -> None: filter_specs = declarations.get("filters") if not filter_specs: continue + if (resource, method) not in SUPPORTED_CUSTOM_FILTER_ROUTES: + supported = ", ".join( + f"{r}.{m}" for r, m in sorted(SUPPORTED_CUSTOM_FILTER_ROUTES) + ) + raise FilterError( + f'Route "{resource}.{method}" does not support custom filters ' + f"(supported: {supported})", + path=f"routes.{resource}.{method}", + source_value=filter_specs, + ) for filter_name, spec in filter_specs.items(): path = f"routes.{resource}.{method}.filters.{filter_name}" if spec.filter_type not in FILTER_TYPE_SCHEMAS: @@ -386,19 +406,22 @@ def validate_filter_call( spec: Optional[CustomFilterSpec], filter_name: str, value: Any, -) -> DefaultFilter: - """Call-time validator for a single filter value. +) -> tuple[Optional[DefaultFilter], Optional[FilterError]]: + """Call-time validator for a single filter value (fail-soft). Validates ``value`` against the Pydantic model for ``spec.filter_type`` when ``spec`` is provided (registered filter), or against ``DefaultFilter`` when ``spec`` is ``None`` (ad-hoc / escape-hatch filter). - Returns the validated filter as a ``DefaultFilter`` carrying the coerced - operator/value — the wire payload is exactly what passed validation, never - the raw input (lax coercion can differ from the input, e.g. ``"42"`` → ``42``). + Never raises: returns ``(validated_filter, None)`` on success or + ``(None, FilterError)`` on failure. The caller collects the error and skips + the key rather than aborting the whole call. + + On success the returned filter carries the coerced operator/value, never the + raw input — lax coercion can differ from the input (e.g. ``"42"`` → ``42``). Model instances are re-validated via ``model_dump()`` rather than trusted: - the filter models are mutable, so an instance that was valid at construction - may not be valid now. + the filter models are mutable, so an instance valid at construction may not + be valid now. Args: spec: The registered ``CustomFilterSpec`` for this filter, or ``None`` for ad-hoc. @@ -406,19 +429,17 @@ def validate_filter_call( value: The filter value to validate (typically a ``DefaultFilter`` instance). Returns: - The validated filter as a ``DefaultFilter`` with coerced operator/value. - - Raises: - FilterError: On operator/value-shape mismatch (wrapping the pydantic - ``ValidationError`` as ``cause``), or on a ``spec.filter_type`` not - present in ``FILTER_TYPE_SCHEMAS`` — call ``validate_routes(routes)`` - at registration time to catch the latter earlier. + ``(DefaultFilter, None)`` when the value is valid, else ``(None, FilterError)``. + The error wraps the pydantic ``ValidationError`` as ``cause``. An unknown + ``spec.filter_type`` (not in ``FILTER_TYPE_SCHEMAS``) also returns a + ``FilterError`` — call ``validate_routes(routes)`` at registration time to + catch that earlier. """ payload = value.model_dump() if isinstance(value, BaseModel) else value if spec is not None: model_cls = FILTER_TYPE_SCHEMAS.get(spec.filter_type) if model_cls is None: - raise FilterError( + return None, FilterError( f'Unknown filter_type "{spec.filter_type}" for filter ' f'"{filter_name}" — call validate_routes(routes) at ' "registration time to catch this earlier", @@ -428,28 +449,64 @@ def validate_filter_call( try: validated = model_cls.model_validate(payload) except ValidationError as exc: - raise FilterError( + return None, FilterError( f'Filter "{filter_name}" failed validation: ' f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", path=f"filters.{filter_name}", source_value=value, cause=exc, - ) from exc + ) # Re-shape to DefaultFilter so the wire bucket carries the coerced # operator/value. DefaultFilter.value is Any per the core spec, so # nothing the typed model accepted can fail here. - return DefaultFilter.model_validate(validated.model_dump()) + return DefaultFilter.model_validate(validated.model_dump()), None # Ad-hoc / escape-hatch: validate against DefaultFilter shape only try: - return DefaultFilter.model_validate(payload) + return DefaultFilter.model_validate(payload), None except ValidationError as exc: - raise FilterError( + return None, FilterError( f'Ad-hoc filter "{filter_name}" has an invalid DefaultFilter shape: ' f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", path=f"filters.{filter_name}", source_value=value, cause=exc, - ) from exc + ) + + +# --------------------------------------------------------------------------- +# Default-field call-time validator (bucket 1) +# --------------------------------------------------------------------------- + + +def _validate_default_field( + alias_key: str, + value: Any, +) -> tuple[Optional[Any], Optional[FilterError]]: + """Validate one default filter value against its REAL field type (fail-soft). + + Bucket-1 (default) values are validated against the named field's real type + on ``OppDefaultFilters`` (e.g. ``status`` → ``StringArrayFilter``) — stricter + than the permissive ``DefaultFilter`` shape. Each field is validated in + isolation by constructing a single-field ``OppDefaultFilters`` keyed by its + alias, so a malformed default can be dropped on its own without discarding + the other valid default fields. + + Fail-soft: each field is validated on its own, collecting the error and + skipping the key on failure. Returns ``(value, None)`` when valid (the + original value is kept — construction is a validation gate, not a reshape) + or ``(None, FilterError)`` when invalid. + """ + try: + OppDefaultFilters.model_validate({alias_key: value}) + except ValidationError as exc: + return None, FilterError( + f'Default filter "{alias_key}" failed validation: ' + f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", + path=f"filters.{alias_key}", + source_value=value, + cause=exc, + ) + return value, None # --------------------------------------------------------------------------- @@ -465,7 +522,7 @@ def classify_filters( resource: str, method: str, consumer_filters: Mapping[str, Union[BaseModel, dict[str, Any]]], -) -> OppFilters: +) -> ClassifyResult[OppFilters]: """Classify consumer filter dict into the ``OppFilters`` search request body. Three-bucket classification: @@ -475,16 +532,19 @@ def classify_filters( given resource/method → land in ``custom_filters``. - Bucket 3 (ad-hoc): any other key → land in ``custom_filters`` passthrough. - The classifier is opportunity-bound today: default names come from - ``OppDefaultFilters`` and the wire body is ``OppFilters``. A future - revision will derive both from the declared resource. + Validation is **fail-soft**: a key that fails its call-time validation is + dropped from the result and its ``FilterError`` collected into + ``ClassifyResult.errors``; the call never raises on a bad filter value. + (Registration-time validation in ``validate_routes`` still raises.) + + Opportunity-bound — see the limitation note above the function. Registered specs are looked up by the exact ``(resource, method)`` strings declared in ``routes``. A non-matching pair (e.g. a pluralization typo in ``resource``) yields no registered bucket at all: every non-default filter is then validated only against the permissive ``DefaultFilter`` shape, exactly - like ad-hoc input, with no error raised. Call sites must pass the same - resource/method strings the plugin declared. + like ad-hoc input. Call sites must pass the same resource/method strings the + plugin declared. Construction normalizes all default consumer keys to the form that ``OppFilters(**kwargs)`` accepts. Because ``OppDefaultFilters`` does NOT set @@ -504,18 +564,13 @@ def classify_filters( call site (raw ``{"operator": ..., "value": ...}`` dicts also accepted). Returns: - ``OppFilters`` request body. Call - ``.model_dump(by_alias=True, exclude_none=True, mode="json")`` for the JSON - body of the search request — ``mode="json"`` is required because + ``ClassifyResult`` — ``.result`` is the valid-only ``OppFilters`` request + body; ``.errors`` is the collected list of ``FilterError``s for keys that + failed (empty on full success). Call + ``.result.model_dump(by_alias=True, exclude_none=True, mode="json")`` for + the JSON body of the search request — ``mode="json"`` is required because coerced ``date`` objects are not JSON-serializable in the default python mode (operator enums are ``StrEnum`` and serialize fine either way). - - Raises: - FilterError: When any filter value fails validation — registered and ad-hoc - values at classification time, default values at ``OppFilters`` - construction — or when the snake_case and camelCase forms of the same - default filter are both supplied. The error surface is uniform across - all three buckets. """ route_declarations = routes.get(resource, {}).get(method, {}) registered_specs: dict[str, CustomFilterSpec] = route_declarations.get( @@ -524,6 +579,7 @@ def classify_filters( default_fields: dict[str, Any] = {} custom_buckets: dict[str, DefaultFilter] = {} + errors: list[FilterError] = [] for key, value in consumer_filters.items(): if key in DEFAULT_FILTER_NAMES: @@ -532,52 +588,54 @@ def classify_filters( # via **kwargs because populate_by_name is not set on OppDefaultFilters. # Normalize: camelCase aliases stay as-is; snake_case keys are mapped to their # alias; keys with no alias (e.g. "status") are passed through unchanged. - # No validate_filter_call here: default values are validated at the wrapped - # OppFilters construction below, against the named field's REAL type (e.g. + # Validate per-field against the named field's actual type (e.g. # status → StringArrayFilter) — stricter than the permissive DefaultFilter - # check, and the single validation point for this bucket. + # check. Fail-soft: an invalid value is collected and skipped. alias_key = _SNAKE_TO_ALIAS.get(key, key) if alias_key in default_fields: # Snake and camel forms of the same field normalize to one key; # without this guard, plain dict assignment would silently drop - # whichever form the consumer's dict ordered first. - raise FilterError( - f'Default filter "{alias_key}" was supplied more than once ' - "(snake_case and camelCase forms of the same filter)", - path=f"filters.{alias_key}", - source_value=value, + # whichever form the consumer's dict ordered first. Fail-soft: + # keep the first-seen value, drop the duplicate, collect a warning. + errors.append( + FilterError( + f'Default filter "{alias_key}" was supplied more than once ' + "(snake_case and camelCase forms of the same filter)", + path=f"filters.{alias_key}", + source_value=value, + ) ) - default_fields[alias_key] = value + continue + validated, error = _validate_default_field(alias_key, value) + if error is not None: + errors.append(error) + continue + default_fields[alias_key] = validated elif key in registered_specs: # Bucket 2: registered custom filter — ship the validated value, - # never the raw input (see validate_filter_call) + # never the raw input (see validate_filter_call). Fail-soft: collect + # the error and skip the key. spec = registered_specs[key] - custom_buckets[key] = validate_filter_call(spec, key, value) + validated, error = validate_filter_call(spec, key, value) + if error is not None: + errors.append(error) + continue + custom_buckets[key] = validated # type: ignore[assignment] else: - # Bucket 3: ad-hoc / escape-hatch passthrough — ship the validated value - custom_buckets[key] = validate_filter_call(None, key, value) + # Bucket 3: ad-hoc / escape-hatch passthrough — ship the validated value. + # Fail-soft: collect the error and skip the key. + validated, error = validate_filter_call(None, key, value) + if error is not None: + errors.append(error) + continue + custom_buckets[key] = validated # type: ignore[assignment] # OppFilters requires the alias form for construction (populate_by_name is not set). # Use "customFilters" (the alias) rather than "custom_filters" (the field name). - # This construction is the validation point for bucket-1 (default) values — wrap - # pydantic failures in FilterError so the error contract is uniform across all - # three buckets (consumers catch `except FilterError` regardless of bucket). - try: - return OppFilters( - **default_fields, - customFilters=custom_buckets if custom_buckets else None, - ) - except ValidationError as exc: - # Name the failing field(s) so bucket-1 errors are as pinpointed as the - # filters.<name> paths raised for buckets 2/3. The loc values are the - # alias keys used at construction (e.g. "closeDateRange"); customFilters - # values were already validated above, so only default fields fail here. - failed = sorted({str(err["loc"][0]) for err in exc.errors() if err.get("loc")}) - field_list = ", ".join(failed) if failed else "<unknown>" - raise FilterError( - f"Filter(s) {field_list} failed validation: " - f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", - path=f"filters.{failed[0]}" if len(failed) == 1 else "filters", - source_value=default_fields, - cause=exc, - ) from exc + # Every value in default_fields / custom_buckets already passed validation + # above, so this construction cannot fail on a value-shape error. + result = OppFilters( + **default_fields, + customFilters=custom_buckets if custom_buckets else None, + ) + return ClassifyResult(result=result, errors=errors) diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 9f15850be..3527a88db 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -3,7 +3,15 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Callable, Generic, Literal, NotRequired, TypedDict, TypeVar +from typing import ( + Any, + Callable, + Generic, + Literal, + NotRequired, + TypedDict, + TypeVar, +) from pydantic import BaseModel, ConfigDict, Field @@ -17,7 +25,7 @@ # Type aliases Handler = Callable[[Any, Any], Any] -# Route-keyed custom-filter declaration types, mirroring the TS SDK shape: +# Route-keyed custom-filter declaration types: # PluginRoutes = {resourceName: {methodName: RouteDeclarations}}. RouteMethodFilters = dict[str, CustomFilterSpec] # {filterName: spec} @@ -105,6 +113,25 @@ def __init__( self.cause = cause +@dataclass +class ClassifyResult(Generic[T]): + """Fail-soft return shape for ``classify_filters``. + + ``classify_filters`` never raises on a bad call-time filter value: each + failing key is dropped from ``result`` and its :class:`FilterError` collected + into ``errors``. + + result: classified filters, containing only the keys that passed validation. + errors: aggregated ``FilterError``s for keys that failed; empty on success. + + Registration-time validation (``validate_routes``) still raises — a malformed + plugin declaration has no result to return. + """ + + result: T + errors: list[FilterError] + + @dataclass class TransformResult(Generic[T]): """Unconditional return shape for to_common / from_common (ADR-0022 Decision #7). diff --git a/lib/python-sdk/examples/consumer_search_with_filters.py b/lib/python-sdk/examples/consumer_search_with_filters.py index c80c8506f..73942085c 100644 --- a/lib/python-sdk/examples/consumer_search_with_filters.py +++ b/lib/python-sdk/examples/consumer_search_with_filters.py @@ -7,8 +7,9 @@ (``define_plugin(routes=...)`` → ``plugin.routes``, a ``PluginRoutes`` map of ``CustomFilterSpec``), and extends the ``OpportunityFilters`` TypedDict so the call-site filter dict narrows per key. - CONSUMER builds a filter dict with the ``f.*`` builders and calls - ``client.opportunities.search(search=..., filters=..., routes=plugin.routes)``. + CONSUMER constructs the client with the registered routes + (``Client(config, routes=plugin.routes)``), builds a filter dict with the + ``f.*`` builders, and calls ``client.opportunities.search(search=..., filters=...)``. The client runs ``classify_filters`` to build the three-bucket request body (default named fields + ``customFilters`` record), POSTs it, and returns typed ``OpportunityBase`` rows. @@ -95,7 +96,8 @@ def demo() -> None: config = Config( base_url="http://localhost:8000", api_key="two_orgs_user_key", timeout=5.0 ) - client = Client(config) + # Routes are bound to the client once at construction (not per search call). + client = Client(config, routes=plugin.routes) # A filter dict built with the f.* helpers. The builders return the precise # per-key models, so each value narrows to exactly the type OppSearchFilters @@ -112,7 +114,6 @@ def demo() -> None: response = client.opportunities.search( search="conservation", filters=filters, - routes=plugin.routes, ) print(f" -> {len(response.items)} opportunities returned (typed OpportunityBase):") diff --git a/lib/python-sdk/examples/custom_filters.py b/lib/python-sdk/examples/custom_filters.py index d47db55d3..4addacd7e 100644 --- a/lib/python-sdk/examples/custom_filters.py +++ b/lib/python-sdk/examples/custom_filters.py @@ -19,7 +19,7 @@ from common_grants_sdk.extensions import classify_filters, f from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType -from common_grants_sdk.extensions.types import FilterError, PluginMeta, PluginRoutes +from common_grants_sdk.extensions.types import PluginMeta, PluginRoutes # --------------------------------------------------------------------------- # Route-keyed custom filter specs + plugin metadata @@ -69,9 +69,9 @@ def main() -> None: # Mixing three kinds of filters in a single flat dict: # "status" — default core filter (snake_case key, no alias) # "close_date_range" — default core filter, snake_case key for an ALIASED field: - # the classifier must normalize it to "closeDateRange" in the - # wire body (an unnormalized snake key would be silently - # dropped from the wire body — the alias landmine) + # the classifier must normalize it to "closeDateRange" + # (an unnormalized snake key is silently dropped — + # the alias landmine) # "agency" — registered custom filter (routes.opportunities.search.agency) # "fundingProgram" — registered custom filter (routes.opportunities.search.fundingProgram) # "legacyTag" — ad-hoc passthrough (not registered, not a core default) @@ -83,35 +83,47 @@ def main() -> None: "legacyTag": f.eq("priority"), } - request_body = classify_filters(routes, "opportunities", "search", consumer_filters) + classified = classify_filters(routes, "opportunities", "search", consumer_filters) print("\nRequest body (by_alias=True, exclude_none=True, mode='json'):") print( json.dumps( - request_body.model_dump(by_alias=True, exclude_none=True, mode="json"), + classified.result.model_dump(by_alias=True, exclude_none=True, mode="json"), indent=2, ) ) - # --- FilterError demo — bad call raises and is caught --- - _section("VALIDATION — bad operator raises FilterError (runtime guarantee)") + # --- FilterError demo — bad call is fail-soft (collected, not raised) --- + _section("VALIDATION — bad operator collected, valid filters still applied") # agency is registered as STRING_ARRAY (expects ArrayOperator: in/notIn). - # Passing f.eq(...) (EquivalenceOperator.EQUAL) triggers call-time validation failure. + # Passing f.eq(...) (EquivalenceOperator.EQUAL) fails call-time validation. + # Fail-soft: classify_filters never raises on a bad + # filter value — the bad key is dropped from the result body and a FilterError + # is collected on .errors. A valid sibling filter still classifies normally. bad_filters = { "agency": f.eq( "wrong-operator-for-array-type" ), # eq is not a valid STRING_ARRAY op + "legacyTag": f.eq("priority"), # valid ad-hoc filter — survives } - try: - classify_filters(routes, "opportunities", "search", bad_filters) - except FilterError as exc: - # str(exc) summarizes the first failure; the structured fields carry - # the full detail — exc.path names the failing filter, exc.cause is + classified_bad = classify_filters(routes, "opportunities", "search", bad_filters) + for err in classified_bad.errors: + # str(err) summarizes the first failure; the structured fields carry + # the full detail — err.path names the failing filter, err.cause is # the underlying pydantic ValidationError for programmatic access. - print(f"FilterError caught: {exc}") - print(f" path: {exc.path}") - print(f" cause: {type(exc.cause).__name__}") + print(f"FilterError collected: {err}") + print(f" path: {err.path}") + print(f" cause: {type(err.cause).__name__}") + # The valid filter still made it into the request body. + print( + "\nValid-only request body: " + + json.dumps( + classified_bad.result.model_dump( + by_alias=True, exclude_none=True, mode="json" + ) + ) + ) # --- Plugin metadata --- _section("PLUGIN METADATA") diff --git a/lib/python-sdk/tests/client/test_client.py b/lib/python-sdk/tests/client/test_client.py index 4ca59562a..124577caa 100644 --- a/lib/python-sdk/tests/client/test_client.py +++ b/lib/python-sdk/tests/client/test_client.py @@ -12,6 +12,8 @@ from common_grants_sdk.client.exceptions import APIError from common_grants_sdk.schemas.pydantic.pagination import PaginatedResultsInfo from common_grants_sdk.schemas.pydantic.responses import Paginated +from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType +from common_grants_sdk.extensions.types import FilterError from pydantic import ValidationError @@ -28,6 +30,25 @@ def test_client_initialization_with_config_and_auth(self): assert client.auth == auth assert isinstance(client.opportunities, type(client.opportunities)) + def test_client_initialization_validates_routes(self): + """Client validates routes at construction and raises on a bad declaration.""" + with patch("common_grants_sdk.client.client.httpx.Client"): + config = Config(base_url="https://api.example.com", api_key="test-key") + # "status" collides with the default status filter name. + bad_routes = { + "opportunities": { + "search": { + "filters": { + "status": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + with pytest.raises(FilterError): + Client(config=config, routes=bad_routes) + def test_client_initialization_defaults_auth(self, monkeypatch): """Test client initialization with default auth from config.""" monkeypatch.setenv("CG_API_BASE_URL", "https://api.example.com") @@ -400,7 +421,11 @@ def sample_get_response(sample_item_data): @pytest.fixture def sample_search_response(sample_item_data): - """Create sample search response.""" + """Create sample search response. + + Search responses are Filtered, so they carry the server's sortInfo and + filterInfo envelope alongside items/paginationInfo. + """ return { "status": 200, "message": "Success", @@ -411,6 +436,13 @@ def sample_search_response(sample_item_data): "totalItems": 2, "totalPages": 1, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } @@ -1116,6 +1148,13 @@ def test_search_some_handles_empty_items(self): "totalItems": 0, "totalPages": 1, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } mock_response = Mock() mock_response.raise_for_status = Mock() @@ -1228,6 +1267,13 @@ def test_search_all_multiple_pages(self, sample_item_data): "totalItems": 5, "totalPages": 3, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } page2_response = { "status": 200, @@ -1239,6 +1285,13 @@ def test_search_all_multiple_pages(self, sample_item_data): "totalItems": 5, "totalPages": 3, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } page3_response = { "status": 200, @@ -1250,6 +1303,13 @@ def test_search_all_multiple_pages(self, sample_item_data): "totalItems": 5, "totalPages": 3, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } def mock_post(*args, **kwargs): @@ -1289,6 +1349,13 @@ def test_search_all_empty_result(self): "totalItems": 0, "totalPages": 1, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } mock_response = Mock() mock_response.raise_for_status = Mock() @@ -1341,6 +1408,13 @@ def test_search_all_handles_error_on_subsequent_page(self, sample_item_data): "totalItems": 5, "totalPages": 3, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } error_data = {"status": 500, "message": "Server error", "errors": []} error_response = Mock() diff --git a/lib/python-sdk/tests/client/test_opportunities.py b/lib/python-sdk/tests/client/test_opportunities.py index 5d75ae3ea..858cd63dd 100644 --- a/lib/python-sdk/tests/client/test_opportunities.py +++ b/lib/python-sdk/tests/client/test_opportunities.py @@ -762,6 +762,220 @@ def test_search_opportunities_success( assert call_args[1]["params"]["page"] == 1 assert call_args[1]["params"]["pageSize"] == 100 + def test_search_surfaces_server_filter_and_sort_info( + self, client, mock_httpx_client, sample_search_response + ): + """search() returns the server's real filterInfo/sortInfo, not blanks. + + The server reports non-fatal filtering feedback in ``filterInfo.errors`` + and the resolved sort in ``sortInfo``. Earlier the client + fabricated empty envelopes, dropping that feedback; this asserts it now + reaches the caller. + """ + sample_search_response["filterInfo"] = { + "filters": {}, + "errors": ["filter 'foo' is unsupported and was ignored"], + } + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + response = client.opportunities.search(search="x") + + assert isinstance(response, OpportunitiesSearchResponse) + assert response.filter_info.errors == [ + "filter 'foo' is unsupported and was ignored" + ] + # sort_info reflects the server value, not the blanked "". + assert response.sort_info.sort_by == "lastModifiedAt" + + def test_search_classifies_custom_filter_bag( + self, mock_httpx_client, sample_search_response + ): + """A flat custom-filter bag is classified inside search(), not by the caller.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + routes = { + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + + # routes is client-bound: supplied once at construction, not per call. + auth = Auth.api_key("test-key") + config = Config( + base_url="https://api.example.com", api_key="test-key", timeout=10.0 + ) + client = Client(config=config, auth=auth, routes=routes) + client.http = mock_httpx_client + client.opportunities.http = mock_httpx_client + + client.opportunities.search( + search="conservation", + status=[OppStatusOptions.OPEN], + filters={ + "agency": {"operator": "in", "value": ["HHS", "NSF"]}, + "legacyTag": {"operator": "eq", "value": "conservation-2024"}, + }, + ) + + sent_filters = mock_httpx_client.post.call_args[1]["json"]["filters"] + # default field (status shorthand) -> top-level + assert sent_filters["status"]["operator"] == "in" + # registered custom + ad-hoc -> customFilters + assert "agency" in sent_filters["customFilters"] + assert "legacyTag" in sent_filters["customFilters"] + # separation invariant: status not duplicated under customFilters + assert "status" not in sent_filters["customFilters"] + + def test_search_status_collision_filters_wins_and_warns( + self, client, mock_httpx_client, sample_search_response + ): + """status via both the shorthand arg and the filters argument: filters wins. + + Fail-soft: rather than raising, search() keeps the + ``filters`` value, ignores the shorthand, and appends a warning to the + response's filterInfo.errors. + """ + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + # filters value uses "forecasted"; shorthand uses OPEN — they must differ + # so the assertion on the winning value can actually fail. + response = client.opportunities.search( + search="conservation", + status=[OppStatusOptions.OPEN], + filters={"status": {"operator": "in", "value": ["forecasted"]}}, + ) + + # The SENT body carries the filters value, not the shorthand. + sent_filters = mock_httpx_client.post.call_args[1]["json"]["filters"] + assert sent_filters["status"]["value"] == ["forecasted"] + # The collision warning reached filterInfo.errors. + assert any( + "filters.status" in e + and "specified via both the status shorthand and the filters argument; " + "used the filters value" in e + for e in response.filter_info.errors + ) + + def test_search_invalid_filter_is_dropped_and_warned_no_raise( + self, mock_httpx_client, sample_search_response + ): + """A malformed registered filter is fail-soft: results return, filter dropped. + + A registered stringArray filter given a non-array ``between`` value fails + classify_filters validation. search() does NOT raise. + The invalid filter is absent from the sent body and its error surfaces in + filterInfo.errors; results still return. + """ + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + routes = { + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + + # routes is client-bound: supplied once at construction, not per call. + auth = Auth.api_key("test-key") + config = Config( + base_url="https://api.example.com", api_key="test-key", timeout=10.0 + ) + client = Client(config=config, auth=auth, routes=routes) + client.http = mock_httpx_client + client.opportunities.http = mock_httpx_client + + response = client.opportunities.search( + search="conservation", + filters={"agency": {"operator": "between", "value": 5}}, + ) + + # Results still return (no raise). + assert isinstance(response, OpportunitiesSearchResponse) + assert len(response.items) == 2 + + # The invalid filter is absent from the sent body. With only the one + # invalid filter, no filters key is sent at all. + sent_body = mock_httpx_client.post.call_args[1]["json"] + assert "agency" not in sent_body.get("filters", {}).get("customFilters", {}) + + # Its error surfaced in filterInfo.errors. + assert any("filters.agency" in e for e in response.filter_info.errors) + + def test_search_invalid_filter_client_errors_precede_server_errors( + self, mock_httpx_client, sample_search_response + ): + """Client-side filter errors are merged AHEAD of server-provided errors.""" + sample_search_response["filterInfo"] = { + "filters": {}, + "errors": ["server: something was ignored"], + } + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + routes = { + "opportunities": { + "search": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY + ) + } + } + } + } + auth = Auth.api_key("test-key") + config = Config( + base_url="https://api.example.com", api_key="test-key", timeout=10.0 + ) + client = Client(config=config, auth=auth, routes=routes) + client.http = mock_httpx_client + client.opportunities.http = mock_httpx_client + + response = client.opportunities.search( + search="conservation", + filters={"agency": {"operator": "between", "value": 5}}, + ) + + errors = response.filter_info.errors + # Client error first, server error last. + assert "filters.agency" in errors[0] + assert errors[-1] == "server: something was ignored" + def test_search_opportunities_different_page( self, client, mock_httpx_client, sample_search_response ): @@ -1146,6 +1360,13 @@ def test_search_all_opportunities_error_on_subsequent_page( "totalItems": 5, "totalPages": 3, }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, } error_data = {"status": 500, "message": "Server error", "errors": []} error_response = Mock() @@ -1178,38 +1399,6 @@ def mock_post(*args, **kwargs): ) assert exc_info.value.error.status == 500 - def test_search_routes_custom_filters_into_request_body( - self, client, mock_httpx_client, sample_search_response - ): - """search(filters=, routes=) lands registered custom filters in the customFilters bucket.""" - mock_response = Mock() - mock_response.status_code = 200 - mock_response.text = json.dumps(sample_search_response) - mock_response.json = Mock(return_value=sample_search_response) - mock_response.raise_for_status = Mock() - mock_httpx_client.post = Mock(return_value=mock_response) - - routes = { - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ) - } - } - } - } - client.opportunities.search( - search="local", - filters={"agency": {"operator": "in", "value": ["NSF", "NIH"]}}, - routes=routes, - page=1, - ) - - body = mock_httpx_client.post.call_args[1]["json"] - assert body["filters"]["customFilters"]["agency"]["value"] == ["NSF", "NIH"] - def test_search_date_filter_body_is_json_serializable( self, client, mock_httpx_client, sample_search_response ): diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py index a332b06f2..691472084 100644 --- a/lib/python-sdk/tests/extensions/test_filters.py +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -17,7 +17,7 @@ validate_routes, ) from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType -from common_grants_sdk.extensions.types import FilterError +from common_grants_sdk.extensions.types import ClassifyResult, FilterError from common_grants_sdk.schemas.pydantic.filters.opportunity import OppFilters # --------------------------------------------------------------------------- @@ -87,9 +87,12 @@ def test_f_helper_wire_values(helper, args, operator, value): def test_classify_default_snake_key_lands_in_named_field(): """Default snake_case key (e.g. "status") lands in a named OppFilters field, not customFilters.""" consumer_filters = {"status": f.in_(["open"])} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert isinstance(classified, ClassifyResult) + assert classified.errors == [] + result = classified.result assert isinstance(result, OppFilters) assert result.status is not None assert result.custom_filters is None @@ -98,9 +101,11 @@ def test_classify_default_snake_key_lands_in_named_field(): def test_classify_default_camel_alias_lands_in_named_field(): """THE LANDMINE: camelCase alias "closeDateRange" must land in named field, NOT customFilters.""" consumer_filters = {"closeDateRange": f.between("2026-01-01", "2026-12-31")} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert classified.errors == [] + result = classified.result assert isinstance(result, OppFilters) # The camelCase alias must normalize to the snake_case field assert result.close_date_range is not None @@ -112,9 +117,11 @@ def test_classify_default_camel_alias_lands_in_named_field(): def test_classify_registered_custom_filter_lands_in_custom_filters(): """A registered custom filter (e.g. "agency") lands in OppFilters.custom_filters.""" consumer_filters = {"agency": f.in_(["NSF", "NIH"])} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert classified.errors == [] + result = classified.result assert isinstance(result, OppFilters) assert result.custom_filters is not None assert "agency" in result.custom_filters @@ -123,9 +130,11 @@ def test_classify_registered_custom_filter_lands_in_custom_filters(): def test_classify_adhoc_unregistered_filter_lands_in_custom_filters(): """An unregistered ad-hoc key (e.g. "legacyTag") passes through to customFilters.""" consumer_filters = {"legacyTag": f.eq("priority")} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert classified.errors == [] + result = classified.result assert isinstance(result, OppFilters) assert result.custom_filters is not None assert "legacyTag" in result.custom_filters @@ -134,9 +143,11 @@ def test_classify_adhoc_unregistered_filter_lands_in_custom_filters(): def test_classify_escape_hatch_key_lands_in_custom_filters(): """gov.<system>@<filter> escape-hatch keys pass through to customFilters.""" consumer_filters = {"gov.someSystem@someFilter": f.eq("test")} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert classified.errors == [] + result = classified.result assert isinstance(result, OppFilters) assert result.custom_filters is not None assert "gov.someSystem@someFilter" in result.custom_filters @@ -157,10 +168,11 @@ def test_request_body_has_default_fields_at_top_level_and_custom_filters_nested( "agency": f.in_(["NSF"]), "legacyTag": f.eq("priority"), } - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) - request_body = result.model_dump(by_alias=True, exclude_none=True) + assert classified.errors == [] + request_body = classified.result.model_dump(by_alias=True, exclude_none=True) # Default field appears at top level assert "status" in request_body @@ -175,10 +187,11 @@ def test_request_body_has_default_fields_at_top_level_and_custom_filters_nested( def test_request_body_no_custom_filters_key_when_all_defaults(): """customFilters key is absent from the request body when all filters are default fields.""" consumer_filters = {"status": f.in_(["open"])} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) - request_body = result.model_dump(by_alias=True, exclude_none=True) + assert classified.errors == [] + request_body = classified.result.model_dump(by_alias=True, exclude_none=True) assert "customFilters" not in request_body @@ -199,9 +212,11 @@ def test_oppfilters_mixed_case_roundtrip(): "2026-01-01", "2026-12-31" ), # camelCase alias default } - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert classified.errors == [] + result = classified.result assert isinstance(result, OppFilters) # camelCase alias must land in named field @@ -274,6 +289,29 @@ def test_validate_routes_collision_with_camel_alias_raises(): validate_routes(routes) +def test_validate_routes_unsupported_route_raises(): + """validate_routes raises FilterError for custom filters on a non-filterable route. + + Only routes whose core operation declares a ``filters`` parameter can carry + custom filters. ``opportunities.list`` has none, so a filter declared there is + a registration-time error rather than a silently-ignored declaration. + """ + routes = { + "opportunities": { + "list": { + "filters": { + "agency": CustomFilterSpec( + filter_type=CustomFilterType.STRING_ARRAY, + description="Not supported on list", + ), + } + } + } + } + with pytest.raises(FilterError, match="does not support custom filters"): + validate_routes(routes) + + def test_validate_routes_valid_routes_do_not_raise(): """validate_routes does not raise for a fully valid routes dict.""" # Should not raise @@ -285,31 +323,41 @@ def test_validate_routes_valid_routes_do_not_raise(): # --------------------------------------------------------------------------- -def test_validate_filter_call_registered_bad_operator_raises(): - """validate_filter_call raises FilterError when a registered filter has an operator/value mismatch.""" - # AGENCY_SPEC is STRING_ARRAY — an "eq" with a scalar value is wrong for StringArrayFilter +def test_validate_filter_call_registered_bad_operator_returns_error(): + """validate_filter_call returns (None, FilterError) for an operator/value mismatch. + + A bad registered filter yields a FilterError, not an exception. AGENCY_SPEC + is STRING_ARRAY — an "eq" with a scalar value is wrong for it. + """ bad_filter = f.eq("not-an-array") - with pytest.raises(FilterError): - validate_filter_call(AGENCY_SPEC, "agency", bad_filter) + value, error = validate_filter_call(AGENCY_SPEC, "agency", bad_filter) + assert value is None + assert isinstance(error, FilterError) + assert error.path == "filters.agency" -def test_validate_filter_call_adhoc_invalid_shape_raises(): - """validate_filter_call raises FilterError when an ad-hoc filter has an invalid DefaultFilter shape.""" +def test_validate_filter_call_adhoc_invalid_shape_returns_error(): + """validate_filter_call returns a FilterError when an ad-hoc filter has an invalid shape.""" # Pass None as spec (ad-hoc), with something that isn't a DefaultFilter class _BadShape: operator = "not_a_real_operator" value = object() # not a valid value type - with pytest.raises(FilterError): - validate_filter_call(None, "legacyTag", _BadShape()) # type: ignore[arg-type] + value, error = validate_filter_call(None, "legacyTag", _BadShape()) # type: ignore[arg-type] + assert value is None + assert isinstance(error, FilterError) + assert error.path == "filters.legacyTag" -def test_validate_filter_call_valid_registered_does_not_raise(): - """validate_filter_call does not raise for a valid registered filter call.""" +def test_validate_filter_call_valid_registered_returns_value_and_no_error(): + """validate_filter_call returns (DefaultFilter, None) for a valid registered filter call.""" valid_filter = f.in_(["NSF", "NIH"]) - # Should not raise — agency is STRING_ARRAY, in_ with list is valid - validate_filter_call(AGENCY_SPEC, "agency", valid_filter) + # agency is STRING_ARRAY, in_ with list is valid + value, error = validate_filter_call(AGENCY_SPEC, "agency", valid_filter) + assert error is None + assert value is not None + assert value.operator == "in" def test_validate_filter_call_money_comparison_passes_valid_money(): @@ -320,36 +368,40 @@ def test_validate_filter_call_money_comparison_passes_valid_money(): compile-error tests; covered here at the runtime layer. """ spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) - validate_filter_call( + value, error = validate_filter_call( spec, "awardFloor", f.gt({"amount": "1000000", "currency": "USD"}) ) + assert error is None + assert value is not None def test_validate_filter_call_money_comparison_rejects_array_operator(): - """A registered moneyComparison filter raises FilterError for an array operator.""" + """A registered moneyComparison filter returns a FilterError for an array operator.""" spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) - with pytest.raises(FilterError): - validate_filter_call( - spec, "awardFloor", f.in_([{"amount": "1000000", "currency": "USD"}]) - ) + value, error = validate_filter_call( + spec, "awardFloor", f.in_([{"amount": "1000000", "currency": "USD"}]) + ) + assert value is None + assert isinstance(error, FilterError) def test_validate_filter_call_money_comparison_rejects_numeric_amount(): - """A registered moneyComparison filter raises FilterError for a numeric amount. + """A registered moneyComparison filter returns a FilterError for a numeric amount. Money.amount is a DecimalString — a raw number is the wrong shape. """ spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) - with pytest.raises(FilterError): - validate_filter_call( - spec, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) - ) + value, error = validate_filter_call( + spec, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) + ) + assert value is None + assert isinstance(error, FilterError) def test_validate_filter_call_money_range_passes_valid_range(): """A registered moneyRange filter accepts between with {min, max} Money values.""" spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) - validate_filter_call( + value, error = validate_filter_call( spec, "awardRange", f.between( @@ -357,15 +409,18 @@ def test_validate_filter_call_money_range_passes_valid_range(): {"amount": "500000", "currency": "USD"}, ), ) + assert error is None + assert value is not None def test_validate_filter_call_money_range_rejects_comparison_operator(): - """A registered moneyRange filter raises FilterError for a comparison operator.""" + """A registered moneyRange filter returns a FilterError for a comparison operator.""" spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) - with pytest.raises(FilterError): - validate_filter_call( - spec, "awardRange", f.gt({"amount": "10000", "currency": "USD"}) - ) + value, error = validate_filter_call( + spec, "awardRange", f.gt({"amount": "10000", "currency": "USD"}) + ) + assert value is None + assert isinstance(error, FilterError) def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): @@ -378,53 +433,91 @@ def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): The moneyRange analog above is covered; this pins the numeric path. """ spec = CustomFilterSpec(filter_type=CustomFilterType.NUMBER_RANGE) - validated = validate_filter_call(spec, "awardCount", f.between(0, 1000)) - wire = validated.model_dump(by_alias=True, exclude_none=True, mode="json") + value, error = validate_filter_call(spec, "awardCount", f.between(0, 1000)) + assert error is None + wire = value.model_dump(by_alias=True, exclude_none=True, mode="json") assert wire["operator"] == "between" assert wire["value"] == {"min": 0, "max": 1000} -def test_classify_default_wrong_shape_raises_plugin_error(): - """A wrong-shaped DEFAULT filter raises FilterError, not a raw pydantic ValidationError. +def test_classify_default_wrong_shape_collects_error_and_omits_key(): + """A wrong-shaped DEFAULT filter is fail-soft: omitted from result, error collected. "status" is a StringArrayFilter (ArrayOperator + list[str]); f.eq("open") is an - equivalence filter. The error contract must be uniform across all three buckets: - consumers following the documented `except FilterError` pattern must catch this. + equivalence filter. classify_filters no longer raises on a bad call-time filter + value: the bad key is dropped from the result body and a + single FilterError with path "filters.status" is collected. """ - with pytest.raises(FilterError) as exc_info: - classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} - ) - # The underlying pydantic error is preserved as cause for programmatic access - assert isinstance(exc_info.value.__cause__, ValidationError) + classified = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} + ) + # No raise; the invalid default is absent from the result body. + assert classified.result.status is None + # Exactly one collected error, pinpointed to filters.status. + assert len(classified.errors) == 1 + assert classified.errors[0].path == "filters.status" + # The underlying pydantic error is preserved as the structured cause. + assert isinstance(classified.errors[0].cause, ValidationError) def test_validate_filter_call_integer_comparison_validates_as_number(): """A registered integerComparison filter validates against NumberComparisonFilter. The spec defines no integer filter model, so the int constraint is not - schema-enforced (same as the TS SDK); a numeric value passes, a non-numeric + schema-enforced; a numeric value passes, a non-numeric value fails. """ spec = CustomFilterSpec(filter_type=CustomFilterType.INTEGER_COMPARISON) - validate_filter_call(spec, "awardCount", f.gt(100)) - with pytest.raises(FilterError): - validate_filter_call(spec, "awardCount", f.gt("not a number")) + value, error = validate_filter_call(spec, "awardCount", f.gt(100)) + assert error is None + assert value is not None + value, error = validate_filter_call(spec, "awardCount", f.gt("not a number")) + assert value is None + assert isinstance(error, FilterError) -def test_classify_default_camel_alias_wrong_shape_raises_plugin_error(): - """A wrong-shaped default filter via its camelCase alias also raises FilterError. +def test_classify_default_camel_alias_wrong_shape_collects_error(): + """A wrong-shaped default filter via its camelCase alias is fail-soft, omitted from result. "closeDateRange" is a DateRangeFilter; f.eq("2026-01-01") is an equivalence - filter — the alias normalization path must surface the same FilterError. + filter — the alias normalization path must collect a FilterError (not raise) + and omit the key. The error path uses the alias form. """ - with pytest.raises(FilterError): - classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - {"closeDateRange": f.eq("2026-01-01")}, - ) + classified = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"closeDateRange": f.eq("2026-01-01")}, + ) + assert classified.result.close_date_range is None + assert len(classified.errors) == 1 + assert classified.errors[0].path == "filters.closeDateRange" + + +def test_classify_default_money_range_wrong_shape_collects_error(): + """A wrong-shaped MoneyRangeFilter default is dropped and its error collected. + + "totalFundingAvailableRange" is a MoneyRangeFilter (RangeOperator + MoneyRange); + f.eq("100") is an equivalence filter with a scalar value — a valid permissive + DefaultFilter, but not a MoneyRangeFilter. MoneyRangeFilter is the one default + type the sibling tests (status -> StringArrayFilter, closeDateRange -> + DateRangeFilter) do not exercise. The regression this guards: if the money-range + defaults were validated against the permissive DefaultFilter shape instead of + their real type, this f.eq value would pass and a bad body would reach the server. + """ + classified = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"totalFundingAvailableRange": f.eq("100")}, + ) + # Fail-soft: omitted from result, error collected (not raised). + assert classified.result.total_funding_available_range is None + assert len(classified.errors) == 1 + assert classified.errors[0].path == "filters.totalFundingAvailableRange" + # Same uniform error contract as the sibling default tests: the underlying + # pydantic ValidationError is preserved as the structured cause. + assert isinstance(classified.errors[0].cause, ValidationError) # --------------------------------------------------------------------------- @@ -453,69 +546,84 @@ def test_boolean_filter_value_survives_to_wire_as_json_true(): DefaultFilter.value is Any per the core spec (filters/base.tsp `unknown`); a narrowed union without bool lax-coerced True -> 1 and corrupted the wire. """ - result = classify_filters( + classified = classify_filters( WIRE_ROUTES, "opportunities", "search", {"isOpen": f.eq(True)} ) - body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert classified.errors == [] + body = classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") assert body["customFilters"]["isOpen"]["value"] is True def test_registered_filter_ships_validated_value_not_raw_input(): - """The wire body carries the value that passed validation, not the raw input. + """The classifier keeps the value that passed validation, not the raw input. NumberComparisonFilter lax-coerces "42" -> 42 (smart-union resolves the int|float union to int first); shipping the raw string would mean the - payload differs from what validation approved. + value differs from what validation approved. """ - result = classify_filters( + classified = classify_filters( WIRE_ROUTES, "opportunities", "search", {"awardCount": f.gt("42")} ) - body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert classified.errors == [] + body = classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") assert body["customFilters"]["awardCount"]["value"] == 42 assert isinstance(body["customFilters"]["awardCount"]["value"], int) def test_number_comparison_registered_filter_rejects_bool(): - """f.eq(True) on a numberComparison-registered filter raises, never ships 1. + """f.eq(True) on a numberComparison-registered filter is dropped, never ships 1. bool subclasses int; without an explicit rejection the int|float union lax-coerces True -> 1 and the wire silently carries a number for a boolean — the corruption class the DefaultFilter.value widening fixed. + Fail-soft: the bad registered filter is collected as an error and omitted + from customFilters rather than raising. """ - with pytest.raises(FilterError): - classify_filters( - WIRE_ROUTES, "opportunities", "search", {"awardCount": f.eq(True)} - ) + classified = classify_filters( + WIRE_ROUTES, "opportunities", "search", {"awardCount": f.eq(True)} + ) + assert classified.result.custom_filters is None + assert len(classified.errors) == 1 + assert classified.errors[0].path == "filters.awardCount" @pytest.mark.filterwarnings("ignore::UserWarning") # pydantic warns during the -# model_dump of the mutated instance, before re-validation raises -def test_mutated_adhoc_instance_is_revalidated_and_raises(): - """An ad-hoc DefaultFilter mutated after construction raises instead of shipping. +# model_dump of the mutated instance, before re-validation collects the error +def test_mutated_adhoc_instance_is_revalidated_and_collected(): + """An ad-hoc DefaultFilter mutated after construction is collected, not shipped. The filter models are mutable; the ad-hoc branch must re-validate instances - rather than trust isinstance. + rather than trust isinstance. Fail-soft: the error is collected and the key + omitted from customFilters. """ flt = f.eq("x") flt.operator = "bogus" # type: ignore[assignment] - with pytest.raises(FilterError): - classify_filters(SAMPLE_ROUTES, "opportunities", "search", {"legacy": flt}) + classified = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"legacy": flt} + ) + assert classified.result.custom_filters is None + assert len(classified.errors) == 1 + assert classified.errors[0].path == "filters.legacy" -def test_unknown_filter_type_raises_plugin_error_not_key_error(): - """A spec whose filter_type never passed validate_routes raises FilterError. +def test_unknown_filter_type_returns_filter_error_not_key_error(): + """A spec whose filter_type never passed validate_routes returns a FilterError. - The uniform error contract holds even when registration-time validation was - skipped — consumers catching `except FilterError` must not see a KeyError. + The uniform fail-soft contract holds even when registration-time validation + was skipped — the call-time validator must surface a FilterError, never a + KeyError, and never raise. """ spec = CustomFilterSpec(filter_type="bogusType") # type: ignore[arg-type] - with pytest.raises(FilterError): - validate_filter_call(spec, "x", f.eq(1)) + value, error = validate_filter_call(spec, "x", f.eq(1)) + assert value is None + assert isinstance(error, FilterError) def test_validate_filter_call_adhoc_accepts_raw_dict(): - """Ad-hoc validation accepts a raw operator/value dict and returns a DefaultFilter.""" - validated = validate_filter_call(None, "x", {"operator": "eq", "value": "v"}) + """Ad-hoc validation accepts a raw operator/value dict and returns (DefaultFilter, None).""" + validated, error = validate_filter_call(None, "x", {"operator": "eq", "value": "v"}) + assert error is None + assert validated is not None assert validated.operator == "eq" assert validated.value == "v" @@ -533,27 +641,34 @@ def test_classify_default_snake_form_of_aliased_key_normalizes_to_alias(): (populate_by_name is not set) and the field stays None. """ consumer_filters = {"close_date_range": f.between("2026-01-01", "2026-12-31")} - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) - assert result.close_date_range is not None - assert result.custom_filters is None + assert classified.errors == [] + assert classified.result.close_date_range is not None + assert classified.result.custom_filters is None -def test_classify_both_forms_of_same_default_filter_raises(): - """Supplying snake AND camel forms of one default filter raises FilterError. +def test_classify_both_forms_of_same_default_filter_collects_error(): + """Supplying snake AND camel forms of one default filter is fail-soft. - Both keys normalize to "closeDateRange"; without the guard, dict - assignment silently drops whichever range the consumer's dict ordered - first (plausible when merging filter dicts from two naming conventions). + Both keys normalize to "closeDateRange". The first form is kept; the + duplicate is dropped and a FilterError collected (a bad call-time filter + never raises). Without the dedup guard, dict assignment + would silently drop whichever range the consumer's dict ordered first. """ consumer_filters = { "close_date_range": f.between("2026-01-01", "2026-06-30"), "closeDateRange": f.between("2026-07-01", "2026-12-31"), } - with pytest.raises(FilterError, match="more than once") as exc_info: - classify_filters(SAMPLE_ROUTES, "opportunities", "search", consumer_filters) - assert exc_info.value.path == "filters.closeDateRange" + classified = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", consumer_filters + ) + # The first-seen value is kept on the result; the duplicate is dropped. + assert classified.result.close_date_range is not None + assert len(classified.errors) == 1 + assert classified.errors[0].path == "filters.closeDateRange" + assert "more than once" in str(classified.errors[0]) @pytest.mark.parametrize( @@ -573,32 +688,41 @@ def test_classify_unmatched_route_treats_registered_name_as_adhoc(resource, meth regression that flattens or mis-keys it either wrongly applies the spec or wrongly skips it. """ - result = classify_filters(SAMPLE_ROUTES, resource, method, {"agency": f.eq("NSF")}) - assert result.custom_filters is not None - assert result.custom_filters["agency"].value == "NSF" - - # ...and the same filter via the declared pair IS spec-validated and rejected - with pytest.raises(FilterError): - classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} - ) + classified = classify_filters( + SAMPLE_ROUTES, resource, method, {"agency": f.eq("NSF")} + ) + assert classified.errors == [] + assert classified.result.custom_filters is not None + assert classified.result.custom_filters["agency"].value == "NSF" + + # ...and the same filter via the declared pair IS spec-validated and dropped + # fail-soft: omitted from result, error collected (no raise). + declared = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} + ) + assert declared.result.custom_filters is None + assert len(declared.errors) == 1 + assert declared.errors[0].path == "filters.agency" def test_request_body_mode_json_round_trip(): """The documented model_dump(mode="json") call yields a json.dumps-able body. - Coerced date objects only serialize in json mode — this is the ADR-0012 - wire body the classifier exists to produce. + Coerced date objects only serialize in json mode, which is what the + classifier dumps. """ consumer_filters = { "close_date_range": f.between(date(2026, 1, 1), date(2026, 12, 31)), "agency": f.in_(["NSF"]), } - result = classify_filters( + classified = classify_filters( SAMPLE_ROUTES, "opportunities", "search", consumer_filters ) + assert classified.errors == [] body = json.loads( - json.dumps(result.model_dump(by_alias=True, exclude_none=True, mode="json")) + json.dumps( + classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") + ) ) assert body["closeDateRange"]["operator"] == "between" assert body["closeDateRange"]["value"]["min"] == "2026-01-01" @@ -607,9 +731,10 @@ def test_request_body_mode_json_round_trip(): def test_classify_empty_filters_dict_yields_empty_body(): """An empty consumer dict produces an OppFilters with no customFilters entry.""" - result = classify_filters(SAMPLE_ROUTES, "opportunities", "search", {}) - assert result.custom_filters is None - body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + classified = classify_filters(SAMPLE_ROUTES, "opportunities", "search", {}) + assert classified.errors == [] + assert classified.result.custom_filters is None + body = classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") assert "customFilters" not in body @@ -623,34 +748,96 @@ def test_filter_type_schemas_covers_every_custom_filter_type(): assert set(FILTER_TYPE_SCHEMAS) == set(CustomFilterType) -def test_plugin_error_path_is_uniform_across_buckets(): - """All three buckets raise FilterError with a filters.<name> path.""" - with pytest.raises(FilterError) as exc1: - classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} - ) - assert exc1.value.path == "filters.status" +def test_collected_error_path_is_uniform_across_buckets(): + """All three buckets collect a FilterError with a filters.<name> path, none raise.""" + c1 = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} + ) + assert len(c1.errors) == 1 + assert c1.errors[0].path == "filters.status" - with pytest.raises(FilterError) as exc2: - classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} - ) - assert exc2.value.path == "filters.agency" + c2 = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} + ) + assert len(c2.errors) == 1 + assert c2.errors[0].path == "filters.agency" - with pytest.raises(FilterError) as exc3: - classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"adhoc": {"operator": "bogus"}} - ) - assert exc3.value.path == "filters.adhoc" + c3 = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"adhoc": {"operator": "bogus"}} + ) + assert len(c3.errors) == 1 + assert c3.errors[0].path == "filters.adhoc" -def test_multiple_failing_defaults_use_collective_path(): - """Two failing default filters produce the collective path "filters".""" - with pytest.raises(FilterError) as exc_info: - classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - {"status": f.eq("open"), "closeDateRange": f.eq("x")}, - ) - assert exc_info.value.path == "filters" +def test_multiple_failing_defaults_collect_one_error_per_key(): + """Two failing default filters each collect their own FilterError, none raise. + + Per-key fail-soft: each invalid default is dropped and pinpointed + individually (no single collective "filters" path), so a consumer sees + exactly which keys failed. + """ + classified = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"status": f.eq("open"), "closeDateRange": f.eq("x")}, + ) + assert classified.result.status is None + assert classified.result.close_date_range is None + paths = sorted(e.path for e in classified.errors) + assert paths == ["filters.closeDateRange", "filters.status"] + + +def test_classify_invalid_registered_and_invalid_adhoc_both_collected(): + """An invalid registered filter AND an invalid ad-hoc filter are both collected. + + Neither is shipped: both keys are omitted from customFilters and each yields + its own FilterError. "agency" is STRING_ARRAY (f.eq scalar is wrong); the + ad-hoc "adhoc" key has a bogus operator. + """ + classified = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"agency": f.eq("NSF"), "adhoc": {"operator": "bogus", "value": 1}}, + ) + assert classified.result.custom_filters is None + paths = sorted(e.path for e in classified.errors) + assert paths == ["filters.adhoc", "filters.agency"] + + +def test_classify_mixed_valid_and_invalid_keeps_valid_drops_invalid(): + """Mixed valid + invalid filters: valid keys present in result, invalid absent. + + Discriminating across all three buckets at once — a valid default (status), + a valid registered custom (agency), a valid ad-hoc (legacyTag), plus an + invalid default (closeDateRange) and an invalid registered (fundingProgram, + a stringComparison given an array value). The three valid keys must survive; + the two invalid keys must be omitted and exactly two errors collected. + """ + classified = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "status": f.in_(["open"]), # valid default + "agency": f.in_(["NSF"]), # valid registered (stringArray) + "legacyTag": f.eq("priority"), # valid ad-hoc + "closeDateRange": f.eq("2026-01-01"), # invalid default (not a range) + "fundingProgram": f.in_( + ["a", "b"] + ), # invalid registered (stringComparison) + }, + ) + result = classified.result + # Valid keys present + assert result.status is not None + assert result.custom_filters is not None + assert "agency" in result.custom_filters + assert "legacyTag" in result.custom_filters + # Invalid keys absent + assert result.close_date_range is None + assert "fundingProgram" not in result.custom_filters + # Exactly the two invalid keys collected (count discriminates). + paths = sorted(e.path for e in classified.errors) + assert paths == ["filters.closeDateRange", "filters.fundingProgram"] diff --git a/lib/ts-sdk/__tests__/client/opportunities.spec.ts b/lib/ts-sdk/__tests__/client/opportunities.spec.ts index 74461dc37..76ff131f2 100644 --- a/lib/ts-sdk/__tests__/client/opportunities.spec.ts +++ b/lib/ts-sdk/__tests__/client/opportunities.spec.ts @@ -3,7 +3,8 @@ import { z } from "zod"; import { http, HttpResponse, setupServer, createPaginatedHandler } from "../utils/mock-fetch"; import { Client, Auth } from "../../src/client"; import { OpportunityBaseSchema } from "../../src/schemas"; -import { withCustomFields } from "../../src/extensions"; +import { withCustomFields, F, FilterError } from "../../src/extensions"; +import type { PluginRoutes } from "../../src/extensions"; import { CustomFieldType } from "../../src/constants"; // ============================================================================= @@ -379,6 +380,221 @@ describe("Opportunities", () => { }); }); + it("classifies a flat custom-filter bag into the OppFilters body when routes are supplied", async () => { + let capturedBody: Record<string, unknown> | undefined; + + server.use( + http.post("/common-grants/opportunities/search", async ({ request }) => { + capturedBody = (await request.json()) as Record<string, unknown>; + return HttpResponse.json({ + status: 200, + message: "Success", + items: [createMockOpportunity(OPP_UUID_1, "Conservation Grant", "open")], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {} }, + }); + }) + ); + + // A plugin registering one custom filter on opportunities.search. + const routes: PluginRoutes = { + opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, + }; + + // routes is client-bound: supplied once at construction, not per call. + const routedClient = new Client({ + baseUrl: "https://api.example.org", + auth: Auth.bearer("test-token"), + routes, + }); + + await routedClient.opportunities.search({ + filters: { + status: F.in(["open"]), // default field → top-level + agency: F.in(["HHS", "NSF"]), // registered custom → customFilters + legacyTag: F.eq("conservation-2024"), // ad-hoc → customFilters passthrough + }, + }); + + // Default field stays top-level; registered + ad-hoc land under customFilters. + expect(capturedBody?.filters).toMatchObject({ + status: { operator: "in", value: ["open"] }, + customFilters: { + agency: { operator: "in", value: ["HHS", "NSF"] }, + legacyTag: { operator: "eq", value: "conservation-2024" }, + }, + }); + // A default field must NOT be duplicated into customFilters. + const customFilters = (capturedBody?.filters as { customFilters?: Record<string, unknown> }) + .customFilters; + expect(customFilters).not.toHaveProperty("status"); + }); + + it("resolves a status collision in favor of filters.status and surfaces a warning", async () => { + // `status` given via both the `statuses` shorthand and `filters.status`: + // `filters` wins, the shorthand is ignored, a warning is appended to + // filterInfo.errors, and search() must NOT throw. + let capturedBody: Record<string, unknown> | undefined; + + server.use( + http.post("/common-grants/opportunities/search", async ({ request }) => { + capturedBody = (await request.json()) as Record<string, unknown>; + return HttpResponse.json({ + status: 200, + message: "Success", + items: [createMockOpportunity(OPP_UUID_1, "Closed Grant", "closed")], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {} }, + }); + }) + ); + + const result = await client.opportunities.search({ + statuses: ["open"], + filters: { status: F.in(["closed"]) }, + }); + + // filters.status wins — the shorthand value ["open"] is ignored. + expect((capturedBody?.filters as { status?: { value?: unknown } }).status).toEqual({ + operator: "in", + value: ["closed"], + }); + + // The collision warning is surfaced (not thrown). + expect(result.filterInfo.errors).toEqual( + expect.arrayContaining([expect.stringContaining("filters.status")]) + ); + }); + + it("drops an invalid filter from the request body and surfaces its error (no throw)", async () => { + // A registered stringArray filter given a non-array `between` value fails + // classifyFilters validation. Fail-soft: results still return, the invalid + // filter is absent from the sent body, and its message lands in + // filterInfo.errors — search() does not throw. + let capturedBody: Record<string, unknown> | undefined; + + server.use( + http.post("/common-grants/opportunities/search", async ({ request }) => { + capturedBody = (await request.json()) as Record<string, unknown>; + return HttpResponse.json({ + status: 200, + message: "Success", + items: [createMockOpportunity(OPP_UUID_1, "Conservation Grant", "open")], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {} }, + }); + }) + ); + + const routes: PluginRoutes = { + opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, + }; + // routes is client-bound: supplied once at construction, not per call. + const routedClient = new Client({ + baseUrl: "https://api.example.org", + auth: Auth.bearer("test-token"), + routes, + }); + + const result = await routedClient.opportunities.search({ + filters: { agency: { operator: "between", value: 5 } }, + }); + + // Results still come back. + expect(result.items).toHaveLength(1); + + // The invalid filter is absent from the sent body (no customFilters.agency, + // and no malformed filters payload shipped). + const sentFilters = capturedBody?.filters as + | { customFilters?: Record<string, unknown> } + | undefined; + expect(sentFilters?.customFilters).toBeUndefined(); + + // Its error message is surfaced client-side. + expect(result.filterInfo.errors).toEqual( + expect.arrayContaining([expect.stringContaining("filters.agency")]) + ); + }); + + it("stays fail-soft when a filter is dropped and the server omits filterInfo (auto-paginate)", async () => { + // Auto-pagination returns the raw server envelope; if it omits filterInfo, + // merging the dropped-filter error must still surface, not throw. + server.use( + http.post("/common-grants/opportunities/search", () => { + return HttpResponse.json({ + status: 200, + message: "Success", + items: [createMockOpportunity(OPP_UUID_1, "Conservation Grant", "open")], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + // No filterInfo on the response. + }); + }) + ); + + const routes: PluginRoutes = { + opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, + }; + const routedClient = new Client({ + baseUrl: "https://api.example.org", + auth: Auth.bearer("test-token"), + routes, + }); + + const result = await routedClient.opportunities.search({ + filters: { agency: { operator: "between", value: 5 } }, + }); + + expect(result.items).toHaveLength(1); + expect(result.filterInfo.errors).toEqual( + expect.arrayContaining([expect.stringContaining("filters.agency")]) + ); + }); + + it("validates routes at construction and throws on a default-name collision", () => { + const badRoutes: PluginRoutes = { + opportunities: { search: { filters: { status: { filterType: "stringArray" } } } }, + }; + expect(() => new Client({ baseUrl: "https://api.example.org", routes: badRoutes })).toThrow( + FilterError + ); + }); + + it("merges client-side filter errors ahead of server-provided errors", async () => { + server.use( + http.post("/common-grants/opportunities/search", () => { + return HttpResponse.json({ + status: 200, + message: "Success", + items: [createMockOpportunity(OPP_UUID_1, "Conservation Grant", "open")], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {}, errors: ["server: something was ignored"] }, + }); + }) + ); + + const routes: PluginRoutes = { + opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, + }; + const routedClient = new Client({ + baseUrl: "https://api.example.org", + auth: Auth.bearer("test-token"), + routes, + }); + + const result = await routedClient.opportunities.search({ + filters: { agency: { operator: "between", value: 5 } }, + }); + + const errors = result.filterInfo.errors ?? []; + expect(errors[0]).toContain("filters.agency"); + expect(errors[errors.length - 1]).toBe("server: something was ignored"); + }); + it("searches with only query parameter", async () => { server.use( http.post("/common-grants/opportunities/search", () => { diff --git a/lib/ts-sdk/__tests__/client/search-filters-types.ts b/lib/ts-sdk/__tests__/client/search-filters-types.ts new file mode 100644 index 000000000..b47ec46ba --- /dev/null +++ b/lib/ts-sdk/__tests__/client/search-filters-types.ts @@ -0,0 +1,72 @@ +/** + * Compile-time assertions for the routes-driven `search({ filters })` narrowing. + * + * Checked by `tsc --noEmit` only — there is no runtime test. Each `@ts-expect-error` + * IS an assertion: it guards a line that must fail to compile. + * + * Why the narrowing is partial: because `definePlugin` preserves the literal + * `routes` type (its `const TRoutes` generic), declared filter names get autocomplete + * and their values are envelope-checked. But the spec supports AD-HOC filters (an open + * key set), so an unknown key cannot be rejected — a typo on a declared name is + * structurally an intentional ad-hoc filter. Narrowing therefore gives autocomplete + + * filter-envelope checking (`{ operator, value }` shape), NOT typo-rejection and NOT + * per-`filterType` value validation (that runs at runtime in `classifyFilters`). The + * single `@ts-expect-error` below pins the part that DOES hold (the `{ operator, value }` + * envelope); the un-guarded "typo" line documents what does not. + */ + +import { definePlugin, F } from "@/extensions"; +import { Client } from "@/client"; + +const plugin = definePlugin({ + routes: { + opportunities: { + search: { + filters: { + fundingMax: { filterType: "numberRange" }, + agency: { filterType: "stringArray" }, + }, + }, + }, + }, +} as const); + +// routes is client-bound: supplied once at construction, so `search` narrows +// filter names from the client's `R` generic. Compile-only — baseUrl unused. +const client = new Client({ routes: plugin.routes }); + +// Compile-only — never executed. +async function _assertions(): Promise<void> { + // Declared filter names accepted, values built with the F.* helpers. + await client.opportunities.search({ + filters: { + fundingMax: F.between(0, 100), + agency: F.in(["HHS", "NSF"]), + }, + }); + + // Ad-hoc (unregistered) key accepted — spec escape hatch, classifyFilters bucket 3. + await client.opportunities.search({ + filters: { legacyTag: F.eq("conservation-2024") }, + }); + + // Intentionally NO @ts-expect-error: a typo on a declared name compiles, + // because it is indistinguishable from an intentional ad-hoc key. This is the + // documented limitation of narrowing against an open (ad-hoc-supporting) key set. + await client.opportunities.search({ + filters: { fundingMaxx: F.between(0, 100) }, + }); + + // The `{ operator, value }` envelope is enforced even though keys are open. + await client.opportunities.search({ + filters: { + // @ts-expect-error — a filter value must be `{ operator, value }`, not a bare string. + fundingMax: "not-a-filter", + }, + }); + + // Works with no filters (back-compat: R defaults to PluginRoutes). + await client.opportunities.search({ query: "education", statuses: ["open"] }); +} + +void _assertions; diff --git a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts index 1f2227001..6d28aff9b 100644 --- a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; -import { classifyFilters, F, validateFilterCall, validateRoutes } from "@/extensions"; +import { classifyFilters, F, validateRoutes } from "@/extensions"; +import { validateFilterCall } from "@/extensions/custom-filters"; import { FilterError } from "@/extensions"; import type { PluginRoutes } from "@/extensions"; import { OppFiltersSchema } from "@/schemas/zod/models"; @@ -51,33 +52,36 @@ describe("classifyFilters", () => { describe("three-bucket classification", () => { it("routes default filters to top-level named request-body fields", () => { - const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { status: { operator: "in", value: ["open"] }, }); expect(result.status).toEqual({ operator: "in", value: ["open"] }); expect(result.customFilters).toBeUndefined(); + expect(errors).toEqual([]); }); it("routes pre-registered custom filters to customFilters record", () => { - const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { agency: { operator: "in", value: ["HHS"] }, }); expect(result.customFilters?.agency).toEqual({ operator: "in", value: ["HHS"] }); expect(result.status).toBeUndefined(); + expect(errors).toEqual([]); }); it("routes ad-hoc filters to customFilters passthrough (no registration required)", () => { - const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { legacyTag: { operator: "eq", value: "legacy-2024" }, }); expect(result.customFilters?.legacyTag).toEqual({ operator: "eq", value: "legacy-2024" }); + expect(errors).toEqual([]); }); it("builds exact ADR-0012 OppFilters request body for mixed default + custom + ad-hoc input", () => { - const result = classifyFilters( + const { result, errors } = classifyFilters( grantsGovRoutes, "opportunities", "search", @@ -96,10 +100,11 @@ describe("classifyFilters", () => { }; expect(result).toEqual(expected); + expect(errors).toEqual([]); }); it("passes gov.<system>@<filterName> namespaced keys through to customFilters verbatim", () => { - const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + const { result } = classifyFilters(grantsGovRoutes, "opportunities", "search", { "gov.grants@announcementType": { operator: "eq", value: "NOFO" }, }); @@ -110,7 +115,7 @@ describe("classifyFilters", () => { }); it("returns only top-level fields when no custom or ad-hoc filters are provided", () => { - const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + const { result } = classifyFilters(grantsGovRoutes, "opportunities", "search", { status: { operator: "in", value: ["open"] }, }); @@ -118,9 +123,74 @@ describe("classifyFilters", () => { }); it("handles an empty filters object gracefully", () => { - const result = classifyFilters(grantsGovRoutes, "opportunities", "search", {}); + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", {}); expect(result).toEqual({}); + expect(errors).toEqual([]); + }); + + // ########################################################################## + // Fail-soft validation (collect, don't throw) + // ########################################################################## + + it("collects a FilterError and omits the key when a default filter violates its real field type (status)", () => { + // `status` is a StringArrayFilter (operator in/notIn, value string[]). + // `{ operator: "gt", value: 5 }` is a structurally valid DefaultFilter + // (passes the permissive shape check) but invalid for `status`. Default + // filters are validated against their real field type — a malformed one is + // fail-soft: dropped from `result`, surfaced in `errors`, never thrown. + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { + status: { operator: "gt", value: 5 }, + }); + + expect(result).not.toHaveProperty("status"); + expect(errors).toHaveLength(1); + expect(errors[0]).toBeInstanceOf(FilterError); + expect(errors[0].path).toBe("filters.status"); + }); + + it("collects errors for invalid registered and ad-hoc filters, omitting both from result", () => { + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { + // registered stringArray given a non-array value → invalid + agency: { operator: "in", value: "not-an-array" }, + // ad-hoc with a missing operator → invalid shape + legacyTag: { value: "no-operator" }, + }); + + expect(result.customFilters).toBeUndefined(); + expect(errors).toHaveLength(2); + expect(errors.map(e => e.path).sort()).toEqual(["filters.agency", "filters.legacyTag"]); + }); + + it("keeps valid keys and drops only invalid ones for mixed valid+invalid input", () => { + const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { + // valid default + status: { operator: "in", value: ["open"] }, + // invalid default (wrong operator/value for status) + closeDateRange: { operator: "in", value: ["nope"] }, + // valid registered custom + agency: { operator: "in", value: ["HHS"] }, + // invalid registered custom (stringArray with non-array value) + fundingProgram: { operator: "like", value: 12345 }, + // valid ad-hoc + legacyTag: { operator: "eq", value: "legacy-2024" }, + }); + + // Valid keys are present. + expect(result.status).toEqual({ operator: "in", value: ["open"] }); + expect(result.customFilters?.agency).toEqual({ operator: "in", value: ["HHS"] }); + expect(result.customFilters?.legacyTag).toEqual({ operator: "eq", value: "legacy-2024" }); + + // Invalid keys are absent. + expect(result).not.toHaveProperty("closeDateRange"); + expect(result.customFilters).not.toHaveProperty("fundingProgram"); + + // errors count discriminates: exactly the two invalid keys. + expect(errors).toHaveLength(2); + expect(errors.map(e => e.path).sort()).toEqual([ + "filters.closeDateRange", + "filters.fundingProgram", + ]); }); }); @@ -196,6 +266,20 @@ describe("classifyFilters", () => { expect(() => validateRoutes(collidingRoutes)).toThrow(FilterError); }); + it("throws FilterError when custom filters are declared on an unsupported route (list)", () => { + const unsupportedRoutes: PluginRoutes = { + opportunities: { + list: { + filters: { + agency: { filterType: "stringArray" }, + }, + }, + }, + }; + + expect(() => validateRoutes(unsupportedRoutes)).toThrow(FilterError); + }); + it("does not throw for valid routes", () => { expect(() => validateRoutes(grantsGovRoutes)).not.toThrow(); }); @@ -206,83 +290,72 @@ describe("classifyFilters", () => { // ############################################################################ describe("call-time validation", () => { - it("throws FilterError on operator/filterType mismatch for a registered filter", () => { + it("returns a FilterError on operator/filterType mismatch for a registered filter", () => { // `like` operator is not valid for numberComparison (only gt/gte/lt/lte/eq/neq) const spec = { filterType: "numberComparison" } as const; - expect(() => validateFilterCall(spec, "amount", { operator: "like", value: "100" })).toThrow( - FilterError - ); + const err = validateFilterCall(spec, "amount", { operator: "like", value: "100" }); + expect(err).toBeInstanceOf(FilterError); }); - it("throws FilterError on value-shape mismatch for a registered stringArray filter", () => { + it("returns a FilterError on value-shape mismatch for a registered stringArray filter", () => { // stringArray requires value to be string[]; passing a plain string fails const spec = { filterType: "stringArray" } as const; - expect(() => - validateFilterCall(spec, "agency", { operator: "in", value: "not-an-array" }) - ).toThrow(FilterError); + const err = validateFilterCall(spec, "agency", { operator: "in", value: "not-an-array" }); + expect(err).toBeInstanceOf(FilterError); }); - it("throws FilterError on value-shape mismatch for a registered numberComparison filter", () => { + it("returns a FilterError on value-shape mismatch for a registered numberComparison filter", () => { // numberComparison requires value to be a number; passing a string fails const spec = { filterType: "numberComparison" } as const; - expect(() => - validateFilterCall(spec, "amount", { operator: "eq", value: "not-a-number" }) - ).toThrow(FilterError); + const err = validateFilterCall(spec, "amount", { operator: "eq", value: "not-a-number" }); + expect(err).toBeInstanceOf(FilterError); }); - it("passes a valid integerComparison filter without throwing", () => { + it("returns undefined for a valid integerComparison filter (no throw)", () => { const spec = { filterType: "integerComparison" } as const; - expect(() => - validateFilterCall(spec, "awardCount", { operator: "eq", value: 2 }) - ).not.toThrow(); + expect(validateFilterCall(spec, "awardCount", { operator: "eq", value: 2 })).toBeUndefined(); }); - it("FilterError path includes the filter name", () => { + it("returned FilterError path includes the filter name", () => { const spec = { filterType: "stringArray" } as const; - try { - validateFilterCall(spec, "agency", { operator: "in", value: "wrong" }); - expect.fail("Expected FilterError to be thrown"); - } catch (err) { - expect(err).toBeInstanceOf(FilterError); - expect((err as FilterError).path).toBe("filters.agency"); - } + const err = validateFilterCall(spec, "agency", { operator: "in", value: "wrong" }); + expect(err).toBeInstanceOf(FilterError); + expect(err?.path).toBe("filters.agency"); }); - it("passes a valid registered filter without throwing", () => { + it("returns undefined for a valid registered filter (no throw)", () => { const spec = { filterType: "stringArray" } as const; - expect(() => + expect( validateFilterCall(spec, "agency", { operator: "in", value: ["HHS", "DOE"] }) - ).not.toThrow(); + ).toBeUndefined(); }); - it("passes an ad-hoc filter through with only a shape check (no operator enforcement)", () => { + it("returns undefined for an ad-hoc filter with a valid shape (no operator enforcement)", () => { // Ad-hoc (spec=undefined) — any valid DefaultFilter shape passes - expect(() => + expect( validateFilterCall(undefined, "legacyTag", { operator: "eq", value: "legacy-2024" }) - ).not.toThrow(); + ).toBeUndefined(); }); - it("throws FilterError for an ad-hoc filter with an invalid shape", () => { + it("returns a FilterError for an ad-hoc filter with an invalid shape", () => { // Missing `operator` key — fails DefaultFilterSchema shape check (operator is required/enum) - expect(() => validateFilterCall(undefined, "badFilter", { value: "something" })).toThrow( - FilterError - ); + const err = validateFilterCall(undefined, "badFilter", { value: "something" }); + expect(err).toBeInstanceOf(FilterError); }); - it("throws FilterError for an ad-hoc filter with an unknown operator", () => { + it("returns a FilterError for an ad-hoc filter with an unknown operator", () => { // `superCustomOp` is not in AllOperatorsEnum — fails DefaultFilterSchema - expect(() => - validateFilterCall(undefined, "badFilter", { - operator: "superCustomOp", - value: "x", - }) - ).toThrow(FilterError); + const err = validateFilterCall(undefined, "badFilter", { + operator: "superCustomOp", + value: "x", + }); + expect(err).toBeInstanceOf(FilterError); }); }); }); diff --git a/lib/ts-sdk/examples/custom-filters.ts b/lib/ts-sdk/examples/custom-filters.ts index edb9e9018..10721be73 100644 --- a/lib/ts-sdk/examples/custom-filters.ts +++ b/lib/ts-sdk/examples/custom-filters.ts @@ -113,7 +113,10 @@ console.log(JSON.stringify(searchParams.filters, null, 2)); // non-optional literal object. The `!` assertion removes the `undefined` from // the union type that `Plugin.routes?:` introduces (routes is optional in the // interface to support plugins that don't declare filters). -const requestBody = classifyFilters( +// `classifyFilters` is fail-soft: it returns `{ result, errors }`. `result` is +// the OppFilters request body (valid keys only); `errors` collects any invalid +// filters that were dropped. For this all-valid input, `errors` is empty. +const { result: requestBody, errors } = classifyFilters( grantsGovPlugin.routes!, "opportunities", "search", @@ -132,6 +135,9 @@ function fail(message: string): never { process.exit(1); } +// All filters in this example are valid — no errors collected. +if (errors.length > 0) fail(`expected no classification errors, got ${errors.length}`); + // Default filters must appear as named top-level fields if (!requestBody.status) fail("status should be a top-level field (default filter bucket)"); if (!requestBody.closeDateRange) diff --git a/lib/ts-sdk/src/client/client.ts b/lib/ts-sdk/src/client/client.ts index 22ecaea98..2667612af 100644 --- a/lib/ts-sdk/src/client/client.ts +++ b/lib/ts-sdk/src/client/client.ts @@ -5,7 +5,9 @@ import { type ClientConfig, type ResolvedConfig, resolveConfig } from "./config"; import { Auth, buildAuthHeaders, type AuthMethod } from "./auth"; import { Opportunities } from "./opportunities"; +import { validateRoutes } from "../extensions/custom-filters"; import type { Paginated } from "../types"; +import type { PluginRoutes } from "../extensions/types"; // ============================================================================= // Options interfaces @@ -67,23 +69,27 @@ export interface FetchManyOptions<T = unknown> { * const list = await client.opportunities.list({ page: 1 }); * ``` */ -export class Client { +export class Client<R extends PluginRoutes = PluginRoutes> { private readonly config: ResolvedConfig; private readonly auth: AuthMethod; /** Opportunities resource namespace */ - public readonly opportunities: Opportunities; + public readonly opportunities: Opportunities<R>; // ============================================================================= // Client constructor // ============================================================================= - constructor(options: ClientConfig & { auth?: AuthMethod }) { + constructor(options: ClientConfig & { auth?: AuthMethod; routes?: R }) { this.config = resolveConfig(options); this.auth = options.auth ?? Auth.none(); - // Initialize resource namespaces - this.opportunities = new Opportunities(this); + // `routes` is client-bound (fixed plugin config supplied once here), so it + // validates at construction and drives the filter-name narrowing on + // `opportunities.search`. The `this as Client` cast avoids circular-generic + // variance friction between `Client<R>` and `Opportunities<R>`. + if (options.routes) validateRoutes(options.routes); + this.opportunities = new Opportunities<R>(this as Client, options.routes); } // ============================================================================= diff --git a/lib/ts-sdk/src/client/opportunities.ts b/lib/ts-sdk/src/client/opportunities.ts index be64cd9bd..cf4041246 100644 --- a/lib/ts-sdk/src/client/opportunities.ts +++ b/lib/ts-sdk/src/client/opportunities.ts @@ -20,6 +20,32 @@ import { OppFiltersSchema, } from "../schemas"; import { ArrayOperator } from "../constants"; +import { classifyFilters } from "../extensions/custom-filters"; +import { FilterError } from "../extensions/types"; +import type { PluginRoutes } from "../extensions/types"; + +// ============================================================================= +// Client-side filter-error merge +// ============================================================================= + +/** + * Merges client-side filter errors into a search response's `filterInfo.errors`, + * flattened to `"{path}: {message}"` and ordered before any existing entries. + * Mutates and returns `response` for call-site convenience. + */ +function mergeFilterErrors<T, F>( + response: Filtered<T, F>, + clientErrors: FilterError[] +): Filtered<T, F> { + if (clientErrors.length === 0) return response; + + const flattened = clientErrors.map(e => `${e.path}: ${e.message}`); + // Auto-pagination returns the raw server envelope, so filterInfo can be absent + // on a non-conformant response; initialize rather than throw (stay fail-soft). + const filterInfo = (response.filterInfo ??= { filters: {} as F }); + filterInfo.errors = [...flattened, ...(filterInfo.errors ?? [])]; + return response; +} // ============================================================================= // Schema type constraint @@ -38,6 +64,39 @@ import { ArrayOperator } from "../constants"; */ type OppSchema = z.ZodType<OpportunityBase, z.ZodTypeDef, unknown>; +// ============================================================================= +// Custom-filter bag typing (routes-driven) +// ============================================================================= + +/** Raw `{ operator, value }` filter object, as produced by the `F.*` helpers. */ +type RawFilter = { operator: string; value: unknown }; + +/** + * The declared custom-filter names for `opportunities.search` in a routes type. + * `definePlugin` preserves the literal `routes` type (its `const TRoutes` generic), + * so a plugin defined inline yields concrete filter-name literals here. + */ +type CustomFilterNames<R extends PluginRoutes> = R extends { + opportunities: { search: { filters: infer Fs } }; +} + ? Extract<keyof Fs, string> + : never; + +/** + * Typed filter bag for `search({ filters })`. + * + * Declared filter names surface in editor autocomplete with a typed value, while + * arbitrary keys remain accepted — the spec supports ad-hoc (escape-hatch) filters + * (bucket 3 of `classifyFilters`), so an unknown key cannot be rejected at the type + * level without dropping ad-hoc support. Net: autocomplete + filter-envelope + * checking (`{ operator, value }` shape) for declared filters — per-`filterType` + * value validation runs at runtime in `classifyFilters`, not at the type level; + * NO typo-rejection on filter names (a typo is structurally an intentional ad-hoc key). + */ +type CustomFilterBag<R extends PluginRoutes> = { + [K in CustomFilterNames<R>]?: RawFilter; +} & Record<string, RawFilter>; + // ============================================================================= // Options types (schema in options for consistent API) // ============================================================================= @@ -59,11 +118,19 @@ export interface ListOptions< /** Options for searching opportunities */ export interface SearchOptions< S extends OppSchema = typeof OpportunityBaseSchema, + R extends PluginRoutes = PluginRoutes, > extends FetchManyOptions<z.infer<S>> { /** Text query to search for in opportunity titles and descriptions */ query?: string; /** Filter by opportunity statuses */ statuses?: OppStatusOptions[]; + /** + * Flat custom-filter bag (filter name → `{ operator, value }`, e.g. built with `F.*`). + * Classified into the `OppFilters` request body via `classifyFilters` when present. + * Registered custom filters get typed names and validate against the specs declared + * in the client's `routes` (bound once at client construction). + */ + filters?: CustomFilterBag<R>; /** Zod schema to parse and type each item. Defaults to `OpportunityBaseSchema`. */ schema?: S; } @@ -82,12 +149,14 @@ export interface SearchOptions< * const list = await client.opportunities.list(); * ``` */ -export class Opportunities { +export class Opportunities<R extends PluginRoutes = PluginRoutes> { private readonly client: Client; private readonly basePath = "/common-grants/opportunities"; + private readonly routes: R | undefined; - constructor(client: Client) { + constructor(client: Client, routes?: R) { this.client = client; + this.routes = routes; } // ############################################################################ @@ -236,22 +305,24 @@ export class Opportunities { * ``` */ async search<S extends OppSchema = typeof OpportunityBaseSchema>( - options?: SearchOptions<S> + options?: SearchOptions<S, R> ): Promise<Filtered<z.infer<S>, OppFilters>> { const schema = options?.schema ?? (OpportunityBaseSchema as unknown as S); - // Build the base search body (without pagination) - const searchBody = this.buildSearchBody(options); + // Build the base search body (without pagination). Client-side filter + // problems are collected fail-soft into `filterErrors` rather than thrown. + const { body: searchBody, errors: filterErrors } = this.buildSearchBody(options); // If page is specified, fetch only that page if (options?.page !== undefined) { - return this.fetchSearchPage( + const page = await this.fetchSearchPage( searchBody, options.page, options.pageSize, options.signal, schema ); + return mergeFilterErrors(page, filterErrors); } // Auto-paginate using fetchMany with POST method. @@ -264,34 +335,72 @@ export class Opportunities { schema, }); - return result as Filtered<z.infer<S>, OppFilters>; + return mergeFilterErrors(result as Filtered<z.infer<S>, OppFilters>, filterErrors); } // ############################################################################ // Private helpers // ############################################################################ - /** Builds the search request body from options */ - private buildSearchBody(options?: SearchOptions<OppSchema>): OppSearchRequest { + /** + * Builds the search request body from options. + * + * Returns the wire `body` alongside the `errors` collected from client-side + * filter classification (fail-soft — never throws on a bad filter). `search()` + * merges these into the response's `filterInfo.errors`. + */ + private buildSearchBody(options?: SearchOptions<OppSchema, R>): { + body: OppSearchRequest; + errors: FilterError[]; + } { const body: OppSearchRequest = {}; + const errors: FilterError[] = []; if (options?.query) { body.search = options.query; } - // Build filters from statuses shorthand and/or explicit filters - if (options?.statuses?.length) { - const filters: OppFilters = {}; + // `routes` is bound once at client construction, so it's read off the + // instance. Fail-soft: invalid keys are dropped and their errors collected. + let filters: OppFilters | undefined; + if (options?.filters) { + const classified = classifyFilters( + this.routes ?? {}, + "opportunities", + "search", + options.filters + ); + errors.push(...classified.errors); + // Cast bridges the Zod-inferred OppFilters to the hand-authored OppFilters + // type alias; they are structurally the same shape. + filters = classified.result as OppFilters; + } - filters.status = { - operator: ArrayOperator.in, - value: options.statuses, - }; + // statuses shorthand → status default field (augments any classified filters) + if (options?.statuses?.length) { + if (filters?.status !== undefined) { + // `status` given via both the shorthand and `filters`: `filters` wins, + // the shorthand is ignored, and a warning is collected (not thrown). + errors.push( + new FilterError( + "specified via both the statuses shorthand and the filters argument; used the filters value", + { path: "filters.status", sourceValue: options.statuses } + ) + ); + } else { + filters = filters ?? {}; + filters.status = { + operator: ArrayOperator.in, + value: options.statuses, + }; + } + } + if (filters) { body.filters = filters; } - return body; + return { body, errors }; } /** Fetches a single search page */ diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index 3e861f617..dd47e8fdb 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -740,23 +740,17 @@ For a complete runnable example with assertions, see [`examples/custom-filters.t - A filter spec uses an unknown `filterType` value. - A custom filter name collides with a default filter field name (e.g. registering `"status"` would shadow the protocol's standard `status` filter). -`validateFilterCall()` validates individual filters at call time. For registered filters it validates the operator and value shape against the declared `filterType`. For ad-hoc filters it applies a shape-only check (`DefaultFilterSchema`). `classifyFilters()` already runs it for every key — you only call it directly when validating a single filter outside classification. +Call-time validation runs automatically inside `classifyFilters()` for every filter key — registered filters are checked for operator/value shape against the declared `filterType`, ad-hoc filters get a shape-only check (`DefaultFilterSchema`). It is not a separate public entry point; `classifyFilters()` (invoked by each `search()`) is the call-time path. -In the PoC, `definePlugin()` passes `routes` through **unvalidated** — the full SDK client will run `validateRoutes()` when a plugin is attached and call-time validation inside each search. Until then, call `validateRoutes()` yourself after defining a plugin if you want registration-time errors surfaced early. +`definePlugin()` passes `routes` through **unvalidated**; the `Client` runs `validateRoutes()` for you when constructed with `routes` (and call-time validation runs inside each search). To surface registration errors without constructing a client, call `validateRoutes()` yourself after defining a plugin. > **PII note:** as with transforms, `FilterError.sourceValue` carries the raw input — here, the consumer's filter value. The [PII warning](#error-handling) above applies equally; log a redacted projection. ```typescript -import { validateRoutes, validateFilterCall } from "@common-grants/sdk/extensions"; +import { validateRoutes } from "@common-grants/sdk/extensions"; // Registration-time — throws FilterError on unknown filterType or collision validateRoutes(grantsGovPlugin.routes!); - -// Call-time — throws FilterError on operator/value mismatch for registered filters. -// Pass the filter's CustomFilterSpec (looked up from the route-method's filters), -// its name, and the value. -const agencySpec = grantsGovPlugin.routes!.opportunities.search.filters!.agency; -validateFilterCall(agencySpec, "agency", F.in(["HHS"])); // valid: in operator + string array value match the stringArray filterType ``` ### The `as const` trap @@ -899,16 +893,14 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, ### Custom filters (PoC) -| Export | Kind | Description | Demonstrated in | -| --------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| [`classifyFilters()`](./custom-filters.ts) | function | Three-bucket classifier. Maps a flat consumer `filters` object to the ADR-0012 `OppFilters` request body: default fields → top-level named fields; registered custom + ad-hoc → `customFilters` record. | [Classifying consumer filters](#classifying-consumer-filters-into-the-request-body) | -| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `FilterError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | -| [`validateFilterCall()`](./custom-filters.ts) | function | Call-time validator. Validates a single filter against its declared `filterType` schema (registered) or shape-only (ad-hoc). Throws `FilterError` on mismatch. | [Validation](#validation--registration-time-and-call-time) | -| [`F`](./custom-filters.ts) | namespace | Helper namespace. `F.eq`, `F.neq`, `F.gt`, `F.gte`, `F.lt`, `F.lte`, `F.in`, `F.notIn`, `F.like`, `F.notLike`, `F.between`, `F.outside` — each compiles to `{ operator, value }`. Note: `F.in` is `"in"` as an object property key. | [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) | -| [`CustomFilterSpec`](./types.ts) | interface | Per-filter declaration: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType`; no `value` field. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | -| [`CustomFilterType`](./types.ts) | type | 11-value literal union: `stringComparison \| stringArray \| numberComparison \| numberArray \| numberRange \| integerComparison \| booleanComparison \| dateComparison \| dateRange \| moneyComparison \| moneyRange`. | [Filter-type catalog](#filter-type-catalog-and-the-f-helpers) | -| [`PluginRoutes`](./types.ts) | type | `Record<string, Record<string, RouteDeclarations>>` — the `routes` value on `DefinePluginOptions`. Keys are resource name → method name → `RouteDeclarations`. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | -| [`RouteDeclarations`](./types.ts) | interface | Per-method filter map: `{ filters?: Record<string, CustomFilterSpec> }`. | | +| Export | Kind | Description | Demonstrated in | +| ------------------------------------------ | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | --- | -------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| [`classifyFilters()`](./custom-filters.ts) | function | Three-bucket classifier. Maps a flat consumer `filters` object to the ADR-0012 `OppFilters` request body: default fields → top-level named fields; registered custom + ad-hoc → `customFilters` record. | [Classifying consumer filters](#classifying-consumer-filters-into-the-request-body) | +| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `FilterError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | | [`F`](./custom-filters.ts) | namespace | Helper namespace. `F.eq`, `F.neq`, `F.gt`, `F.gte`, `F.lt`, `F.lte`, `F.in`, `F.notIn`, `F.like`, `F.notLike`, `F.between`, `F.outside` — each compiles to `{ operator, value }`. Note: `F.in` is `"in"` as an object property key. | [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) | +| [`CustomFilterSpec`](./types.ts) | interface | Per-filter declaration: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType`; no `value` field. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | +| [`CustomFilterType`](./types.ts) | type | 11-value literal union: `stringComparison \| stringArray \| numberComparison \| numberArray \| numberRange \| integerComparison \| booleanComparison \| dateComparison \| dateRange \| moneyComparison \| moneyRange`. | [Filter-type catalog](#filter-type-catalog-and-the-f-helpers) | +| [`PluginRoutes`](./types.ts) | type | `Record<string, Record<string, RouteDeclarations>>` — the `routes` value on `DefinePluginOptions`. Keys are resource name → method name → `RouteDeclarations`. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | +| [`RouteDeclarations`](./types.ts) | interface | Per-method filter map: `{ filters?: Record<string, CustomFilterSpec> }`. | | ### Shared types diff --git a/lib/ts-sdk/src/extensions/custom-filters.ts b/lib/ts-sdk/src/extensions/custom-filters.ts index 8e37014c7..0d7215d0e 100644 --- a/lib/ts-sdk/src/extensions/custom-filters.ts +++ b/lib/ts-sdk/src/extensions/custom-filters.ts @@ -73,6 +73,14 @@ const VALID_FILTER_TYPES = new Set<string>(Object.keys(FILTER_TYPE_SCHEMAS)); */ const DEFAULT_FILTER_NAMES = new Set<string>(Object.keys(OppDefaultFiltersSchema.shape)); +/** + * `resource.method` routes whose custom filters this client classifies. A route + * is filter-capable when its core operation declares a `filters` parameter + * (lib/core routes); this set hardcodes that subset. As more routes gain filter + * support, derive it from the contract rather than extending this literal by hand. + */ +const SUPPORTED_CUSTOM_FILTER_ROUTES = new Set<string>(["opportunities.search"]); + // ############################################################################ // Public — F helpers // ############################################################################ @@ -136,6 +144,9 @@ export const F = { * 2. A custom filter name that collides with a default-filter field name * (`status`, `closeDateRange`, `totalFundingAvailableRange`, * `minAwardAmountRange`, `maxAwardAmountRange`) + * 3. Filters declared on a route that does not support custom filters — a + * `resource.method` not in `SUPPORTED_CUSTOM_FILTER_ROUTES` (e.g. + * `opportunities.list`, whose core operation declares no `filters`) * * Duplicate filter names within a route-method need no check: filter names are * object keys, and JS object literals cannot represent duplicate keys. @@ -151,6 +162,14 @@ export function validateRoutes(routes: PluginRoutes): void { const filters = (declarations as RouteDeclarations).filters; if (!filters) continue; + if (!SUPPORTED_CUSTOM_FILTER_ROUTES.has(`${resourceKey}.${methodKey}`)) { + const supported = [...SUPPORTED_CUSTOM_FILTER_ROUTES].join(", "); + throw new FilterError( + `Route "${resourceKey}.${methodKey}" does not support custom filters (supported: ${supported})`, + { path: `routes.${resourceKey}.${methodKey}`, sourceValue: filters } + ); + } + for (const [filterName, spec] of Object.entries(filters)) { const path = `routes.${resourceKey}.${methodKey}.filters.${filterName}`; @@ -189,35 +208,39 @@ export function validateRoutes(routes: PluginRoutes): void { * - For AD-HOC filters (spec is undefined): shape-only check against * `DefaultFilterSchema` (no operator/filterType enforcement — accepted trade-off). * + * Fail-soft: returns a `FilterError` describing the problem, or `undefined` + * when the value is valid. The caller (`classifyFilters`) collects returned + * errors rather than aborting the whole call. + * * @param spec - The registered `CustomFilterSpec` for this filter, or `undefined` for ad-hoc * @param filterName - The filter key (used in error `path`) * @param filterValue - The raw filter value from the consumer `filters` object - * @throws {FilterError} on operator/filterType mismatch or value-shape mismatch + * @returns A `FilterError` on operator/filterType mismatch or value-shape mismatch, else `undefined` */ export function validateFilterCall( spec: CustomFilterSpec | undefined, filterName: string, filterValue: unknown -): void { +): FilterError | undefined { const path = `filters.${filterName}`; if (spec === undefined) { // Ad-hoc filter — shape-only check against DefaultFilterSchema const result = DefaultFilterSchema.safeParse(filterValue); if (!result.success) { - throw new FilterError( + return new FilterError( `Ad-hoc filter "${filterName}" has an invalid shape: ${result.error.message}`, { path, sourceValue: filterValue } ); } - return; + return undefined; } // Registered filter — validate against the filterType's schema const schema = FILTER_TYPE_SCHEMAS[spec.filterType]; if (!schema) { // Should not reach here if validateRoutes was called first, but guard anyway - throw new FilterError( + return new FilterError( `Unknown filterType "${spec.filterType}" for registered filter "${filterName}"`, { path, sourceValue: filterValue } ); @@ -226,17 +249,30 @@ export function validateFilterCall( // One parse validates both the operator enum and the value shape const result = schema.safeParse(filterValue); if (!result.success) { - throw new FilterError( + return new FilterError( `Filter "${filterName}" (filterType: "${spec.filterType}") failed validation: ${result.error.message}`, { path, sourceValue: filterValue } ); } + + return undefined; } // ############################################################################ // Public — classifyFilters (three-bucket classifier) // ############################################################################ +/** + * Fail-soft result of `classifyFilters`. + * + * `result` holds only the keys that passed validation; `errors` aggregates the + * `FilterError`s for keys that failed (those keys are omitted, not thrown on). + */ +export interface ClassifyResult<T = z.infer<typeof OppFiltersSchema>> { + result: T; + errors: FilterError[]; +} + /** * Classifies a flat consumer `filters` object into the ADR-0012 `OppFilters` request body. * @@ -251,52 +287,71 @@ export function validateFilterCall( * `gov.<system>@<filterName>` namespaced keys are treated as ad-hoc custom * filter keys and flow into `customFilters` verbatim — no auto-migration. * - * Call-time validation (`validateFilterCall`) is run for each key during classification. + * Validation runs for each key during classification: default fields are checked + * against their real field type (`OppDefaultFiltersSchema`), registered and ad-hoc + * keys via `validateFilterCall`. Validation is **fail-soft**: a key that fails + * is dropped from `result` and its `FilterError` is pushed onto `errors`; the + * call never throws on a bad filter. Valid keys classify normally. * * @param routes - The `PluginRoutes` from the plugin definition * @param resourceKey - The resource name (e.g. `"opportunities"`) * @param methodKey - The method name (e.g. `"search"`) * @param consumerFilters - The flat consumer-facing filters object - * @returns The classified `OppFilters` request body + * @returns `{ result, errors }` — the valid-only request body and the collected errors */ export function classifyFilters( routes: PluginRoutes, resourceKey: string, methodKey: string, consumerFilters: Record<string, unknown> -): z.infer<typeof OppFiltersSchema> { +): ClassifyResult { // Resolve registered filter specs for this route-method const registeredFilters: Record<string, CustomFilterSpec> = routes[resourceKey]?.[methodKey]?.filters ?? {}; const defaultFields: Partial<z.infer<typeof OppDefaultFiltersSchema>> = {}; const customFilters: Record<string, z.infer<typeof DefaultFilterSchema>> = {}; + const errors: FilterError[] = []; for (const [key, value] of Object.entries(consumerFilters)) { // Look up registered spec (undefined for ad-hoc and gov.* namespaced keys) const spec = registeredFilters[key] as CustomFilterSpec | undefined; if (DEFAULT_FILTER_NAMES.has(key)) { - // Bucket 1: default filter → top-level named field. - // Default keys get shape-only validation via DefaultFilterSchema (the same - // treatment as ad-hoc keys); per-field type enforcement against - // OppDefaultFiltersSchema is intentionally not applied here. The server - // validates default fields and reports any it cannot apply. - validateFilterCall(undefined, key, value); + // Bucket 1: default filter → top-level named field, validated against its + // real type from OppDefaultFiltersSchema (e.g. status → StringArrayFilter). + // safeParse doesn't reshape, so the original value is assigned unchanged. + // An invalid value is skipped and its error collected. + const fieldSchema = (OppDefaultFiltersSchema.shape as Record<string, z.ZodTypeAny>)[key]; + const result = fieldSchema.safeParse(value); + if (!result.success) { + errors.push( + new FilterError(`Default filter "${key}" failed validation: ${result.error.message}`, { + path: `filters.${key}`, + sourceValue: value, + }) + ); + continue; + } (defaultFields as Record<string, unknown>)[key] = value; } else { // Bucket 2 (registered custom) or Bucket 3 (ad-hoc / gov.* namespaced) - // Run call-time validation — passes spec if registered, undefined if ad-hoc - validateFilterCall(spec, key, value); + // Run call-time validation — passes spec if registered, undefined if ad-hoc. + // Fail-soft: collect the returned error and skip the key. + const error = validateFilterCall(spec, key, value); + if (error) { + errors.push(error); + continue; + } customFilters[key] = value as z.infer<typeof DefaultFilterSchema>; } } // Build request body — omit customFilters key entirely when empty (match nullish shape) - const requestBody: z.infer<typeof OppFiltersSchema> = { + const result: z.infer<typeof OppFiltersSchema> = { ...defaultFields, ...(Object.keys(customFilters).length > 0 ? { customFilters } : {}), }; - return requestBody; + return { result, errors }; } diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index ef6edeadd..8878e9c0d 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -16,7 +16,8 @@ export { definePlugin } from "./define-plugin"; // Custom filters — route-keyed filter registration + classification export type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; export { FilterError } from "./types"; -export { classifyFilters, validateRoutes, validateFilterCall, F } from "./custom-filters"; +export type { ClassifyResult } from "./custom-filters"; +export { classifyFilters, validateRoutes, F } from "./custom-filters"; // Schema utilities — lower-level tools for working with custom fields export type { WithCustomFieldsResult } from "./with-custom-fields"; From c861b407cbc92c025cc147c639b03f7ed141a0b4 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Fri, 26 Jun 2026 12:33:19 -0700 Subject: [PATCH 39/51] feat(core): add boolean and integer comparison filter models (#953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(core): add boolean and integer comparison filter models Add BooleanComparisonFilter (eq/neq against a boolean) and IntegerComparisonFilter (comparison + equivalence operators against an integer), exported via filters/index.tsp at protocol v0.3.0. Refs #895. * docs: correct changeset — comparison filters emit to JSON Schema only Filter models aren't route-referenced, so the openapi3 emitter omits them; only the json-schema emitter outputs them. The changeset claimed both. * fix(core): version new comparison filters at v0.4 v0.3 is already published, so BooleanComparisonFilter and IntegerComparisonFilter first ship in v0.4. Add v0_4 to the Versions enum and retag both models; correct the changeset to match. * chore: remove accidentally committed .venv and gitignore it The lib/python-sdk/.venv virtualenv (1,039 files) was swept in by an overly broad git add. Untrack it and add .venv/ to the python-sdk .gitignore so it can't recur. --- .../boolean-integer-comparison-filters.md | 8 ++++++++ lib/core/lib/core/filters/boolean.tsp | 18 ++++++++++++++++++ lib/core/lib/core/filters/index.tsp | 1 + lib/core/lib/core/filters/numeric.tsp | 15 +++++++++++++++ lib/core/lib/main.tsp | 1 + lib/python-sdk/.gitignore | 4 ++++ 6 files changed, 47 insertions(+) create mode 100644 .changeset/boolean-integer-comparison-filters.md create mode 100644 lib/core/lib/core/filters/boolean.tsp diff --git a/.changeset/boolean-integer-comparison-filters.md b/.changeset/boolean-integer-comparison-filters.md new file mode 100644 index 000000000..af4de730a --- /dev/null +++ b/.changeset/boolean-integer-comparison-filters.md @@ -0,0 +1,8 @@ +--- +"@common-grants/core": minor +--- + +Add `BooleanComparisonFilter` and `IntegerComparisonFilter` to the core filter catalog (#895). +`BooleanComparisonFilter` compares a boolean value with `eq`/`neq`; `IntegerComparisonFilter` +compares an integer value with the full comparison + equivalence operator set. Both are added at +protocol version 0.4.0 and emitted to JSON Schema output. diff --git a/lib/core/lib/core/filters/boolean.tsp b/lib/core/lib/core/filters/boolean.tsp new file mode 100644 index 000000000..ee884f5e8 --- /dev/null +++ b/lib/core/lib/core/filters/boolean.tsp @@ -0,0 +1,18 @@ +import "./base.tsp"; + +namespace CommonGrants.Filters; + +// ############################################################################ +// Boolean comparison filter +// ############################################################################ + +/** A filter that applies a comparison to a boolean value */ +@Versioning.added(CommonGrants.Versions.v0_4) +model BooleanComparisonFilter { + /** The operator to apply to the filter value */ + operator: EquivalenceOperators; + + /** The value to use for the filter operation */ + @example(true) + value: boolean; +} diff --git a/lib/core/lib/core/filters/index.tsp b/lib/core/lib/core/filters/index.tsp index c111b1844..c85dd2a17 100644 --- a/lib/core/lib/core/filters/index.tsp +++ b/lib/core/lib/core/filters/index.tsp @@ -1,4 +1,5 @@ import "./base.tsp"; +import "./boolean.tsp"; import "./date.tsp"; import "./numeric.tsp"; import "./money.tsp"; diff --git a/lib/core/lib/core/filters/numeric.tsp b/lib/core/lib/core/filters/numeric.tsp index b01933e23..6cfbf743f 100644 --- a/lib/core/lib/core/filters/numeric.tsp +++ b/lib/core/lib/core/filters/numeric.tsp @@ -20,6 +20,21 @@ model NumberComparisonFilter { value: numeric; } +// ############################################################################ +// Integer comparison filter +// ############################################################################ + +/** Filters by comparing a field to an integer value */ +@Versioning.added(CommonGrants.Versions.v0_4) +model IntegerComparisonFilter { + /** The comparison operator to apply to the filter value */ + operator: ComparisonOperators | EquivalenceOperators; + + /** The value to use for the filter operation */ + @example(100) + value: integer; +} + // ############################################################################ // Number range filter // ############################################################################ diff --git a/lib/core/lib/main.tsp b/lib/core/lib/main.tsp index 8d270ac86..80c27f8da 100644 --- a/lib/core/lib/main.tsp +++ b/lib/core/lib/main.tsp @@ -19,4 +19,5 @@ enum Versions { v0_1: "0.1.0", v0_2: "0.2.0", v0_3: "0.3.0", + v0_4: "0.4.0", } diff --git a/lib/python-sdk/.gitignore b/lib/python-sdk/.gitignore index a4db3a2c8..c2ace7c4e 100644 --- a/lib/python-sdk/.gitignore +++ b/lib/python-sdk/.gitignore @@ -11,5 +11,9 @@ __pycache__/ .pytest_cache/ .coverage +# Virtual environments +.venv/ +venv/ + #Generated Schema objects generated/ \ No newline at end of file From 3569e148db11d41412f635a8e199c8daadb32c53 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:52:29 -0700 Subject: [PATCH 40/51] feat(py-sdk): typed custom-filters authoring surface and client (#966) (#970) * chore(python-sdk): switch type gate from mypy to pyright (basic) * feat(python-sdk): make OpportunityFilters an open TypedDict (extra_items) * feat(python-sdk): add SearchResult/ListResult with per-row parse error partitioning * feat(python-sdk): typed custom-filters authoring surface and client * feat(python-sdk): add typed custom-filters example, type fixtures, and test * fix(python-sdk): export parse_batch from the client package * fix(python-sdk): address self-review findings (sort_info, collision guard, pytest pythonpath) * ci: use full clone in bump-version workflow to prevent changeset hang actions/checkout defaulted to a shallow depth-1 clone. When a PR's changesets were added in commits before HEAD, `changeset version` deepens the clone via `git fetch --deepen` to find each adding commit, and that fetch hangs the job until the 6h runner cap. fetch-depth: 0 gives full history so the deepen path is never entered; timeout-minutes bounds any future hang. * docs(py-sdk): fix stale review comments; normalize status shorthand echo - Correct results.py/opportunities.py comments: page-aggregation preserves the first page's sort/filter info via model_copy (the getattr fallbacks only fire on the empty-result path that returns a plain Paginated), not "aggregation drops them". - Update three _FHelpers comments that referenced the deleted filter_type machinery (CustomFilterSpec / FILTER_TYPE_SCHEMAS) to the current route-TypedDict value-model mechanism. - status shorthand now stores plain str values so filter_info.filters echoes the same shape as the classified path; the wire request is unchanged. * feat(py-sdk): raise on invalid standard/registered filter values client.opportunities.search() now raises FilterError on an invalid value for a standard or registered filter (and on a conflicting status shorthand) before sending the request, instead of collecting it into filter_info.errors. Ad-hoc (unregistered) keys stay best-effort and are dropped if unusable; filter_info.errors now carries server-returned errors only. classify_filters stays fail-soft (still collects, never raises) and tags standard/registered failures as strict so the resource client can act on them. Adds a runnable runtime unhappy-path demo to the typed-custom-filters example. Part of #966. * ci(py-sdk): conform Node setup to repo standard (setup-node@v6 + .nvmrc) The pyright type gate's Node step used actions/setup-node@v4 with a hardcoded node-version "20", diverging from every other workflow (setup-node@v6 + node-version-file: .nvmrc, which pins 22) and undershooting the repo's node engines floor (>=22). Match the standard. * docs(py-sdk): note deprecation of the status shorthand on search() The status param duplicates the status key in filters; flag it as deprecated so consumers get a heads-up before it is removed. --- .changeset/python-typed-custom-filters.md | 9 +- .github/workflows/ci-bump-version.yml | 5 + .github/workflows/ci-lib-pysdk.yml | 5 + lib/python-sdk/DEVELOPMENT.md | 4 +- lib/python-sdk/Makefile | 2 +- .../common_grants_sdk/client/__init__.py | 8 +- .../common_grants_sdk/client/client.py | 193 ++++++------- .../common_grants_sdk/client/opportunities.py | 253 +++++++++-------- .../common_grants_sdk/client/results.py | 81 ++++++ .../common_grants_sdk/extensions/__init__.py | 6 +- .../common_grants_sdk/extensions/filters.py | 225 +++++++-------- .../common_grants_sdk/extensions/plugin.py | 92 +++++- .../common_grants_sdk/extensions/schema.py | 14 +- .../common_grants_sdk/extensions/specs.py | 30 -- .../common_grants_sdk/extensions/types.py | 45 ++- .../schemas/pydantic/filters/opportunity.py | 21 +- lib/python-sdk/examples/__init__.py | 0 .../examples/consumer_search_with_filters.py | 32 +-- lib/python-sdk/examples/custom_filters.py | 73 ++--- .../examples/typed_custom_filters.py | 133 +++++++++ .../examples/typed_custom_filters_failures.py | 61 ++++ lib/python-sdk/poetry.lock | 219 +++------------ lib/python-sdk/pyproject.toml | 8 +- lib/python-sdk/pyrightconfig.json | 6 + lib/python-sdk/tests/client/test_client.py | 53 +++- .../tests/client/test_opportunities.py | 264 ++++++++++-------- lib/python-sdk/tests/client/test_results.py | 27 ++ .../test_typed_custom_filters_example.py | 99 +++++++ .../tests/extensions/test_filters.py | 264 ++++++++---------- .../tests/extensions/test_plugin.py | 42 +-- 30 files changed, 1288 insertions(+), 986 deletions(-) create mode 100644 lib/python-sdk/common_grants_sdk/client/results.py create mode 100644 lib/python-sdk/examples/__init__.py create mode 100644 lib/python-sdk/examples/typed_custom_filters.py create mode 100644 lib/python-sdk/examples/typed_custom_filters_failures.py create mode 100644 lib/python-sdk/pyrightconfig.json create mode 100644 lib/python-sdk/tests/client/test_results.py create mode 100644 lib/python-sdk/tests/client/test_typed_custom_filters_example.py diff --git a/.changeset/python-typed-custom-filters.md b/.changeset/python-typed-custom-filters.md index 517d7d740..dd19473ef 100644 --- a/.changeset/python-typed-custom-filters.md +++ b/.changeset/python-typed-custom-filters.md @@ -2,8 +2,9 @@ "common-grants-sdk": minor --- -Add codegen-free typed custom-filter authoring and client-side filtered search to the Python SDK. +Typed custom-filter authoring and consumption for the Python SDK. -- `OpportunityFilters` (a `TypedDict`) is the typed authoring surface for opportunity-search filters: a consumer annotates their filter dict with it — or a `total=False` subclass — to get per-key autocomplete and value-type narrowing, and to give each custom filter its own typed key. Clean value aliases (`StringArray`, `NumberComparison`, `DateComparison`, …) read at the call site. -- `define_plugin(routes=...)` accepts a route-keyed `PluginRoutes` map of `CustomFilterSpec` (`routes[resource][method].filters[name]`, per ADR-0022 Decision #10) and threads it onto `plugin.routes`. -- `Opportunities.search()` gains optional `filters` and `routes` parameters: the consumer's filter dict is classified via `classify_filters` into the three-bucket search body (default named fields + a `customFilters` record) and POSTed, returning typed `OpportunityBase` rows. `status` is now an optional shorthand merged into the filter set. +- `OpportunityFilters` is now an open `TypedDict` (PEP 728 `extra_items`): standard filter keys are typed to their value models, a `total=False` subclass gives each registered custom filter its own typed key, and unregistered keys still pass through. One source of truth for both registration and the consumer's `search(filters=...)` call site. Clean value aliases (`StringArray`, `NumberComparison`, `DateComparison`, …) read at the call site. +- `define_plugin(routes=...)` takes typed route carriers — `PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters))` — so a misspelled route/method is a type error, replacing the stringly-typed route dict. +- `Plugin.get_client(config)` returns a client already scoped with the plugin's routes and schemas: `opportunities.search()` / `.list()` parse responses with the plugin's custom fields by default and return `SearchResult` / `ListResult` that partition successfully parsed `items` from per-row parse `errors`, so one malformed row no longer fails the batch. +- `opportunities.search()` raises `FilterError` on an invalid value for a standard or registered filter (or a conflicting `status` shorthand) before sending the request; ad-hoc (unregistered) keys stay best-effort and are dropped if unusable. `filter_info.errors` now carries server-returned errors only. diff --git a/.github/workflows/ci-bump-version.yml b/.github/workflows/ci-bump-version.yml index 2743249a1..01757256c 100644 --- a/.github/workflows/ci-bump-version.yml +++ b/.github/workflows/ci-bump-version.yml @@ -12,11 +12,16 @@ permissions: jobs: version: runs-on: ubuntu-latest + timeout-minutes: 10 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout repo uses: actions/checkout@v6 + with: + # Full history: Changesets deepens a shallow clone (git fetch --deepen) + # to find each changeset's adding commit, which can hang in CI. + fetch-depth: 0 - name: Install Node.js uses: actions/setup-node@v6 diff --git a/.github/workflows/ci-lib-pysdk.yml b/.github/workflows/ci-lib-pysdk.yml index cf86b389b..b9c808751 100644 --- a/.github/workflows/ci-lib-pysdk.yml +++ b/.github/workflows/ci-lib-pysdk.yml @@ -24,6 +24,11 @@ jobs: - name: Install Poetry uses: Gr1N/setup-poetry@v9 + - name: Install Node + uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + - name: Install dependencies run: make install diff --git a/lib/python-sdk/DEVELOPMENT.md b/lib/python-sdk/DEVELOPMENT.md index 157ae5d43..f06009d64 100644 --- a/lib/python-sdk/DEVELOPMENT.md +++ b/lib/python-sdk/DEVELOPMENT.md @@ -51,9 +51,9 @@ poetry run ruff check . #### Type Checking -Verify types with MyPy: +Verify types with pyright (basic mode; tests are excluded from the gate — see `pyrightconfig.json`): ```bash -poetry run mypy . +poetry run pyright ``` ## Release runbook diff --git a/lib/python-sdk/Makefile b/lib/python-sdk/Makefile index 043a6fdbf..9bbb287f9 100644 --- a/lib/python-sdk/Makefile +++ b/lib/python-sdk/Makefile @@ -23,6 +23,6 @@ check-lint: $(RUNTIME_PREFIX) ruff check . check-types: - $(RUNTIME_PREFIX) mypy . + $(RUNTIME_PREFIX) pyright checks: check-format check-lint check-types diff --git a/lib/python-sdk/common_grants_sdk/client/__init__.py b/lib/python-sdk/common_grants_sdk/client/__init__.py index 13bf62e2c..5341e6180 100644 --- a/lib/python-sdk/common_grants_sdk/client/__init__.py +++ b/lib/python-sdk/common_grants_sdk/client/__init__.py @@ -1,11 +1,17 @@ """Client module for the CommonGrants API.""" from .auth import Auth -from .client import Client +from .client import BaseClient, Client from .config import Config +from .results import ListResult, ParseFailure, SearchResult, parse_batch __all__ = [ "Auth", + "BaseClient", "Client", "Config", + "ListResult", + "ParseFailure", + "SearchResult", + "parse_batch", ] diff --git a/lib/python-sdk/common_grants_sdk/client/client.py b/lib/python-sdk/common_grants_sdk/client/client.py index ce50f7936..9cd7bb3d4 100644 --- a/lib/python-sdk/common_grants_sdk/client/client.py +++ b/lib/python-sdk/common_grants_sdk/client/client.py @@ -1,9 +1,21 @@ -"""Main HTTP client for the CommonGrants API.""" +"""HTTP client for the CommonGrants API. + +``BaseClient`` is the transport plumbing (auth headers, paginated GET/POST). The +generic ``Client`` layers the typed resource facade on top: it binds a plugin's +route filters (``FiltersT``) and Opportunity schema (``ItemT``) so +``client.opportunities.search(filters=...)`` is typed and responses parse with the +plugin's custom fields by default. Construct one directly, or via +``plugin.get_client(...)``. +""" + +from __future__ import annotations import httpx -from typing import Any, Optional, cast +from typing import Any, Generic, Optional, cast from uuid import UUID +import typing_extensions as te + from .auth import Auth from .config import Config from .response import SuccessResponse @@ -12,54 +24,53 @@ from .pagination import pagination from .types import ItemsT from ..extensions.filters import validate_routes -from ..extensions.types import PluginRoutes +from ..extensions.plugin import PluginSchemas +from ..extensions.types import FiltersT, PluginRoutes, ResourceRoutes +from ..schemas.pydantic.models import OpportunityBase from ..schemas.pydantic.responses import Filtered, Paginated +# Bound is OpportunityBase[Any] (the custom-fields parameter is invariant, so a +# bare OpportunityBase bound would reject OpportunityBase[OppCustomFields]). +ItemT = te.TypeVar( + "ItemT", bound="OpportunityBase[Any]", default="OpportunityBase[Any]" +) + + +def _resolve_opportunity_schema( + schemas: Optional[PluginSchemas[Any]], +) -> type[OpportunityBase]: + """The Opportunity model a client parses into by default. + + Reads the common model bound to the plugin's Opportunity schema slot; falls + back to the base ``OpportunityBase`` when no schemas are bound. + """ + if schemas is None: + return OpportunityBase + common = getattr(getattr(schemas, "Opportunity", None), "common_schema", None) + return common if isinstance(common, type) else OpportunityBase + -class Client: - """HTTP client for interacting with the CommonGrants API.""" +class BaseClient: + """Transport plumbing for the CommonGrants API (auth + paginated GET/POST).""" def __init__( self, config: Optional[Config] = None, auth: Optional[Auth] = None, - routes: Optional[PluginRoutes] = None, ): - """Initialize the CommonGrants client. + """Initialize the transport layer. Args: config: Optional Config instance. If None, a default Config is created. auth: Optional Auth instance. If None, API key authentication is used with the key from config. - routes: Optional plugin ``routes`` declaration (fixed plugin config). - Bound and validated once here; used to classify registered custom - filters in ``opportunities.search``. - - Raises: - FilterError: If ``routes`` declares an unknown ``filter_type``, a - custom filter name that collides with a default filter name, or - custom filters on a route that does not support them (e.g. - ``opportunities.list``). """ self.config = config or Config() self.auth = auth or Auth.api_key(self.config.api_key) self.http = httpx.Client(timeout=self.config.timeout) - self.routes = routes or {} - validate_routes(self.routes) - self.opportunities = Opportunities(client=self) def post(self, path: str, **kwargs) -> httpx.Response: - """Simple wrapper around self.http.post. - - Automatically adds authentication headers to the request. - - Args: - path: end point path (should start with /) - **kwargs: Additional arguments to pass to httpx.post (e.g., json, data, params) - - Returns: - httpx.Response instance - """ + """Wrapper around ``self.http.post`` that adds auth headers.""" return self.http.post( self.url(path), headers=self.auth.get_headers(), @@ -67,17 +78,7 @@ def post(self, path: str, **kwargs) -> httpx.Response: ) def get(self, path: str, **kwargs) -> httpx.Response: - """Simple wrapper around self.http.get. - - Automatically adds authentication headers to the request. - - Args: - path: end point path (should start with /) - **kwargs: Additional arguments to pass to httpx.get (e.g., params, headers) - - Returns: - httpx.Response instance - """ + """Wrapper around ``self.http.get`` that adds auth headers.""" return self.http.get( self.url(path), headers=self.auth.get_headers(), @@ -89,16 +90,7 @@ def get_item( path: str, item_id: str | UUID, ) -> SuccessResponse: - """Get a specific item by ID from an endpoint. - - Makes a GET request to {path}/{item_id}. - - Args: - path: end point path (should start with /) - item_id: The item ID (string or UUID) - - Returns: - SuccessResponse instance with status, message, and data fields. + """Get a specific item by ID (GET ``{path}/{item_id}``). Raises: APIError: If the API request fails @@ -121,23 +113,9 @@ def list( page_size: int | None = None, params: dict[str, Any] | None = None, ) -> Paginated[ItemsT]: - """Fetch a set of items from an endpoint using a GET request. + """Fetch a set of items via GET, with pagination. - Args: - path: end point path (should start with /) - page: Page number (1-indexed). If None, method will fetch all - items across all pages and aggregate them into a single response. - When fetching all pages, the number of items is limited by - config.list_items_limit. - page_size: Number of items per page. If None, uses the default from - client config. - params: Additional parameters to pass to the API. These are merged - with pagination parameters (page, pageSize). - - Returns: - Paginated[ItemsT] instance. When page is None, the response contains - all items aggregated from all pages (up to list_items_limit), with - pagination_info summarizing the aggregated result. + When page is None, aggregates all pages up to ``config.list_items_limit``. Raises: APIError: If the API request fails @@ -168,21 +146,9 @@ def search( page: int | None = None, page_size: int | None = None, ) -> Paginated[ItemsT]: - """Fetch a set of items from an endpoint based on request data using a POST request. + """Fetch a set of items via POST, with pagination. - Args: - path: end point path (should start with /) - request_data: search specific request data (sent in request body as form data) - page: Page number (1-indexed). If None, will fetch all items across all pages - and aggregate them into a single response. When fetching all pages, the - number of items is limited by config.list_items_limit. - page_size: Number of items per page. If None, uses the default from - client config. - - Returns: - Paginated[ItemsT] instance. When page is None, the response contains - all items aggregated from all pages (up to list_items_limit), with - pagination_info summarizing the aggregated result. + When page is None, aggregates all pages up to ``config.list_items_limit``. Raises: APIError: If the API request fails @@ -209,17 +175,7 @@ def search( return result def url(self, path: str) -> str: - """Construct a full URL from base URL and path. - - Trailing slashes are automatically stripped from the base URL before - concatenation. - - Args: - path: end point path (should start with /) - - Returns: - Complete URL string - """ + """Construct a full URL from base URL and path (trailing slash stripped).""" base = self.config.base_url.rstrip("/") return f"{base}{path}" @@ -228,16 +184,53 @@ def close(self): self.http.close() def __enter__(self): - """Context manager entry. - - Returns: - The Client instance - """ + """Context manager entry; returns the client instance.""" return self def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit. + """Context manager exit; closes the HTTP client.""" + self.close() + + +class Client(BaseClient, Generic[FiltersT, ItemT]): + """Typed resource facade over :class:`BaseClient`. - Automatically closes the HTTP client when exiting the context. + Binds a plugin's route filters and Opportunity schema, exposing the typed + ``opportunities`` resource. Prefer ``plugin.get_client(...)`` over constructing + this directly. + """ + + def __init__( + self, + config: Optional[Config] = None, + auth: Optional[Auth] = None, + routes: Optional[PluginRoutes[Any]] = None, + schemas: Optional[PluginSchemas[Any]] = None, + ): + """Initialize the client. + + Args: + config: Optional Config instance. + auth: Optional Auth instance. + routes: Optional typed ``PluginRoutes`` carrier (fixed plugin config). + Bound and validated once here; used to classify registered custom + filters in ``opportunities.search``. + schemas: Optional ``PluginSchemas``; its Opportunity common model + becomes the default parse schema for ``opportunities`` responses. + + Raises: + FilterError: If ``routes`` registers a filter whose value type is not a + filter value model. """ - self.close() + super().__init__(config=config, auth=auth) + self.routes: PluginRoutes[Any] = ( + routes + if routes is not None + else PluginRoutes(opportunities=ResourceRoutes()) + ) + validate_routes(self.routes) + self.schemas = schemas + self._opportunity_schema: type[OpportunityBase] = _resolve_opportunity_schema( + schemas + ) + self.opportunities: Opportunities[FiltersT, ItemT] = Opportunities(client=self) diff --git a/lib/python-sdk/common_grants_sdk/client/opportunities.py b/lib/python-sdk/common_grants_sdk/client/opportunities.py index 334a0afb1..4e9731f33 100644 --- a/lib/python-sdk/common_grants_sdk/client/opportunities.py +++ b/lib/python-sdk/common_grants_sdk/client/opportunities.py @@ -1,32 +1,40 @@ -"""Opportunity namespace for the CommonGrants API.""" +"""Opportunity resource for the CommonGrants API.""" -import json -from typing import TYPE_CHECKING, Type +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Generic, List, Optional, cast from uuid import UUID +import typing_extensions as te + +from ..extensions.filters import classify_filters +from ..extensions.types import FilterError, FiltersT from ..schemas.pydantic.models import OpportunityBase -from ..schemas.pydantic.requests.opportunity import OpportunitySearchRequest -from ..schemas.pydantic.responses import ( - OpportunitiesListResponse, - OpportunitiesSearchResponse, - OpportunityResponse, - Paginated, -) from ..schemas.pydantic.models.opp_status import OppStatusOptions -from ..extensions.filters import classify_filters -from ..extensions.types import FilterError -from .types import ItemsT -from typing import Any, List, Mapping +from ..schemas.pydantic.requests.opportunity import OpportunitySearchRequest +from ..schemas.pydantic.responses.success import FilterInfo +from .results import ListResult, SearchResult, parse_batch if TYPE_CHECKING: from .client import Client +# Bound is OpportunityBase[Any] (the custom-fields parameter is invariant, so a +# bare OpportunityBase bound would reject OpportunityBase[OppCustomFields]). +ItemT = te.TypeVar( + "ItemT", bound="OpportunityBase[Any]", default="OpportunityBase[Any]" +) + + +class Opportunities(Generic[FiltersT, ItemT]): + """Fetch opportunity data from the CommonGrants API. -class Opportunities: - """Class for fetching opportunity data from CommonGrants API.""" + Bound (via ``plugin.get_client``) to a plugin's registered filter TypedDict + (``FiltersT``) and Opportunity schema (``ItemT``): ``search(filters=...)`` is + typed by the registered filters, and responses parse into ``ItemT`` by default. + """ - def __init__(self, client: "Client"): - """Initialize the Opportunity namespace. + def __init__(self, client: "Client[FiltersT, ItemT]"): + """Initialize the Opportunity resource. Args: client: Client instance for making API requests @@ -38,116 +46,113 @@ def path(self) -> str: """Return the API path for opportunities.""" return "/common-grants/opportunities" + def _schema( + self, override: Optional[type[OpportunityBase]] + ) -> type[OpportunityBase]: + """The schema to parse into: a per-call override, else the bound default.""" + return override if override is not None else self.client._opportunity_schema + def list( self, page: int | None = None, page_size: int | None = None, - schema: Type[OpportunityBase] | None = OpportunityBase, - ) -> OpportunitiesListResponse: + schema: Optional[type[OpportunityBase]] = None, + ) -> ListResult[ItemT]: """Fetch a set of opportunities. Args: - page: Page number (1-indexed). If None, method will fetch all - items across all pages and aggregate them into a single response. - page_size: Number of items per page. If None, uses the default from - client config. - schema: OpportunityBase to support custom fields from caller. + page: Page number (1-indexed). If None, fetches all pages. + page_size: Number of items per page. If None, uses the client default. + schema: Per-call parse-schema override; defaults to the plugin's + Opportunity schema (or ``OpportunityBase`` when unbound). Returns: - OpportunitiesListResponse instance. When page is None, the response - contains all items aggregated from all pages, with pagination_info - summarizing the aggregated result. + ``ListResult`` — parsed ``items`` plus per-row parse ``errors``. Raises: APIError: If the API request fails """ - # Call client method to get paginated response - paginated_response: Paginated[ItemsT] = self.client.list( # type: ignore[valid-type] - self.path, page=page, page_size=page_size + resolved = self._schema(schema) + paginated = self.client.list(self.path, page=page, page_size=page_size) + items, errors = parse_batch( + cast("list[dict[str, Any]]", list(paginated.items)), resolved + ) + return ListResult( + items=cast("list[ItemT]", items), + errors=errors, + pagination_info=paginated.pagination_info, ) - - # Hydrate OpportunityBase models from items dict - items = [ - schema.model_validate_json(json.dumps(item)) # type: ignore[union-attr] - for item in paginated_response.items - ] - - # Convert paginated_response to dict and replace items with hydrated models - response_data = paginated_response.model_dump(by_alias=True) - response_data["items"] = items - - # Hydrate OpportunitiesListResponse from response data - return OpportunitiesListResponse.model_validate(response_data) def get( - self, opp_id: str | UUID, schema: Type[OpportunityBase] | None = OpportunityBase - ) -> OpportunityBase: + self, + opp_id: str | UUID, + schema: Optional[type[OpportunityBase]] = None, + ) -> ItemT: """Get a specific opportunity by ID. Args: opp_id: The opportunity ID - schema: OpportunityBase to support custom fields from caller. + schema: Per-call parse-schema override; defaults to the plugin's + Opportunity schema (or ``OpportunityBase`` when unbound). Returns: - OpportunityBase instance + The parsed opportunity (``ItemT``). Raises: APIError: If the API request fails """ - # Call client method to get SuccessResponse + resolved = self._schema(schema) success_response = self.client.get_item(self.path, opp_id) - - # Hydrate OpportunityBase from response - response_data = success_response.model_dump(by_alias=True) - response_data["data"] = schema.from_dict(success_response.data) # type: ignore[union-attr] - - # Hydrate OpportunityResponse from response data - opportunity_response = OpportunityResponse.model_validate(response_data) - - # Return the OpportunityBase from the response - return opportunity_response.data + return cast("ItemT", resolved.model_validate(success_response.data)) def search( self, - search: str, + search: str = "", status: List[OppStatusOptions] | None = None, page: int | None = None, page_size: int | None = None, - schema: Type[OpportunityBase] | None = OpportunityBase, - filters: Mapping[str, Any] | None = None, - ) -> OpportunitiesSearchResponse: - """Search for opportunties by a query string + schema: Optional[type[OpportunityBase]] = None, + filters: Optional[FiltersT] = None, + ) -> SearchResult[ItemT]: + """Search for opportunities. Args: - search: The string to search for. - status: List of statuses to search on (shorthand for the ``status`` - default filter). - page: Page number (1-indexed). If None, method will fetch all - items across all pages and aggregate them into a single response. - page_size: Number of items per page. If None, uses the default from - client config. - schema: OpportunityBase to support custom fields added by the caller. - filters: Flat custom-filter dict (``{name: {"operator", "value"}}``, - e.g. built with the ``f`` helper), classified via - ``classify_filters``. Registered custom filters validate against - the specs declared in the client's ``routes`` (bound at construction). + search: The query string. + status: Statuses to filter on (shorthand for the ``status`` filter). + Deprecated — pass status through ``filters`` instead; this + shorthand will be removed in a future release. + page: Page number (1-indexed). If None, fetches all pages. + page_size: Number of items per page. If None, uses the client default. + schema: Per-call parse-schema override; defaults to the plugin's + Opportunity schema (or ``OpportunityBase`` when unbound). + filters: Typed filter dict — registered keys are validated against the + plugin's route filters and standard keys against their models. An + invalid value on a standard or registered filter raises + ``FilterError`` before any request; ad-hoc (unregistered) keys are + best-effort and dropped if unusable. Returns: - OpportunitiesSearchResponse with items and pagination info + ``SearchResult`` — parsed ``items``, per-row parse ``errors``, and + ``filter_info`` carrying the server's filter feedback. - Raises: - APIError: if the API request fails + Raises: + APIError: If the API request fails. + FilterError: If a standard or registered filter — or the ``status`` + shorthand — is given an invalid or conflicting value. """ + resolved = self._schema(schema) - # Classify the custom-filter dict. Fail-soft: invalid filters are dropped - # and their errors collected here rather than raised. - filters_body: dict = {} - filter_errors: list[FilterError] = [] + # Raise on an invalid standard/registered filter value rather than + # collecting it: filter_info.errors is reserved for server errors. The + # classifier stays fail-soft; ad-hoc keys are best-effort (dropped). + filters_body: dict[str, Any] = {} if filters: classified = classify_filters( - self.client.routes or {}, "opportunities", "search", filters + self.client.routes, "opportunities", "search", filters ) - filter_errors.extend(classified.errors) + strict_error = next((e for e in classified.errors if e.strict), None) + if strict_error is not None: + raise strict_error filters_body = classified.result.model_dump( by_alias=True, exclude_none=True, mode="json" ) @@ -155,57 +160,59 @@ def search( if status: if "status" in filters_body: # ``status`` given via both the shorthand and ``filters``: - # ``filters`` wins, the shorthand is ignored, a warning collected. - filter_errors.append( - FilterError( - "specified via both the status shorthand and the filters " - "argument; used the filters value", - path="filters.status", - source_value=status, - ) + # conflicting input to a standard filter — raise, don't guess. + raise FilterError( + "status specified via both the status shorthand and the " + "filters argument; pass it through only one of them", + path="filters.status", + source_value=status, ) - else: - filters_body["status"] = {"operator": "in", "value": status} - - request: dict = { + # str values (not enum members) so filter_info.filters echoes + # the same shape as the classified path. + filters_body["status"] = { + "operator": "in", + "value": [s.value for s in status], + } + + request: dict[str, Any] = { "pagination": {"page": 1, "pageSize": 10}, "search": search, "sorting": {"sortBy": "lastModifiedAt", "sortOrder": "desc"}, } - # Only set the filters key when non-empty. if filters_body: request["filters"] = filters_body request_data = OpportunitySearchRequest.model_validate(request) - # Call client method to get paginated response. mode="json" so date filter - # values serialize to ISO strings — httpx encodes json= with stdlib json, - # which rejects the datetime.date objects pydantic parses dates into. - paginated_response: Paginated[ItemsT] = self.client.search( # type: ignore[valid-type] + # mode="json" so date filter values serialize to ISO strings — httpx encodes + # json= with stdlib json, which rejects the date objects pydantic parses to. + paginated = self.client.search( f"{self.path}/search", request_data.model_dump(by_alias=True, exclude_unset=True, mode="json"), page=page, page_size=page_size, ) - # Hydrate OpportunityBase models from items dict - items = [ - schema.model_validate_json(json.dumps(item)) # type: ignore[union-attr] - for item in paginated_response.items - ] - - # Replace items with the hydrated models. - response_data = paginated_response.model_dump(by_alias=True) - response_data["items"] = items - - # Merge collected client-side errors into filterInfo.errors, flattened to - # "{path}: {message}" and ordered before any existing entries. - if filter_errors: - filter_info = response_data.setdefault("filterInfo", {}) - server_errors = filter_info.get("errors") or [] - filter_info["errors"] = [ - f"{e.path}: {e}" for e in filter_errors - ] + server_errors - - # Hydrate OpportunitiesSearchResponse from response data - return OpportunitiesSearchResponse.model_validate(response_data) + items, parse_errors = parse_batch( + cast("list[dict[str, Any]]", list(paginated.items)), resolved + ) + + # filter_info carries the server's filter feedback only — client-side + # filter problems already raised above. sort/filter info are preserved + # through page-aggregation (pagination copies the first page's Filtered + # envelope); the getattr fallbacks fire only on the empty-result path, + # which returns a plain Paginated. + server_filter_info = getattr(paginated, "filter_info", None) + server_errors = list(getattr(server_filter_info, "errors", None) or []) + filter_info: FilterInfo[Any] = FilterInfo( + filters=filters_body, + errors=server_errors, + ) + + return SearchResult( + items=cast("list[ItemT]", items), + errors=parse_errors, + pagination_info=paginated.pagination_info, + filter_info=filter_info, + sort_info=getattr(paginated, "sort_info", None), + ) diff --git a/lib/python-sdk/common_grants_sdk/client/results.py b/lib/python-sdk/common_grants_sdk/client/results.py new file mode 100644 index 000000000..f185937be --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/client/results.py @@ -0,0 +1,81 @@ +"""Typed result containers for the CommonGrants client. + +``SearchResult`` / ``ListResult`` partition a batch response into the rows that +parsed (``items``) and the rows that did not (``errors``), so one malformed row +never fails the whole response. This is distinct from ``filter_info.errors``, +which carries filter-validation and server-reported errors. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Generic, Optional + +from pydantic import BaseModel, ValidationError +from typing_extensions import TypeVar + +from ..schemas.pydantic.pagination import PaginatedResultsInfo +from ..schemas.pydantic.responses.success import FilterInfo +from ..schemas.pydantic.sorting import SortedResultsInfo + +ItemT = TypeVar("ItemT", bound=BaseModel) + + +@dataclass +class ParseFailure: + """A single row that failed to parse into the target schema. + + ``index`` is the row's position in the response batch; ``raw`` is the + unparsed row so a consumer can inspect or re-process it. + """ + + index: int + message: str + raw: dict[str, Any] + + +@dataclass +class ListResult(Generic[ItemT]): + """A parsed list response: successfully parsed ``items`` plus per-row ``errors``.""" + + items: list[ItemT] + errors: list[ParseFailure] + pagination_info: PaginatedResultsInfo + + +@dataclass +class SearchResult(Generic[ItemT]): + """A parsed search response. + + ``items`` are the rows that parsed; ``errors`` are per-row ``ParseFailure``s. + ``filter_info.errors`` is a separate channel carrying filter-validation and + server-reported errors (fail-soft), not row parse failures. + """ + + items: list[ItemT] + errors: list[ParseFailure] + pagination_info: PaginatedResultsInfo + filter_info: FilterInfo[Any] + # None only when the response carried no sort envelope (the empty-result + # path returns a plain Paginated); page-aggregation preserves the first + # page's sort_info via model_copy. + sort_info: Optional[SortedResultsInfo] = None + + +def parse_batch( + rows: list[dict[str, Any]], schema: type[ItemT] +) -> tuple[list[ItemT], list[ParseFailure]]: + """Validate each row into ``schema``, collecting failures instead of raising. + + Returns ``(items, errors)``: ``items`` are the rows that validated, in order; + ``errors`` are ``ParseFailure``s for the rows that did not. A single bad row + never aborts the batch. + """ + items: list[ItemT] = [] + errors: list[ParseFailure] = [] + for index, row in enumerate(rows): + try: + items.append(schema.model_validate(row)) + except ValidationError as exc: + errors.append(ParseFailure(index=index, message=str(exc), raw=row)) + return items, errors diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index d51efc5e7..012be99fe 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -21,8 +21,6 @@ from .specs import ( ConflictStrategy, CustomFieldSpec, - CustomFilterSpec, - CustomFilterType, PluginCustomFieldSpec, SchemaExtensions, ) @@ -34,6 +32,7 @@ PassthroughModel, PluginCapability, PluginRoutes, + ResourceRoutes, TransformError, TransformResult, ) @@ -66,9 +65,8 @@ "ClassifyResult", "FilterError", "PluginRoutes", + "ResourceRoutes", "classify_filters", - "CustomFilterSpec", - "CustomFilterType", "f", "validate_routes", ] diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index 789ce77ad..a4eeef2e6 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -1,15 +1,17 @@ """Pure-runtime filter engine for the CommonGrants Python SDK. Provides: -- ``f`` helper singleton for building DefaultFilter instances. -- ``FILTER_TYPE_SCHEMAS`` — map from CustomFilterType to the Pydantic validation model. +- ``f`` helper singleton for building filter value models. - ``DEFAULT_FILTER_NAMES`` — frozenset of all core default-filter field names (snake + alias). - ``validate_routes(routes)`` — registration-time validator; raises FilterError. -- ``validate_filter_call(spec, filter_name, value)`` — call-time validator; fail-soft, - returns ``(wire-ready DefaultFilter, None)`` on success or ``(None, FilterError)``. +- ``validate_filter_call(model_cls, filter_name, value)`` — call-time validator; fail-soft. + Validates against ``model_cls`` (a registered filter's value model) or ``DefaultFilter`` + when ``model_cls`` is ``None``; returns ``(wire-ready DefaultFilter, None)`` on success + or ``(None, FilterError)``. - ``classify_filters(routes, resource, method, consumer_filters)`` — fail-soft classifier producing a ``ClassifyResult`` (the valid-only ``OppFilters`` search request body plus - the collected ``FilterError``s; never raises on a bad call-time filter value). + the collected ``FilterError``s; never raises on a bad call-time filter value). Registered + custom filters are recovered from the route's TypedDict via ``get_type_hints``. No generate.py / codegen dependency. Correctness is enforced at runtime by Pydantic v2. """ @@ -17,11 +19,12 @@ from __future__ import annotations from collections.abc import Mapping -from types import MappingProxyType -from typing import Any, Optional, Union, overload +from dataclasses import fields +from typing import Any, Optional, Union, get_type_hints, overload from pydantic import BaseModel, ValidationError +from common_grants_sdk.schemas.pydantic.base import CommonGrantsBaseModel from common_grants_sdk.schemas.pydantic.filters.base import ( ArrayOperator, ComparisonOperator, @@ -31,14 +34,6 @@ StringOperator, ) from common_grants_sdk.schemas.pydantic.filters.boolean import BooleanComparisonFilter -from common_grants_sdk.schemas.pydantic.filters.date import ( - DateComparisonFilter, - DateRangeFilter, -) -from common_grants_sdk.schemas.pydantic.filters.money import ( - MoneyComparisonFilter, - MoneyRangeFilter, -) from common_grants_sdk.schemas.pydantic.filters.numeric import ( NumberArrayFilter, NumberComparisonFilter, @@ -48,13 +43,13 @@ from common_grants_sdk.schemas.pydantic.filters.opportunity import ( OppDefaultFilters, OppFilters, + OpportunityFilters, ) from common_grants_sdk.schemas.pydantic.filters.string import ( StringArrayFilter, StringComparisonFilter, ) -from .specs import CustomFilterSpec, CustomFilterType from .types import ClassifyResult, FilterError, PluginRoutes # --------------------------------------------------------------------------- @@ -164,7 +159,7 @@ def _comparison(self, operator: ComparisonOperator, value: Any) -> BaseModel: # Numbers map to NumberComparisonFilter; bool is excluded (it is not a # comparable number here and NumberComparisonFilter rejects it). Money # dicts and ISODate strings fall through to DefaultFilter, where the - # registered filter_type model validates them at call time. + # registered filter's value model validates them at call time. if not isinstance(value, bool) and isinstance(value, (int, float)): return NumberComparisonFilter(operator=operator, value=value) return DefaultFilter(operator=operator, value=value) @@ -198,7 +193,7 @@ def _array(self, operator: ArrayOperator, value: list[Any]) -> BaseModel: # An all-string list is a StringArrayFilter; an all-number (non-bool) list # is a NumberArrayFilter. Mixed / Money-dict lists fall through to # DefaultFilter. An empty list is treated as a string array (the common - # case; the registered filter_type re-validates either way). + # case; the registered filter's value model re-validates either way). if all(isinstance(v, str) for v in value): return StringArrayFilter(operator=operator, value=value) if all(not isinstance(v, bool) and isinstance(v, (int, float)) for v in value): @@ -238,7 +233,7 @@ def outside(self, min: Any, max: Any) -> BaseModel: def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: # A numeric min+max is a NumberRangeFilter; date / Money ranges (whose # value sub-models differ) fall through to DefaultFilter, where the - # registered filter_type (DATE_RANGE / MONEY_RANGE) validates the shape. + # registered filter's value model validates the shape. numeric = all( not isinstance(v, bool) and isinstance(v, (int, float)) for v in (min, max) ) @@ -252,33 +247,6 @@ def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: #: Module-level singleton — use as ``f.eq(...)``, ``f.in_([...])``, etc. f = _FHelpers() -# --------------------------------------------------------------------------- -# FILTER_TYPE_SCHEMAS — call-time validation map -# --------------------------------------------------------------------------- - -#: Maps each CustomFilterType to the Pydantic model used to validate operator/value shape. -#: ``booleanComparison`` uses the SDK-level ``BooleanComparisonFilter`` model (the spec -#: defines no boolean filter model). -#: Read-only: the catalog is closed — registering new filter types is a spec/SDK -#: change (extend CustomFilterType + this map together), not a runtime extension point. -FILTER_TYPE_SCHEMAS: Mapping[CustomFilterType, type[BaseModel]] = MappingProxyType( - { - CustomFilterType.STRING_COMPARISON: StringComparisonFilter, - CustomFilterType.STRING_ARRAY: StringArrayFilter, - CustomFilterType.NUMBER_COMPARISON: NumberComparisonFilter, - CustomFilterType.NUMBER_ARRAY: NumberArrayFilter, - CustomFilterType.NUMBER_RANGE: NumberRangeFilter, - # integerComparison reuses NumberComparisonFilter — the spec defines no - # integer filter model, so the int constraint is not schema-enforced - CustomFilterType.INTEGER_COMPARISON: NumberComparisonFilter, - CustomFilterType.BOOLEAN_COMPARISON: BooleanComparisonFilter, - CustomFilterType.DATE_COMPARISON: DateComparisonFilter, - CustomFilterType.DATE_RANGE: DateRangeFilter, - CustomFilterType.MONEY_COMPARISON: MoneyComparisonFilter, - CustomFilterType.MONEY_RANGE: MoneyRangeFilter, - } -) - # --------------------------------------------------------------------------- # DEFAULT_FILTER_NAMES — must include BOTH snake_case field names AND camelCase aliases # --------------------------------------------------------------------------- @@ -319,69 +287,79 @@ def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: } # --------------------------------------------------------------------------- -# validate_routes — registration-time validator +# Registered-filter recovery + validate_routes (registration-time validator) # --------------------------------------------------------------------------- -# (resource, method) pairs whose custom filters this client classifies. A route -# is filter-capable when its core operation declares a ``filters`` parameter -# (lib/core routes); this set hardcodes that subset. As more routes gain filter -# support, derive it from the contract rather than extending this literal by hand. -SUPPORTED_CUSTOM_FILTER_ROUTES: set[tuple[str, str]] = {("opportunities", "search")} +def _registered_filter_models(route_td: Any) -> dict[str, type[BaseModel]]: + """Recover a route's registered custom filters from its filter TypedDict. -def validate_routes(routes: PluginRoutes) -> None: - """Registration-time validator for a plugin's route filter declarations. + ``route_td`` is the TypedDict class an author put in the route slot (e.g. + ``OppSearchFilters``), or ``None``. Returns ``{filterName: value model}`` for + every key the author declared beyond the standard ``OpportunityFilters`` keys. + Non-model annotations are skipped here; ``validate_routes`` (run at ``Client`` + construction) rejects them, so a caller who invokes ``classify_filters`` directly + on unvalidated routes gets silent skipping rather than a raise. + """ + if route_td is None: + return {} + standard = set(get_type_hints(OpportunityFilters)) + return { + name: ann + for name, ann in get_type_hints(route_td).items() + if name not in standard and isinstance(ann, type) and issubclass(ann, BaseModel) + } - Iterates every filter spec in ``routes`` and raises ``FilterError`` on: - 1. Unknown ``filter_type`` (not in ``FILTER_TYPE_SCHEMAS``). - 2. Filter name that collides with a core default-filter name in - ``DEFAULT_FILTER_NAMES`` (the escape-hatch collision check; a namespaced - key such as ``gov.<system>@<filterName>`` passes through as ad-hoc instead). - 3. Filters declared on a route that does not support custom filters, i.e. a - ``(resource, method)`` not in ``SUPPORTED_CUSTOM_FILTER_ROUTES`` (e.g. - ``opportunities.list``, whose core operation declares no ``filters``). - Duplicate custom-filter names need no check: ``routes`` is dict-keyed, so a - duplicate name cannot reach this validator (a duplicated literal key collapses - silently to its last occurrence at dict construction, before this runs). +def validate_routes(routes: PluginRoutes[Any]) -> None: + """Registration-time validator for a plugin's typed route carriers. - Methods whose ``RouteDeclarations`` carry no ``filters`` key are skipped — - declaring a method with no filters is valid. + The typed carriers make a misspelled resource/method a *static* error. Two + runtime checks remain: + + 1. A registered custom filter's value type must be a filter value model (a + ``CommonGrantsBaseModel`` subclass) — a mistyped TypedDict value (e.g. + ``region: int``) would otherwise fail opaquely at call time. + 2. A route TypedDict must not redeclare a standard ``OpportunityFilters`` key + with a different value type. Authors get the standard keys for free; a + re-typed standard key (e.g. ``status: StringComparisonFilter``) creates a + static/runtime mismatch — the call site sees the override, but + classification validates against the real standard field type. Args: - routes: Route-keyed filter declarations as ``PluginRoutes``. + routes: Typed route registration (``PluginRoutes``). Raises: - FilterError: On the first invalid declaration found. + FilterError: On the first offending declaration. """ - for resource, methods in routes.items(): - for method, declarations in methods.items(): - filter_specs = declarations.get("filters") - if not filter_specs: + standard_hints = get_type_hints(OpportunityFilters) + for resource_field in fields(routes): + resource_routes = getattr(routes, resource_field.name) + for method_field in fields(resource_routes): + route_td = getattr(resource_routes, method_field.name) + if route_td is None: continue - if (resource, method) not in SUPPORTED_CUSTOM_FILTER_ROUTES: - supported = ", ".join( - f"{r}.{m}" for r, m in sorted(SUPPORTED_CUSTOM_FILTER_ROUTES) - ) - raise FilterError( - f'Route "{resource}.{method}" does not support custom filters ' - f"(supported: {supported})", - path=f"routes.{resource}.{method}", - source_value=filter_specs, - ) - for filter_name, spec in filter_specs.items(): - path = f"routes.{resource}.{method}.filters.{filter_name}" - if spec.filter_type not in FILTER_TYPE_SCHEMAS: - raise FilterError( - f'Unknown filter_type "{spec.filter_type}" for filter "{filter_name}"', - path=path, - source_value=spec, - ) - if filter_name in DEFAULT_FILTER_NAMES: + path_prefix = f"routes.{resource_field.name}.{method_field.name}" + for name, ann in get_type_hints(route_td).items(): + if name in standard_hints: + if ann is not standard_hints[name]: + raise FilterError( + f'Filter "{name}" on ' + f"{resource_field.name}.{method_field.name} redeclares a " + "standard filter with a different type", + path=f"{path_prefix}.{name}", + source_value=ann, + ) + continue + if not ( + isinstance(ann, type) and issubclass(ann, CommonGrantsBaseModel) + ): raise FilterError( - f'Filter name "{filter_name}" collides with a default filter name', - path=path, - source_value=filter_name, + f'Registered filter "{name}" on ' + f"{resource_field.name}.{method_field.name} must be a filter " + "value model (a CommonGrantsBaseModel subclass)", + path=f"{path_prefix}.{name}", + source_value=ann, ) @@ -403,15 +381,15 @@ def _first_error_detail(exc: ValidationError) -> str: def validate_filter_call( - spec: Optional[CustomFilterSpec], + model_cls: Optional[type[BaseModel]], filter_name: str, value: Any, ) -> tuple[Optional[DefaultFilter], Optional[FilterError]]: """Call-time validator for a single filter value (fail-soft). - Validates ``value`` against the Pydantic model for ``spec.filter_type`` when - ``spec`` is provided (registered filter), or against ``DefaultFilter`` when - ``spec`` is ``None`` (ad-hoc / escape-hatch filter). + Validates ``value`` against ``model_cls`` when provided (a registered filter's + value model, recovered from the route's TypedDict), or against ``DefaultFilter`` + when ``model_cls`` is ``None`` (ad-hoc / escape-hatch filter). Never raises: returns ``(validated_filter, None)`` on success or ``(None, FilterError)`` on failure. The caller collects the error and skips @@ -424,28 +402,16 @@ def validate_filter_call( be valid now. Args: - spec: The registered ``CustomFilterSpec`` for this filter, or ``None`` for ad-hoc. + model_cls: The registered filter's value model, or ``None`` for ad-hoc. filter_name: The filter name (used in FilterError path). value: The filter value to validate (typically a ``DefaultFilter`` instance). Returns: ``(DefaultFilter, None)`` when the value is valid, else ``(None, FilterError)``. - The error wraps the pydantic ``ValidationError`` as ``cause``. An unknown - ``spec.filter_type`` (not in ``FILTER_TYPE_SCHEMAS``) also returns a - ``FilterError`` — call ``validate_routes(routes)`` at registration time to - catch that earlier. + The error wraps the pydantic ``ValidationError`` as ``cause``. """ payload = value.model_dump() if isinstance(value, BaseModel) else value - if spec is not None: - model_cls = FILTER_TYPE_SCHEMAS.get(spec.filter_type) - if model_cls is None: - return None, FilterError( - f'Unknown filter_type "{spec.filter_type}" for filter ' - f'"{filter_name}" — call validate_routes(routes) at ' - "registration time to catch this earlier", - path=f"filters.{filter_name}", - source_value=spec, - ) + if model_cls is not None: try: validated = model_cls.model_validate(payload) except ValidationError as exc: @@ -518,7 +484,7 @@ def _validate_default_field( # deriving both from the declared resource is tracked in # https://github.com/HHS/simpler-grants-protocol/issues/896. def classify_filters( - routes: PluginRoutes, + routes: PluginRoutes[Any], resource: str, method: str, consumer_filters: Mapping[str, Union[BaseModel, dict[str, Any]]], @@ -528,8 +494,8 @@ def classify_filters( Three-bucket classification: - Bucket 1 (default): key is in ``DEFAULT_FILTER_NAMES`` (snake or camelCase alias) → normalize to the camelCase alias form, land in a named OppFilters field. - - Bucket 2 (registered custom): key matches a registered ``CustomFilterSpec`` for the - given resource/method → land in ``custom_filters``. + - Bucket 2 (registered custom): key matches a filter registered on the route's + TypedDict for the given resource/method → land in ``custom_filters``. - Bucket 3 (ad-hoc): any other key → land in ``custom_filters`` passthrough. Validation is **fail-soft**: a key that fails its call-time validation is @@ -572,10 +538,9 @@ def classify_filters( coerced ``date`` objects are not JSON-serializable in the default python mode (operator enums are ``StrEnum`` and serialize fine either way). """ - route_declarations = routes.get(resource, {}).get(method, {}) - registered_specs: dict[str, CustomFilterSpec] = route_declarations.get( - "filters", {} - ) + route = getattr(routes, resource, None) + route_td = getattr(route, method, None) if route is not None else None + registered = _registered_filter_models(route_td) default_fields: dict[str, Any] = {} custom_buckets: dict[str, DefaultFilter] = {} @@ -603,27 +568,31 @@ def classify_filters( "(snake_case and camelCase forms of the same filter)", path=f"filters.{alias_key}", source_value=value, + strict=True, ) ) continue validated, error = _validate_default_field(alias_key, value) if error is not None: + # Standard filter — the SDK owns its contract; mark strict so the + # resource client raises rather than fail-softing. + error.strict = True errors.append(error) continue default_fields[alias_key] = validated - elif key in registered_specs: - # Bucket 2: registered custom filter — ship the validated value, - # never the raw input (see validate_filter_call). Fail-soft: collect - # the error and skip the key. - spec = registered_specs[key] - validated, error = validate_filter_call(spec, key, value) + elif key in registered: + # Bucket 2: registered custom filter — validate against the value + # model recovered from the route's TypedDict; ship the validated + # value, never the raw input. Fail-soft here; strict for the client. + validated, error = validate_filter_call(registered[key], key, value) if error is not None: + error.strict = True errors.append(error) continue custom_buckets[key] = validated # type: ignore[assignment] else: # Bucket 3: ad-hoc / escape-hatch passthrough — ship the validated value. - # Fail-soft: collect the error and skip the key. + # Fail-soft: collect the (non-strict) error and skip the key. validated, error = validate_filter_call(None, key, value) if error is not None: errors.append(error) diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 17f0b50a7..39f3c1e27 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -2,15 +2,19 @@ A plugin maps the schema extensions an author builds with ``schema(...)`` onto the registered extensible schemas, keyed by registry name, and (optionally) registers -per-route custom filters via ``routes`` (a ``PluginRoutes`` map). Schemas a plugin +per-route custom filters via ``routes`` (a ``PluginRoutes`` carrier). Schemas a plugin does not extend fall back to the base schema (a ``SchemaOnly``), never ``None``, so consumers get fully-typed, non-optional dot access: ``plugin.schemas.Opportunity``. + +``plugin.get_client(...)`` returns a client already scoped with the plugin's routes and +schemas: ``client.opportunities.search(filters=...)`` is typed by the registered filter +TypedDict, and responses parse with the plugin's Opportunity schema by default. """ from __future__ import annotations from dataclasses import dataclass, field, fields -from typing import Any, Generic, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, Optional, cast, overload import typing_extensions as te @@ -21,9 +25,24 @@ SchemaWithTransforms, schema, ) -from .types import PluginMeta, PluginRoutes - -SchemasT = TypeVar("SchemasT") +from .types import FiltersT, PluginMeta, PluginRoutes, ResourceRoutes + +if TYPE_CHECKING: + from ..client.auth import Auth + from ..client.client import Client + from ..client.config import Config + +SchemasT = te.TypeVar("SchemasT") + +# The item model a get_client()-scoped client parses responses into: the common +# model bound to the plugin's Opportunity schema slot (OpportunityBase or a +# custom-fields parametrization), defaulting to the base when unextended. +# Bound is OpportunityBase[Any] (not OpportunityBase) because the custom-fields +# type parameter is invariant — a bare OpportunityBase bound would reject +# OpportunityBase[OppCustomFields]. +ItemT = te.TypeVar( + "ItemT", bound="OpportunityBase[Any]", default="OpportunityBase[Any]" +) # The fallback for a schema a plugin does not extend: the base schema, no custom # fields, no transforms. A SchemaOnly type, so unextended slots have no @@ -56,34 +75,74 @@ class PluginSchemas(Generic[_TOpportunity]): @dataclass(frozen=True) -class Plugin(Generic[SchemasT]): +class Plugin(Generic[SchemasT, FiltersT]): """The plugin singleton consumers import. ``schemas`` is a typed frozen dataclass, so ``plugin.schemas.Opportunity`` is - fully typed (dot access). ``routes`` is the route-keyed custom-filter - registration (``PluginRoutes``) a consumer passes to ``classify_filters``. + fully typed (dot access). ``routes`` is the typed custom-filter registration + (``PluginRoutes``); its filter TypedDict flows through ``get_client`` to type + ``client.opportunities.search(filters=...)``. """ schemas: SchemasT - routes: PluginRoutes + routes: PluginRoutes[FiltersT] meta: PluginMeta + @overload + def get_client( + self: "Plugin[PluginSchemas[SchemaOnly[ItemT]], FiltersT]", + config: Optional[Config] = ..., + auth: Optional[Auth] = ..., + ) -> "Client[FiltersT, ItemT]": ... + + @overload + def get_client( + self: "Plugin[PluginSchemas[SchemaWithTransforms[Any, ItemT]], FiltersT]", + config: Optional[Config] = ..., + auth: Optional[Auth] = ..., + ) -> "Client[FiltersT, ItemT]": ... + + def get_client( + self, + config: "Optional[Config]" = None, + auth: "Optional[Auth]" = None, + ) -> "Client[Any, Any]": + """Return a client pre-scoped with this plugin's routes and schemas. + + Consumers call ``plugin.get_client(config)`` instead of constructing a + client and passing ``routes=`` / ``schemas=`` by hand. The returned + client types ``opportunities.search(filters=...)`` by the plugin's + registered filters and parses responses with its Opportunity schema. + """ + # Local import: the client imports from extensions, so importing it at + # module scope here would create a cycle. + from ..client.client import Client + + # self.schemas/self.routes are opaque here (SchemasT); the overloads above + # carry the precise Client[FiltersT, ItemT] the caller sees. + return Client( + config=config, + auth=auth, + routes=cast("Any", self.routes), + schemas=cast("Any", self.schemas), + ) + def define_plugin( schemas: SchemasT, *, - routes: PluginRoutes | None = None, + routes: PluginRoutes[FiltersT] | None = None, meta: PluginMeta, -) -> Plugin[SchemasT]: +) -> Plugin[SchemasT, FiltersT]: """Assemble the plugin from schema extensions, optional route registrations, and metadata. Each schema attribute name must equal the entry's ``schema_name``, so ``schemas.Opportunity`` really holds the Opportunity extensible schema. - ``routes`` is the route-keyed custom-filter registration (``PluginRoutes``), - threaded onto the returned plugin so a consumer can pass ``plugin.routes`` to - ``classify_filters``. Omitted, it defaults to an empty map. + ``routes`` is the typed custom-filter registration (``PluginRoutes``), threaded + onto the returned plugin so ``plugin.get_client()`` can classify and type custom + filters. Omitted, it defaults to a carrier with no registered filters. Raises: PluginDefinitionError: If any slot does not hold a schema extension, or holds @@ -101,4 +160,7 @@ def define_plugin( ) if errors: raise PluginDefinitionError("plugin", errors) - return Plugin(schemas=schemas, routes=routes or {}, meta=meta) + resolved_routes: PluginRoutes[Any] = ( + routes if routes is not None else PluginRoutes(opportunities=ResourceRoutes()) + ) + return Plugin(schemas=schemas, routes=resolved_routes, meta=meta) diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py index c128a8868..6e848e2c9 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/schema.py +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -23,6 +23,7 @@ Optional, TypeGuard, TypeVar, + cast, get_args, get_origin, overload, @@ -390,16 +391,23 @@ def schema( common_schema=common_schema, source_schema=source_schema, ) + # impl params are Any by design — the overloads above are the static + # contract; cast the ctor args pyright can't reconcile with its generics. return SchemaWithTransforms( - schema_name, common_schema, custom_fields, source_schema, to_fn, from_fn + schema_name, + common_schema, + custom_fields, + cast(Any, source_schema), + to_fn, + cast(Any, from_fn), ) if to_common is not None: return SchemaWithTransforms( schema_name, common_schema, custom_fields, - source_schema, + cast(Any, source_schema), to_common, - from_common, + cast(Any, from_common), ) return SchemaOnly(schema_name, common_schema, custom_fields) diff --git a/lib/python-sdk/common_grants_sdk/extensions/specs.py b/lib/python-sdk/common_grants_sdk/extensions/specs.py index 766e5824a..a91dd925c 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/specs.py +++ b/lib/python-sdk/common_grants_sdk/extensions/specs.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum from typing import Any, Literal, Optional, TypedDict from ..schemas.pydantic.fields.custom import CustomFieldType @@ -11,35 +10,6 @@ ConflictStrategy = Literal["error", "first_wins", "last_wins"] -class CustomFilterType(StrEnum): - """Catalog of registerable filter types. Uses *Comparison/*Array/*Range wire values.""" - - STRING_COMPARISON = "stringComparison" - STRING_ARRAY = "stringArray" - NUMBER_COMPARISON = "numberComparison" - NUMBER_ARRAY = "numberArray" - NUMBER_RANGE = "numberRange" - INTEGER_COMPARISON = "integerComparison" - BOOLEAN_COMPARISON = "booleanComparison" - DATE_COMPARISON = "dateComparison" - DATE_RANGE = "dateRange" - MONEY_COMPARISON = "moneyComparison" - MONEY_RANGE = "moneyRange" - - -@dataclass(frozen=True) -class CustomFilterSpec: - """Per-filter declaration: filter_type constrains to a known type; description is optional. - - Frozen: a spec is an immutable declaration; mutating one after registration - would bypass validate_routes. No ``value`` field — the allowed operator set - is enforced by the filter_type's validation model at call time. - """ - - filter_type: CustomFilterType - description: Optional[str] = None - - @dataclass class CustomFieldSpec: """Runtime custom-field declaration consumed by the registration path. diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 3527a88db..76d26abad 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -8,14 +8,13 @@ Callable, Generic, Literal, - NotRequired, - TypedDict, - TypeVar, + Mapping, ) from pydantic import BaseModel, ConfigDict, Field +from typing_extensions import TypeVar -from .specs import CustomFilterSpec +from ..schemas.pydantic.filters.opportunity import OpportunityFilters T = TypeVar("T") @@ -25,24 +24,34 @@ # Type aliases Handler = Callable[[Any, Any], Any] -# Route-keyed custom-filter declaration types: -# PluginRoutes = {resourceName: {methodName: RouteDeclarations}}. -RouteMethodFilters = dict[str, CustomFilterSpec] # {filterName: spec} +# Custom-filter registration is a typed carrier: an author writes +# ``PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters))`` where the +# ``search`` slot holds the filter TypedDict class directly. A misspelled +# resource/method is a type error; registered keys are recovered at runtime with +# ``get_type_hints(route.search)``. +FiltersT = TypeVar("FiltersT", bound=Mapping[str, Any], default=OpportunityFilters) -class RouteDeclarations(TypedDict): - """Filter declarations for a single route method (e.g. ``search``). +@dataclass(frozen=True) +class ResourceRoutes(Generic[FiltersT]): + """Custom-filter registration for one resource's routes. - ``filters`` maps filter name → ``CustomFilterSpec`` and is optional: a - method may appear in the route map with no declarations. + ``search`` holds the filter TypedDict class an author defined (e.g. + ``OppSearchFilters``), or ``None`` when the resource registers no custom + filters. A single resource (opportunities) is supported today. """ - filters: NotRequired[RouteMethodFilters] + search: type[FiltersT] | None = None -PluginRoutes = dict[ - str, dict[str, RouteDeclarations] -] # {resource: {method: declarations}} +@dataclass(frozen=True) +class PluginRoutes(Generic[FiltersT]): + """Typed route registration passed to ``define_plugin(routes=...)``. + + One slot per filter-capable resource (opportunities today). + """ + + opportunities: ResourceRoutes[FiltersT] class PassthroughModel(BaseModel): @@ -95,6 +104,10 @@ class FilterError(Exception): Note: source_value may contain PII. Adopters are responsible for redacting it before logging or re-raising. The SDK does not redact by default. + + ``strict`` marks a failure on a typed filter — a standard or registered + custom filter, whose contract the SDK owns. The resource client raises on + strict failures; non-strict (ad-hoc / passthrough) failures stay fail-soft. """ def __init__( @@ -105,12 +118,14 @@ def __init__( handler: str | None = None, source_value: Any = None, cause: BaseException | None = None, + strict: bool = False, ) -> None: super().__init__(message) self.path = path self.handler = handler self.source_value = source_value self.cause = cause + self.strict = strict @dataclass diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py index bb6cab981..dc2a04050 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py @@ -1,7 +1,10 @@ """Schemas for the CommonGrants API.""" -from typing import Optional, TypedDict +from typing import Optional +from typing_extensions import TypedDict + +from ..base import CommonGrantsBaseModel from .base import DefaultFilter from .boolean import BooleanComparisonFilter from .date import DateComparisonFilter, DateRangeFilter @@ -65,20 +68,22 @@ class OppFilters(OppDefaultFilters): ) -class OpportunityFilters(TypedDict, total=False): +class OpportunityFilters(TypedDict, total=False, extra_items=CommonGrantsBaseModel): """Typed authoring surface for opportunity filters — the dict consumers annotate, extended to give each custom filter its own typed key. - Keys are the wire names a consumer passes to ``classify_filters`` (the + Standard keys are the wire names a consumer passes to the client (the camelCase aliases of :class:`OppDefaultFilters`), each typed to its value - model. A plugin author extends this to register custom filters for the route:: + model. A plugin author extends this to register custom filters for a route:: class OppSearchFilters(OpportunityFilters, total=False): - agency: StringArray + region: StringArray - ``total=False`` so every standard key is optional. This is the static - authoring surface only; the throw-based ``classify_filters`` runtime is - unchanged and does not consume it. + Open via PEP 728 ``extra_items``: any key beyond those declared is typed + ``CommonGrantsBaseModel`` (the common base of every filter value model), so + a registered custom key gets exact typing through the subclass while an + unregistered ad-hoc key still passes. ``total=False`` makes every standard + key optional. """ status: StringArray diff --git a/lib/python-sdk/examples/__init__.py b/lib/python-sdk/examples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/python-sdk/examples/consumer_search_with_filters.py b/lib/python-sdk/examples/consumer_search_with_filters.py index 73942085c..ee0c8deed 100644 --- a/lib/python-sdk/examples/consumer_search_with_filters.py +++ b/lib/python-sdk/examples/consumer_search_with_filters.py @@ -3,10 +3,11 @@ The full downstream flow an adopter writes against the SDK: - AUTHOR registers the custom filters a route accepts on the plugin - (``define_plugin(routes=...)`` → ``plugin.routes``, a ``PluginRoutes`` map - of ``CustomFilterSpec``), and extends the ``OpportunityFilters`` TypedDict - so the call-site filter dict narrows per key. + AUTHOR registers the custom filters a route accepts on the plugin by extending + the ``OpportunityFilters`` TypedDict (one typed key per custom filter) and + passing the carrier as ``define_plugin(routes=...)`` → ``plugin.routes``, a + ``PluginRoutes`` carrier. The typed dict also narrows the call-site filter + dict per key. CONSUMER constructs the client with the registered routes (``Client(config, routes=plugin.routes)``), builds a filter dict with the ``f.*`` builders, and calls ``client.opportunities.search(search=..., filters=...)``. @@ -35,10 +36,10 @@ from common_grants_sdk.client import Client from common_grants_sdk.client.config import Config from common_grants_sdk.extensions import ( - CustomFilterSpec, - CustomFilterType, PluginMeta, + PluginRoutes, PluginSchemas, + ResourceRoutes, define_plugin, f, schema, @@ -65,24 +66,11 @@ class OppSearchFilters(OpportunityFilters, total=False): # Route registration the client's classifier consumes (see define_plugin(routes=...)). -# Each custom filter's value shape is enforced at call time by its filter_type model. +# The OppSearchFilters TypedDict IS the registration: each custom filter's value +# shape is its declared annotation, enforced against the call-site value at call time. plugin = define_plugin( PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), - routes={ - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ), - "awardCount": CustomFilterSpec( - filter_type=CustomFilterType.NUMBER_COMPARISON, - description="Number of awards expected", - ), - } - } - } - }, + routes=PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)), meta=PluginMeta(name="grants.gov", source_system="grants.gov"), ) diff --git a/lib/python-sdk/examples/custom_filters.py b/lib/python-sdk/examples/custom_filters.py index 4addacd7e..1b484ad7b 100644 --- a/lib/python-sdk/examples/custom_filters.py +++ b/lib/python-sdk/examples/custom_filters.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Custom filters example — codegen-free request-body demo. -Demonstrates route-keyed custom filter declaration, flat call-site classification, -and the three-bucket ``OppFilters`` request body (default named fields + customFilters record) -using the grants.gov canonical example from #646/#869. +Demonstrates typed route registration, flat call-site classification, and the +three-bucket ``OppFilters`` request body (default named fields + customFilters +record) using the grants.gov canonical example from #646/#869. No code generation (custom filters are a pure-runtime classifier). Routes are a -plain ``PluginRoutes`` declaration passed directly to ``classify_filters`` — the -plugin framework (``define_plugin``) does not need to carry them. +typed ``PluginRoutes`` carrier — the author extends ``OpportunityFilters`` with +one typed key per custom filter, so ``classify_filters`` recovers each key's +value model from the TypedDict. Run (from lib/python-sdk/): poetry run python examples/custom_filters.py @@ -17,12 +18,21 @@ import json -from common_grants_sdk.extensions import classify_filters, f -from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType -from common_grants_sdk.extensions.types import PluginMeta, PluginRoutes +from common_grants_sdk.extensions import ( + PluginRoutes, + ResourceRoutes, + classify_filters, + f, +) +from common_grants_sdk.extensions.types import PluginMeta +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OpportunityFilters, + StringArray, + StringComparison, +) # --------------------------------------------------------------------------- -# Route-keyed custom filter specs + plugin metadata +# Typed route registration + plugin metadata # --------------------------------------------------------------------------- meta = PluginMeta( @@ -32,19 +42,21 @@ capabilities=["customFilters"], ) -routes: PluginRoutes = { - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec(filter_type=CustomFilterType.STRING_ARRAY), - "fundingProgram": CustomFilterSpec( - filter_type=CustomFilterType.STRING_COMPARISON, - description="Program name filter", - ), - } - } - }, -} + +class OppSearchFilters(OpportunityFilters, total=False): + """The custom filters this plugin accepts on opportunities.search. + + Each key's annotation *is* its Pydantic value model, so the call-site value + validates against it and the classifier recovers it from the TypedDict. + """ + + agency: StringArray + fundingProgram: StringComparison + + +routes: PluginRoutes = PluginRoutes( + opportunities=ResourceRoutes(search=OppSearchFilters) +) def _section(title: str) -> None: @@ -56,12 +68,9 @@ def _section(title: str) -> None: def main() -> None: # --- Declared routes --- _section("PLUGIN ROUTES (declared)") - for resource, methods in routes.items(): - for method, declarations in methods.items(): - print(f" {resource}.{method}:") - for name, spec in declarations.get("filters", {}).items(): - desc = f" — {spec.description}" if spec.description else "" - print(f" {name}: {spec.filter_type.value}{desc}") + print(" opportunities.search custom filters:") + print(" agency: StringArray") + print(" fundingProgram: StringComparison — Program name filter") # --- Classify: default + registered custom + ad-hoc --- _section("CLASSIFY FILTERS — default + custom + ad-hoc (canonical grants.gov demo)") @@ -72,8 +81,8 @@ def main() -> None: # the classifier must normalize it to "closeDateRange" # (an unnormalized snake key is silently dropped — # the alias landmine) - # "agency" — registered custom filter (routes.opportunities.search.agency) - # "fundingProgram" — registered custom filter (routes.opportunities.search.fundingProgram) + # "agency" — registered custom filter (opportunities.search.agency) + # "fundingProgram" — registered custom filter (opportunities.search.fundingProgram) # "legacyTag" — ad-hoc passthrough (not registered, not a core default) consumer_filters = { "status": f.in_(["open", "forecasted"]), @@ -96,7 +105,7 @@ def main() -> None: # --- FilterError demo — bad call is fail-soft (collected, not raised) --- _section("VALIDATION — bad operator collected, valid filters still applied") - # agency is registered as STRING_ARRAY (expects ArrayOperator: in/notIn). + # agency is registered as StringArray (expects ArrayOperator: in/notIn). # Passing f.eq(...) (EquivalenceOperator.EQUAL) fails call-time validation. # Fail-soft: classify_filters never raises on a bad # filter value — the bad key is dropped from the result body and a FilterError @@ -104,7 +113,7 @@ def main() -> None: bad_filters = { "agency": f.eq( "wrong-operator-for-array-type" - ), # eq is not a valid STRING_ARRAY op + ), # eq is not a valid StringArray op "legacyTag": f.eq("priority"), # valid ad-hoc filter — survives } classified_bad = classify_filters(routes, "opportunities", "search", bad_filters) diff --git a/lib/python-sdk/examples/typed_custom_filters.py b/lib/python-sdk/examples/typed_custom_filters.py new file mode 100644 index 000000000..c17b57e4c --- /dev/null +++ b/lib/python-sdk/examples/typed_custom_filters.py @@ -0,0 +1,133 @@ +"""Typed custom filters — the plugin-author and plugin-consumer experience. + +Shows the whole surface end to end: an author declares custom fields and a custom +filter once, assembles a plugin, and a consumer gets a fully-typed client from +``plugin.get_client()`` — registered filters type the ``search(filters=...)`` call +site, and responses parse with the custom fields by default. + +The ``_typecheck`` function below is never executed; it exists so pyright verifies +the consumer's typing (``assert_type``). The unhappy path has two halves: +- static (a route typo, a wrong filter value) lives in + ``typed_custom_filters_failures.py`` — run pyright against it. +- runtime (an invalid value on a standard/registered filter raises before any + request) is ``demo_invalid_filter_raises`` below — runnable without an API:: + + poetry run python -c "import examples.typed_custom_filters as e; e.demo_invalid_filter_raises()" + +Run live against an API: ``poetry run python examples/typed_custom_filters.py``. +""" + +from __future__ import annotations + +from typing import Optional, cast + +from pydantic import Field +from typing_extensions import assert_type + +from common_grants_sdk.client import Client, Config, SearchResult +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PluginMeta, + PluginRoutes, + PluginSchemas, + ResourceRoutes, + define_plugin, + f, + schema, +) +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OpportunityFilters, + StringArray, +) +from common_grants_sdk.schemas.pydantic.models import OpportunityBase + + +# 1. CUSTOM FIELDS — declared once as a typed model. Each field's value type flows +# through to ``opp.custom_fields.<field>.value`` on every parsed response row. +class OppCustomFields(CustomFieldSet): + program_code: Optional[CustomField[str]] = Field( + default=None, description="Program code carried from the source system." + ) + legacy_id: Optional[CustomField[int]] = Field( + default=None, description="Legacy integer identifier." + ) + + +# 2. CUSTOM FILTERS — declared once by subclassing the SDK's filter TypedDict. This +# same type registers the filter (via the route below) AND types the consumer's +# ``search(filters=...)`` call site — no second declaration to drift. +class OppSearchFilters(OpportunityFilters, total=False): + region: StringArray + + +# 3. ASSEMBLE — bind the custom-field model to the Opportunity schema slot and the +# custom-filter type to the opportunities/search route. One call, two axes. +opportunity_plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OppCustomFields])), + routes=PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)), + meta=PluginMeta(name="grants-gov adapter", source_system="grants.gov"), +) + + +def _typecheck() -> None: + """Static proof of the consumer typing (checked by pyright, never run).""" + client = opportunity_plugin.get_client() + assert_type(client, Client[OppSearchFilters, OpportunityBase[OppCustomFields]]) + + # "region" is typed; f.eq(...) here would be a pyright error (see the failures + # fixture). "status" is a standard filter; an ad-hoc key still passes through. + result = client.opportunities.search( + search="ai", filters={"region": f.in_(["US-CA", "US-NY"])} + ) + assert_type(result, SearchResult[OpportunityBase[OppCustomFields]]) + assert_type(result.items, list[OpportunityBase[OppCustomFields]]) + + for opp in result.items: + fields = opp.custom_fields + assert_type(fields, Optional[OppCustomFields]) + if fields is not None and fields.program_code is not None: + assert_type( + fields.program_code.value, str + ) # typed str from CustomField[str] + + +def demo_invalid_filter_raises() -> None: + """Runtime unhappy-path check: an invalid registered-filter value raises. + + Needs no API — the client validates and raises before any request goes out. + ``region`` is a StringArray filter, so a scalar ``between`` value is invalid. + """ + from common_grants_sdk.extensions import FilterError + + # Dummy config is never used to connect — the guard raises before any request. + client = opportunity_plugin.get_client( + Config(base_url="https://example.invalid", api_key="unused") + ) + # cast: a real consumer's dynamic/untyped filter dict bypasses the static + # check; the runtime guard is the backstop that still catches the bad value. + bad_filters = cast( + OppSearchFilters, {"region": {"operator": "between", "value": 5}} + ) + try: + client.opportunities.search(filters=bad_filters) + except FilterError as exc: + print("raised as expected:", exc.path, "-", exc) + return + raise AssertionError("expected FilterError on an invalid registered filter value") + + +def main() -> None: + """Run a live search against a configured API.""" + client = opportunity_plugin.get_client(Config()) + result = client.opportunities.search( + search="ai", filters={"region": f.in_(["US-CA", "US-NY"])} + ) + for opp in result.items: + print(opp.id, opp.custom_fields) + for err in result.errors: + print("parse error at row", err.index, "-", err.message) + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/examples/typed_custom_filters_failures.py b/lib/python-sdk/examples/typed_custom_filters_failures.py new file mode 100644 index 000000000..a928f81bf --- /dev/null +++ b/lib/python-sdk/examples/typed_custom_filters_failures.py @@ -0,0 +1,61 @@ +"""Negative type fixtures — these SHOULD fail pyright. + +Reviewers validate the unhappy path (the guards the happy-path example relies on) +by running pyright against this file directly:: + + cd lib/python-sdk && poetry run pyright examples/typed_custom_filters_failures.py + +Expected: pyright reports errors on the lines marked ``# EXPECT-ERROR`` — a route +typo and a wrong filter value. This file is excluded from the type gate so its +intentional errors do not fail CI; it exists to prove the type guards fire. +""" + +from __future__ import annotations + +from typing import Optional + +from pydantic import Field + +from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + PluginMeta, + PluginRoutes, + PluginSchemas, + ResourceRoutes, + define_plugin, + f, + schema, +) +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OpportunityFilters, + StringArray, +) +from common_grants_sdk.schemas.pydantic.models import OpportunityBase + + +class OppCustomFields(CustomFieldSet): + program_code: Optional[CustomField[str]] = Field(default=None) + + +class OppSearchFilters(OpportunityFilters, total=False): + region: StringArray + + +# 1. Route typo: PluginRoutes has no "opportunites" slot (only "opportunities"). +bad_routes = PluginRoutes( + opportunites=ResourceRoutes(search=OppSearchFilters) # EXPECT-ERROR: no such field +) + +plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OppCustomFields])), + routes=PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)), + meta=PluginMeta(name="grants-gov adapter", source_system="grants.gov"), +) +client = plugin.get_client() + +# 2. Wrong filter value: "region" is a StringArray filter (needs f.in_([...])); an +# f.eq(...) is a StringComparison and is rejected at the call site. +client.opportunities.search( + filters={"region": f.eq("US-CA")} # EXPECT-ERROR: region is a StringArray filter +) diff --git a/lib/python-sdk/poetry.lock b/lib/python-sdk/poetry.lock index 7eed682bb..41ff914b0 100644 --- a/lib/python-sdk/poetry.lock +++ b/lib/python-sdk/poetry.lock @@ -435,107 +435,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "librt" -version = "0.9.0" -description = "Mypyc runtime library" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -markers = "platform_python_implementation != \"PyPy\"" -files = [ - {file = "librt-0.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f8e12706dcb8ff6b3ed57514a19e45c49ad00bcd423e87b2b2e4b5f64578443"}, - {file = "librt-0.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4e3dda8345307fd7306db0ed0cb109a63a2c85ba780eb9dc2d09b2049a931f9c"}, - {file = "librt-0.9.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:de7dac64e3eb832ffc7b840eb8f52f76420cde1b845be51b2a0f6b870890645e"}, - {file = "librt-0.9.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22a904cbdb678f7cb348c90d543d3c52f581663d687992fee47fd566dcbf5285"}, - {file = "librt-0.9.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:224b9727eb8bc188bc3bcf29d969dba0cd61b01d9bac80c41575520cc4baabb2"}, - {file = "librt-0.9.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e94cbc6ad9a6aeea46d775cbb11f361022f778a9cc8cc90af653d3a594b057ce"}, - {file = "librt-0.9.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7bc30ad339f4e1a01d4917d645e522a0bc0030644d8973f6346397c93ba1503f"}, - {file = "librt-0.9.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:56d65b583cf43b8cf4c8fbe1e1da20fa3076cc32a1149a141507af1062718236"}, - {file = "librt-0.9.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0a1be03168b2691ba61927e299b352a6315189199ca18a57b733f86cb3cc8d38"}, - {file = "librt-0.9.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:63c12efcd160e1d14da11af0c46c0217473e1e0d2ae1acbccc83f561ea4c2a7b"}, - {file = "librt-0.9.0-cp310-cp310-win32.whl", hash = "sha256:e9002e98dcb1c0a66723592520decd86238ddcef168b37ff6cfb559200b4b774"}, - {file = "librt-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:9fcb461fbf70654a52a7cc670e606f04449e2374c199b1825f754e16dacfedd8"}, - {file = "librt-0.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90904fac73c478f4b83f4ed96c99c8208b75e6f9a8a1910548f69a00f1eaa671"}, - {file = "librt-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:789fff71757facc0738e8d89e3b84e4f0251c1c975e85e81b152cdaca927cc2d"}, - {file = "librt-0.9.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1bf465d1e5b0a27713862441f6467b5ab76385f4ecf8f1f3a44f8aa3c695b4b6"}, - {file = "librt-0.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f819e0c6413e259a17a7c0d49f97f405abadd3c2a316a3b46c6440b7dbbedbb1"}, - {file = "librt-0.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0785c2fb4a81e1aece366aa3e2e039f4a4d7d21aaaded5227d7f3c703427882"}, - {file = "librt-0.9.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80b25c7b570a86c03b5da69e665809deb39265476e8e21d96a9328f9762f9990"}, - {file = "librt-0.9.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d4d16b608a1c43d7e33142099a75cd93af482dadce0bf82421e91cad077157f4"}, - {file = "librt-0.9.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:194fc1a32e1e21fe809d38b5faea66cc65eaa00217c8901fbdb99866938adbdb"}, - {file = "librt-0.9.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8c6bc1384d9738781cfd41d09ad7f6e8af13cfea2c75ece6bd6d2566cdea2076"}, - {file = "librt-0.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15cb151e52a044f06e54ac7f7b47adbfc89b5c8e2b63e1175a9d587c43e8942a"}, - {file = "librt-0.9.0-cp311-cp311-win32.whl", hash = "sha256:f100bfe2acf8a3689af9d0cc660d89f17286c9c795f9f18f7b62dd1a6b247ae6"}, - {file = "librt-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:0b73e4266307e51c95e09c0750b7ec383c561d2e97d58e473f6f6a209952fbb8"}, - {file = "librt-0.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:bc5518873822d2faa8ebdd2c1a4d7c8ef47b01a058495ab7924cb65bdbf5fc9a"}, - {file = "librt-0.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9b3e3bc363f71bda1639a4ee593cb78f7fbfeacc73411ec0d4c92f00730010a4"}, - {file = "librt-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a09c2f5869649101738653a9b7ab70cf045a1105ac66cbb8f4055e61df78f2d"}, - {file = "librt-0.9.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5ca8e133d799c948db2ab1afc081c333a825b5540475164726dcbf73537e5c2f"}, - {file = "librt-0.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:603138ee838ee1583f1b960b62d5d0007845c5c423feb68e44648b1359014e27"}, - {file = "librt-0.9.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4003f70c56a5addd6aa0897f200dd59afd3bf7bcd5b3cce46dd21f925743bc2"}, - {file = "librt-0.9.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78042f6facfd98ecb25e9829c7e37cce23363d9d7c83bc5f72702c5059eb082b"}, - {file = "librt-0.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a361c9434a64d70a7dbb771d1de302c0cc9f13c0bffe1cf7e642152814b35265"}, - {file = "librt-0.9.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:dd2c7e082b0b92e1baa4da28163a808672485617bc855cc22a2fd06978fa9084"}, - {file = "librt-0.9.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7e6274fd33fc5b2a14d41c9119629d3ff395849d8bcbc80cf637d9e8d2034da8"}, - {file = "librt-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5093043afb226ecfa1400120d1ebd4442b4f99977783e4f4f7248879009b227f"}, - {file = "librt-0.9.0-cp312-cp312-win32.whl", hash = "sha256:9edcc35d1cae9fd5320171b1a838c7da8a5c968af31e82ecc3dff30b4be0957f"}, - {file = "librt-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc2917258e131ae5f958a4d872e07555b51cb7466a43433218061c74ef33745"}, - {file = "librt-0.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:90e6d5420fc8a300518d4d2288154ff45005e920425c22cbbfe8330f3f754bd9"}, - {file = "librt-0.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29b68cd9714531672db62cc54f6e8ff981900f824d13fa0e00749189e13778e"}, - {file = "librt-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d5c8a5929ac325729f6119802070b561f4db793dffc45e9ac750992a4ed4d22"}, - {file = "librt-0.9.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:756775d25ec8345b837ab52effee3ad2f3b2dfd6bbee3e3f029c517bd5d8f05a"}, - {file = "librt-0.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b8f5d00b49818f4e2b1667db994488b045835e0ac16fe2f924f3871bd2b8ac5"}, - {file = "librt-0.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c81aef782380f0f13ead670aae01825eb653b44b046aa0e5ebbb79f76ed4aa11"}, - {file = "librt-0.9.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66b58fed90a545328e80d575467244de3741e088c1af928f0b489ebec3ef3858"}, - {file = "librt-0.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e78fb7419e07d98c2af4b8567b72b3eaf8cb05caad642e9963465569c8b2d87e"}, - {file = "librt-0.9.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c3786f0f4490a5cd87f1ed6cefae833ad6b1060d52044ce0434a2e85893afd0"}, - {file = "librt-0.9.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8494cfc61e03542f2d381e71804990b3931175a29b9278fdb4a5459948778dc2"}, - {file = "librt-0.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:07cf11f769831186eeac424376e6189f20ace4f7263e2134bdb9757340d84d4d"}, - {file = "librt-0.9.0-cp313-cp313-win32.whl", hash = "sha256:850d6d03177e52700af605fd60db7f37dcb89782049a149674d1a9649c2138fd"}, - {file = "librt-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:a5af136bfba820d592f86c67affcef9b3ff4d4360ac3255e341e964489b48519"}, - {file = "librt-0.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:4c4d0440a3a8e31d962340c3e1cc3fc9ee7febd34c8d8f770d06adb947779ea5"}, - {file = "librt-0.9.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:3f05d145df35dca5056a8bc3838e940efebd893a54b3e19b2dda39ceaa299bcb"}, - {file = "librt-0.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1c587494461ebd42229d0f1739f3aa34237dd9980623ecf1be8d3bcba79f4499"}, - {file = "librt-0.9.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b0a2040f801406b93657a70b72fa12311063a319fee72ce98e1524da7200171f"}, - {file = "librt-0.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f38bc489037eca88d6ebefc9c4d41a4e07c8e8b4de5188a9e6d290273ad7ebb1"}, - {file = "librt-0.9.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3fd278f5e6bf7c75ccd6d12344eb686cc020712683363b66f46ac79d37c799f"}, - {file = "librt-0.9.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fcbdf2a9ca24e87bbebb47f1fe34e531ef06f104f98c9ccfc953a3f3344c567a"}, - {file = "librt-0.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e306d956cfa027fe041585f02a1602c32bfa6bb8ebea4899d373383295a6c62f"}, - {file = "librt-0.9.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:465814ab157986acb9dfa5ccd7df944be5eefc0d08d31ec6e8d88bc71251d845"}, - {file = "librt-0.9.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:703f4ae36d6240bfe24f542bac784c7e4194ec49c3ba5a994d02891649e2d85b"}, - {file = "librt-0.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3be322a15ee5e70b93b7a59cfd074614f22cc8c9ff18bd27f474e79137ea8d3b"}, - {file = "librt-0.9.0-cp314-cp314-win32.whl", hash = "sha256:b8da9f8035bb417770b1e1610526d87ad4fc58a2804dc4d79c53f6d2cf5a6eb9"}, - {file = "librt-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8bd70d5d816566a580d193326912f4a76ec2d28a97dc4cd4cc831c0af8e330e"}, - {file = "librt-0.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:fc5758e2b7a56532dc33e3c544d78cbaa9ecf0a0f2a2da2df882c1d6b99a317f"}, - {file = "librt-0.9.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f24b90b0e0c8cc9491fb1693ae91fe17cb7963153a1946395acdbdd5818429a4"}, - {file = "librt-0.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3fe56e80badb66fdcde06bef81bbaa5bfcf6fbd7aefb86222d9e369c38c6b228"}, - {file = "librt-0.9.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:527b5b820b47a09e09829051452bb0d1dd2122261254e2a6f674d12f1d793d54"}, - {file = "librt-0.9.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d429bdd4ac0ab17c8e4a8af0ed2a7440b16eba474909ab357131018fe8c7e71"}, - {file = "librt-0.9.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7202bdcac47d3a708271c4304a474a8605a4a9a4a709e954bf2d3241140aa938"}, - {file = "librt-0.9.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0d620e74897f8c2613b3c4e2e9c1e422eb46d2ddd07df540784d44117836af3"}, - {file = "librt-0.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d69fc39e627908f4c03297d5a88d9284b73f4d90b424461e32e8c2485e21c283"}, - {file = "librt-0.9.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:c2640e23d2b7c98796f123ffd95cf2022c7777aa8a4a3b98b36c570d37e85eee"}, - {file = "librt-0.9.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:451daa98463b7695b0a30aa56bf637831ea559e7b8101ac2ef6382e8eb15e29c"}, - {file = "librt-0.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:928bd06eca2c2bbf4349e5b817f837509b0604342e65a502de1d50a7570afd15"}, - {file = "librt-0.9.0-cp314-cp314t-win32.whl", hash = "sha256:a9c63e04d003bc0fb6a03b348018b9a3002f98268200e22cc80f146beac5dc40"}, - {file = "librt-0.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f162af66a2ed3f7d1d161a82ca584efd15acd9c1cff190a373458c32f7d42118"}, - {file = "librt-0.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a4b25c6c25cac5d0d9d6d6da855195b254e0021e513e0249f0e3b444dc6e0e61"}, - {file = "librt-0.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5112c2fb7c2eefefaeaf5c97fec81343ef44ee86a30dcfaa8223822fba6467b4"}, - {file = "librt-0.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a81eea9b999b985e4bacc650c4312805ea7008fd5e45e1bf221310176a7bcb3a"}, - {file = "librt-0.9.0-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:eea1b54943475f51698f85fa230c65ccac769f1e603b981be060ac5763d90927"}, - {file = "librt-0.9.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:81107843ed1836874b46b310f9b1816abcb89912af627868522461c3b7333c0f"}, - {file = "librt-0.9.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa95738a68cedd3a6f5492feddc513e2e166b50602958139e47bbdd82da0f5a7"}, - {file = "librt-0.9.0-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6788207daa0c19955d2b668f3294a368d19f67d9b5f274553fd073c1260cbb9f"}, - {file = "librt-0.9.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f48c963a76d71b9d7927eb817b543d0dccd52ab6648b99d37bd54f4cd475d856"}, - {file = "librt-0.9.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:42ff8a962554c350d4a83cf47d9b7b78b0e6ff7943e87df7cdfc97c07f3c016f"}, - {file = "librt-0.9.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:657f8ba7b9eaaa82759a104137aed2a3ef7bc46ccfd43e0d89b04005b3e0a4cc"}, - {file = "librt-0.9.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2d03fa4fd277a7974c1978c92c374c57f44edeee163d147b477b143446ad1bf6"}, - {file = "librt-0.9.0-cp39-cp39-win32.whl", hash = "sha256:d9da80e5b04acce03ced8ba6479a71c2a2edf535c2acc0d09c80d2f80f3bad15"}, - {file = "librt-0.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:54d412e47c21b85865676ed0724e37a89e9593c2eee1e7367adf85bfad56ffb1"}, - {file = "librt-0.9.0.tar.gz", hash = "sha256:a0951822531e7aee6e0dfb556b30d5ee36bbe234faf60c20a16c01be3530869d"}, -] - [[package]] name = "markupsafe" version = "3.0.3" @@ -667,77 +566,6 @@ files = [ {file = "more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd"}, ] -[[package]] -name = "mypy" -version = "1.20.2" -description = "Optional static typing for Python" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4"}, - {file = "mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997"}, - {file = "mypy-1.20.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97d7b9a485b40f8ca425460e89bf1da2814625b2da627c0dcc6aa46c92631d14"}, - {file = "mypy-1.20.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e1c12f6d2db3d78b909b5f77513c11eb7f2dd2782b96a3ab6dffc7d44575c99"}, - {file = "mypy-1.20.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89dce27e142d25ffbc154c1819383b69f2e9234dc4ed4766f42e0e8cb264ab5c"}, - {file = "mypy-1.20.2-cp310-cp310-win_amd64.whl", hash = "sha256:f376e37f9bf2a946872fc5fd1199c99310748e3c26c7a26683f13f8bdb756cbd"}, - {file = "mypy-1.20.2-cp310-cp310-win_arm64.whl", hash = "sha256:6e2b469efd811707bc530fd1effef0f5d6eebcb7fe376affae69025da4b979a2"}, - {file = "mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c"}, - {file = "mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3"}, - {file = "mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254"}, - {file = "mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98"}, - {file = "mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac"}, - {file = "mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67"}, - {file = "mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100"}, - {file = "mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b"}, - {file = "mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4"}, - {file = "mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6"}, - {file = "mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066"}, - {file = "mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102"}, - {file = "mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9"}, - {file = "mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58"}, - {file = "mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026"}, - {file = "mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943"}, - {file = "mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517"}, - {file = "mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15"}, - {file = "mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee"}, - {file = "mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f"}, - {file = "mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330"}, - {file = "mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30"}, - {file = "mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924"}, - {file = "mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb"}, - {file = "mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc"}, - {file = "mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558"}, - {file = "mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8"}, - {file = "mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3"}, - {file = "mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609"}, - {file = "mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2"}, - {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c"}, - {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744"}, - {file = "mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6"}, - {file = "mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec"}, - {file = "mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382"}, - {file = "mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563"}, - {file = "mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665"}, -] - -[package.dependencies] -librt = {version = ">=0.8.0", markers = "platform_python_implementation != \"PyPy\""} -mypy_extensions = ">=1.0.0" -pathspec = ">=1.0.0" -typing_extensions = [ - {version = ">=4.6.0", markers = "python_version < \"3.15\""}, - {version = ">=4.14.0", markers = "python_version >= \"3.15\""}, -] - -[package.extras] -dmypy = ["psutil (>=4.0)"] -faster-cache = ["orjson"] -install-types = ["pip"] -mypyc = ["setuptools (>=50)"] -native-parser = ["ast-serialize (>=0.1.1,<1.0.0)"] -reports = ["lxml"] - [[package]] name = "mypy-extensions" version = "1.1.0" @@ -750,6 +578,18 @@ files = [ {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +files = [ + {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, + {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, +] + [[package]] name = "packaging" version = "25.0" @@ -983,6 +823,27 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyright" +version = "1.1.410" +description = "Command line wrapper for pyright" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "pyright-1.1.410-py3-none-any.whl", hash = "sha256:5e961bed37cacf96b3f7cd7b1da39b350a9239aa2e69138d0e88f728cfaf296c"}, + {file = "pyright-1.1.410.tar.gz", hash = "sha256:07a073b8ba6749826773c1269773efa11b93440d9a6aa60419d9a3172d6dc488"}, +] + +[package.dependencies] +nodeenv = ">=1.6.0" +typing-extensions = ">=4.1" + +[package.extras] +all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] +nodejs = ["nodejs-wheel-binaries"] + [[package]] name = "pytest" version = "8.4.2" @@ -1232,18 +1093,6 @@ files = [ [package.dependencies] typing_extensions = ">=4.14.0" -[[package]] -name = "types-pyyaml" -version = "6.0.12.20260518" -description = "Typing stubs for PyYAML" -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd"}, - {file = "types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466"}, -] - [[package]] name = "typing-extensions" version = "4.15.0" @@ -1310,4 +1159,4 @@ tests = ["build", "coverage", "mypy", "ruff", "wheel"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "3f77f590c479648c52657dd69e21f4bba3d82ad7566c160e0c37b1013feb205c" +content-hash = "c4cd76d6f4bec4d0bb0c9dbd30f453ba97aa3a773bbc24a981802940d4b1a4d0" diff --git a/lib/python-sdk/pyproject.toml b/lib/python-sdk/pyproject.toml index 3c8a1e731..11006d4eb 100644 --- a/lib/python-sdk/pyproject.toml +++ b/lib/python-sdk/pyproject.toml @@ -11,19 +11,23 @@ python = "^3.11" pydantic = "^2.11.0" marshmallow = "^3.20.0" httpx = ">=0.27,<0.29" +typing-extensions = ">=4.13" [tool.poetry.group.dev.dependencies] black = "^26.3.1" -mypy = "^1.4.1" +pyright = "1.1.410" pytest = "^8.0.0" pytest-cov = "^5.0.0" ruff = ">=0.11,<0.16" tomlkit = ">=0.13.3,<0.16.0" tomlq = "^0.1.0" -types-pyyaml = "^6.0.12.20250915" datamodel-code-generator = ">=0.43.1,<0.65.0" +[tool.pytest.ini_options] +# Put the package root on sys.path so tests can import the runnable examples/. +pythonpath = ["."] + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/lib/python-sdk/pyrightconfig.json b/lib/python-sdk/pyrightconfig.json new file mode 100644 index 000000000..2ae24a65f --- /dev/null +++ b/lib/python-sdk/pyrightconfig.json @@ -0,0 +1,6 @@ +{ + "include": ["common_grants_sdk", "examples/typed_custom_filters.py"], + "exclude": ["tests", "**/__pycache__", "generated"], + "typeCheckingMode": "basic", + "pythonVersion": "3.11" +} diff --git a/lib/python-sdk/tests/client/test_client.py b/lib/python-sdk/tests/client/test_client.py index 124577caa..abfbce070 100644 --- a/lib/python-sdk/tests/client/test_client.py +++ b/lib/python-sdk/tests/client/test_client.py @@ -12,8 +12,12 @@ from common_grants_sdk.client.exceptions import APIError from common_grants_sdk.schemas.pydantic.pagination import PaginatedResultsInfo from common_grants_sdk.schemas.pydantic.responses import Paginated -from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType +from common_grants_sdk.extensions import PluginRoutes, ResourceRoutes from common_grants_sdk.extensions.types import FilterError +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OpportunityFilters, + StringArray, +) from pydantic import ValidationError @@ -31,23 +35,42 @@ def test_client_initialization_with_config_and_auth(self): assert isinstance(client.opportunities, type(client.opportunities)) def test_client_initialization_validates_routes(self): - """Client validates routes at construction and raises on a bad declaration.""" + """Client validates routes at construction and raises on a bad declaration. + + The typed carrier makes a misspelled resource/method a static error, so the + only remaining runtime check is that a registered custom filter's value type + is a filter value model. A TypedDict whose custom key is annotated with a + non-model type (``region: int``) is the genuinely-invalid registration. + """ with patch("common_grants_sdk.client.client.httpx.Client"): config = Config(base_url="https://api.example.com", api_key="test-key") - # "status" collides with the default status filter name. - bad_routes = { - "opportunities": { - "search": { - "filters": { - "status": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ) - } - } - } - } + + class BadFilters(OpportunityFilters, total=False): + region: int # not a filter value model + with pytest.raises(FilterError): - Client(config=config, routes=bad_routes) + Client( + config=config, + routes=PluginRoutes( + opportunities=ResourceRoutes(search=BadFilters) + ), + ) + + def test_client_initialization_accepts_valid_routes(self): + """A valid typed routes carrier is accepted at construction (no raise).""" + with patch("common_grants_sdk.client.client.httpx.Client"): + config = Config(base_url="https://api.example.com", api_key="test-key") + + class OppSearchFilters(OpportunityFilters, total=False): + region: StringArray + + client = Client( + config=config, + routes=PluginRoutes( + opportunities=ResourceRoutes(search=OppSearchFilters) + ), + ) + assert client.routes.opportunities.search is OppSearchFilters def test_client_initialization_defaults_auth(self, monkeypatch): """Test client initialization with default auth from config.""" diff --git a/lib/python-sdk/tests/client/test_opportunities.py b/lib/python-sdk/tests/client/test_opportunities.py index 858cd63dd..a493a652d 100644 --- a/lib/python-sdk/tests/client/test_opportunities.py +++ b/lib/python-sdk/tests/client/test_opportunities.py @@ -9,21 +9,33 @@ import httpx from pydantic import ValidationError -from common_grants_sdk.client import Client, Auth +from common_grants_sdk.client import ( + Client, + Auth, + ListResult, + ParseFailure, + SearchResult, +) from common_grants_sdk.client.config import Config from common_grants_sdk.client.exceptions import APIError +from common_grants_sdk.extensions import FilterError, PluginRoutes, ResourceRoutes from common_grants_sdk.schemas.pydantic.models import OpportunityBase from common_grants_sdk.schemas.pydantic.fields import CustomFieldType -from common_grants_sdk.schemas.pydantic.responses import ( - OpportunitiesListResponse, - OpportunitiesSearchResponse, +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OpportunityFilters, + StringArray, ) from common_grants_sdk.schemas.pydantic.models.opp_status import OppStatusOptions -from common_grants_sdk.extensions.specs import ( - CustomFieldSpec, - CustomFilterSpec, - CustomFilterType, -) +from common_grants_sdk.extensions.specs import CustomFieldSpec + + +class OppSearchFilters(OpportunityFilters, total=False): + """Route filter TypedDict registering ``agency`` as a stringArray custom filter.""" + + agency: StringArray + + +AGENCY_ROUTES = PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)) @pytest.fixture @@ -276,7 +288,7 @@ def test_list_opportunities_success( response = client.opportunities.list(page=1, schema=opp_base) - assert isinstance(response, OpportunitiesListResponse) + assert isinstance(response, ListResult) assert len(response.items) == 2 assert all(isinstance(item, OpportunityBase) for item in response.items) assert response.pagination_info.page == 1 @@ -450,7 +462,7 @@ def test_list_opportunities_401(self, client, mock_httpx_client): def test_list_opportunities_validation_error(self, client, mock_httpx_client): """Test listing opportunities with validation error.""" - # Valid JSON but doesn't match OpportunitiesListResponse schema + # Valid JSON but doesn't match the paginated list response schema invalid_data = {"invalid": "data"} mock_response = Mock() mock_response.status_code = 200 @@ -483,7 +495,7 @@ def test_list_all_opportunities_single_page( mock_httpx_client.get = Mock(return_value=mock_response) response = client.opportunities.list(page=None) - assert isinstance(response, OpportunitiesListResponse) + assert isinstance(response, ListResult) assert len(response.items) == 2 assert all(isinstance(item, OpportunityBase) for item in response.items) # When fetching all, pagination info should reflect aggregated result @@ -560,7 +572,7 @@ def mock_get(*args, **kwargs): mock_httpx_client.get = Mock(side_effect=mock_get) response = client.opportunities.list(page=None) - assert isinstance(response, OpportunitiesListResponse) + assert isinstance(response, ListResult) # Should have all 5 items from all 3 pages assert len(response.items) == 5 assert all(isinstance(item, OpportunityBase) for item in response.items) @@ -622,7 +634,7 @@ def mock_get(*args, **kwargs): mock_httpx_client.get = Mock(side_effect=mock_get) response = client.opportunities.list(page=None, page_size=3) - assert isinstance(response, OpportunitiesListResponse) + assert isinstance(response, ListResult) assert len(response.items) == 4 assert response.pagination_info.page_size == 4 @@ -653,7 +665,7 @@ def test_list_all_opportunities_empty_result(self, client, mock_httpx_client): mock_httpx_client.get = Mock(return_value=mock_response) response = client.opportunities.list(page=None) - assert isinstance(response, OpportunitiesListResponse) + assert isinstance(response, ListResult) assert len(response.items) == 0 assert response.pagination_info.total_items == 0 assert response.pagination_info.total_pages == 1 @@ -748,7 +760,7 @@ def test_search_opportunities_success( search="local", status=[OppStatusOptions.OPEN], schema=opp_base ) - assert isinstance(response, OpportunitiesSearchResponse) + assert isinstance(response, SearchResult) assert len(response.items) == 2 assert all(isinstance(item, OpportunityBase) for item in response.items) assert response.items[0].get_custom_field_value("legacy_id", int) == 12345 @@ -762,15 +774,40 @@ def test_search_opportunities_success( assert call_args[1]["params"]["page"] == 1 assert call_args[1]["params"]["pageSize"] == 100 - def test_search_surfaces_server_filter_and_sort_info( + def test_search_partial_parse_partitions_items_and_errors( + self, client, mock_httpx_client, sample_search_response + ): + """A malformed row is collected into result.errors (not raised); the valid + rows still return in result.items — per-row fail-soft parsing.""" + good_row = sample_search_response["items"][0] + sample_search_response["items"] = [good_row, {"id": "not-a-uuid"}] + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + response = client.opportunities.search(search="x", page=1) + + assert isinstance(response, SearchResult) + assert len(response.items) == 1 + assert len(response.errors) == 1 + assert isinstance(response.errors[0], ParseFailure) + assert response.errors[0].index == 1 + # sort_info survives a single-page fetch (regression guard: it must not be + # silently dropped from SearchResult). + assert response.sort_info is not None + assert response.sort_info.sort_by == "lastModifiedAt" + + def test_search_surfaces_server_filter_info( self, client, mock_httpx_client, sample_search_response ): - """search() returns the server's real filterInfo/sortInfo, not blanks. + """search() returns the server's real filterInfo.errors, not blanks. - The server reports non-fatal filtering feedback in ``filterInfo.errors`` - and the resolved sort in ``sortInfo``. Earlier the client - fabricated empty envelopes, dropping that feedback; this asserts it now - reaches the caller. + The server reports non-fatal filtering feedback in ``filterInfo.errors``. + Earlier the client fabricated an empty envelope, dropping that feedback; + this asserts it now reaches the caller via ``SearchResult.filter_info``. """ sample_search_response["filterInfo"] = { "filters": {}, @@ -785,12 +822,10 @@ def test_search_surfaces_server_filter_and_sort_info( response = client.opportunities.search(search="x") - assert isinstance(response, OpportunitiesSearchResponse) + assert isinstance(response, SearchResult) assert response.filter_info.errors == [ "filter 'foo' is unsupported and was ignored" ] - # sort_info reflects the server value, not the blanked "". - assert response.sort_info.sort_by == "lastModifiedAt" def test_search_classifies_custom_filter_bag( self, mock_httpx_client, sample_search_response @@ -803,24 +838,12 @@ def test_search_classifies_custom_filter_bag( mock_response.raise_for_status = Mock() mock_httpx_client.post = Mock(return_value=mock_response) - routes = { - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ) - } - } - } - } - # routes is client-bound: supplied once at construction, not per call. auth = Auth.api_key("test-key") config = Config( base_url="https://api.example.com", api_key="test-key", timeout=10.0 ) - client = Client(config=config, auth=auth, routes=routes) + client = Client(config=config, auth=auth, routes=AGENCY_ROUTES) client.http = mock_httpx_client client.opportunities.http = mock_httpx_client @@ -842,100 +865,109 @@ def test_search_classifies_custom_filter_bag( # separation invariant: status not duplicated under customFilters assert "status" not in sent_filters["customFilters"] - def test_search_status_collision_filters_wins_and_warns( + def test_search_status_collision_raises( self, client, mock_httpx_client, sample_search_response ): - """status via both the shorthand arg and the filters argument: filters wins. + """status via both the shorthand arg and the filters argument raises. - Fail-soft: rather than raising, search() keeps the - ``filters`` value, ignores the shorthand, and appends a warning to the - response's filterInfo.errors. + Conflicting input to the standard ``status`` filter is a client bug: + search() raises a ``FilterError`` rather than silently picking one and + stashing a warning in ``filter_info.errors`` (reserved for server errors). + No request is sent. """ - mock_response = Mock() - mock_response.status_code = 200 - mock_response.text = json.dumps(sample_search_response) - mock_response.json = Mock(return_value=sample_search_response) - mock_response.raise_for_status = Mock() - mock_httpx_client.post = Mock(return_value=mock_response) + mock_httpx_client.post = Mock() - # filters value uses "forecasted"; shorthand uses OPEN — they must differ - # so the assertion on the winning value can actually fail. - response = client.opportunities.search( - search="conservation", - status=[OppStatusOptions.OPEN], - filters={"status": {"operator": "in", "value": ["forecasted"]}}, - ) + with pytest.raises(FilterError, match="specified via both"): + client.opportunities.search( + search="conservation", + status=[OppStatusOptions.OPEN], + filters={"status": {"operator": "in", "value": ["forecasted"]}}, + ) - # The SENT body carries the filters value, not the shorthand. - sent_filters = mock_httpx_client.post.call_args[1]["json"]["filters"] - assert sent_filters["status"]["value"] == ["forecasted"] - # The collision warning reached filterInfo.errors. - assert any( - "filters.status" in e - and "specified via both the status shorthand and the filters argument; " - "used the filters value" in e - for e in response.filter_info.errors - ) + # The raise happens before any request goes out. + mock_httpx_client.post.assert_not_called() - def test_search_invalid_filter_is_dropped_and_warned_no_raise( + def test_search_invalid_registered_filter_raises( self, mock_httpx_client, sample_search_response ): - """A malformed registered filter is fail-soft: results return, filter dropped. + """An invalid value on a registered custom filter raises (AC6). A registered stringArray filter given a non-array ``between`` value fails - classify_filters validation. search() does NOT raise. - The invalid filter is absent from the sent body and its error surfaces in - filterInfo.errors; results still return. + classify validation. search() raises rather than dropping it into + ``filter_info.errors``, and sends no request. """ - mock_response = Mock() - mock_response.status_code = 200 - mock_response.text = json.dumps(sample_search_response) - mock_response.json = Mock(return_value=sample_search_response) - mock_response.raise_for_status = Mock() - mock_httpx_client.post = Mock(return_value=mock_response) - - routes = { - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ) - } - } - } - } + mock_httpx_client.post = Mock() # routes is client-bound: supplied once at construction, not per call. auth = Auth.api_key("test-key") config = Config( base_url="https://api.example.com", api_key="test-key", timeout=10.0 ) - client = Client(config=config, auth=auth, routes=routes) + client = Client(config=config, auth=auth, routes=AGENCY_ROUTES) client.http = mock_httpx_client client.opportunities.http = mock_httpx_client + with pytest.raises(FilterError) as exc_info: + client.opportunities.search( + search="conservation", + filters={"agency": {"operator": "between", "value": 5}}, + ) + assert exc_info.value.path == "filters.agency" + mock_httpx_client.post.assert_not_called() + + def test_search_invalid_standard_filter_raises( + self, client, mock_httpx_client, sample_search_response + ): + """An invalid value on a standard filter raises (AC6). + + The standard ``status`` filter is a stringArray; a ``between`` operator + with a scalar value fails validation and search() raises — no request. + """ + mock_httpx_client.post = Mock() + + with pytest.raises(FilterError) as exc_info: + client.opportunities.search( + search="conservation", + filters={"status": {"operator": "between", "value": 5}}, + ) + assert exc_info.value.path == "filters.status" + mock_httpx_client.post.assert_not_called() + + def test_search_invalid_adhoc_filter_passes_through_no_raise( + self, client, mock_httpx_client, sample_search_response + ): + """A malformed ad-hoc (unregistered) filter is best-effort, not fatal (AC6). + + Ad-hoc keys live in the untyped escape hatch: an unusable value is dropped + rather than raised (only standard/registered filters raise). Results return. + """ + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(sample_search_response) + mock_response.json = Mock(return_value=sample_search_response) + mock_response.raise_for_status = Mock() + mock_httpx_client.post = Mock(return_value=mock_response) + + # No routes: "legacyTag" is neither standard nor registered -> ad-hoc. response = client.opportunities.search( search="conservation", - filters={"agency": {"operator": "between", "value": 5}}, + filters={"legacyTag": "not-a-filter"}, ) - # Results still return (no raise). - assert isinstance(response, OpportunitiesSearchResponse) + assert isinstance(response, SearchResult) assert len(response.items) == 2 - - # The invalid filter is absent from the sent body. With only the one - # invalid filter, no filters key is sent at all. + # Dropped from the sent body (never reached the wire). sent_body = mock_httpx_client.post.call_args[1]["json"] - assert "agency" not in sent_body.get("filters", {}).get("customFilters", {}) - - # Its error surfaced in filterInfo.errors. - assert any("filters.agency" in e for e in response.filter_info.errors) + assert "legacyTag" not in sent_body.get("filters", {}).get("customFilters", {}) - def test_search_invalid_filter_client_errors_precede_server_errors( + def test_search_filter_info_errors_are_server_only( self, mock_httpx_client, sample_search_response ): - """Client-side filter errors are merged AHEAD of server-provided errors.""" + """``filter_info.errors`` carries server-returned errors only. + + With a valid registered filter and server-side filter feedback, the client + surfaces the server errors verbatim and injects none of its own. + """ sample_search_response["filterInfo"] = { "filters": {}, "errors": ["server: something was ignored"], @@ -947,34 +979,20 @@ def test_search_invalid_filter_client_errors_precede_server_errors( mock_response.raise_for_status = Mock() mock_httpx_client.post = Mock(return_value=mock_response) - routes = { - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ) - } - } - } - } auth = Auth.api_key("test-key") config = Config( base_url="https://api.example.com", api_key="test-key", timeout=10.0 ) - client = Client(config=config, auth=auth, routes=routes) + client = Client(config=config, auth=auth, routes=AGENCY_ROUTES) client.http = mock_httpx_client client.opportunities.http = mock_httpx_client response = client.opportunities.search( search="conservation", - filters={"agency": {"operator": "between", "value": 5}}, + filters={"agency": {"operator": "in", "value": ["HHS", "NSF"]}}, ) - errors = response.filter_info.errors - # Client error first, server error last. - assert "filters.agency" in errors[0] - assert errors[-1] == "server: something was ignored" + assert response.filter_info.errors == ["server: something was ignored"] def test_search_opportunities_different_page( self, client, mock_httpx_client, sample_search_response @@ -1152,7 +1170,7 @@ def test_search_opportunities_401(self, client, mock_httpx_client): def test_search_opportunities_validation_error(self, client, mock_httpx_client): """Test searching opportunities with validation error.""" - # Valid JSON but doesn't match OpportunitiesListResponse schema + # Valid JSON but doesn't match the paginated list response schema invalid_data = {"invalid": "data"} mock_response = Mock() mock_response.status_code = 200 @@ -1193,7 +1211,7 @@ def test_search_all_opportunities_single_page( response = client.opportunities.search( search="local", status=[OppStatusOptions.OPEN], page=None ) - assert isinstance(response, OpportunitiesSearchResponse) + assert isinstance(response, SearchResult) assert len(response.items) == 2 assert all(isinstance(item, OpportunityBase) for item in response.items) # When fetching all, pagination info should reflect aggregated result @@ -1293,7 +1311,7 @@ def mock_post(*args, **kwargs): response = client.opportunities.search( search="local", status=[OppStatusOptions.OPEN], page=None ) - assert isinstance(response, OpportunitiesSearchResponse) + assert isinstance(response, SearchResult) # Should have all 5 items from all 3 pages assert len(response.items) == 5 assert all(isinstance(item, OpportunityBase) for item in response.items) @@ -1341,7 +1359,7 @@ def test_search_all_opportunities_empty_result(self, client, mock_httpx_client): response = client.opportunities.search( search="local", status=[OppStatusOptions.OPEN], page=None ) - assert isinstance(response, OpportunitiesSearchResponse) + assert isinstance(response, SearchResult) assert len(response.items) == 0 assert response.pagination_info.total_items == 0 assert response.pagination_info.total_pages == 1 diff --git a/lib/python-sdk/tests/client/test_results.py b/lib/python-sdk/tests/client/test_results.py new file mode 100644 index 000000000..ce0c31ec4 --- /dev/null +++ b/lib/python-sdk/tests/client/test_results.py @@ -0,0 +1,27 @@ +"""Tests for parse_batch row-level error partitioning.""" + +from pydantic import BaseModel + +from common_grants_sdk.client import ParseFailure, parse_batch + + +class _Row(BaseModel): + n: int + + +def test_parse_batch_partitions_good_and_bad_rows(): + items, errors = parse_batch([{"n": 1}, {"n": "not-an-int"}, {"n": 3}], _Row) + + # one bad row is collected, not raised; the good rows still come through + assert [item.n for item in items] == [1, 3] + assert len(errors) == 1 + assert isinstance(errors[0], ParseFailure) + assert errors[0].index == 1 + assert errors[0].raw == {"n": "not-an-int"} + assert errors[0].message # non-empty + + +def test_parse_batch_all_valid_has_no_errors(): + items, errors = parse_batch([{"n": 1}, {"n": 2}], _Row) + assert [item.n for item in items] == [1, 2] + assert errors == [] diff --git a/lib/python-sdk/tests/client/test_typed_custom_filters_example.py b/lib/python-sdk/tests/client/test_typed_custom_filters_example.py new file mode 100644 index 000000000..7eba84bbb --- /dev/null +++ b/lib/python-sdk/tests/client/test_typed_custom_filters_example.py @@ -0,0 +1,99 @@ +"""Happy-path test for the typed_custom_filters example. + +Exercises the example's plugin from the consumer side: ``get_client()`` parses the +plugin's custom fields with no per-call ``schema=``, and a malformed row is +partitioned into ``result.errors`` rather than raising. +""" + +import json +from datetime import UTC, datetime +from unittest.mock import Mock + +import httpx + +from common_grants_sdk.client import Config, ParseFailure, SearchResult +from common_grants_sdk.extensions import f +from examples.typed_custom_filters import opportunity_plugin + + +def _opportunity_row(): + return { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "title": "AI research grant", + "description": "Funding for applied AI research.", + "status": {"value": "open", "description": "Open"}, + "createdAt": datetime.now(UTC).isoformat(), + "lastModifiedAt": datetime.now(UTC).isoformat(), + "customFields": { + "programCode": { + "name": "programCode", + "fieldType": "string", + "value": "EDU-1", + }, + "legacyId": {"name": "legacyId", "fieldType": "integer", "value": 42}, + }, + } + + +def _search_response(items): + return { + "status": 200, + "message": "Success", + "items": items, + "paginationInfo": { + "page": 1, + "pageSize": 100, + "totalItems": 2, + "totalPages": 1, + }, + "sortInfo": {"sortBy": "lastModifiedAt", "sortOrder": "desc", "errors": []}, + "filterInfo": {"filters": {}, "errors": []}, + } + + +def _client_with_response(payload): + client = opportunity_plugin.get_client( + Config(base_url="https://api.example.com", api_key="test-key") + ) + mock_http = Mock(spec=httpx.Client) + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(payload) + mock_response.json = Mock(return_value=payload) + mock_response.raise_for_status = Mock() + mock_http.post = Mock(return_value=mock_response) + client.http = mock_http + return client + + +def test_get_client_parses_custom_fields_without_per_call_schema(): + """The plugin-bound client parses custom fields with no ``schema=`` argument.""" + client = _client_with_response(_search_response([_opportunity_row()])) + + result = client.opportunities.search( + search="ai", filters={"region": f.in_(["US-CA", "US-NY"])}, page=1 + ) + + assert isinstance(result, SearchResult) + assert len(result.items) == 1 + opp = result.items[0] + assert opp.custom_fields is not None + assert opp.custom_fields.program_code is not None + assert opp.custom_fields.program_code.value == "EDU-1" + assert opp.custom_fields.legacy_id is not None + assert opp.custom_fields.legacy_id.value == 42 + assert result.errors == [] + + +def test_search_partitions_a_malformed_row(): + """A malformed row lands in result.errors; valid rows still return.""" + client = _client_with_response( + _search_response([_opportunity_row(), {"id": "not-a-uuid"}]) + ) + + result = client.opportunities.search(search="ai", page=1) + + assert len(result.items) == 1 + assert len(result.errors) == 1 + assert isinstance(result.errors[0], ParseFailure) + assert result.errors[0].index == 1 diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py index 691472084..7a333c9ad 100644 --- a/lib/python-sdk/tests/extensions/test_filters.py +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -9,37 +9,49 @@ import pytest from pydantic import ValidationError +from common_grants_sdk.extensions import PluginRoutes, ResourceRoutes from common_grants_sdk.extensions.filters import ( - FILTER_TYPE_SCHEMAS, classify_filters, f, validate_filter_call, validate_routes, ) -from common_grants_sdk.extensions.specs import CustomFilterSpec, CustomFilterType from common_grants_sdk.extensions.types import ClassifyResult, FilterError -from common_grants_sdk.schemas.pydantic.filters.opportunity import OppFilters +from common_grants_sdk.schemas.pydantic.filters.money import ( + MoneyComparisonFilter, + MoneyRangeFilter, +) +from common_grants_sdk.schemas.pydantic.filters.numeric import ( + NumberComparisonFilter, + NumberRangeFilter, +) +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + BooleanComparison, + NumberComparison, + OpportunityFilters, + OppFilters, + StringArray, + StringComparison, +) +from common_grants_sdk.schemas.pydantic.filters.string import StringArrayFilter # --------------------------------------------------------------------------- # Shared fixtures # --------------------------------------------------------------------------- -AGENCY_SPEC = CustomFilterSpec(filter_type=CustomFilterType.STRING_ARRAY) -FUNDING_PROGRAM_SPEC = CustomFilterSpec( - filter_type=CustomFilterType.STRING_COMPARISON, - description="Program name filter", -) -SAMPLE_ROUTES = { - "opportunities": { - "search": { - "filters": { - "agency": AGENCY_SPEC, - "fundingProgram": FUNDING_PROGRAM_SPEC, - } - } - } -} +class OppSearchFilters(OpportunityFilters, total=False): + """The registered custom filters for opportunities.search. + + ``agency`` is a stringArray filter; ``fundingProgram`` a stringComparison — + each key's value model *is* its declared type, recovered at classify time. + """ + + agency: StringArray + fundingProgram: StringComparison + + +SAMPLE_ROUTES = PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)) # --------------------------------------------------------------------------- @@ -238,99 +250,72 @@ def test_oppfilters_mixed_case_roundtrip(): # --------------------------------------------------------------------------- -def test_validate_routes_unknown_filter_type_raises(): - """validate_routes raises FilterError when filter_type is not in FILTER_TYPE_SCHEMAS.""" - # Dataclasses don't validate field types at runtime, so a bogus filter_type - # can be passed directly (the annotation is for type checkers only). - bad_spec = CustomFilterSpec(filter_type="unknownType") # type: ignore[arg-type] - - routes = {"opportunities": {"search": {"filters": {"myFilter": bad_spec}}}} - with pytest.raises(FilterError, match="Unknown filter_type"): - validate_routes(routes) +def test_validate_routes_non_model_filter_type_raises(): + """validate_routes raises FilterError when a registered custom filter's value + type is not a filter value model. + The typed carrier makes a misspelled resource/method a *static* error, so the + only meaningful runtime check left is that each registered key is annotated + with a ``CommonGrantsBaseModel`` subclass. A ``region: int`` annotation is the + genuinely-invalid registration this surfaces at registration time. + """ -def test_validate_routes_collision_with_default_filter_name_raises(): - """validate_routes raises FilterError when a custom filter name collides with a CORE DEFAULT name. + class BadFilters(OpportunityFilters, total=False): + region: int # not a filter value model - This is the escape-hatch collision check. E.g. naming a custom filter "status" - would shadow the core default "status" field — must be caught at registration time. - """ - routes = { - "opportunities": { - "search": { - "filters": { - "status": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY, - description="Should collide with default", - ), - } - } - } - } - with pytest.raises(FilterError, match="collides"): + routes = PluginRoutes(opportunities=ResourceRoutes(search=BadFilters)) + with pytest.raises(FilterError): validate_routes(routes) -def test_validate_routes_collision_with_camel_alias_raises(): - """validate_routes raises FilterError for camelCase alias collision (e.g. "closeDateRange").""" - routes = { - "opportunities": { - "search": { - "filters": { - "closeDateRange": CustomFilterSpec( - filter_type=CustomFilterType.DATE_RANGE, - description="Should collide with default alias", - ), - } - } - } - } - with pytest.raises(FilterError, match="collides"): +def test_validate_routes_redeclared_standard_key_wrong_type_raises(): + """validate_routes raises when a route TypedDict redeclares a standard key with + a different value type — the call site would see the override while classify + validates against the real standard field type (a static/runtime mismatch).""" + + class RedeclaresStatus(OpportunityFilters, total=False): + status: StringComparison # standard "status" is a StringArray filter + + routes = PluginRoutes(opportunities=ResourceRoutes(search=RedeclaresStatus)) + with pytest.raises(FilterError): validate_routes(routes) -def test_validate_routes_unsupported_route_raises(): - """validate_routes raises FilterError for custom filters on a non-filterable route. +def test_validate_routes_redeclared_standard_key_same_type_ok(): + """Redeclaring a standard key with its SAME type is a harmless no-op override.""" - Only routes whose core operation declares a ``filters`` parameter can carry - custom filters. ``opportunities.list`` has none, so a filter declared there is - a registration-time error rather than a silently-ignored declaration. - """ - routes = { - "opportunities": { - "list": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY, - description="Not supported on list", - ), - } - } - } - } - with pytest.raises(FilterError, match="does not support custom filters"): - validate_routes(routes) + class RedeclaresStatusSame(OpportunityFilters, total=False): + status: StringArray # same as the standard type + + validate_routes( + PluginRoutes(opportunities=ResourceRoutes(search=RedeclaresStatusSame)) + ) def test_validate_routes_valid_routes_do_not_raise(): - """validate_routes does not raise for a fully valid routes dict.""" + """validate_routes does not raise for a fully valid typed routes carrier.""" # Should not raise validate_routes(SAMPLE_ROUTES) +def test_validate_routes_empty_carrier_does_not_raise(): + """The empty carrier (no registered filters) passes validation silently.""" + validate_routes(PluginRoutes(opportunities=ResourceRoutes())) + + # --------------------------------------------------------------------------- -# Call-time validation (validate_filter_call) — RAISES FilterError +# Call-time validation (validate_filter_call) — fail-soft # --------------------------------------------------------------------------- def test_validate_filter_call_registered_bad_operator_returns_error(): """validate_filter_call returns (None, FilterError) for an operator/value mismatch. - A bad registered filter yields a FilterError, not an exception. AGENCY_SPEC - is STRING_ARRAY — an "eq" with a scalar value is wrong for it. + A bad registered filter yields a FilterError, not an exception. agency is + validated against StringArrayFilter — an "eq" with a scalar value is wrong for it. """ bad_filter = f.eq("not-an-array") - value, error = validate_filter_call(AGENCY_SPEC, "agency", bad_filter) + value, error = validate_filter_call(StringArrayFilter, "agency", bad_filter) assert value is None assert isinstance(error, FilterError) assert error.path == "filters.agency" @@ -339,7 +324,7 @@ def test_validate_filter_call_registered_bad_operator_returns_error(): def test_validate_filter_call_adhoc_invalid_shape_returns_error(): """validate_filter_call returns a FilterError when an ad-hoc filter has an invalid shape.""" - # Pass None as spec (ad-hoc), with something that isn't a DefaultFilter + # Pass None as model_cls (ad-hoc), with something that isn't a DefaultFilter class _BadShape: operator = "not_a_real_operator" value = object() # not a valid value type @@ -353,56 +338,56 @@ class _BadShape: def test_validate_filter_call_valid_registered_returns_value_and_no_error(): """validate_filter_call returns (DefaultFilter, None) for a valid registered filter call.""" valid_filter = f.in_(["NSF", "NIH"]) - # agency is STRING_ARRAY, in_ with list is valid - value, error = validate_filter_call(AGENCY_SPEC, "agency", valid_filter) + # agency is StringArrayFilter, in_ with list is valid + value, error = validate_filter_call(StringArrayFilter, "agency", valid_filter) assert error is None assert value is not None assert value.operator == "in" def test_validate_filter_call_money_comparison_passes_valid_money(): - """A registered moneyComparison filter accepts a comparison operator and Money value. + """A moneyComparison filter accepts a comparison operator and Money value. Money.amount is a decimal STRING ("1000000"), not a number — the shape that drifted in the TS compile-time filter map and was locked there with compile-error tests; covered here at the runtime layer. """ - spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) value, error = validate_filter_call( - spec, "awardFloor", f.gt({"amount": "1000000", "currency": "USD"}) + MoneyComparisonFilter, + "awardFloor", + f.gt({"amount": "1000000", "currency": "USD"}), ) assert error is None assert value is not None def test_validate_filter_call_money_comparison_rejects_array_operator(): - """A registered moneyComparison filter returns a FilterError for an array operator.""" - spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) + """A moneyComparison filter returns a FilterError for an array operator.""" value, error = validate_filter_call( - spec, "awardFloor", f.in_([{"amount": "1000000", "currency": "USD"}]) + MoneyComparisonFilter, + "awardFloor", + f.in_([{"amount": "1000000", "currency": "USD"}]), ) assert value is None assert isinstance(error, FilterError) def test_validate_filter_call_money_comparison_rejects_numeric_amount(): - """A registered moneyComparison filter returns a FilterError for a numeric amount. + """A moneyComparison filter returns a FilterError for a numeric amount. Money.amount is a DecimalString — a raw number is the wrong shape. """ - spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_COMPARISON) value, error = validate_filter_call( - spec, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) + MoneyComparisonFilter, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) ) assert value is None assert isinstance(error, FilterError) def test_validate_filter_call_money_range_passes_valid_range(): - """A registered moneyRange filter accepts between with {min, max} Money values.""" - spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) + """A moneyRange filter accepts between with {min, max} Money values.""" value, error = validate_filter_call( - spec, + MoneyRangeFilter, "awardRange", f.between( {"amount": "10000", "currency": "USD"}, @@ -414,17 +399,16 @@ def test_validate_filter_call_money_range_passes_valid_range(): def test_validate_filter_call_money_range_rejects_comparison_operator(): - """A registered moneyRange filter returns a FilterError for a comparison operator.""" - spec = CustomFilterSpec(filter_type=CustomFilterType.MONEY_RANGE) + """A moneyRange filter returns a FilterError for a comparison operator.""" value, error = validate_filter_call( - spec, "awardRange", f.gt({"amount": "10000", "currency": "USD"}) + MoneyRangeFilter, "awardRange", f.gt({"amount": "10000", "currency": "USD"}) ) assert value is None assert isinstance(error, FilterError) def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): - """A registered numberRange filter round-trips its NumberRange sub-model to wire dict. + """A numberRange filter round-trips its NumberRange sub-model to wire dict. f.between(int, int) returns a NumberRangeFilter whose ``.value`` is a NumberRange sub-model (not a plain dict); validation must accept it and model_dump must recurse @@ -432,8 +416,9 @@ def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): NumberRange object — or that dropped the int payload — would ship a non-JSON body. The moneyRange analog above is covered; this pins the numeric path. """ - spec = CustomFilterSpec(filter_type=CustomFilterType.NUMBER_RANGE) - value, error = validate_filter_call(spec, "awardCount", f.between(0, 1000)) + value, error = validate_filter_call( + NumberRangeFilter, "awardCount", f.between(0, 1000) + ) assert error is None wire = value.model_dump(by_alias=True, exclude_none=True, mode="json") assert wire["operator"] == "between" @@ -460,18 +445,17 @@ def test_classify_default_wrong_shape_collects_error_and_omits_key(): assert isinstance(classified.errors[0].cause, ValidationError) -def test_validate_filter_call_integer_comparison_validates_as_number(): - """A registered integerComparison filter validates against NumberComparisonFilter. +def test_validate_filter_call_number_comparison_validates_as_number(): + """A numberComparison filter validates against NumberComparisonFilter. - The spec defines no integer filter model, so the int constraint is not - schema-enforced; a numeric value passes, a non-numeric - value fails. + A numeric value passes; a non-numeric value fails. """ - spec = CustomFilterSpec(filter_type=CustomFilterType.INTEGER_COMPARISON) - value, error = validate_filter_call(spec, "awardCount", f.gt(100)) + value, error = validate_filter_call(NumberComparisonFilter, "awardCount", f.gt(100)) assert error is None assert value is not None - value, error = validate_filter_call(spec, "awardCount", f.gt("not a number")) + value, error = validate_filter_call( + NumberComparisonFilter, "awardCount", f.gt("not a number") + ) assert value is None assert isinstance(error, FilterError) @@ -524,20 +508,15 @@ def test_classify_default_money_range_wrong_shape_collects_error(): # Wire-body integrity: the value that passed validation is the value shipped # --------------------------------------------------------------------------- -WIRE_ROUTES = { - "opportunities": { - "search": { - "filters": { - "isOpen": CustomFilterSpec( - filter_type=CustomFilterType.BOOLEAN_COMPARISON - ), - "awardCount": CustomFilterSpec( - filter_type=CustomFilterType.NUMBER_COMPARISON - ), - } - } - } -} + +class WireFilters(OpportunityFilters, total=False): + """Registered wire-integrity filters: a boolean and a number comparison.""" + + isOpen: BooleanComparison + awardCount: NumberComparison + + +WIRE_ROUTES = PluginRoutes(opportunities=ResourceRoutes(search=WireFilters)) def test_boolean_filter_value_survives_to_wire_as_json_true(): @@ -606,19 +585,6 @@ def test_mutated_adhoc_instance_is_revalidated_and_collected(): assert classified.errors[0].path == "filters.legacy" -def test_unknown_filter_type_returns_filter_error_not_key_error(): - """A spec whose filter_type never passed validate_routes returns a FilterError. - - The uniform fail-soft contract holds even when registration-time validation - was skipped — the call-time validator must surface a FilterError, never a - KeyError, and never raise. - """ - spec = CustomFilterSpec(filter_type="bogusType") # type: ignore[arg-type] - value, error = validate_filter_call(spec, "x", f.eq(1)) - assert value is None - assert isinstance(error, FilterError) - - def test_validate_filter_call_adhoc_accepts_raw_dict(): """Ad-hoc validation accepts a raw operator/value dict and returns (DefaultFilter, None).""" validated, error = validate_filter_call(None, "x", {"operator": "eq", "value": "v"}) @@ -681,9 +647,9 @@ def test_classify_both_forms_of_same_default_filter_collects_error(): def test_classify_unmatched_route_treats_registered_name_as_adhoc(resource, method): """A (resource, method) pair with no routes entry has NO registered bucket. - "agency" is registered as STRING_ARRAY under opportunities.search only; + "agency" is registered as a stringArray under opportunities.search only; via any other route pair it is validated as permissive ad-hoc, so - f.eq("NSF") (invalid for STRING_ARRAY) passes through to customFilters. + f.eq("NSF") (invalid for stringArray) passes through to customFilters. Discriminates both levels of the routes[resource][method] lookup — a regression that flattens or mis-keys it either wrongly applies the spec or wrongly skips it. @@ -738,16 +704,6 @@ def test_classify_empty_filters_dict_yields_empty_body(): assert "customFilters" not in body -def test_filter_type_schemas_covers_every_custom_filter_type(): - """Every CustomFilterType member has a validation model. - - A catalog member without a FILTER_TYPE_SCHEMAS entry would reject valid - registrations in validate_routes — this assert turns that drift into a - CI failure at the moment the enum and the map diverge. - """ - assert set(FILTER_TYPE_SCHEMAS) == set(CustomFilterType) - - def test_collected_error_path_is_uniform_across_buckets(): """All three buckets collect a FilterError with a filters.<name> path, none raise.""" c1 = classify_filters( @@ -792,7 +748,7 @@ def test_classify_invalid_registered_and_invalid_adhoc_both_collected(): """An invalid registered filter AND an invalid ad-hoc filter are both collected. Neither is shipped: both keys are omitted from customFilters and each yields - its own FilterError. "agency" is STRING_ARRAY (f.eq scalar is wrong); the + its own FilterError. "agency" is a stringArray (f.eq scalar is wrong); the ad-hoc "adhoc" key has a bogus operator. """ classified = classify_filters( diff --git a/lib/python-sdk/tests/extensions/test_plugin.py b/lib/python-sdk/tests/extensions/test_plugin.py index c5db350f7..ca05a6865 100644 --- a/lib/python-sdk/tests/extensions/test_plugin.py +++ b/lib/python-sdk/tests/extensions/test_plugin.py @@ -8,19 +8,22 @@ from common_grants_sdk.extensions import ( CustomField, CustomFieldSet, - CustomFilterSpec, - CustomFilterType, PassthroughModel, Plugin, PluginMeta, PluginRoutes, PluginSchemas, + ResourceRoutes, SchemaOnly, SchemaWithTransforms, define_plugin, schema, ) from common_grants_sdk.extensions.schema import PluginDefinitionError +from common_grants_sdk.schemas.pydantic.filters.opportunity import ( + OpportunityFilters, + StringArray, +) from common_grants_sdk.schemas.pydantic.models import OpportunityBase @@ -109,32 +112,33 @@ def test_plugin_is_frozen(): # --------------------------------------------------------------------------- +class OppSearchFilters(OpportunityFilters, total=False): + """A route filter TypedDict registering one custom filter (agency).""" + + agency: StringArray + + def test_define_plugin_threads_routes_onto_plugin(): - """The PluginRoutes map passed to define_plugin is threaded onto the plugin. + """The PluginRoutes carrier passed to define_plugin is threaded onto the plugin. - Regression: if define_plugin dropped ``routes=``, plugin.routes would be empty - and the registered custom filters would be lost. + Regression: if define_plugin dropped ``routes=``, plugin.routes would default + to the empty carrier and the registered custom filters would be lost. """ - registered: PluginRoutes = { - "opportunities": { - "search": { - "filters": { - "agency": CustomFilterSpec( - filter_type=CustomFilterType.STRING_ARRAY - ) - } - } - } - } + registered: PluginRoutes = PluginRoutes( + opportunities=ResourceRoutes(search=OppSearchFilters) + ) plugin = define_plugin( PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), routes=registered, meta=_meta(), ) assert plugin.routes is registered + assert plugin.routes.opportunities.search is OppSearchFilters -def test_define_plugin_defaults_routes_to_empty_map(): - """Omitting ``routes`` yields a concrete, non-optional empty map (never None).""" +def test_define_plugin_defaults_routes_to_empty_carrier(): + """Omitting ``routes`` yields a concrete, non-optional empty carrier (never None).""" plugin = define_plugin(PluginSchemas(), meta=_meta()) - assert plugin.routes == {} + assert isinstance(plugin.routes, PluginRoutes) + # The empty carrier registers no custom filters. + assert plugin.routes.opportunities.search is None From 6674fcd120c7ae66ed01bd51393fa445bbef9061 Mon Sep 17 00:00:00 2001 From: Billy Daly <billy.daly@agile6.com> Date: Tue, 7 Jul 2026 10:07:06 -0400 Subject: [PATCH 41/51] refactor(py-sdk): remove `routes` and `schemas` args from `Client` (#978) * fix(py-sdk): use get_client() in the custom-filters consumer example Client(config, routes=...) leaves the Client generics unbound, so search(filters=...) fell back to OpportunityFilters and rejected the OppSearchFilters filters input. This switches the example to use plugin.get_client(), which binds FiltersT/ItemT, and adds /examples to pyright to catch similar type issues earlier. * refactor(py-sdk): remove routes and schemas param from Client This makes `Plugin.get_client()` the single correct path for accessing a Client that has been scoped with custom filters and custom fields from a Plugin, and avoids the type inference errors that stemmed from a plugin consumer directly calling `Client(routes=..., schemas=...)` * fix(py-sdk): type hints in examples/custom_filters.py Applies pyright checks to examples (excluding intentional failures file) and fixes a lingering IDE-level error in custom_filters.py by slotting OppSearchFilters in PluginRoutes * docs(py-sdk): fix inconsistent client docstrings --- .../common_grants_sdk/client/client.py | 63 +++++++++++-------- .../common_grants_sdk/client/opportunities.py | 2 +- .../common_grants_sdk/extensions/filters.py | 7 ++- .../common_grants_sdk/extensions/plugin.py | 12 ++-- .../examples/consumer_search_with_filters.py | 19 +++--- lib/python-sdk/examples/custom_filters.py | 2 +- lib/python-sdk/pyrightconfig.json | 12 +++- lib/python-sdk/tests/client/test_client.py | 38 ++++++----- .../tests/client/test_opportunities.py | 24 +++++-- 9 files changed, 113 insertions(+), 66 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/client/client.py b/lib/python-sdk/common_grants_sdk/client/client.py index 9cd7bb3d4..c5dc963eb 100644 --- a/lib/python-sdk/common_grants_sdk/client/client.py +++ b/lib/python-sdk/common_grants_sdk/client/client.py @@ -1,11 +1,12 @@ """HTTP client for the CommonGrants API. ``BaseClient`` is the transport plumbing (auth headers, paginated GET/POST). The -generic ``Client`` layers the typed resource facade on top: it binds a plugin's -route filters (``FiltersT``) and Opportunity schema (``ItemT``) so +generic ``Client`` layers the typed resource facade on top. Scope it to a plugin +with ``plugin.get_client(...)``: that binds the plugin's route filters +(``FiltersT``) and Opportunity schema (``ItemT``) so ``client.opportunities.search(filters=...)`` is typed and responses parse with the -plugin's custom fields by default. Construct one directly, or via -``plugin.get_client(...)``. +plugin's custom fields by default. Constructing ``Client`` directly gives an +unscoped client (standard filters only, base ``OpportunityBase`` rows). """ from __future__ import annotations @@ -196,41 +197,53 @@ class Client(BaseClient, Generic[FiltersT, ItemT]): """Typed resource facade over :class:`BaseClient`. Binds a plugin's route filters and Opportunity schema, exposing the typed - ``opportunities`` resource. Prefer ``plugin.get_client(...)`` over constructing - this directly. + ``opportunities`` resource. Construct one via ``plugin.get_client(...)``: that + binds the plugin's registered filters (``FiltersT``) and Opportunity schema + (``ItemT``) so ``opportunities.search`` is typed and responses parse with the + plugin's custom fields. Constructing ``Client`` directly leaves those generics + unbound (standard filters only, base ``OpportunityBase`` rows). """ def __init__( self, config: Optional[Config] = None, auth: Optional[Auth] = None, - routes: Optional[PluginRoutes[Any]] = None, - schemas: Optional[PluginSchemas[Any]] = None, ): """Initialize the client. Args: config: Optional Config instance. auth: Optional Auth instance. - routes: Optional typed ``PluginRoutes`` carrier (fixed plugin config). - Bound and validated once here; used to classify registered custom - filters in ``opportunities.search``. - schemas: Optional ``PluginSchemas``; its Opportunity common model - becomes the default parse schema for ``opportunities`` responses. - - Raises: - FilterError: If ``routes`` registers a filter whose value type is not a - filter value model. """ super().__init__(config=config, auth=auth) - self.routes: PluginRoutes[Any] = ( - routes - if routes is not None - else PluginRoutes(opportunities=ResourceRoutes()) - ) - validate_routes(self.routes) - self.schemas = schemas + self._routes: PluginRoutes[Any] = PluginRoutes(opportunities=ResourceRoutes()) + self._schemas: Optional[PluginSchemas[Any]] = None self._opportunity_schema: type[OpportunityBase] = _resolve_opportunity_schema( - schemas + None ) self.opportunities: Opportunities[FiltersT, ItemT] = Opportunities(client=self) + + def _bind_routes(self, routes: PluginRoutes[Any]) -> None: + """Scope this client to a plugin's registered custom filters. + + Internal hook called by ``plugin.get_client``: it validates the route + registration and stores it for ``opportunities.search`` to classify + registered custom filters. Build scoped clients via ``get_client`` rather + than calling this directly. + + Raises: + FilterError: If ``routes`` registers a filter whose value type is not a + filter value model. + """ + validate_routes(routes) + self._routes = routes + + def _bind_schemas(self, schemas: Optional[PluginSchemas[Any]]) -> None: + """Bind a plugin's schema extensions as the default parse schemas. + + Internal hook called by ``plugin.get_client``: the Opportunity common + model becomes the default schema ``opportunities`` responses parse into. + Build scoped clients via ``get_client`` rather than calling this directly. + """ + self._schemas = schemas + self._opportunity_schema = _resolve_opportunity_schema(schemas) diff --git a/lib/python-sdk/common_grants_sdk/client/opportunities.py b/lib/python-sdk/common_grants_sdk/client/opportunities.py index 4e9731f33..223e88bb7 100644 --- a/lib/python-sdk/common_grants_sdk/client/opportunities.py +++ b/lib/python-sdk/common_grants_sdk/client/opportunities.py @@ -148,7 +148,7 @@ def search( filters_body: dict[str, Any] = {} if filters: classified = classify_filters( - self.client.routes, "opportunities", "search", filters + self.client._routes, "opportunities", "search", filters ) strict_error = next((e for e in classified.errors if e.strict), None) if strict_error is not None: diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index a4eeef2e6..50e534a49 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -297,9 +297,10 @@ def _registered_filter_models(route_td: Any) -> dict[str, type[BaseModel]]: ``route_td`` is the TypedDict class an author put in the route slot (e.g. ``OppSearchFilters``), or ``None``. Returns ``{filterName: value model}`` for every key the author declared beyond the standard ``OpportunityFilters`` keys. - Non-model annotations are skipped here; ``validate_routes`` (run at ``Client`` - construction) rejects them, so a caller who invokes ``classify_filters`` directly - on unvalidated routes gets silent skipping rather than a raise. + Non-model annotations are skipped here; ``validate_routes`` (run when + ``plugin.get_client`` binds routes) rejects them, so a caller who invokes + ``classify_filters`` directly on unvalidated routes gets silent skipping rather + than a raise. """ if route_td is None: return {} diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 39f3c1e27..19733f1fd 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -110,7 +110,7 @@ def get_client( """Return a client pre-scoped with this plugin's routes and schemas. Consumers call ``plugin.get_client(config)`` instead of constructing a - client and passing ``routes=`` / ``schemas=`` by hand. The returned + client and binding the plugin's schemas/routes by hand. The returned client types ``opportunities.search(filters=...)`` by the plugin's registered filters and parses responses with its Opportunity schema. """ @@ -120,12 +120,10 @@ def get_client( # self.schemas/self.routes are opaque here (SchemasT); the overloads above # carry the precise Client[FiltersT, ItemT] the caller sees. - return Client( - config=config, - auth=auth, - routes=cast("Any", self.routes), - schemas=cast("Any", self.schemas), - ) + client = Client(config=config, auth=auth) + client._bind_schemas(cast("Any", self.schemas)) + client._bind_routes(cast("Any", self.routes)) + return client def define_plugin( diff --git a/lib/python-sdk/examples/consumer_search_with_filters.py b/lib/python-sdk/examples/consumer_search_with_filters.py index ee0c8deed..8240efcde 100644 --- a/lib/python-sdk/examples/consumer_search_with_filters.py +++ b/lib/python-sdk/examples/consumer_search_with_filters.py @@ -8,12 +8,13 @@ passing the carrier as ``define_plugin(routes=...)`` → ``plugin.routes``, a ``PluginRoutes`` carrier. The typed dict also narrows the call-site filter dict per key. - CONSUMER constructs the client with the registered routes - (``Client(config, routes=plugin.routes)``), builds a filter dict with the - ``f.*`` builders, and calls ``client.opportunities.search(search=..., filters=...)``. - The client runs ``classify_filters`` to build the three-bucket request body - (default named fields + ``customFilters`` record), POSTs it, and returns - typed ``OpportunityBase`` rows. + CONSUMER gets a pre-scoped client from ``plugin.get_client(config)`` (routes and + schemas already bound), builds a filter dict with the ``f.*`` builders, and + calls ``client.opportunities.search(search=..., filters=...)``. The client + runs ``classify_filters`` to build the three-bucket request body (default + named fields + ``customFilters`` record), POSTs it, and returns typed + ``OpportunityBase`` rows. Because ``get_client`` binds the plugin's registered + ``OppSearchFilters``, ``search(filters=...)`` narrows to it per key. The ``f.*`` builders return the precise filter model per value type (``f.in_([...])`` -> ``StringArrayFilter``, ``f.gte(3)`` -> ``NumberComparisonFilter``), @@ -33,7 +34,6 @@ from typing_extensions import assert_type -from common_grants_sdk.client import Client from common_grants_sdk.client.config import Config from common_grants_sdk.extensions import ( PluginMeta, @@ -84,8 +84,9 @@ def demo() -> None: config = Config( base_url="http://localhost:8000", api_key="two_orgs_user_key", timeout=5.0 ) - # Routes are bound to the client once at construction (not per search call). - client = Client(config, routes=plugin.routes) + # get_client binds the plugin's routes and schemas: search(filters=...) is typed + # by OppSearchFilters and responses parse with the plugin's Opportunity schema. + client = plugin.get_client(config) # A filter dict built with the f.* helpers. The builders return the precise # per-key models, so each value narrows to exactly the type OppSearchFilters diff --git a/lib/python-sdk/examples/custom_filters.py b/lib/python-sdk/examples/custom_filters.py index 1b484ad7b..d6412b58c 100644 --- a/lib/python-sdk/examples/custom_filters.py +++ b/lib/python-sdk/examples/custom_filters.py @@ -54,7 +54,7 @@ class OppSearchFilters(OpportunityFilters, total=False): fundingProgram: StringComparison -routes: PluginRoutes = PluginRoutes( +routes: PluginRoutes[OppSearchFilters] = PluginRoutes( opportunities=ResourceRoutes(search=OppSearchFilters) ) diff --git a/lib/python-sdk/pyrightconfig.json b/lib/python-sdk/pyrightconfig.json index 2ae24a65f..79a5d27f1 100644 --- a/lib/python-sdk/pyrightconfig.json +++ b/lib/python-sdk/pyrightconfig.json @@ -1,6 +1,14 @@ { - "include": ["common_grants_sdk", "examples/typed_custom_filters.py"], - "exclude": ["tests", "**/__pycache__", "generated"], + "include": ["common_grants_sdk", "examples"], + "exclude": [ + "tests", + "**/__pycache__", + "generated", + "examples/typed_custom_filters_failures.py", + "**/node_modules", + "**/.*", + ".venv" + ], "typeCheckingMode": "basic", "pythonVersion": "3.11" } diff --git a/lib/python-sdk/tests/client/test_client.py b/lib/python-sdk/tests/client/test_client.py index abfbce070..90ae38cbc 100644 --- a/lib/python-sdk/tests/client/test_client.py +++ b/lib/python-sdk/tests/client/test_client.py @@ -12,8 +12,16 @@ from common_grants_sdk.client.exceptions import APIError from common_grants_sdk.schemas.pydantic.pagination import PaginatedResultsInfo from common_grants_sdk.schemas.pydantic.responses import Paginated -from common_grants_sdk.extensions import PluginRoutes, ResourceRoutes +from common_grants_sdk.extensions import ( + PluginMeta, + PluginRoutes, + PluginSchemas, + ResourceRoutes, + define_plugin, + schema, +) from common_grants_sdk.extensions.types import FilterError +from common_grants_sdk.schemas.pydantic.models import OpportunityBase from common_grants_sdk.schemas.pydantic.filters.opportunity import ( OpportunityFilters, StringArray, @@ -34,8 +42,8 @@ def test_client_initialization_with_config_and_auth(self): assert client.auth == auth assert isinstance(client.opportunities, type(client.opportunities)) - def test_client_initialization_validates_routes(self): - """Client validates routes at construction and raises on a bad declaration. + def test_get_client_validates_routes(self): + """get_client validates routes when binding and raises on a bad declaration. The typed carrier makes a misspelled resource/method a static error, so the only remaining runtime check is that a registered custom filter's value type @@ -48,29 +56,31 @@ def test_client_initialization_validates_routes(self): class BadFilters(OpportunityFilters, total=False): region: int # not a filter value model + plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), + routes=PluginRoutes(opportunities=ResourceRoutes(search=BadFilters)), + meta=PluginMeta(name="t", source_system="t"), + ) with pytest.raises(FilterError): - Client( - config=config, - routes=PluginRoutes( - opportunities=ResourceRoutes(search=BadFilters) - ), - ) + plugin.get_client(config) - def test_client_initialization_accepts_valid_routes(self): - """A valid typed routes carrier is accepted at construction (no raise).""" + def test_get_client_accepts_valid_routes(self): + """A valid typed routes carrier binds through get_client (no raise).""" with patch("common_grants_sdk.client.client.httpx.Client"): config = Config(base_url="https://api.example.com", api_key="test-key") class OppSearchFilters(OpportunityFilters, total=False): region: StringArray - client = Client( - config=config, + plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), routes=PluginRoutes( opportunities=ResourceRoutes(search=OppSearchFilters) ), + meta=PluginMeta(name="t", source_system="t"), ) - assert client.routes.opportunities.search is OppSearchFilters + client = plugin.get_client(config) + assert client._routes.opportunities.search is OppSearchFilters def test_client_initialization_defaults_auth(self, monkeypatch): """Test client initialization with default auth from config.""" diff --git a/lib/python-sdk/tests/client/test_opportunities.py b/lib/python-sdk/tests/client/test_opportunities.py index a493a652d..fc6c9a03b 100644 --- a/lib/python-sdk/tests/client/test_opportunities.py +++ b/lib/python-sdk/tests/client/test_opportunities.py @@ -18,7 +18,15 @@ ) from common_grants_sdk.client.config import Config from common_grants_sdk.client.exceptions import APIError -from common_grants_sdk.extensions import FilterError, PluginRoutes, ResourceRoutes +from common_grants_sdk.extensions import ( + FilterError, + PluginMeta, + PluginRoutes, + PluginSchemas, + ResourceRoutes, + define_plugin, + schema, +) from common_grants_sdk.schemas.pydantic.models import OpportunityBase from common_grants_sdk.schemas.pydantic.fields import CustomFieldType from common_grants_sdk.schemas.pydantic.filters.opportunity import ( @@ -37,6 +45,14 @@ class OppSearchFilters(OpportunityFilters, total=False): AGENCY_ROUTES = PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)) +# A plugin scoped to AGENCY_ROUTES: get_client() is the public path to a client +# whose opportunities.search classifies the registered ``agency`` custom filter. +AGENCY_PLUGIN = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), + routes=AGENCY_ROUTES, + meta=PluginMeta(name="test", source_system="test"), +) + @pytest.fixture def sample_opportunity_data(): @@ -843,7 +859,7 @@ def test_search_classifies_custom_filter_bag( config = Config( base_url="https://api.example.com", api_key="test-key", timeout=10.0 ) - client = Client(config=config, auth=auth, routes=AGENCY_ROUTES) + client = AGENCY_PLUGIN.get_client(config, auth) client.http = mock_httpx_client client.opportunities.http = mock_httpx_client @@ -903,7 +919,7 @@ def test_search_invalid_registered_filter_raises( config = Config( base_url="https://api.example.com", api_key="test-key", timeout=10.0 ) - client = Client(config=config, auth=auth, routes=AGENCY_ROUTES) + client = AGENCY_PLUGIN.get_client(config, auth) client.http = mock_httpx_client client.opportunities.http = mock_httpx_client @@ -983,7 +999,7 @@ def test_search_filter_info_errors_are_server_only( config = Config( base_url="https://api.example.com", api_key="test-key", timeout=10.0 ) - client = Client(config=config, auth=auth, routes=AGENCY_ROUTES) + client = AGENCY_PLUGIN.get_client(config, auth) client.http = mock_httpx_client client.opportunities.http = mock_httpx_client From f909175089fd22be0f64c1b78f4d2e1d16fc4812 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:30:36 -0700 Subject: [PATCH 42/51] feat(ts-sdk): typed custom-filters authoring surface and client (#967) (#975) * feat(ts-sdk): close PluginRoutes keys to registered resource and method names * feat(ts-sdk): type registered custom-filter values by their declared filterType * feat(ts-sdk): add parseBatch row partitioning with ParseFailure detail * feat(ts-sdk): add fail-fast categorizeFilters that throws before the request * feat(ts-sdk): partition fetch helpers via parseBatch and add put/patch transport verbs * feat(ts-sdk)!: registry-driven resources, fail-fast filters, partitioned results * feat(ts-sdk)!: remove fail-soft classifyFilters in favor of categorizeFilters * feat(ts-sdk): plugin.getClient() builds a client typed by the plugin's schemas and routes * docs(ts-sdk): getClient consumer examples, fail-fast filter docs, changeset * refactor(ts-sdk): review polish and statuses deprecation - Drop unused write-verb transport surface (Client.put/patch, mutate) and the one-entry resource registry; Opportunities constructed directly - Shrink the public barrel (RESOURCE_NAMES, FILTER_TYPE_SCHEMAS removed) - encodeURIComponent the id path segment; auth-passthrough test for getClient - PII caveats on ParseFailure.raw / FilterError.sourceValue; redacted log patterns in README and example - Deprecate the statuses search shorthand in favor of filters.status - Honest optional filterInfo typing, undefined-spec guard in validateRoutes, example output-ordering fix, doc/comment accuracy * refactor(ts-sdk): keep the classifyFilters name for the fail-fast classifier Nothing consumes this surface yet and the Python SDK will converge on the same erroring pattern under classify_filters, so keep the name consistent across SDKs instead of renaming to categorizeFilters alongside the fail-soft removal. --- .changeset/ts-typed-custom-filters.md | 13 + lib/ts-sdk/__tests__/client/client.spec.ts | 142 ++++++++-- .../__tests__/client/opportunities.spec.ts | 216 +++++++------- lib/ts-sdk/__tests__/client/resources.spec.ts | 71 +++++ lib/ts-sdk/__tests__/client/results.spec.ts | 43 +++ .../__tests__/client/search-filters-types.ts | 47 +-- .../extensions/custom-filters.spec.ts | 267 ++++++++---------- .../__tests__/extensions/get-client.spec.ts | 181 ++++++++++++ .../extensions/plugin-routes-types.ts | 105 +++++++ lib/ts-sdk/examples/custom-filters.ts | 83 +++++- lib/ts-sdk/src/client/README.md | 10 +- lib/ts-sdk/src/client/client.ts | 97 +++++-- lib/ts-sdk/src/client/index.ts | 8 +- lib/ts-sdk/src/client/resources/base.ts | 42 +++ lib/ts-sdk/src/client/resources/builder.ts | 71 +++++ .../client/{ => resources}/opportunities.ts | 251 ++++++++-------- lib/ts-sdk/src/client/resources/registry.ts | 24 ++ lib/ts-sdk/src/client/results.ts | 71 +++++ lib/ts-sdk/src/extensions/README.md | 76 ++--- lib/ts-sdk/src/extensions/custom-filters.ts | 128 ++++----- lib/ts-sdk/src/extensions/define-plugin.ts | 26 +- lib/ts-sdk/src/extensions/index.ts | 12 +- lib/ts-sdk/src/extensions/types.ts | 14 +- 23 files changed, 1433 insertions(+), 565 deletions(-) create mode 100644 .changeset/ts-typed-custom-filters.md create mode 100644 lib/ts-sdk/__tests__/client/resources.spec.ts create mode 100644 lib/ts-sdk/__tests__/client/results.spec.ts create mode 100644 lib/ts-sdk/__tests__/extensions/get-client.spec.ts create mode 100644 lib/ts-sdk/__tests__/extensions/plugin-routes-types.ts create mode 100644 lib/ts-sdk/src/client/resources/base.ts create mode 100644 lib/ts-sdk/src/client/resources/builder.ts rename lib/ts-sdk/src/client/{ => resources}/opportunities.ts (58%) create mode 100644 lib/ts-sdk/src/client/resources/registry.ts create mode 100644 lib/ts-sdk/src/client/results.ts diff --git a/.changeset/ts-typed-custom-filters.md b/.changeset/ts-typed-custom-filters.md new file mode 100644 index 000000000..41b0a00f5 --- /dev/null +++ b/.changeset/ts-typed-custom-filters.md @@ -0,0 +1,13 @@ +--- +"@common-grants/sdk": minor +--- + +Typed custom-filters authoring surface and plugin-scoped client for the TypeScript SDK: + +- `definePlugin()` route keys are closed unions: a misspelled resource or method is a compile error, and an invalid registration throws `FilterError` at definition time. +- `plugin.getClient(config)` returns a client pre-bound to the plugin's schemas and registered filters; responses parse with the plugin schema by default (per-call `schema` override still wins). +- Filter validation is fail-fast: an invalid value on any filter — standard, registered custom, or ad-hoc — throws `FilterError` before the request is sent (compile error where the type is known). Well-formed ad-hoc filters still pass through; `filterInfo.errors` now carries server-returned errors only. +- `search()`/`list()` return `{ items, errors }`: valid rows in `items`, per-row `ParseFailure` (`index`, `raw`, `error`) in `errors`; `onParseError: "throw"` opts into fail-hard, and `get()` remains fail-hard. +- Breaking: `new Client(...)` no longer accepts `routes` (bind them via `plugin.getClient()`); `classifyFilters` now throws `FilterError` instead of returning collected errors, and `ClassifyResult` is removed. +- Breaking: the `Opportunities` constructor is now `(client, boundSchema?, routes?)` — `routes` moved from the second to the third parameter (a compile error at existing two-argument call sites). +- Breaking: the `schema` option on `Client.fetchMany` requires a `.safeParse()` method (any Zod schema qualifies); bare `{ parse }` objects are no longer accepted. diff --git a/lib/ts-sdk/__tests__/client/client.spec.ts b/lib/ts-sdk/__tests__/client/client.spec.ts index 54e41a3b4..faf2ad609 100644 --- a/lib/ts-sdk/__tests__/client/client.spec.ts +++ b/lib/ts-sdk/__tests__/client/client.spec.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest"; +import { z } from "zod"; import { http, HttpResponse, setupServer, createPaginatedHandler } from "../utils/mock-fetch"; -import { Client, Auth } from "../../src/client"; +import { Client, Auth, BatchParseError } from "../../src/client"; // ============================================================================= // Mock data helpers @@ -190,6 +191,114 @@ describe("Client", () => { }); }); + // ============================================================================= + // Client.fetchMany row partitioning + // ============================================================================= + + describe("fetchMany row partitioning", () => { + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + const rowSchema = z.object({ id: z.string(), amount: z.number() }); + + const mixedRowsHandler = () => + HttpResponse.json({ + status: 200, + message: "Success", + items: [ + { id: "a", amount: 1 }, + { id: "b", amount: "bad" }, + { id: "c", amount: 3 }, + ], + paginationInfo: { page: 1, pageSize: 3, totalItems: 3, totalPages: 1 }, + }); + + it("partitions bad rows into errors instead of throwing", async () => { + server.use(http.get("/rows", mixedRowsHandler)); + + const result = await defaultClient.fetchMany("/rows", { schema: rowSchema }); + + expect(result.items).toEqual([ + { id: "a", amount: 1 }, + { id: "c", amount: 3 }, + ]); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].index).toBe(1); + expect(result.errors[0].raw).toEqual({ id: "b", amount: "bad" }); + }); + + it('throws BatchParseError under onParseError: "throw"', async () => { + server.use(http.get("/rows", mixedRowsHandler)); + + await expect( + defaultClient.fetchMany("/rows", { schema: rowSchema, onParseError: "throw" }) + ).rejects.toThrow(BatchParseError); + }); + + it("uses the raw row count, not the parsed count, to decide the last page", async () => { + let requestCount = 0; + + server.use( + http.get("/rows", ({ request }) => { + requestCount++; + const url = new URL(request.url); + const page = Number(url.searchParams.get("page") ?? "1"); + const pages: Record<number, unknown[]> = { + 1: [ + { id: "a", amount: 1 }, + { id: "bad", amount: "x" }, + { id: "c", amount: 3 }, + ], + 2: [{ id: "d", amount: 4 }], + }; + return HttpResponse.json({ + status: 200, + message: "Success", + items: pages[page] ?? [], + paginationInfo: { page, pageSize: 3, totalItems: 4, totalPages: 2 }, + }); + }) + ); + + const result = await defaultClient.fetchMany("/rows", { pageSize: 3, schema: rowSchema }); + + // A parse-shrunk page (2 of 3 rows valid) must not read as the last page. + expect(requestCount).toBe(2); + expect(result.items.map(i => i.id)).toEqual(["a", "c", "d"]); + expect(result.errors).toHaveLength(1); + }); + + it("maxItems truncates items only; errors report every fetched row", async () => { + server.use( + http.get("/rows", () => + HttpResponse.json({ + status: 200, + message: "Success", + items: [ + { id: "a", amount: 1 }, + { id: "b", amount: 2 }, + { id: "c", amount: 3 }, + { id: "d", amount: "bad" }, + { id: "e", amount: "bad" }, + ], + paginationInfo: { page: 1, pageSize: 5, totalItems: 5, totalPages: 1 }, + }) + ) + ); + + const result = await defaultClient.fetchMany("/rows", { + pageSize: 5, + maxItems: 2, + schema: rowSchema, + }); + + expect(result.items.map(i => i.id)).toEqual(["a", "b"]); + // Rows past the cap were still fetched, so their failures still surface. + expect(result.errors.map(e => e.index)).toEqual([3, 4]); + }); + }); + // ============================================================================= // Client.fetchMany (auto-pagination) // ============================================================================= @@ -374,7 +483,7 @@ describe("Client", () => { await expect(defaultClient.fetchMany("/test-items", { pageSize: 5 })).rejects.toThrow("500"); }); - it("calls schema.parse for each item when provided", async () => { + it("calls schema.safeParse for each item when provided", async () => { const parseCalls: unknown[] = []; server.use( @@ -391,9 +500,12 @@ describe("Client", () => { "/test-items", { schema: { - parse: (item: unknown) => { + safeParse: (item: unknown) => { parseCalls.push(item); - return { ...(item as { id: string; name: string }), parsed: true as const }; + return { + success: true as const, + data: { ...(item as { id: string; name: string }), parsed: true as const }, + }; }, }, } @@ -438,27 +550,5 @@ describe("Client", () => { expect((result as Record<string, unknown>).extraField).toBe("preserved"); expect(result.items).toHaveLength(3); }); - - it("throws when schema.parse throws (validation failure)", async () => { - server.use( - http.get( - "/test-items", - createPaginatedHandler({ - items: generateMockItems(3), - defaultPageSize: 10, - }) - ) - ); - - await expect( - defaultClient.fetchMany("/test-items", { - schema: { - parse: () => { - throw new Error("Validation failed"); - }, - }, - }) - ).rejects.toThrow("Validation failed"); - }); }); }); diff --git a/lib/ts-sdk/__tests__/client/opportunities.spec.ts b/lib/ts-sdk/__tests__/client/opportunities.spec.ts index 76ff131f2..5b6673623 100644 --- a/lib/ts-sdk/__tests__/client/opportunities.spec.ts +++ b/lib/ts-sdk/__tests__/client/opportunities.spec.ts @@ -1,9 +1,10 @@ import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest"; import { z } from "zod"; import { http, HttpResponse, setupServer, createPaginatedHandler } from "../utils/mock-fetch"; -import { Client, Auth } from "../../src/client"; +import { Client, Auth, Opportunities, BatchParseError } from "../../src/client"; +import type { CustomFilterBag } from "../../src/client"; import { OpportunityBaseSchema } from "../../src/schemas"; -import { withCustomFields, F, FilterError } from "../../src/extensions"; +import { withCustomFields, F, FilterError, validateRoutes } from "../../src/extensions"; import type { PluginRoutes } from "../../src/extensions"; import { CustomFieldType } from "../../src/constants"; @@ -111,6 +112,14 @@ const client = new Client({ auth: Auth.bearer("test-token"), }); +/** Opportunities bound to routes over a fresh authed client (the plugin.getClient() shape). */ +const makeRoutedOpportunities = (routes: PluginRoutes) => + new Opportunities( + new Client({ baseUrl: "https://api.example.org", auth: Auth.bearer("test-token") }), + undefined, + routes + ); + // ============================================================================= // Opportunities resource tests // ============================================================================= @@ -374,7 +383,7 @@ describe("Opportunities", () => { expect(result.items[0].title).toBe("Education Grant"); expect(result.paginationInfo.totalItems).toBe(2); expect(result.sortInfo.sortBy).toBe("lastModifiedAt"); - expect(result.filterInfo.filters.status).toEqual({ + expect(result.filterInfo?.filters.status).toEqual({ operator: "in", value: ["open", "forecasted"], }); @@ -402,14 +411,11 @@ describe("Opportunities", () => { opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, }; - // routes is client-bound: supplied once at construction, not per call. - const routedClient = new Client({ - baseUrl: "https://api.example.org", - auth: Auth.bearer("test-token"), - routes, - }); + // routes are resource-bound: supplied once at construction, not per call + // (plugin.getClient() does this wiring for consumers). + const routedOpps = makeRoutedOpportunities(routes); - await routedClient.opportunities.search({ + await routedOpps.search({ filters: { status: F.in(["open"]), // default field → top-level agency: F.in(["HHS", "NSF"]), // registered custom → customFilters @@ -431,58 +437,49 @@ describe("Opportunities", () => { expect(customFilters).not.toHaveProperty("status"); }); - it("resolves a status collision in favor of filters.status and surfaces a warning", async () => { - // `status` given via both the `statuses` shorthand and `filters.status`: - // `filters` wins, the shorthand is ignored, a warning is appended to - // filterInfo.errors, and search() must NOT throw. - let capturedBody: Record<string, unknown> | undefined; + it("throws FilterError when status is given via both the shorthand and filters", async () => { + // `status` given via both the `statuses` shorthand and `filters.status` is + // a conflicting instruction: search() throws before any request is sent. + let requested = false; server.use( - http.post("/common-grants/opportunities/search", async ({ request }) => { - capturedBody = (await request.json()) as Record<string, unknown>; + http.post("/common-grants/opportunities/search", () => { + requested = true; return HttpResponse.json({ status: 200, message: "Success", - items: [createMockOpportunity(OPP_UUID_1, "Closed Grant", "closed")], - paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + items: [], + paginationInfo: { page: 1, pageSize: 25, totalItems: 0, totalPages: 1 }, sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, filterInfo: { filters: {} }, }); }) ); - const result = await client.opportunities.search({ - statuses: ["open"], - filters: { status: F.in(["closed"]) }, - }); - - // filters.status wins — the shorthand value ["open"] is ignored. - expect((capturedBody?.filters as { status?: { value?: unknown } }).status).toEqual({ - operator: "in", - value: ["closed"], - }); + await expect( + client.opportunities.search({ + statuses: ["open"], + filters: { status: F.in(["closed"]) }, + }) + ).rejects.toThrow(FilterError); - // The collision warning is surfaced (not thrown). - expect(result.filterInfo.errors).toEqual( - expect.arrayContaining([expect.stringContaining("filters.status")]) - ); + expect(requested).toBe(false); }); - it("drops an invalid filter from the request body and surfaces its error (no throw)", async () => { + it("rejects with FilterError on an invalid registered filter value before any request", async () => { // A registered stringArray filter given a non-array `between` value fails - // classifyFilters validation. Fail-soft: results still return, the invalid - // filter is absent from the sent body, and its message lands in - // filterInfo.errors — search() does not throw. - let capturedBody: Record<string, unknown> | undefined; + // classifyFilters validation. Fail-fast: search() throws and no HTTP + // request is made. + let requested = false; server.use( - http.post("/common-grants/opportunities/search", async ({ request }) => { - capturedBody = (await request.json()) as Record<string, unknown>; + http.post("/common-grants/opportunities/search", () => { + requested = true; return HttpResponse.json({ status: 200, message: "Success", - items: [createMockOpportunity(OPP_UUID_1, "Conservation Grant", "open")], - paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + items: [], + paginationInfo: { page: 1, pageSize: 25, totalItems: 0, totalPages: 1 }, sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, filterInfo: { filters: {} }, }); @@ -492,36 +489,24 @@ describe("Opportunities", () => { const routes: PluginRoutes = { opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, }; - // routes is client-bound: supplied once at construction, not per call. - const routedClient = new Client({ - baseUrl: "https://api.example.org", - auth: Auth.bearer("test-token"), - routes, - }); - - const result = await routedClient.opportunities.search({ - filters: { agency: { operator: "between", value: 5 } }, - }); - - // Results still come back. - expect(result.items).toHaveLength(1); - - // The invalid filter is absent from the sent body (no customFilters.agency, - // and no malformed filters payload shipped). - const sentFilters = capturedBody?.filters as - | { customFilters?: Record<string, unknown> } - | undefined; - expect(sentFilters?.customFilters).toBeUndefined(); + const routedOpps = makeRoutedOpportunities(routes); + + await expect( + routedOpps.search({ + // Wrong value family on a registered filter is a compile error too; + // cast to exercise the runtime backstop plain-JS callers hit. + filters: { agency: { operator: "between", value: 5 } } as unknown as CustomFilterBag< + typeof routes + >, + }) + ).rejects.toThrow(FilterError); - // Its error message is surfaced client-side. - expect(result.filterInfo.errors).toEqual( - expect.arrayContaining([expect.stringContaining("filters.agency")]) - ); + expect(requested).toBe(false); }); - it("stays fail-soft when a filter is dropped and the server omits filterInfo (auto-paginate)", async () => { - // Auto-pagination returns the raw server envelope; if it omits filterInfo, - // merging the dropped-filter error must still surface, not throw. + it("passes the server envelope through unchanged when filterInfo is absent (auto-paginate)", async () => { + // Auto-pagination returns the raw server envelope; the client no longer + // injects its own filterInfo, so an absent filterInfo stays absent. server.use( http.post("/common-grants/opportunities/search", () => { return HttpResponse.json({ @@ -538,32 +523,31 @@ describe("Opportunities", () => { const routes: PluginRoutes = { opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, }; - const routedClient = new Client({ - baseUrl: "https://api.example.org", - auth: Auth.bearer("test-token"), - routes, - }); + const routedOpps = makeRoutedOpportunities(routes); - const result = await routedClient.opportunities.search({ - filters: { agency: { operator: "between", value: 5 } }, + const result = await routedOpps.search({ + filters: { agency: F.in(["HHS"]) }, }); expect(result.items).toHaveLength(1); - expect(result.filterInfo.errors).toEqual( - expect.arrayContaining([expect.stringContaining("filters.agency")]) - ); + expect(result.filterInfo).toBeUndefined(); }); - it("validates routes at construction and throws on a default-name collision", () => { + it("validateRoutes throws on a default-name collision (runtime backstop)", () => { const badRoutes: PluginRoutes = { opportunities: { search: { filters: { status: { filterType: "stringArray" } } } }, }; - expect(() => new Client({ baseUrl: "https://api.example.org", routes: badRoutes })).toThrow( - FilterError - ); + expect(() => validateRoutes(badRoutes)).toThrow(FilterError); }); - it("merges client-side filter errors ahead of server-provided errors", async () => { + it("throws FilterError on direct construction with invalid routes (bypassing definePlugin)", () => { + const badRoutes: PluginRoutes = { + opportunities: { search: { filters: { status: { filterType: "stringArray" } } } }, + }; + expect(() => makeRoutedOpportunities(badRoutes)).toThrow(FilterError); + }); + + it("passes server-returned filterInfo.errors through unmodified", async () => { server.use( http.post("/common-grants/opportunities/search", () => { return HttpResponse.json({ @@ -580,19 +564,59 @@ describe("Opportunities", () => { const routes: PluginRoutes = { opportunities: { search: { filters: { agency: { filterType: "stringArray" } } } }, }; - const routedClient = new Client({ - baseUrl: "https://api.example.org", - auth: Auth.bearer("test-token"), - routes, - }); + const routedOpps = makeRoutedOpportunities(routes); - const result = await routedClient.opportunities.search({ - filters: { agency: { operator: "between", value: 5 } }, + const result = await routedOpps.search({ + filters: { agency: F.in(["HHS"]) }, }); - const errors = result.filterInfo.errors ?? []; - expect(errors[0]).toContain("filters.agency"); - expect(errors[errors.length - 1]).toBe("server: something was ignored"); + // filterInfo.errors carries server-returned errors only, untouched. + expect(result.filterInfo?.errors).toEqual(["server: something was ignored"]); + }); + + it("partitions a malformed row into result.errors and keeps valid rows in items", async () => { + const goodOpp = createMockOpportunity(OPP_UUID_1, "Conservation Grant", "open"); + const badRow = { id: "not-a-uuid", title: 42 }; + + server.use( + http.post("/common-grants/opportunities/search", () => { + return HttpResponse.json({ + status: 200, + message: "Success", + items: [goodOpp, badRow], + paginationInfo: { page: 1, pageSize: 25, totalItems: 2, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {} }, + }); + }) + ); + + const result = await client.opportunities.search({ page: 1 }); + + expect(result.items).toHaveLength(1); + expect(result.items[0].title).toBe("Conservation Grant"); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].index).toBe(1); + expect(result.errors[0].raw).toEqual(badRow); + }); + + it('rejects with BatchParseError on a malformed row under onParseError: "throw"', async () => { + server.use( + http.post("/common-grants/opportunities/search", () => { + return HttpResponse.json({ + status: 200, + message: "Success", + items: [{ id: "not-a-uuid", title: 42 }], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {} }, + }); + }) + ); + + await expect(client.opportunities.search({ page: 1, onParseError: "throw" })).rejects.toThrow( + BatchParseError + ); }); it("searches with only query parameter", async () => { @@ -657,7 +681,7 @@ describe("Opportunities", () => { const result = await client.opportunities.search({ statuses: ["open"] }); expect(result.items).toHaveLength(1); - expect(result.filterInfo.filters.status).toEqual({ + expect(result.filterInfo?.filters.status).toEqual({ operator: "in", value: ["open"], }); @@ -816,7 +840,7 @@ describe("Opportunities", () => { // Should preserve sortInfo and filterInfo from first page expect(result.sortInfo.sortBy).toBe("lastModifiedAt"); - expect(result.filterInfo.filters.status).toEqual({ + expect(result.filterInfo?.filters.status).toEqual({ operator: "in", value: ["open"], }); diff --git a/lib/ts-sdk/__tests__/client/resources.spec.ts b/lib/ts-sdk/__tests__/client/resources.spec.ts new file mode 100644 index 000000000..e331d5ec0 --- /dev/null +++ b/lib/ts-sdk/__tests__/client/resources.spec.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest"; +import { z } from "zod"; +import { http, HttpResponse, setupServer } from "../utils/mock-fetch"; +import { Client, Resource } from "../../src/client"; + +const server = setupServer(); +const client = new Client({ baseUrl: "https://api.example.org" }); + +// Test-only subclass proving the write-verb mechanism: a bespoke verb is a +// one-liner over the base `mutate` primitive, carrying its own schemas. +class Widgets extends Resource<{ id: string }> { + replace(id: string, body: unknown) { + return this.mutate("put", `/widgets/${id}`, body, { + requestSchema: z.object({ name: z.string() }), + responseSchema: z.object({ id: z.string() }), + }); + } +} + +const WidgetItemSchema = z.object({ id: z.string() }); + +describe("Resource.mutate", () => { + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + it("round-trips a valid body through the transport verb", async () => { + let capturedBody: unknown; + + server.use( + http.put("/widgets/w-1", async ({ request }) => { + capturedBody = await request.json(); + return HttpResponse.json({ id: "w-1" }); + }) + ); + + const widgets = new Widgets(client, WidgetItemSchema); + const result = await widgets.replace("w-1", { name: "New Widget" }); + + expect(capturedBody).toEqual({ name: "New Widget" }); + expect(result).toEqual({ id: "w-1" }); + }); + + it("validates the body before the request (fail-fast)", async () => { + let requested = false; + + server.use( + http.put("/widgets/w-1", () => { + requested = true; + return HttpResponse.json({ id: "w-1" }); + }) + ); + + const widgets = new Widgets(client, WidgetItemSchema); + + await expect(widgets.replace("w-1", { name: 42 })).rejects.toThrow(z.ZodError); + expect(requested).toBe(false); + }); + + it("parses the response fail-hard", async () => { + server.use( + http.put("/widgets/w-1", () => { + return HttpResponse.json({ wrong: "shape" }); + }) + ); + + const widgets = new Widgets(client, WidgetItemSchema); + + await expect(widgets.replace("w-1", { name: "New Widget" })).rejects.toThrow(z.ZodError); + }); +}); diff --git a/lib/ts-sdk/__tests__/client/results.spec.ts b/lib/ts-sdk/__tests__/client/results.spec.ts new file mode 100644 index 000000000..07b143078 --- /dev/null +++ b/lib/ts-sdk/__tests__/client/results.spec.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { BatchParseError, parseBatch } from "../../src/client/results"; + +const rowSchema = z.object({ id: z.string(), amount: z.number() }); +const good = { id: "a", amount: 1 }; +const bad = { id: "b", amount: "not-a-number" }; + +describe("parseBatch", () => { + it("partitions valid rows into items and failures into errors", () => { + const { items, errors } = parseBatch(rowSchema, [good, bad, { id: "c", amount: 3 }]); + expect(items).toEqual([good, { id: "c", amount: 3 }]); + expect(errors).toHaveLength(1); + expect(errors[0].index).toBe(1); + expect(errors[0].raw).toEqual(bad); + expect(errors[0].error).toBeInstanceOf(z.ZodError); + }); + + it("returns empty errors for an all-valid batch", () => { + const { items, errors } = parseBatch(rowSchema, [good]); + expect(items).toHaveLength(1); + expect(errors).toHaveLength(0); + }); + + it("returns empty items and errors for an empty batch", () => { + const { items, errors } = parseBatch(rowSchema, []); + expect(items).toHaveLength(0); + expect(errors).toHaveLength(0); + }); + + it('throws BatchParseError on the first failure under "throw"', () => { + expect(() => parseBatch(rowSchema, [good, bad], "throw")).toThrow(BatchParseError); + try { + parseBatch(rowSchema, [good, bad], "throw"); + expect.unreachable("parseBatch should have thrown"); + } catch (e) { + if (!(e instanceof BatchParseError)) throw e; + expect(e.failure.index).toBe(1); + expect(e.failure.raw).toEqual(bad); + expect(e.failure.error).toBeInstanceOf(z.ZodError); + } + }); +}); diff --git a/lib/ts-sdk/__tests__/client/search-filters-types.ts b/lib/ts-sdk/__tests__/client/search-filters-types.ts index b47ec46ba..2c044add6 100644 --- a/lib/ts-sdk/__tests__/client/search-filters-types.ts +++ b/lib/ts-sdk/__tests__/client/search-filters-types.ts @@ -6,17 +6,16 @@ * * Why the narrowing is partial: because `definePlugin` preserves the literal * `routes` type (its `const TRoutes` generic), declared filter names get autocomplete - * and their values are envelope-checked. But the spec supports AD-HOC filters (an open - * key set), so an unknown key cannot be rejected — a typo on a declared name is - * structurally an intentional ad-hoc filter. Narrowing therefore gives autocomplete + - * filter-envelope checking (`{ operator, value }` shape), NOT typo-rejection and NOT - * per-`filterType` value validation (that runs at runtime in `classifyFilters`). The - * single `@ts-expect-error` below pins the part that DOES hold (the `{ operator, value }` - * envelope); the un-guarded "typo" line documents what does not. + * and their values are typed by their declared `filterType` (a wrong value family is + * a compile error). But the spec supports AD-HOC filters (an open key set), so an + * unknown key cannot be rejected — a typo on a declared name is structurally an + * intentional ad-hoc filter. Narrowing therefore gives autocomplete + per-`filterType` + * value checking on declared names, NOT typo-rejection (that ends up ad-hoc, validated + * at runtime by `classifyFilters`). The un-guarded "typo" line documents the limit. */ import { definePlugin, F } from "@/extensions"; -import { Client } from "@/client"; +import { Client, Opportunities } from "@/client"; const plugin = definePlugin({ routes: { @@ -31,41 +30,55 @@ const plugin = definePlugin({ }, } as const); -// routes is client-bound: supplied once at construction, so `search` narrows -// filter names from the client's `R` generic. Compile-only — baseUrl unused. -const client = new Client({ routes: plugin.routes }); +// routes are resource-bound: supplied once at construction, so `search` narrows +// filter names from the resource's `R` generic (plugin.getClient() does this +// wiring for consumers). Compile-only — baseUrl unused. +const client = new Client({ baseUrl: "http://localhost" }); +const opportunities = new Opportunities( + client, + undefined, + plugin.routes as NonNullable<typeof plugin.routes> +); // Compile-only — never executed. async function _assertions(): Promise<void> { // Declared filter names accepted, values built with the F.* helpers. - await client.opportunities.search({ + await opportunities.search({ filters: { fundingMax: F.between(0, 100), agency: F.in(["HHS", "NSF"]), }, }); - // Ad-hoc (unregistered) key accepted — spec escape hatch, classifyFilters bucket 3. - await client.opportunities.search({ + // Ad-hoc (unregistered) key accepted — spec escape hatch (classifyFilters bucket 3). + await opportunities.search({ filters: { legacyTag: F.eq("conservation-2024") }, }); // Intentionally NO @ts-expect-error: a typo on a declared name compiles, // because it is indistinguishable from an intentional ad-hoc key. This is the // documented limitation of narrowing against an open (ad-hoc-supporting) key set. - await client.opportunities.search({ + await opportunities.search({ filters: { fundingMaxx: F.between(0, 100) }, }); + // Wrong value family on a declared name is a compile error. + await opportunities.search({ + filters: { + // @ts-expect-error — fundingMax is a numberRange filter; eq is not valid for it + fundingMax: F.eq(100), + }, + }); + // The `{ operator, value }` envelope is enforced even though keys are open. - await client.opportunities.search({ + await opportunities.search({ filters: { // @ts-expect-error — a filter value must be `{ operator, value }`, not a bare string. fundingMax: "not-a-filter", }, }); - // Works with no filters (back-compat: R defaults to PluginRoutes). + // Works with no filters, on the client's default resource slot. await client.opportunities.search({ query: "education", statuses: ["open"] }); } diff --git a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts index 6d28aff9b..dac615fbf 100644 --- a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts @@ -42,158 +42,10 @@ const mixedConsumerFilters: Record<string, unknown> = { }; // ############################################################################ -// classifyFilters tests +// validateRoutes / validateFilterCall tests // ############################################################################ -describe("classifyFilters", () => { - // ############################################################################ - // Three-bucket classification - // ############################################################################ - - describe("three-bucket classification", () => { - it("routes default filters to top-level named request-body fields", () => { - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - status: { operator: "in", value: ["open"] }, - }); - - expect(result.status).toEqual({ operator: "in", value: ["open"] }); - expect(result.customFilters).toBeUndefined(); - expect(errors).toEqual([]); - }); - - it("routes pre-registered custom filters to customFilters record", () => { - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - agency: { operator: "in", value: ["HHS"] }, - }); - - expect(result.customFilters?.agency).toEqual({ operator: "in", value: ["HHS"] }); - expect(result.status).toBeUndefined(); - expect(errors).toEqual([]); - }); - - it("routes ad-hoc filters to customFilters passthrough (no registration required)", () => { - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - legacyTag: { operator: "eq", value: "legacy-2024" }, - }); - - expect(result.customFilters?.legacyTag).toEqual({ operator: "eq", value: "legacy-2024" }); - expect(errors).toEqual([]); - }); - - it("builds exact ADR-0012 OppFilters request body for mixed default + custom + ad-hoc input", () => { - const { result, errors } = classifyFilters( - grantsGovRoutes, - "opportunities", - "search", - mixedConsumerFilters - ); - - // Assert the exact request body shape - const expected: ReturnType<typeof OppFiltersSchema.parse> = { - status: { operator: "in", value: ["open", "closed"] }, - closeDateRange: { operator: "between", value: { min: "2025-01-01", max: "2025-12-31" } }, - customFilters: { - agency: { operator: "in", value: ["HHS", "DOE"] }, - fundingProgram: { operator: "like", value: "SBIR%" }, - legacyTag: { operator: "eq", value: "legacy-2024" }, - }, - }; - - expect(result).toEqual(expected); - expect(errors).toEqual([]); - }); - - it("passes gov.<system>@<filterName> namespaced keys through to customFilters verbatim", () => { - const { result } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - "gov.grants@announcementType": { operator: "eq", value: "NOFO" }, - }); - - expect(result.customFilters?.["gov.grants@announcementType"]).toEqual({ - operator: "eq", - value: "NOFO", - }); - }); - - it("returns only top-level fields when no custom or ad-hoc filters are provided", () => { - const { result } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - status: { operator: "in", value: ["open"] }, - }); - - expect(result).not.toHaveProperty("customFilters"); - }); - - it("handles an empty filters object gracefully", () => { - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", {}); - - expect(result).toEqual({}); - expect(errors).toEqual([]); - }); - - // ########################################################################## - // Fail-soft validation (collect, don't throw) - // ########################################################################## - - it("collects a FilterError and omits the key when a default filter violates its real field type (status)", () => { - // `status` is a StringArrayFilter (operator in/notIn, value string[]). - // `{ operator: "gt", value: 5 }` is a structurally valid DefaultFilter - // (passes the permissive shape check) but invalid for `status`. Default - // filters are validated against their real field type — a malformed one is - // fail-soft: dropped from `result`, surfaced in `errors`, never thrown. - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - status: { operator: "gt", value: 5 }, - }); - - expect(result).not.toHaveProperty("status"); - expect(errors).toHaveLength(1); - expect(errors[0]).toBeInstanceOf(FilterError); - expect(errors[0].path).toBe("filters.status"); - }); - - it("collects errors for invalid registered and ad-hoc filters, omitting both from result", () => { - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - // registered stringArray given a non-array value → invalid - agency: { operator: "in", value: "not-an-array" }, - // ad-hoc with a missing operator → invalid shape - legacyTag: { value: "no-operator" }, - }); - - expect(result.customFilters).toBeUndefined(); - expect(errors).toHaveLength(2); - expect(errors.map(e => e.path).sort()).toEqual(["filters.agency", "filters.legacyTag"]); - }); - - it("keeps valid keys and drops only invalid ones for mixed valid+invalid input", () => { - const { result, errors } = classifyFilters(grantsGovRoutes, "opportunities", "search", { - // valid default - status: { operator: "in", value: ["open"] }, - // invalid default (wrong operator/value for status) - closeDateRange: { operator: "in", value: ["nope"] }, - // valid registered custom - agency: { operator: "in", value: ["HHS"] }, - // invalid registered custom (stringArray with non-array value) - fundingProgram: { operator: "like", value: 12345 }, - // valid ad-hoc - legacyTag: { operator: "eq", value: "legacy-2024" }, - }); - - // Valid keys are present. - expect(result.status).toEqual({ operator: "in", value: ["open"] }); - expect(result.customFilters?.agency).toEqual({ operator: "in", value: ["HHS"] }); - expect(result.customFilters?.legacyTag).toEqual({ operator: "eq", value: "legacy-2024" }); - - // Invalid keys are absent. - expect(result).not.toHaveProperty("closeDateRange"); - expect(result.customFilters).not.toHaveProperty("fundingProgram"); - - // errors count discriminates: exactly the two invalid keys. - expect(errors).toHaveLength(2); - expect(errors.map(e => e.path).sort()).toEqual([ - "filters.closeDateRange", - "filters.fundingProgram", - ]); - }); - }); - +describe("route and filter-call validation", () => { // ############################################################################ // Registration-time validation (validateRoutes) // ############################################################################ @@ -267,7 +119,9 @@ describe("classifyFilters", () => { }); it("throws FilterError when custom filters are declared on an unsupported route (list)", () => { - const unsupportedRoutes: PluginRoutes = { + // `list` is no longer expressible in PluginRoutes (closed RouteMethod union); + // cast through unknown to exercise the runtime backstop plain-JS callers hit. + const unsupportedRoutes = { opportunities: { list: { filters: { @@ -275,7 +129,7 @@ describe("classifyFilters", () => { }, }, }, - }; + } as unknown as PluginRoutes; expect(() => validateRoutes(unsupportedRoutes)).toThrow(FilterError); }); @@ -283,6 +137,21 @@ describe("classifyFilters", () => { it("does not throw for valid routes", () => { expect(() => validateRoutes(grantsGovRoutes)).not.toThrow(); }); + + it("skips explicitly-undefined resource, method, and filter-spec values (no raw TypeError)", () => { + // Partial<Record<...>> admits explicit undefined; a conditional-route + // pattern like `{ opportunities: hasFilters ? {...} : undefined }` must + // not crash Object.entries. + expect(() => validateRoutes({ opportunities: undefined })).not.toThrow(); + expect(() => validateRoutes({ opportunities: { search: undefined } })).not.toThrow(); + // Filter-spec values are only undefined for plain-JS callers (the record + // value type is not Partial), but the backstop must not raw-TypeError. + expect(() => + validateRoutes({ + opportunities: { search: { filters: { agency: undefined } } }, + } as unknown as PluginRoutes) + ).not.toThrow(); + }); }); // ############################################################################ @@ -426,3 +295,97 @@ describe("F helpers", () => { expect(() => DefaultFilterSchema.parse(F.between(100, 500))).not.toThrow(); }); }); + +// ############################################################################ +// classifyFilters (fail-fast classifier) +// ############################################################################ + +describe("classifyFilters", () => { + it("classifies defaults top-level and registered/ad-hoc under customFilters", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + status: { operator: "in", value: ["open"] }, + agency: F.in(["NSF"]), + adHocKey: F.eq("x"), + }); + expect(result.status).toEqual({ operator: "in", value: ["open"] }); + expect(result.customFilters).toMatchObject({ + agency: { operator: "in", value: ["NSF"] }, + adHocKey: { operator: "eq", value: "x" }, + }); + }); + + it("throws FilterError on an invalid standard filter value", () => { + expect(() => + classifyFilters(grantsGovRoutes, "opportunities", "search", { + status: { operator: "bogus", value: 1 }, + }) + ).toThrow(FilterError); + }); + + it("throws FilterError on a wrong-typed registered filter value", () => { + expect(() => + classifyFilters(grantsGovRoutes, "opportunities", "search", { + agency: { operator: "eq", value: "NSF" }, + }) + ).toThrow(FilterError); + }); + + it("throws FilterError on a malformed ad-hoc filter value", () => { + expect(() => + classifyFilters(grantsGovRoutes, "opportunities", "search", { + adHocKey: { notAFilter: true }, + }) + ).toThrow(FilterError); + }); + + it("passes well-formed ad-hoc filters through without throwing", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + adHocKey: F.between(1, 10), + }); + expect(result.customFilters?.adHocKey).toBeDefined(); + }); + + it("omits customFilters entirely when only defaults are present", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + status: { operator: "in", value: ["open"] }, + }); + expect(result.customFilters).toBeUndefined(); + }); + + it("builds the exact ADR-0012 OppFilters request body for mixed default + custom + ad-hoc input", () => { + const result = classifyFilters( + grantsGovRoutes, + "opportunities", + "search", + mixedConsumerFilters + ); + + const expected: ReturnType<typeof OppFiltersSchema.parse> = { + status: { operator: "in", value: ["open", "closed"] }, + closeDateRange: { operator: "between", value: { min: "2025-01-01", max: "2025-12-31" } }, + customFilters: { + agency: { operator: "in", value: ["HHS", "DOE"] }, + fundingProgram: { operator: "like", value: "SBIR%" }, + legacyTag: { operator: "eq", value: "legacy-2024" }, + }, + }; + + expect(result).toEqual(expected); + }); + + it("passes gov.<system>@<filterName> namespaced keys through to customFilters verbatim", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { + "gov.grants@announcementType": { operator: "eq", value: "NOFO" }, + }); + + expect(result.customFilters?.["gov.grants@announcementType"]).toEqual({ + operator: "eq", + value: "NOFO", + }); + }); + + it("handles an empty filters object gracefully", () => { + const result = classifyFilters(grantsGovRoutes, "opportunities", "search", {}); + expect(result).toEqual({}); + }); +}); diff --git a/lib/ts-sdk/__tests__/extensions/get-client.spec.ts b/lib/ts-sdk/__tests__/extensions/get-client.spec.ts new file mode 100644 index 000000000..12e9e9e7c --- /dev/null +++ b/lib/ts-sdk/__tests__/extensions/get-client.spec.ts @@ -0,0 +1,181 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest"; +import { z } from "zod"; +import { http, HttpResponse, setupServer } from "../utils/mock-fetch"; +import { definePlugin, F, FilterError } from "../../src/extensions"; +import { Auth } from "../../src/client"; +import type { CustomFilterBag, ListResult } from "../../src/client"; +import { OpportunityBaseSchema } from "../../src/schemas"; +import { withCustomFields } from "../../src/extensions"; + +const server = setupServer(); + +const plugin = definePlugin({ + schemas: { + Opportunity: { + customFields: { + programCode: { fieldType: "string", value: z.string() }, + }, + }, + }, + routes: { + opportunities: { search: { filters: { region: { filterType: "stringArray" } } } }, + }, + meta: { name: "grants-gov adapter", sourceSystem: "grants.gov" }, +} as const); + +const mockOpp = { + id: "550e8400-e29b-41d4-a716-446655440001", + title: "STEM Grant", + status: { value: "open" }, + description: "A STEM education grant", + createdAt: "2024-01-15T10:30:00Z", + lastModifiedAt: "2024-06-01T14:22:00Z", + customFields: { + programCode: { name: "programCode", fieldType: "string", value: "STEM-ED" }, + }, +}; + +const searchEnvelope = (items: unknown[]) => ({ + status: 200, + message: "Success", + items, + paginationInfo: { page: 1, pageSize: 25, totalItems: items.length, totalPages: 1 }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + filterInfo: { filters: {} }, +}); + +describe("plugin.getClient", () => { + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + it("returns a client whose search parses with the plugin schema by default", async () => { + let capturedBody: Record<string, unknown> | undefined; + + server.use( + http.post("/common-grants/opportunities/search", async ({ request }) => { + capturedBody = (await request.json()) as Record<string, unknown>; + return HttpResponse.json(searchEnvelope([mockOpp])); + }) + ); + + const client = plugin.getClient({ baseUrl: "https://api.example.org" }); + const result = await client.opportunities.search({ + page: 1, + filters: { region: F.in(["US-CA"]) }, + }); + + // No per-call schema, yet the plugin's custom field is parsed and typed. + expect(result.items).toHaveLength(1); + expect(result.items[0].customFields?.programCode?.value).toBe("STEM-ED"); + + // Registered filter lands under customFilters on the wire. + expect(capturedBody?.filters).toMatchObject({ + customFilters: { region: { operator: "in", value: ["US-CA"] } }, + }); + }); + + it("binds the plugin schema for get() and list() as well", async () => { + server.use( + http.get("/common-grants/opportunities/:id", () => + HttpResponse.json({ status: 200, message: "Success", data: mockOpp }) + ), + http.get("/common-grants/opportunities", () => { + return HttpResponse.json({ + status: 200, + message: "Success", + items: [mockOpp], + paginationInfo: { page: 1, pageSize: 25, totalItems: 1, totalPages: 1 }, + }); + }) + ); + + const client = plugin.getClient({ baseUrl: "https://api.example.org" }); + + const opp = await client.opportunities.get(mockOpp.id); + expect(opp.customFields?.programCode?.value).toBe("STEM-ED"); + + const list: ListResult<typeof opp> = await client.opportunities.list({ page: 1 }); + expect(list.items[0].customFields?.programCode?.value).toBe("STEM-ED"); + }); + + it("passes auth through to the underlying transport", async () => { + let capturedAuth: string | null = null; + + server.use( + http.get("/common-grants/opportunities/:id", ({ request }) => { + capturedAuth = request.headers.get("authorization"); + return HttpResponse.json({ status: 200, message: "Success", data: mockOpp }); + }) + ); + + const client = plugin.getClient({ + baseUrl: "https://api.example.org", + auth: Auth.bearer("test-token"), + }); + await client.opportunities.get(mockOpp.id); + + expect(capturedAuth).toBe("Bearer test-token"); + }); + + it("throws FilterError on an invalid registered filter value before any request", async () => { + let requested = false; + + server.use( + http.post("/common-grants/opportunities/search", () => { + requested = true; + return HttpResponse.json(searchEnvelope([])); + }) + ); + + const client = plugin.getClient({ baseUrl: "https://api.example.org" }); + + await expect( + client.opportunities.search({ + // Wrong value family is a compile error too; cast to hit the runtime backstop. + filters: { region: { operator: "eq", value: "US-CA" } } as unknown as CustomFilterBag< + NonNullable<typeof plugin.routes> + >, + }) + ).rejects.toThrow(FilterError); + + expect(requested).toBe(false); + }); + + it("definePlugin throws FilterError on a bad route at definition time (runtime backstop)", () => { + expect(() => + definePlugin({ + routes: { + opportunities: { + // `status` collides with a default-filter name — invalid registration. + search: { filters: { status: { filterType: "stringArray" } } }, + }, + }, + } as const) + ).toThrow(FilterError); + }); + + it("definePlugin accepts an explicitly-undefined resource route value", () => { + expect(() => definePlugin({ routes: { opportunities: undefined } } as const)).not.toThrow(); + }); + + it("per-call schema override still wins over the plugin-bound default", async () => { + server.use( + http.post("/common-grants/opportunities/search", () => + HttpResponse.json(searchEnvelope([mockOpp])) + ) + ); + + const overrideSchema = withCustomFields(OpportunityBaseSchema, { + legacyId: { fieldType: "integer", value: z.number().int() }, + } as const); + + const client = plugin.getClient({ baseUrl: "https://api.example.org" }); + const result = await client.opportunities.search({ page: 1, schema: overrideSchema }); + + // Parsed under the override: legacyId slot is typed (absent here), and + // programCode is no longer a typed member of customFields. + expect(result.items).toHaveLength(1); + expect(result.items[0].customFields?.legacyId?.value).toBeUndefined(); + }); +}); diff --git a/lib/ts-sdk/__tests__/extensions/plugin-routes-types.ts b/lib/ts-sdk/__tests__/extensions/plugin-routes-types.ts new file mode 100644 index 000000000..f80fd00fa --- /dev/null +++ b/lib/ts-sdk/__tests__/extensions/plugin-routes-types.ts @@ -0,0 +1,105 @@ +/** + * Compile-time assertions for the closed `PluginRoutes` key space. + * + * Checked by `tsc --noEmit` only (`pnpm --filter @common-grants/sdk run check:types`) + * — there is no runtime test. Each `@ts-expect-error` directive IS the assertion: + * it guards a line that must fail to compile while route keys are closed unions. + * If the key space ever reopens, the guarded line compiles, the directive becomes + * unused (ts2578), and the type-check gate fails. + */ + +import { definePlugin, F } from "@/extensions"; +import type { CustomFilterBag } from "@/client"; + +// A misspelled resource key is a compile error. +definePlugin({ + routes: { + // @ts-expect-error - "oportunites" is not a registered resource name + oportunites: { + search: { filters: { agency: { filterType: "stringArray" } } }, + }, + }, +} as const); + +// A misspelled method key is a compile error. +definePlugin({ + routes: { + opportunities: { + // @ts-expect-error - "serch" is not a supported route method + serch: { filters: { agency: { filterType: "stringArray" } } }, + }, + }, +} as const); + +// Correct spelling compiles clean. +definePlugin({ + routes: { + opportunities: { + search: { filters: { agency: { filterType: "stringArray" } } }, + }, + }, +} as const); + +// ############################################################################ +// Registered filter values are typed by their declared filterType +// ############################################################################ + +const regionPlugin = definePlugin({ + routes: { + opportunities: { + search: { filters: { region: { filterType: "stringArray" } } }, + }, + }, +} as const); + +type RegionBag = CustomFilterBag<NonNullable<typeof regionPlugin.routes>>; + +// Registered key accepts its declared value family. +const ok: RegionBag = { region: F.in(["US-CA", "US-NY"]) }; + +// @ts-expect-error - region is a stringArray filter; eq is an equivalence operator +const wrongOperator: RegionBag = { region: F.eq("US-CA") }; + +// Ad-hoc keys still pass through (typed as the raw `{operator, value}` shape). +const adHoc: RegionBag = { somethingElse: F.eq("x") }; + +void regionPlugin; +void ok; +void wrongOperator; +void adHoc; + +// ############################################################################ +// getClient projection: item + filter types flow from the plugin +// ############################################################################ + +const typedPlugin = definePlugin({ + schemas: { + Opportunity: { customFields: { programCode: { fieldType: "string" } } }, + }, + routes: { + opportunities: { + search: { filters: { region: { filterType: "stringArray" } } }, + }, + }, +} as const); +const builtClient = typedPlugin.getClient({ baseUrl: "http://localhost" }); + +declare const searchResult: Awaited<ReturnType<typeof builtClient.opportunities.search>>; +const item = searchResult.items[0]; + +// Item type is projected from the plugin's custom fields (string), not any. +// If the projection ever collapses to `any`, this directive reports unused +// and the type gate fails. +// @ts-expect-error - programCode value is a string, not a number +const asNumber: number = item.customFields?.programCode?.value; +void asNumber; + +// Registered filter keys are typed on the built client: +void builtClient.opportunities.search({ filters: { region: F.in(["US-CA"]) } }); +// @ts-expect-error - region is a stringArray filter; eq is not valid for it +void builtClient.opportunities.search({ filters: { region: F.eq("US-CA") } }); + +// Per-row failures are typed on the result: +declare const failure: (typeof searchResult.errors)[number]; +const failureIndex: number = failure.index; +void failureIndex; diff --git a/lib/ts-sdk/examples/custom-filters.ts b/lib/ts-sdk/examples/custom-filters.ts index 10721be73..4119f3ef9 100644 --- a/lib/ts-sdk/examples/custom-filters.ts +++ b/lib/ts-sdk/examples/custom-filters.ts @@ -7,8 +7,12 @@ * fundingProgram), and an ad-hoc filter (legacyTag) — using `F.*` helpers. * Control fields (query, maxResults, signal, schema) stay OUTSIDE `filters`. * 3. Calling `classifyFilters` to produce the ADR-0012 request body: - * default fields at top-level, custom + ad-hoc under `customFilters`. - * 4. A COMMENT block (not executed) demonstrating the `as const` widening + * default fields at top-level, custom + ad-hoc under `customFilters` — + * an invalid value on any key throws `FilterError` before a body is built. + * 4. Building a plugin-bound client via `plugin.getClient()` and showing the + * fail-fast guard: an invalid registered filter value rejects BEFORE any + * HTTP request (so this step needs no server). + * 5. A COMMENT block (not executed) demonstrating the `as const` widening * trap — see custom-filters-types.ts for the compile-time narrowing assertions. * * Run with: `pnpm example:custom-filters` @@ -23,7 +27,7 @@ * send it over the wire — transport is not handled here. */ -import { classifyFilters, definePlugin, F } from "../src/extensions"; +import { classifyFilters, definePlugin, F, FilterError } from "../src/extensions"; // ############################################################################ // Step 1 — Define the grants.gov plugin with route-keyed custom filters @@ -113,10 +117,9 @@ console.log(JSON.stringify(searchParams.filters, null, 2)); // non-optional literal object. The `!` assertion removes the `undefined` from // the union type that `Plugin.routes?:` introduces (routes is optional in the // interface to support plugins that don't declare filters). -// `classifyFilters` is fail-soft: it returns `{ result, errors }`. `result` is -// the OppFilters request body (valid keys only); `errors` collects any invalid -// filters that were dropped. For this all-valid input, `errors` is empty. -const { result: requestBody, errors } = classifyFilters( +// `classifyFilters` is fail-fast: an invalid value on any key — standard, +// registered, or ad-hoc — throws `FilterError` before a request body exists. +const requestBody = classifyFilters( grantsGovPlugin.routes!, "opportunities", "search", @@ -135,9 +138,6 @@ function fail(message: string): never { process.exit(1); } -// All filters in this example are valid — no errors collected. -if (errors.length > 0) fail(`expected no classification errors, got ${errors.length}`); - // Default filters must appear as named top-level fields if (!requestBody.status) fail("status should be a top-level field (default filter bucket)"); if (!requestBody.closeDateRange) @@ -156,6 +156,19 @@ if (!requestBody.customFilters.legacyTag) if ((requestBody.customFilters as Record<string, unknown>).status) fail("status must NOT be in customFilters — it is a default filter"); +// Fail-fast demo: a wrong value family on a registered filter throws +// FilterError before any request body is produced. +try { + classifyFilters(grantsGovPlugin.routes!, "opportunities", "search", { + agency: { operator: "eq", value: 42 }, // agency is a stringArray filter + }); + fail("expected classifyFilters to throw FilterError for an invalid registered value"); +} catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log(`\n=== Fail-fast demo ===`); + console.log(`FilterError (expected): ${e.message.split("\n")[0]}`); +} + console.log("\n=== Assertions passed ==="); console.log(" status → top-level (default filter bucket)"); console.log(" closeDateRange → top-level (default filter bucket)"); @@ -164,7 +177,46 @@ console.log(" fundingProgram → customFilters (registered custom filter)"); console.log(" legacyTag → customFilters (ad-hoc passthrough)"); // ############################################################################ -// Step 4 — The `as const` widening trap (comment block — not executed) +// Step 5 — plugin.getClient(): the consumer path +// ############################################################################ + +// The client is pre-bound to the plugin: responses parse with the plugin's +// compiled schema by default, and `search({ filters })` types the registered +// filter names. Against a live API (e.g. `pnpm example:server`), the consumer +// flow looks like: +// +// const result = await client.opportunities.search({ +// filters: { agency: F.in(["HHS"]) }, +// }); +// for (const opp of result.items) console.log(opp.title); // valid rows +// // ParseFailure rows — err.raw may carry PII; log a redacted projection +// for (const err of result.errors) console.log(err.index, err.error.message); +// +// (Executed versions live in __tests__/extensions/get-client.spec.ts.) +const client = grantsGovPlugin.getClient({ baseUrl: "http://localhost:8000" }); + +// Fail-fast without a server: an invalid registered value throws BEFORE any +// HTTP request is made, so this rejects even though nothing is listening. +// (Async so this file compiles under CommonJS module settings — no top-level +// await; the final logs chain off it below so output order matches source order.) +async function step5FailFastDemo(): Promise<void> { + try { + await client.opportunities.search({ + // Wrong value family for a stringArray filter — also a compile error; + // the cast simulates a plain-JS caller hitting the runtime backstop. + filters: { agency: { operator: "eq", value: 42 } } as never, + }); + fail("expected search() to reject with FilterError before any request"); + } catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log("\n=== Step 5: getClient fail-fast demo ==="); + console.log(`search() rejected before any request (expected): ${e.message.split("\n")[0]}`); + console.log("\n✓ getClient consumer path complete"); + } +} + +// ############################################################################ +// Step 6 — The `as const` widening trap (comment block — not executed) // ############################################################################ // If you forget `as const` on the definePlugin call, TypeScript widens the @@ -194,5 +246,10 @@ console.log(" legacyTag → customFilters (ad-hoc passthrough)"); // // // → literal filterTypes are lost, so per-key narrowing is impossible // // // → unknown keys, wrong operators, and wrong value shapes silently accepted -console.log("\n✓ custom-filters example complete"); -console.log(" See __tests__/extensions/custom-filters-types.ts for compile-time narrowing proof"); +// Chained after Step 5 settles: the completion banner provably prints last. +void step5FailFastDemo().then(() => { + console.log("\n✓ custom-filters example complete"); + console.log( + " See __tests__/extensions/custom-filters-types.ts for compile-time narrowing proof" + ); +}); diff --git a/lib/ts-sdk/src/client/README.md b/lib/ts-sdk/src/client/README.md index 9374f37e3..b0d4fb9a7 100644 --- a/lib/ts-sdk/src/client/README.md +++ b/lib/ts-sdk/src/client/README.md @@ -204,11 +204,11 @@ const response = await client.fetch("/custom/endpoint", { ### Opportunities resource -| Method | Route | Description | -| -------------------------------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------- | -| [`client.opportunities.get(id, options?)`](./opportunities.ts) | `GET /common-grants/opportunities/{id}` | Fetch a single opportunity by ID. Accepts an optional `schema` for typed custom fields. | -| [`client.opportunities.list(options?)`](./opportunities.ts) | `GET /common-grants/opportunities` | List opportunities with auto-pagination. Pass `page` to fetch a specific page instead. | -| [`client.opportunities.search(options?)`](./opportunities.ts) | `POST /common-grants/opportunities/search` | Search with text query, status filters, and auto-pagination. | +| Method | Route | Description | +| ------------------------------------------------------------------------ | ------------------------------------------ | --------------------------------------------------------------------------------------- | +| [`client.opportunities.get(id, options?)`](./resources/opportunities.ts) | `GET /common-grants/opportunities/{id}` | Fetch a single opportunity by ID. Accepts an optional `schema` for typed custom fields. | +| [`client.opportunities.list(options?)`](./resources/opportunities.ts) | `GET /common-grants/opportunities` | List opportunities with auto-pagination. Pass `page` to fetch a specific page instead. | +| [`client.opportunities.search(options?)`](./resources/opportunities.ts) | `POST /common-grants/opportunities/search` | Search with text query, status filters, and auto-pagination. | ### Auth namespace diff --git a/lib/ts-sdk/src/client/client.ts b/lib/ts-sdk/src/client/client.ts index 2667612af..e21ea3fb9 100644 --- a/lib/ts-sdk/src/client/client.ts +++ b/lib/ts-sdk/src/client/client.ts @@ -4,10 +4,11 @@ import { type ClientConfig, type ResolvedConfig, resolveConfig } from "./config"; import { Auth, buildAuthHeaders, type AuthMethod } from "./auth"; -import { Opportunities } from "./opportunities"; -import { validateRoutes } from "../extensions/custom-filters"; +import { Opportunities } from "./resources/opportunities"; +import { EXTENSIBLE_SCHEMA_MAP } from "../extensions/types"; +import { parseBatch, type OnParseError, type ParseFailure } from "./results"; import type { Paginated } from "../types"; -import type { PluginRoutes } from "../extensions/types"; +import type { z } from "zod"; // ============================================================================= // Options interfaces @@ -41,8 +42,10 @@ export interface FetchManyOptions<T = unknown> { method?: "GET" | "POST"; /** Request body for POST requests (pagination will be merged in) */ body?: Record<string, unknown>; - /** Schema to parse/validate each item (any object with a `.parse()` method, e.g. a Zod schema) */ - schema?: { parse: (data: unknown) => T }; + /** Schema to parse/validate each item (any object with a `.safeParse()` method, e.g. a Zod schema) */ + schema?: { safeParse: (data: unknown) => z.SafeParseReturnType<unknown, T> }; + /** Row-parse failure handling: partition into `errors` (default) or throw on first failure */ + onParseError?: OnParseError; } // ============================================================================= @@ -69,27 +72,23 @@ export interface FetchManyOptions<T = unknown> { * const list = await client.opportunities.list({ page: 1 }); * ``` */ -export class Client<R extends PluginRoutes = PluginRoutes> { +export class Client { private readonly config: ResolvedConfig; private readonly auth: AuthMethod; - /** Opportunities resource namespace */ - public readonly opportunities: Opportunities<R>; + /** Opportunities resource namespace (base schema; plugin-typed via plugin.getClient()) */ + // resource-slot + public readonly opportunities: Opportunities; // ============================================================================= // Client constructor // ============================================================================= - constructor(options: ClientConfig & { auth?: AuthMethod; routes?: R }) { + constructor(options: ClientConfig & { auth?: AuthMethod }) { this.config = resolveConfig(options); this.auth = options.auth ?? Auth.none(); - // `routes` is client-bound (fixed plugin config supplied once here), so it - // validates at construction and drives the filter-name narrowing on - // `opportunities.search`. The `this as Client` cast avoids circular-generic - // variance friction between `Client<R>` and `Opportunities<R>`. - if (options.routes) validateRoutes(options.routes); - this.opportunities = new Opportunities<R>(this as Client, options.routes); + this.opportunities = new Opportunities(this, EXTENSIBLE_SCHEMA_MAP.Opportunity); } // ============================================================================= @@ -194,6 +193,42 @@ export class Client<R extends PluginRoutes = PluginRoutes> { }); } + // ============================================================================= + // Client.put / Client.patch - write request helpers + // ============================================================================= + + /** + * Makes an authenticated PUT request to the API. + * + * @param path - API path (will be appended to baseUrl) + * @param body - Request body (will be JSON stringified) + * @param options - Request options + * @returns Fetch Response + */ + async put(path: string, body: unknown, options?: PostOptions): Promise<Response> { + return this.fetch(path, { + method: "PUT", + body: JSON.stringify(body), + signal: options?.signal, + }); + } + + /** + * Makes an authenticated PATCH request to the API. + * + * @param path - API path (will be appended to baseUrl) + * @param body - Request body (will be JSON stringified) + * @param options - Request options + * @returns Fetch Response + */ + async patch(path: string, body: unknown, options?: PostOptions): Promise<Response> { + return this.fetch(path, { + method: "PATCH", + body: JSON.stringify(body), + signal: options?.signal, + }); + } + // ============================================================================= // Client.fetchMany - auto-pagination // ============================================================================= @@ -217,7 +252,10 @@ export class Client<R extends PluginRoutes = PluginRoutes> { * }); * ``` */ - async fetchMany<T>(path: string, options?: FetchManyOptions<T>): Promise<Paginated<T>> { + async fetchMany<T>( + path: string, + options?: FetchManyOptions<T> + ): Promise<Paginated<T> & { errors: ParseFailure[] }> { // Set defaults. const pageSize = options?.pageSize ?? this.config.pageSize; const maxItems = options?.maxItems ?? this.config.maxItems; @@ -228,6 +266,10 @@ export class Client<R extends PluginRoutes = PluginRoutes> { const firstResult = await this.fetchOnePage<T>(path, method, startPage, pageSize, options); const firstPageJson = firstResult.json; const allItems: T[] = [...firstResult.items.slice(0, maxItems)]; + // Per-row parse failures aggregate across pages; each failure's `index` is + // relative to its own page's rows. Failures are reported for every fetched + // row — including rows past the maxItems cap, which truncates items only. + const allErrors: ParseFailure[] = [...firstResult.errors]; // Fetch remaining pages, up to maxItems. let currentPage = startPage + 1; @@ -237,6 +279,7 @@ export class Client<R extends PluginRoutes = PluginRoutes> { // Add items up to maxItems limit. const remainingCapacity = maxItems - allItems.length; allItems.push(...result.items.slice(0, remainingCapacity)); + allErrors.push(...result.errors); // Stop if we've fetched all available items. if (result.isLastPage || allItems.length >= maxItems) break; @@ -247,12 +290,13 @@ export class Client<R extends PluginRoutes = PluginRoutes> { return { ...firstPageJson, items: allItems, + errors: allErrors, paginationInfo: { ...firstPageJson.paginationInfo, page: 1, pageSize: allItems.length, }, - } as Paginated<T>; + } as Paginated<T> & { errors: ParseFailure[] }; } // ============================================================================= @@ -272,6 +316,7 @@ export class Client<R extends PluginRoutes = PluginRoutes> { ): Promise<{ json: Paginated<unknown>; items: T[]; + errors: ParseFailure[]; isLastPage: boolean; totalPages: number | undefined; }> { @@ -298,22 +343,24 @@ export class Client<R extends PluginRoutes = PluginRoutes> { throw new Error(`Failed to fetch ${path}: ${response.status} ${response.statusText}`); } - // Parse/validate items if schema is provided + // Parse/validate items if schema is provided; valid rows partition into + // `items`, per-row failures into `errors` (or throw under "throw"). const json = (await response.json()) as Paginated<unknown>; const { items: rawItems, paginationInfo } = json; - const items: T[] = options?.schema - ? rawItems.map(item => options.schema!.parse(item)) - : (rawItems as T[]); + const { items, errors } = options?.schema + ? parseBatch(options.schema, rawItems, options?.onParseError ?? "collect") + : { items: rawItems as T[], errors: [] as ParseFailure[] }; - // Determine if this is the last page. + // Determine if this is the last page from the RAW row count — a + // parse-shrunk page must not read as the last page. const totalPages = paginationInfo.totalPages ?? undefined; const isLastPage = - items.length < pageSize || + rawItems.length < pageSize || (totalPages !== undefined && currentPage >= totalPages) || - items.length === 0; + rawItems.length === 0; // Return the results. - return { json, items, isLastPage, totalPages }; + return { json, items, errors, isLastPage, totalPages }; } /** Constructs the full URL for an API path. */ diff --git a/lib/ts-sdk/src/client/index.ts b/lib/ts-sdk/src/client/index.ts index bbe6a5a29..5ab308a77 100644 --- a/lib/ts-sdk/src/client/index.ts +++ b/lib/ts-sdk/src/client/index.ts @@ -17,9 +17,15 @@ export { Client, type FetchManyOptions } from "./client"; export { Opportunities, + type CustomFilterBag, type GetOptions, type ListOptions, type SearchOptions, -} from "./opportunities"; +} from "./resources/opportunities"; +export type { BuiltClient } from "./resources/builder"; +export { Resource } from "./resources/base"; +export { RESOURCE_REGISTRY } from "./resources/registry"; export { Auth, type AuthMethod } from "./auth"; export { type ClientConfig, type ResolvedConfig } from "./config"; +export { parseBatch, BatchParseError } from "./results"; +export type { ParseFailure, OnParseError, ListResult, SearchResult } from "./results"; diff --git a/lib/ts-sdk/src/client/resources/base.ts b/lib/ts-sdk/src/client/resources/base.ts new file mode 100644 index 000000000..ef789e97f --- /dev/null +++ b/lib/ts-sdk/src/client/resources/base.ts @@ -0,0 +1,42 @@ +/** + * Shared base for API resource namespaces (opportunities today; organizations, + * applications, awards as they land). + */ + +import { z } from "zod"; +import type { Client } from "../client"; +import type { PluginRoutes } from "../../extensions/types"; + +export abstract class Resource<TItem> { + constructor( + protected readonly client: Client, + protected readonly boundSchema: z.ZodType<TItem, z.ZodTypeDef, unknown>, + protected readonly routes?: PluginRoutes + ) {} + + /** + * Shared write primitive: the body is validated before the request (throws), + * the single response is parsed fail-hard. Write/action verbs on subclasses + * (e.g. `replace()`, `submit()`) are one-liners over this. + */ + protected async mutate<TRes>( + method: "post" | "put" | "patch", + path: string, + body: unknown, + opts: { + requestSchema?: z.ZodTypeAny; + responseSchema: z.ZodType<TRes, z.ZodTypeDef, unknown>; + } + ): Promise<TRes> { + const payload: unknown = opts.requestSchema ? opts.requestSchema.parse(body) : body; + const response = await this.client[method](path, payload); + + if (!response.ok) { + throw new Error( + `Failed to ${method.toUpperCase()} ${path}: ${response.status} ${response.statusText}` + ); + } + + return opts.responseSchema.parse(await response.json()); + } +} diff --git a/lib/ts-sdk/src/client/resources/builder.ts b/lib/ts-sdk/src/client/resources/builder.ts new file mode 100644 index 000000000..e30381fc5 --- /dev/null +++ b/lib/ts-sdk/src/client/resources/builder.ts @@ -0,0 +1,71 @@ +/** + * Assembles a plugin-typed client: a transport `Client` whose resource slots + * are bound to the plugin's compiled schemas and registered filter routes. + * Consumed by `plugin.getClient()`. + */ + +import { z } from "zod"; +import { Client } from "../client"; +import { Opportunities } from "./opportunities"; +import { RESOURCE_REGISTRY } from "./registry"; +import type { PluginRoutes, ResourceName } from "../../extensions/types"; +import type { Plugin, PluginSchemasInput } from "../../extensions/define-plugin"; +import type { ClientConfig } from "../config"; +import type { AuthMethod } from "../auth"; +import type { OpportunityBase } from "../../types"; + +/** Item type projected from the plugin's compiled schema for a model. */ +type ResolvedItemType<T extends PluginSchemasInput> = z.infer< + Plugin<T>["schemas"]["Opportunity"]["commonSchema"] +> & + OpportunityBase; + +/** + * A `Client` whose resource slots carry the plugin's item and filter types: + * `search({ filters })` autocompletes the registered filter names and + * responses parse with the plugin's schema by default. + * + * The default resource slots are `Omit`ted (not intersected) — an intersection + * would let `Client`'s looser default `opportunities` signature win call + * resolution and erase the plugin's filter typing. + */ +export type BuiltClient<T extends PluginSchemasInput, TRoutes extends PluginRoutes> = Omit< + Client, + ResourceName +> & { + // resource-slot: typed projection per resource + opportunities: Opportunities<TRoutes, ResolvedItemType<T>>; +}; + +/** + * Builds a client from a plugin: constructs the transport `Client`, then + * overlays each resource slot with a plugin-bound instance (the plugin's + * compiled schema as the default parse schema, its routes for filter + * classification). + */ +export function buildClientForPlugin<T extends PluginSchemasInput, TRoutes extends PluginRoutes>( + plugin: Plugin<T, TRoutes>, + config: ClientConfig & { auth?: AuthMethod } = {} +): BuiltClient<T, TRoutes> { + const client = new Client(config); + + // resource-slot: build each slot from the registry, so adding a resource + // (organizations, applications, awards) is a registry entry + a typed slot. + // definePlugin populates a compiled schema for every extensible model, so the + // registry's schema binding always resolves here. + const { resourceClass, schemaName } = RESOURCE_REGISTRY.opportunities; + const oppSchema = plugin.schemas[schemaName].commonSchema as z.ZodType< + OpportunityBase, + z.ZodTypeDef, + unknown + >; + (client as unknown as { opportunities: Opportunities }).opportunities = new resourceClass( + client, + oppSchema, + plugin.routes + ); + + // Second of the two cast points (the slot assignment above is the other): + // BuiltClient projects the plugin's item + filter types. + return client as unknown as BuiltClient<T, TRoutes>; +} diff --git a/lib/ts-sdk/src/client/opportunities.ts b/lib/ts-sdk/src/client/resources/opportunities.ts similarity index 58% rename from lib/ts-sdk/src/client/opportunities.ts rename to lib/ts-sdk/src/client/resources/opportunities.ts index cf4041246..81f203c51 100644 --- a/lib/ts-sdk/src/client/opportunities.ts +++ b/lib/ts-sdk/src/client/resources/opportunities.ts @@ -3,49 +3,23 @@ */ import { z } from "zod"; -import type { Client, FetchManyOptions } from "./client"; -import type { - OpportunityBase, - OppStatusOptions, - OppFilters, - OppSearchRequest, - Paginated, - Filtered, -} from "../types"; +import type { Client, FetchManyOptions } from "../client"; +import type { OpportunityBase, OppStatusOptions, OppFilters, OppSearchRequest } from "../../types"; import { OkSchema, PaginatedSchema, FilteredSchema, OpportunityBaseSchema, OppFiltersSchema, -} from "../schemas"; -import { ArrayOperator } from "../constants"; -import { classifyFilters } from "../extensions/custom-filters"; -import { FilterError } from "../extensions/types"; -import type { PluginRoutes } from "../extensions/types"; - -// ============================================================================= -// Client-side filter-error merge -// ============================================================================= - -/** - * Merges client-side filter errors into a search response's `filterInfo.errors`, - * flattened to `"{path}: {message}"` and ordered before any existing entries. - * Mutates and returns `response` for call-site convenience. - */ -function mergeFilterErrors<T, F>( - response: Filtered<T, F>, - clientErrors: FilterError[] -): Filtered<T, F> { - if (clientErrors.length === 0) return response; - - const flattened = clientErrors.map(e => `${e.path}: ${e.message}`); - // Auto-pagination returns the raw server envelope, so filterInfo can be absent - // on a non-conformant response; initialize rather than throw (stay fail-soft). - const filterInfo = (response.filterInfo ??= { filters: {} as F }); - filterInfo.errors = [...flattened, ...(filterInfo.errors ?? [])]; - return response; -} +} from "../../schemas"; +import { ArrayOperator } from "../../constants"; +import { classifyFilters, validateRoutes } from "../../extensions/custom-filters"; +import type { CustomFilterInput } from "../../extensions/custom-filters"; +import { FilterError } from "../../extensions/types"; +import type { CustomFilterType, PluginRoutes } from "../../extensions/types"; +import { parseBatch } from "../results"; +import type { ListResult, OnParseError, SearchResult } from "../results"; +import { Resource } from "./base"; // ============================================================================= // Schema type constraint @@ -72,29 +46,35 @@ type OppSchema = z.ZodType<OpportunityBase, z.ZodTypeDef, unknown>; type RawFilter = { operator: string; value: unknown }; /** - * The declared custom-filter names for `opportunities.search` in a routes type. + * The declared filter specs for `opportunities.search` in a routes type. * `definePlugin` preserves the literal `routes` type (its `const TRoutes` generic), - * so a plugin defined inline yields concrete filter-name literals here. + * so a plugin defined inline yields concrete filter-name and filterType literals here. */ -type CustomFilterNames<R extends PluginRoutes> = R extends { +type RouteFilters<R extends PluginRoutes> = R extends { opportunities: { search: { filters: infer Fs } }; } - ? Extract<keyof Fs, string> + ? Fs : never; +/** The declared custom-filter names for `opportunities.search` in a routes type. */ +type CustomFilterNames<R extends PluginRoutes> = Extract<keyof RouteFilters<R>, string>; + /** * Typed filter bag for `search({ filters })`. * - * Declared filter names surface in editor autocomplete with a typed value, while - * arbitrary keys remain accepted — the spec supports ad-hoc (escape-hatch) filters - * (bucket 3 of `classifyFilters`), so an unknown key cannot be rejected at the type - * level without dropping ad-hoc support. Net: autocomplete + filter-envelope - * checking (`{ operator, value }` shape) for declared filters — per-`filterType` - * value validation runs at runtime in `classifyFilters`, not at the type level; - * NO typo-rejection on filter names (a typo is structurally an intentional ad-hoc key). + * Declared filter names surface in editor autocomplete with the value typed by + * their declared `filterType` (a wrong value family is a compile error), while + * arbitrary keys remain accepted — the spec supports ad-hoc (escape-hatch) filters, + * so an unknown key cannot be rejected at the type level without dropping ad-hoc + * support (a typo is structurally an intentional ad-hoc key). Runtime validation + * backstops both cases. */ -type CustomFilterBag<R extends PluginRoutes> = { - [K in CustomFilterNames<R>]?: RawFilter; +export type CustomFilterBag<R extends PluginRoutes> = { + [K in CustomFilterNames<R>]?: RouteFilters<R>[K] extends { + filterType: infer FT extends CustomFilterType; + } + ? CustomFilterInput<FT> + : RawFilter; } & Record<string, RawFilter>; // ============================================================================= @@ -103,7 +83,7 @@ type CustomFilterBag<R extends PluginRoutes> = { /** Options for getting a single opportunity */ export interface GetOptions<S extends OppSchema = typeof OpportunityBaseSchema> { - /** Zod schema to parse and type the response. Defaults to `OpportunityBaseSchema`. */ + /** Zod schema to parse and type the response. Defaults to the bound (plugin or base) schema. */ schema?: S; } @@ -111,7 +91,7 @@ export interface GetOptions<S extends OppSchema = typeof OpportunityBaseSchema> export interface ListOptions< S extends OppSchema = typeof OpportunityBaseSchema, > extends FetchManyOptions<z.infer<S>> { - /** Zod schema to parse and type each item. Defaults to `OpportunityBaseSchema`. */ + /** Zod schema to parse and type each item. Defaults to the bound (plugin or base) schema. */ schema?: S; } @@ -122,16 +102,19 @@ export interface SearchOptions< > extends FetchManyOptions<z.infer<S>> { /** Text query to search for in opportunity titles and descriptions */ query?: string; - /** Filter by opportunity statuses */ + /** + * Filter by opportunity statuses (shorthand for the `status` filter). + * @deprecated Pass status through `filters` instead; this shorthand will be + * removed in a future release. + */ statuses?: OppStatusOptions[]; /** * Flat custom-filter bag (filter name → `{ operator, value }`, e.g. built with `F.*`). - * Classified into the `OppFilters` request body via `classifyFilters` when present. - * Registered custom filters get typed names and validate against the specs declared - * in the client's `routes` (bound once at client construction). + * Classified into the `OppFilters` request body via `classifyFilters` when present; + * an invalid value on any key throws `FilterError` before the request is sent. */ filters?: CustomFilterBag<R>; - /** Zod schema to parse and type each item. Defaults to `OpportunityBaseSchema`. */ + /** Zod schema to parse and type each item. Defaults to the bound (plugin or base) schema. */ schema?: S; } @@ -149,14 +132,27 @@ export interface SearchOptions< * const list = await client.opportunities.list(); * ``` */ -export class Opportunities<R extends PluginRoutes = PluginRoutes> { - private readonly client: Client; +export class Opportunities< + R extends PluginRoutes = PluginRoutes, + TItem extends OpportunityBase = OpportunityBase, +> extends Resource<TItem> { private readonly basePath = "/common-grants/opportunities"; - private readonly routes: R | undefined; - constructor(client: Client, routes?: R) { - this.client = client; - this.routes = routes; + constructor(client: Client, boundSchema?: z.ZodType<TItem, z.ZodTypeDef, unknown>, routes?: R) { + super( + client, + boundSchema ?? (OpportunityBaseSchema as unknown as z.ZodType<TItem, z.ZodTypeDef, unknown>), + routes + ); + // Backstop for direct construction (bypassing definePlugin): an invalid + // registration — e.g. a filter named after a default field — throws here + // instead of silently shadowing the default bucket at classification time. + if (routes) validateRoutes(routes); + } + + /** Per-call override wins; otherwise the bound (plugin or base) schema. */ + private resolveSchema<S extends OppSchema>(override?: S): S { + return override ?? (this.boundSchema as unknown as S); } // ############################################################################ @@ -166,10 +162,13 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { /** * Get a specific opportunity by ID. * + * A single requested entity that does not parse is a real error, so `get()` + * is fail-hard: a malformed response throws. + * * @param id - The opportunity ID * @param options - Optional settings; use `schema` for typed custom field access. * @returns The opportunity data - * @throws {Error} If the request fails + * @throws {Error} If the request fails or the response does not parse * * @example * ```ts @@ -185,12 +184,12 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { * console.log(typed.customFields?.legacyId?.value); // typed as number * ``` */ - async get<S extends OppSchema = typeof OpportunityBaseSchema>( + async get<S extends OppSchema = z.ZodType<TItem, z.ZodTypeDef, unknown>>( id: string, options?: GetOptions<S> ): Promise<z.infer<S>> { - const schema = options?.schema ?? (OpportunityBaseSchema as unknown as S); - const response = await this.client.get(`${this.basePath}/${id}`); + const schema = this.resolveSchema(options?.schema); + const response = await this.client.get(`${this.basePath}/${encodeURIComponent(id)}`); if (!response.ok) { throw new Error(`Failed to get opportunity ${id}: ${response.status} ${response.statusText}`); @@ -209,9 +208,12 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { /** * List opportunities with auto-pagination by default. * + * Rows are parsed individually: valid rows land in `items`, per-row failures + * in `errors` (set `onParseError: "throw"` to fail hard on the first bad row). + * * @param options - Pagination and schema options. If `page` is specified, fetches only that page. * Use `schema` for typed custom field access. - * @returns Paginated list of opportunities + * @returns Paginated list of opportunities plus per-row parse failures * @throws {Error} If the request fails * * @example @@ -229,10 +231,10 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { * const typed = await client.opportunities.list({ schema: OpportunitySchema }); * ``` */ - async list<S extends OppSchema = typeof OpportunityBaseSchema>( + async list<S extends OppSchema = z.ZodType<TItem, z.ZodTypeDef, unknown>>( options?: ListOptions<S> - ): Promise<Paginated<z.infer<S>>> { - const schema = options?.schema ?? (OpportunityBaseSchema as unknown as S); + ): Promise<ListResult<z.infer<S>>> { + const schema = this.resolveSchema(options?.schema); // If page is specified, fetch only that page if (options?.page !== undefined) { @@ -249,14 +251,21 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { } const json = await response.json(); - return PaginatedSchema(schema).parse(json) as Paginated<z.infer<S>>; + // Validate the envelope with raw rows, then partition rows individually. + const envelope = PaginatedSchema(z.unknown()).parse(json); + const { items, errors } = parseBatch( + schema, + envelope.items as unknown[], + options?.onParseError ?? "collect" + ); + return { ...envelope, items, errors } as ListResult<z.infer<S>>; } // Auto-paginate by default return this.client.fetchMany(this.basePath, { ...options, schema, - }); + }) as Promise<ListResult<z.infer<S>>>; } // ############################################################################ @@ -268,8 +277,16 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { * * Supports auto-pagination by default. If `page` is specified, only fetches that page. * + * Filter validation is fail-fast: an invalid value on any filter — standard, + * registered custom, or ad-hoc — throws `FilterError` before the request is + * sent. A caught `FilterError`'s `.sourceValue` may carry PII; redact before + * logging. `filterInfo.errors` on the response carries server-returned errors only. + * Rows are parsed individually: valid rows land in `items`, per-row failures + * in `errors` (set `onParseError: "throw"` to fail hard on the first bad row). + * * @param options - Search options including query text, status filters, pagination, and schema - * @returns Filtered list of opportunities + * @returns Filtered list of opportunities plus per-row parse failures + * @throws {FilterError} If a filter value is invalid (before any request) * @throws {Error} If the request fails * * @example @@ -304,25 +321,25 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { * const typed = await client.opportunities.search({ query: "test", schema: OpportunitySchema }); * ``` */ - async search<S extends OppSchema = typeof OpportunityBaseSchema>( + async search<S extends OppSchema = z.ZodType<TItem, z.ZodTypeDef, unknown>>( options?: SearchOptions<S, R> - ): Promise<Filtered<z.infer<S>, OppFilters>> { - const schema = options?.schema ?? (OpportunityBaseSchema as unknown as S); + ): Promise<SearchResult<z.infer<S>>> { + const schema = this.resolveSchema(options?.schema); - // Build the base search body (without pagination). Client-side filter - // problems are collected fail-soft into `filterErrors` rather than thrown. - const { body: searchBody, errors: filterErrors } = this.buildSearchBody(options); + // Build the base search body (without pagination). Invalid filter values + // throw FilterError here — before any request is sent. + const searchBody = this.buildSearchBody(options); // If page is specified, fetch only that page if (options?.page !== undefined) { - const page = await this.fetchSearchPage( + return this.fetchSearchPage( searchBody, options.page, options.pageSize, options.signal, - schema + schema, + options?.onParseError ); - return mergeFilterErrors(page, filterErrors); } // Auto-paginate using fetchMany with POST method. @@ -335,7 +352,7 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { schema, }); - return mergeFilterErrors(result as Filtered<z.infer<S>, OppFilters>, filterErrors); + return result as unknown as SearchResult<z.infer<S>>; } // ############################################################################ @@ -345,72 +362,52 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { /** * Builds the search request body from options. * - * Returns the wire `body` alongside the `errors` collected from client-side - * filter classification (fail-soft — never throws on a bad filter). `search()` - * merges these into the response's `filterInfo.errors`. + * Filter classification is fail-fast: `classifyFilters` throws `FilterError` + * on the first invalid value, so a body is only produced for valid input. */ - private buildSearchBody(options?: SearchOptions<OppSchema, R>): { - body: OppSearchRequest; - errors: FilterError[]; - } { + private buildSearchBody(options?: SearchOptions<OppSchema, R>): OppSearchRequest { const body: OppSearchRequest = {}; - const errors: FilterError[] = []; if (options?.query) { body.search = options.query; } - // `routes` is bound once at client construction, so it's read off the - // instance. Fail-soft: invalid keys are dropped and their errors collected. let filters: OppFilters | undefined; if (options?.filters) { - const classified = classifyFilters( - this.routes ?? {}, - "opportunities", - "search", - options.filters - ); - errors.push(...classified.errors); - // Cast bridges the Zod-inferred OppFilters to the hand-authored OppFilters - // type alias; they are structurally the same shape. - filters = classified.result as OppFilters; + filters = classifyFilters(this.routes ?? {}, "opportunities", "search", options.filters); } // statuses shorthand → status default field (augments any classified filters) if (options?.statuses?.length) { if (filters?.status !== undefined) { - // `status` given via both the shorthand and `filters`: `filters` wins, - // the shorthand is ignored, and a warning is collected (not thrown). - errors.push( - new FilterError( - "specified via both the statuses shorthand and the filters argument; used the filters value", - { path: "filters.status", sourceValue: options.statuses } - ) + throw new FilterError( + "specified via both the statuses shorthand and the filters argument; pass one or the other", + { path: "filters.status", sourceValue: options.statuses } ); - } else { - filters = filters ?? {}; - filters.status = { - operator: ArrayOperator.in, - value: options.statuses, - }; } + filters = filters ?? {}; + filters.status = { + operator: ArrayOperator.in, + value: options.statuses, + }; } if (filters) { body.filters = filters; } - return { body, errors }; + return body; } /** Fetches a single search page */ private async fetchSearchPage<S extends OppSchema>( searchBody: OppSearchRequest, page: number, - pageSize?: number, - signal?: AbortSignal, - schema: S = OpportunityBaseSchema as unknown as S - ): Promise<Filtered<z.infer<S>, OppFilters>> { + pageSize: number | undefined, + signal: AbortSignal | undefined, + schema: S, + onParseError?: OnParseError + ): Promise<SearchResult<z.infer<S>>> { const requestBody: OppSearchRequest = { ...searchBody, pagination: { @@ -426,6 +423,14 @@ export class Opportunities<R extends PluginRoutes = PluginRoutes> { } const json = await response.json(); - return FilteredSchema(schema, OppFiltersSchema).parse(json) as Filtered<z.infer<S>, OppFilters>; + // Validate the envelope with raw rows, then partition rows individually. + // filterInfo passes through exactly as the server returned it. + const envelope = FilteredSchema(z.unknown(), OppFiltersSchema).parse(json); + const { items, errors } = parseBatch( + schema, + envelope.items as unknown[], + onParseError ?? "collect" + ); + return { ...envelope, items, errors } as unknown as SearchResult<z.infer<S>>; } } diff --git a/lib/ts-sdk/src/client/resources/registry.ts b/lib/ts-sdk/src/client/resources/registry.ts new file mode 100644 index 000000000..194e26f88 --- /dev/null +++ b/lib/ts-sdk/src/client/resources/registry.ts @@ -0,0 +1,24 @@ +/** + * Resource registry: the single map from resource name to its class and + * extensible-schema binding. Adding a resource = one entry here plus its + * typed slots (grep `resource-slot` for the full set). + */ + +import { Opportunities } from "./opportunities"; +import type { ExtensibleSchemaName, ResourceName } from "../../extensions/types"; + +interface ResourceRegistryEntry { + resourceClass: typeof Opportunities; + schemaName: ExtensibleSchemaName; +} + +// resource-slot: add an entry per API resource +export const RESOURCE_REGISTRY = { + opportunities: { resourceClass: Opportunities, schemaName: "Opportunity" }, +} as const satisfies Record<ResourceName, ResourceRegistryEntry>; + +/** Compile-time seam guard: `ResourceName` and this registry's keys must not drift. */ +type Equal<A, B> = + (<T>() => T extends A ? 1 : 2) extends <T>() => T extends B ? 1 : 2 ? true : false; +const _registryMatchesResourceNames: Equal<ResourceName, keyof typeof RESOURCE_REGISTRY> = true; +void _registryMatchesResourceNames; diff --git a/lib/ts-sdk/src/client/results.ts b/lib/ts-sdk/src/client/results.ts new file mode 100644 index 000000000..79584f9aa --- /dev/null +++ b/lib/ts-sdk/src/client/results.ts @@ -0,0 +1,71 @@ +/** + * Per-row parse partitioning for list/search responses. + * + * `parseBatch` validates each response row against a schema and partitions the + * outcome: valid rows land in `items`, failures in `errors` (each carrying its + * row index and raw payload). One malformed row never aborts the batch unless + * the caller opts into `"throw"`. + */ + +import { z } from "zod"; +import { Filtered, OppFilters, Paginated } from "../types"; + +/** + * One response row that failed schema validation, with its position and raw payload. + * + * `index` is the row's position within its own page's raw rows, not the + * aggregated `items` array. + * + * `raw` is the entire unvalidated row and may carry PII (e.g. in + * `customFields`) — redact before logging, as with `TransformError.sourceValue`. + */ +export interface ParseFailure { + index: number; + raw: unknown; + error: z.ZodError; +} + +/** Thrown by `parseBatch` under `onParseError: "throw"`; carries the first failure's detail. */ +export class BatchParseError extends Error { + constructor(public readonly failure: ParseFailure) { + super(`Row ${failure.index} failed to parse: ${failure.error.message}`); + this.name = "BatchParseError"; + } +} + +/** Row-parse failure handling: partition into `errors` (default) or throw on first failure. */ +export type OnParseError = "collect" | "throw"; + +export function parseBatch<T>( + schema: { safeParse(data: unknown): z.SafeParseReturnType<unknown, T> }, + rows: unknown[], + onParseError: OnParseError = "collect" +): { items: T[]; errors: ParseFailure[] } { + const items: T[] = []; + const errors: ParseFailure[] = []; + rows.forEach((raw, index) => { + const parsed = schema.safeParse(raw); + if (parsed.success) { + items.push(parsed.data); + return; + } + const failure: ParseFailure = { index, raw, error: parsed.error }; + if (onParseError === "throw") throw new BatchParseError(failure); + errors.push(failure); + }); + return { items, errors }; +} + +/** Paginated result whose `items` are valid rows only; per-row failures land in `errors`. */ +export type ListResult<T> = Paginated<T> & { errors: ParseFailure[] }; + +/** + * Search result: filtered envelope plus per-row parse failures. + * + * `filterInfo` is optional because auto-pagination passes the server envelope + * through unchanged, and a server may omit it. + */ +export type SearchResult<T, F = OppFilters> = Omit<Filtered<T, F>, "filterInfo"> & { + filterInfo?: Filtered<T, F>["filterInfo"]; + errors: ParseFailure[]; +}; diff --git a/lib/ts-sdk/src/extensions/README.md b/lib/ts-sdk/src/extensions/README.md index dd47e8fdb..9aa3a2f2d 100644 --- a/lib/ts-sdk/src/extensions/README.md +++ b/lib/ts-sdk/src/extensions/README.md @@ -47,7 +47,7 @@ The `@common-grants/sdk/extensions` module provides TypeScript utilities for wor - [Plugin creation](#plugin-creation) - [Schema utilities](#schema-utilities) - [Transforms (PoC)](#transforms-poc) - - [Custom filters (PoC)](#custom-filters-poc) + - [Custom filters](#custom-filters) - [Shared types](#shared-types) ## Key concepts @@ -360,11 +360,11 @@ opp.customFields?.cfda?.value; // string ## Using plugins with the API client -Pass a plugin's extended schema to the API client via the `schema` option. The client uses it to parse API responses into fully typed objects: +Build a client from the plugin with `plugin.getClient()`. The returned client is pre-bound to the plugin: responses parse with the plugin's compiled schemas by default, and `search({ filters })` types the registered filter names — no per-call `schema` needed: ```typescript -import { Client, Auth } from "@common-grants/sdk/client"; -import { definePlugin } from "@common-grants/sdk/extensions"; +import { Auth } from "@common-grants/sdk/client"; +import { definePlugin, F } from "@common-grants/sdk/extensions"; const myPlugin = definePlugin({ schemas: { @@ -375,37 +375,44 @@ const myPlugin = definePlugin({ }, }, }, + routes: { + opportunities: { + search: { filters: { agency: { filterType: "stringArray" } } }, + }, + }, } as const); -const client = new Client({ +const client = myPlugin.getClient({ baseUrl: "https://api.example.gov", auth: Auth.apiKey("your-api-key"), }); -// Get a single opportunity with typed custom fields -const opp = await client.opportunities.get(oppId, { - schema: myPlugin.schemas.Opportunity.commonSchema, -}); +// Get a single opportunity — custom fields are typed, no `schema` option needed +const opp = await client.opportunities.get(oppId); opp.customFields?.legacyId?.value; // typed as number opp.customFields?.category?.value; // typed as string -// List with the same schema -const response = await client.opportunities.list({ - schema: myPlugin.schemas.Opportunity.commonSchema, -}); -for (const opp of response.items) { - console.log(opp.customFields?.category?.value); -} - -// Search with the same schema +// Search with a registered custom filter — `agency` autocompletes and its +// value family is compile-checked const results = await client.opportunities.search({ query: "health", statuses: ["open"], - schema: myPlugin.schemas.Opportunity.commonSchema, + filters: { agency: F.in(["HHS"]) }, }); + +// Valid rows land in `items`; rows that fail schema parsing land in `errors` +// (each with its row `index` and `raw` payload) instead of throwing the page. +for (const opp of results.items) { + console.log(opp.customFields?.category?.value); +} +for (const err of results.errors) { + // `err.raw` is the entire unvalidated row and may carry PII — redact before + // logging (same caveat as `TransformError.sourceValue` above). + console.warn(`row ${err.index} failed to parse: ${err.error.message}`); +} ``` -The `schema` option is accepted by `get()`, `list()`, and `search()`. When omitted, the client falls back to `OpportunityBaseSchema` (with untyped `customFields`). +The per-call `schema` option is still accepted by `get()`, `list()`, and `search()` and overrides the plugin-bound default. A plain `new Client(...)` (no plugin) keeps working with `OpportunityBaseSchema` defaults (untyped `customFields`, ad-hoc filters only). ## Plugin transformations @@ -687,9 +694,9 @@ F.between("2025-01-01", "2025-12-31"); // { operator: "between", value: { min: " ### Classifying consumer filters into the request body -`definePlugin()` registers filter **declarations** — which filter names exist and what type each one is. `classifyFilters()` is the separate **search-time** step: it takes the consumer's actual filter values and produces the body of the request sent to the API. In the full SDK the client will call it internally when you invoke a search method — consumers won't call it directly. The PoC exposes it standalone because client wiring is out of scope (tracked under [#645](https://github.com/HHS/simpler-grants-protocol/issues/645)). +`definePlugin()` registers filter **declarations** — which filter names exist and what type each one is. `classifyFilters()` is the separate **search-time** step: it takes the consumer's actual filter values and produces the body of the request sent to the API. The client calls it internally on every `search()`; consumers normally never call it directly. -`classifyFilters()` accepts the plugin's `routes`, a resource name, a method name, and the consumer's flat `filters` object. It returns an `OppFilters` request body conforming to ADR-0012: +`classifyFilters()` accepts the plugin's `routes`, a resource name, a method name, and the consumer's flat `filters` object. It returns an `OppFilters` request body conforming to ADR-0012 — and it is **fail-fast**: an invalid value on any key (standard, registered, or ad-hoc) throws `FilterError` before a request body exists: ```typescript import { classifyFilters } from "@common-grants/sdk/extensions"; @@ -740,9 +747,9 @@ For a complete runnable example with assertions, see [`examples/custom-filters.t - A filter spec uses an unknown `filterType` value. - A custom filter name collides with a default filter field name (e.g. registering `"status"` would shadow the protocol's standard `status` filter). -Call-time validation runs automatically inside `classifyFilters()` for every filter key — registered filters are checked for operator/value shape against the declared `filterType`, ad-hoc filters get a shape-only check (`DefaultFilterSchema`). It is not a separate public entry point; `classifyFilters()` (invoked by each `search()`) is the call-time path. +Call-time validation runs automatically inside `classifyFilters()` for every filter key — registered filters are checked for operator/value shape against the declared `filterType`, ad-hoc filters get a shape-only check (`DefaultFilterSchema`). Any failure **throws `FilterError` before the request is sent**; `filterInfo.errors` on the response carries server-returned errors only. -`definePlugin()` passes `routes` through **unvalidated**; the `Client` runs `validateRoutes()` for you when constructed with `routes` (and call-time validation runs inside each search). To surface registration errors without constructing a client, call `validateRoutes()` yourself after defining a plugin. +`definePlugin()` runs `validateRoutes()` at definition time, so a misspelled route or an invalid filter registration throws at the definition site (the compile-time closed route keys catch the same mistakes for TypeScript authors). > **PII note:** as with transforms, `FilterError.sourceValue` carries the raw input — here, the consumer's filter value. The [PII warning](#error-handling) above applies equally; log a redacted projection. @@ -891,16 +898,17 @@ The tables below list everything exported from `@common-grants/sdk/extensions`, | [`ToCommon`](./transform-helpers.ts) | type | Helper type for a hand-written `toCommon`. Takes a `TransformTypes` arg; `source` is typed from `sourceSchema`, the return checked against the resolved common **input** type. | | | [`FromCommon`](./transform-helpers.ts) | type | Helper type for a hand-written `fromCommon`. Takes a `TransformTypes` arg; `common` is the resolved common **output** type, the return the source type. | | -### Custom filters (PoC) - -| Export | Kind | Description | Demonstrated in | -| ------------------------------------------ | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | --- | -------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| [`classifyFilters()`](./custom-filters.ts) | function | Three-bucket classifier. Maps a flat consumer `filters` object to the ADR-0012 `OppFilters` request body: default fields → top-level named fields; registered custom + ad-hoc → `customFilters` record. | [Classifying consumer filters](#classifying-consumer-filters-into-the-request-body) | -| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `FilterError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | | [`F`](./custom-filters.ts) | namespace | Helper namespace. `F.eq`, `F.neq`, `F.gt`, `F.gte`, `F.lt`, `F.lte`, `F.in`, `F.notIn`, `F.like`, `F.notLike`, `F.between`, `F.outside` — each compiles to `{ operator, value }`. Note: `F.in` is `"in"` as an object property key. | [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) | -| [`CustomFilterSpec`](./types.ts) | interface | Per-filter declaration: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType`; no `value` field. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | -| [`CustomFilterType`](./types.ts) | type | 11-value literal union: `stringComparison \| stringArray \| numberComparison \| numberArray \| numberRange \| integerComparison \| booleanComparison \| dateComparison \| dateRange \| moneyComparison \| moneyRange`. | [Filter-type catalog](#filter-type-catalog-and-the-f-helpers) | -| [`PluginRoutes`](./types.ts) | type | `Record<string, Record<string, RouteDeclarations>>` — the `routes` value on `DefinePluginOptions`. Keys are resource name → method name → `RouteDeclarations`. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | -| [`RouteDeclarations`](./types.ts) | interface | Per-method filter map: `{ filters?: Record<string, CustomFilterSpec> }`. | | +### Custom filters + +| Export | Kind | Description | Demonstrated in | +| ------------------------------------------ | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| [`classifyFilters()`](./custom-filters.ts) | function | Fail-fast three-bucket classifier. Maps a flat consumer `filters` object to the ADR-0012 `OppFilters` request body (default fields → top-level named fields; registered custom + ad-hoc → `customFilters` record); throws `FilterError` on the first invalid value. | [Classifying consumer filters](#classifying-consumer-filters-into-the-request-body) | +| [`validateRoutes()`](./custom-filters.ts) | function | Registration-time validator. Throws `FilterError` on unknown `filterType` or default-field name collisions. | [Validation](#validation--registration-time-and-call-time) | +| [`F`](./custom-filters.ts) | namespace | Helper namespace. `F.eq`, `F.neq`, `F.gt`, `F.gte`, `F.lt`, `F.lte`, `F.in`, `F.notIn`, `F.like`, `F.notLike`, `F.between`, `F.outside` — each compiles to `{ operator, value }`. Note: `F.in` is `"in"` as an object property key. | [Filter-type catalog and the `F.*` helpers](#filter-type-catalog-and-the-f-helpers) | +| [`CustomFilterSpec`](./types.ts) | interface | Per-filter declaration: `{ filterType: CustomFilterType; description?: string }`. Operators are derived from `filterType`; no `value` field. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | +| [`CustomFilterType`](./types.ts) | type | 11-value literal union: `stringComparison \| stringArray \| numberComparison \| numberArray \| numberRange \| integerComparison \| booleanComparison \| dateComparison \| dateRange \| moneyComparison \| moneyRange`. | [Filter-type catalog](#filter-type-catalog-and-the-f-helpers) | +| [`PluginRoutes`](./types.ts) | type | `Partial<Record<ResourceName, RouteMethods>>` — the `routes` value on `DefinePluginOptions`. Both key levels are closed unions, so a misspelled resource or method is a compile error. | [Declaring custom filters on a route](#declaring-custom-filters-on-a-route) | +| [`RouteDeclarations`](./types.ts) | interface | Per-method filter map: `{ filters?: Record<string, CustomFilterSpec> }`. | | ### Shared types diff --git a/lib/ts-sdk/src/extensions/custom-filters.ts b/lib/ts-sdk/src/extensions/custom-filters.ts index 0d7215d0e..a0821d384 100644 --- a/lib/ts-sdk/src/extensions/custom-filters.ts +++ b/lib/ts-sdk/src/extensions/custom-filters.ts @@ -5,7 +5,8 @@ * * - `classifyFilters` — transforms a flat consumer `filters` object into the * ADR-0012 `OppFilters` request body (three-bucket: default → named top-level fields; - * registered custom → `customFilters`; ad-hoc → `customFilters` passthrough). + * registered custom → `customFilters`; ad-hoc → `customFilters` passthrough), + * throwing `FilterError` on the first invalid value. * - `validateRoutes` — registration-time validation; rejects unknown `filterType` * and custom names that collide with default-filter names. * - `validateFilterCall` — call-time validation; rejects operator/filterType mismatch @@ -34,19 +35,27 @@ import { StringComparisonFilterSchema, } from "../schemas/zod/filters"; import { OppDefaultFiltersSchema, OppFiltersSchema } from "../schemas/zod/models"; -import type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; +import type { + CustomFilterSpec, + CustomFilterType, + PluginRoutes, + ResourceName, + RouteDeclarations, + RouteMethod, +} from "./types"; import { FilterError } from "./types"; // ############################################################################ -// Internal — filter-type schema map +// Public — filter-type schema map // ############################################################################ /** * Maps each CustomFilterType to the Zod schema that validates its * `{operator, value}` pair. Each schema constrains both the allowed operator * enum and the value shape, so a single parse covers both checks. + * Literal-typed (`as const`) so `CustomFilterInput` can project per-type inputs. */ -const FILTER_TYPE_SCHEMAS: Record<CustomFilterType, z.ZodTypeAny> = { +export const FILTER_TYPE_SCHEMAS = { stringComparison: StringComparisonFilterSchema, stringArray: StringArrayFilterSchema, numberComparison: NumberComparisonFilterSchema, @@ -60,7 +69,12 @@ const FILTER_TYPE_SCHEMAS: Record<CustomFilterType, z.ZodTypeAny> = { dateRange: DateRangeFilterSchema, moneyComparison: MoneyComparisonFilterSchema, moneyRange: MoneyRangeFilterSchema, -}; +} as const satisfies Record<CustomFilterType, z.ZodTypeAny>; + +/** Input type accepted for a registered custom filter of the given filterType. */ +export type CustomFilterInput<FT extends CustomFilterType> = z.input< + (typeof FILTER_TYPE_SCHEMAS)[FT] +>; /** * All 11 valid CustomFilterType values — used for unknown-filterType detection. @@ -107,29 +121,29 @@ const SUPPORTED_CUSTOM_FILTER_ROUTES = new Set<string>(["opportunities.search"]) */ export const F = { /** Equals — `{ operator: "eq", value }` */ - eq: (value: unknown) => ({ operator: "eq" as const, value }), + eq: <V>(value: V) => ({ operator: "eq" as const, value }), /** Not equals — `{ operator: "neq", value }` */ - neq: (value: unknown) => ({ operator: "neq" as const, value }), + neq: <V>(value: V) => ({ operator: "neq" as const, value }), /** Greater than — `{ operator: "gt", value }` */ - gt: (value: unknown) => ({ operator: "gt" as const, value }), + gt: <V>(value: V) => ({ operator: "gt" as const, value }), /** Greater than or equal — `{ operator: "gte", value }` */ - gte: (value: unknown) => ({ operator: "gte" as const, value }), + gte: <V>(value: V) => ({ operator: "gte" as const, value }), /** Less than — `{ operator: "lt", value }` */ - lt: (value: unknown) => ({ operator: "lt" as const, value }), + lt: <V>(value: V) => ({ operator: "lt" as const, value }), /** Less than or equal — `{ operator: "lte", value }` */ - lte: (value: unknown) => ({ operator: "lte" as const, value }), + lte: <V>(value: V) => ({ operator: "lte" as const, value }), /** Array inclusion — `{ operator: "in", value: [...] }` */ - in: (value: unknown[]) => ({ operator: "in" as const, value }), + in: <V>(value: V[]) => ({ operator: "in" as const, value }), /** Array exclusion — `{ operator: "notIn", value: [...] }` */ - notIn: (value: unknown[]) => ({ operator: "notIn" as const, value }), + notIn: <V>(value: V[]) => ({ operator: "notIn" as const, value }), /** String pattern match — `{ operator: "like", value }` */ like: (value: string) => ({ operator: "like" as const, value }), /** String pattern non-match — `{ operator: "notLike", value }` */ notLike: (value: string) => ({ operator: "notLike" as const, value }), /** Range (inclusive) — `{ operator: "between", value: { min, max } }` */ - between: (min: unknown, max: unknown) => ({ operator: "between" as const, value: { min, max } }), + between: <V>(min: V, max: V) => ({ operator: "between" as const, value: { min, max } }), /** Range (exclusive) — `{ operator: "outside", value: { min, max } }` */ - outside: (min: unknown, max: unknown) => ({ operator: "outside" as const, value: { min, max } }), + outside: <V>(min: V, max: V) => ({ operator: "outside" as const, value: { min, max } }), }; // ############################################################################ @@ -158,8 +172,10 @@ export const F = { */ export function validateRoutes(routes: PluginRoutes): void { for (const [resourceKey, methods] of Object.entries(routes)) { + // Partial<Record<...>> admits explicitly-undefined values; skip them. + if (!methods) continue; for (const [methodKey, declarations] of Object.entries(methods)) { - const filters = (declarations as RouteDeclarations).filters; + const filters = (declarations as RouteDeclarations | undefined)?.filters; if (!filters) continue; if (!SUPPORTED_CUSTOM_FILTER_ROUTES.has(`${resourceKey}.${methodKey}`)) { @@ -173,6 +189,10 @@ export function validateRoutes(routes: PluginRoutes): void { for (const [filterName, spec] of Object.entries(filters)) { const path = `routes.${resourceKey}.${methodKey}.filters.${filterName}`; + // Plain-JS callers can pass explicitly-undefined specs; skip them like + // the resource and method levels above. + if (!spec) continue; + // Check for unknown filterType if (!VALID_FILTER_TYPES.has(spec.filterType)) { throw new FilterError( @@ -209,7 +229,7 @@ export function validateRoutes(routes: PluginRoutes): void { * `DefaultFilterSchema` (no operator/filterType enforcement — accepted trade-off). * * Fail-soft: returns a `FilterError` describing the problem, or `undefined` - * when the value is valid. The caller (`classifyFilters`) collects returned + * when the value is valid. The caller (`classifyFilters`) throws returned * errors rather than aborting the whole call. * * @param spec - The registered `CustomFilterSpec` for this filter, or `undefined` for ad-hoc @@ -259,99 +279,71 @@ export function validateFilterCall( } // ############################################################################ -// Public — classifyFilters (three-bucket classifier) +// Public — classifyFilters (fail-fast classifier) // ############################################################################ /** - * Fail-soft result of `classifyFilters`. - * - * `result` holds only the keys that passed validation; `errors` aggregates the - * `FilterError`s for keys that failed (those keys are omitted, not thrown on). - */ -export interface ClassifyResult<T = z.infer<typeof OppFiltersSchema>> { - result: T; - errors: FilterError[]; -} - -/** - * Classifies a flat consumer `filters` object into the ADR-0012 `OppFilters` request body. - * - * Three-bucket classification: - * 1. **Default filters** — keys present in `OppDefaultFiltersSchema` (e.g. `status`, - * `closeDateRange`) → top-level named fields on the request body. - * 2. **Registered custom filters** — keys declared in the route-method's `filters` - * spec → `customFilters[name]`. - * 3. **Ad-hoc filters** — unregistered keys (not in defaults, not in spec) → - * `customFilters[name]` passthrough. + * Classifies a flat consumer `filters` object into the ADR-0012 `OppFilters` + * request body, throwing on the first invalid value instead of dropping it. * - * `gov.<system>@<filterName>` namespaced keys are treated as ad-hoc custom - * filter keys and flow into `customFilters` verbatim — no auto-migration. - * - * Validation runs for each key during classification: default fields are checked - * against their real field type (`OppDefaultFiltersSchema`), registered and ad-hoc - * keys via `validateFilterCall`. Validation is **fail-soft**: a key that fails - * is dropped from `result` and its `FilterError` is pushed onto `errors`; the - * call never throws on a bad filter. Valid keys classify normally. + * Three-bucket classification (defaults → top-level + * fields; registered + ad-hoc → `customFilters`), but validation is fail-fast: + * any invalid value — standard, registered, or ad-hoc — throws before a request + * body is produced. Well-formed ad-hoc (unregistered) keys still pass through. * * @param routes - The `PluginRoutes` from the plugin definition * @param resourceKey - The resource name (e.g. `"opportunities"`) * @param methodKey - The method name (e.g. `"search"`) * @param consumerFilters - The flat consumer-facing filters object - * @returns `{ result, errors }` — the valid-only request body and the collected errors + * @returns The classified `OppFilters` request body + * @throws FilterError on the first invalid filter value */ export function classifyFilters( routes: PluginRoutes, resourceKey: string, methodKey: string, consumerFilters: Record<string, unknown> -): ClassifyResult { - // Resolve registered filter specs for this route-method +): z.infer<typeof OppFiltersSchema> { + // Resolve registered filter specs for this route-method. The selectors are + // runtime strings; cast them rather than widening the routes map. const registeredFilters: Record<string, CustomFilterSpec> = - routes[resourceKey]?.[methodKey]?.filters ?? {}; + routes[resourceKey as ResourceName]?.[methodKey as RouteMethod]?.filters ?? {}; const defaultFields: Partial<z.infer<typeof OppDefaultFiltersSchema>> = {}; const customFilters: Record<string, z.infer<typeof DefaultFilterSchema>> = {}; - const errors: FilterError[] = []; for (const [key, value] of Object.entries(consumerFilters)) { - // Look up registered spec (undefined for ad-hoc and gov.* namespaced keys) const spec = registeredFilters[key] as CustomFilterSpec | undefined; if (DEFAULT_FILTER_NAMES.has(key)) { // Bucket 1: default filter → top-level named field, validated against its - // real type from OppDefaultFiltersSchema (e.g. status → StringArrayFilter). - // safeParse doesn't reshape, so the original value is assigned unchanged. - // An invalid value is skipped and its error collected. + // real type from OppDefaultFiltersSchema. An invalid value throws. const fieldSchema = (OppDefaultFiltersSchema.shape as Record<string, z.ZodTypeAny>)[key]; const result = fieldSchema.safeParse(value); if (!result.success) { - errors.push( - new FilterError(`Default filter "${key}" failed validation: ${result.error.message}`, { + throw new FilterError( + `Default filter "${key}" failed validation: ${result.error.message}`, + { path: `filters.${key}`, sourceValue: value, - }) + } ); - continue; } (defaultFields as Record<string, unknown>)[key] = value; } else { - // Bucket 2 (registered custom) or Bucket 3 (ad-hoc / gov.* namespaced) - // Run call-time validation — passes spec if registered, undefined if ad-hoc. - // Fail-soft: collect the returned error and skip the key. + // Bucket 2 (registered custom) or Bucket 3 (ad-hoc / gov.* namespaced). + // An invalid value throws — including a malformed ad-hoc shape. const error = validateFilterCall(spec, key, value); if (error) { - errors.push(error); - continue; + throw error; } customFilters[key] = value as z.infer<typeof DefaultFilterSchema>; } } - // Build request body — omit customFilters key entirely when empty (match nullish shape) - const result: z.infer<typeof OppFiltersSchema> = { + // Omit customFilters key entirely when empty (match nullish shape) + return { ...defaultFields, ...(Object.keys(customFilters).length > 0 ? { customFilters } : {}), }; - - return { result, errors }; } diff --git a/lib/ts-sdk/src/extensions/define-plugin.ts b/lib/ts-sdk/src/extensions/define-plugin.ts index 071613cfd..1c32c2eb1 100644 --- a/lib/ts-sdk/src/extensions/define-plugin.ts +++ b/lib/ts-sdk/src/extensions/define-plugin.ts @@ -18,6 +18,11 @@ import type { import { EXTENSIBLE_SCHEMA_MAP, TransformError } from "./types"; import { withCustomFields, type WithCustomFieldsResult } from "./with-custom-fields"; import { buildTransforms } from "./transforms"; +import { validateRoutes } from "./custom-filters"; +import { buildClientForPlugin } from "../client/resources/builder"; +import type { BuiltClient } from "../client/resources/builder"; +import type { ClientConfig } from "../client/config"; +import type { AuthMethod } from "../client/auth"; // ############################################################################ // Public types - PluginSchemasInput, DefinePluginOptions, Plugin @@ -106,6 +111,12 @@ export interface Plugin< meta?: PluginMeta; /** Route-keyed custom filter declarations, passed through unchanged from `DefinePluginOptions.routes`. */ routes?: TRoutes; + /** + * Builds a client pre-bound to this plugin: responses parse with the plugin's + * compiled schemas by default, and `search({ filters })` types the registered + * filter names — no constructor `routes` or per-call `schema` needed. + */ + getClient(config?: ClientConfig & { auth?: AuthMethod }): BuiltClient<T, TRoutes>; } // ############################################################################ @@ -183,6 +194,11 @@ export function definePlugin< const TRoutes extends PluginRoutes = PluginRoutes, >(options: DefinePluginOptions<T> & { routes?: TRoutes }): Plugin<T, TRoutes> { const { meta, schemas: schemasInput, routes } = options; + + // Runtime backstop for plain-JS authors: a misspelled route or an invalid + // filter registration throws here, at the definition site. + if (routes) validateRoutes(routes); + const schemas: Record<string, object> = {}; for (const [name, extensibleSchema] of Object.entries(EXTENSIBLE_SCHEMA_MAP) as [ @@ -271,7 +287,15 @@ export function definePlugin< // Cast is safe — the runtime loop mirrors the PluginSchemas<T> mapped type, // but TypeScript can't verify that from the dynamic Object.entries() iteration. // The second generic TRoutes preserves the literal routes type from `as const` calls. - return { schemas, meta, routes } as Plugin<T, TRoutes>; + const plugin = { + schemas, + meta, + routes, + getClient(config?: ClientConfig & { auth?: AuthMethod }) { + return buildClientForPlugin(plugin, config); + }, + } as Plugin<T, TRoutes>; + return plugin; } // ############################################################################ diff --git a/lib/ts-sdk/src/extensions/index.ts b/lib/ts-sdk/src/extensions/index.ts index 8878e9c0d..3a45a0ade 100644 --- a/lib/ts-sdk/src/extensions/index.ts +++ b/lib/ts-sdk/src/extensions/index.ts @@ -14,9 +14,17 @@ export type { CustomFieldSpec, HasCustomFields, ExtensibleObject } from "./types export { definePlugin } from "./define-plugin"; // Custom filters — route-keyed filter registration + classification -export type { CustomFilterSpec, CustomFilterType, PluginRoutes, RouteDeclarations } from "./types"; +export type { + CustomFilterSpec, + CustomFilterType, + PluginRoutes, + ResourceName, + RouteDeclarations, + RouteMethod, + RouteMethods, +} from "./types"; export { FilterError } from "./types"; -export type { ClassifyResult } from "./custom-filters"; +export type { CustomFilterInput } from "./custom-filters"; export { classifyFilters, validateRoutes, F } from "./custom-filters"; // Schema utilities — lower-level tools for working with custom fields diff --git a/lib/ts-sdk/src/extensions/types.ts b/lib/ts-sdk/src/extensions/types.ts index d616c863e..541d18169 100644 --- a/lib/ts-sdk/src/extensions/types.ts +++ b/lib/ts-sdk/src/extensions/types.ts @@ -110,11 +110,21 @@ export interface RouteDeclarations { filters?: Record<string, CustomFilterSpec>; } +/** Resource names the SDK exposes. Grows as API resources land. */ +export type ResourceName = "opportunities"; + +/** Route methods that accept custom filters. Widen only when a method actually takes filters. */ +export type RouteMethod = "search"; + +/** Method → declarations map for a single resource. */ +export type RouteMethods = Partial<Record<RouteMethod, RouteDeclarations>>; + /** * Route map for custom filter declarations: resource → method → declarations. * * Passed as `routes` in `definePlugin()`. Filters attach to resource methods - * (which vary asymmetrically across schemas), not to a schema. + * (which vary asymmetrically across schemas), not to a schema. Both key levels + * are closed unions, so a misspelled resource or method is a compile error. * * @example * ```typescript @@ -130,7 +140,7 @@ export interface RouteDeclarations { * }; * ``` */ -export type PluginRoutes = Record<string, Record<string, RouteDeclarations>>; +export type PluginRoutes = Partial<Record<ResourceName, RouteMethods>>; // ############################################################################ // Public types - HasCustomFields, ExtensibleObject From 48ef6a59d725e173324c8ecb2b6c806578c5c3bc Mon Sep 17 00:00:00 2001 From: Billy Daly <billy.daly@agile6.com> Date: Tue, 7 Jul 2026 17:27:02 -0400 Subject: [PATCH 43/51] feat(ts-sdk): strengthen ad hoc filter validation (#982) * feat(ts-sdk): enforce operator/value structure for ad-hoc filters Ad-hoc (unregistered) filters were validated shape-only against DefaultFilterSchema, which accepted any value for a known operator. They now validate against FILTER_TYPE_SCHEMAS, the same map registered filters use: an ad-hoc filter is valid when its {operator, value} is well-formed for some known filterType. This reuses the single source of truth (and picks up new filter types automatically) rather than a parallel schema, and rejects combinations no filter family supports (e.g. `in` with a non-array value, `between` with a scalar). On a mismatch, FilterError carries a targeted, operator-aware message instead of the value passing through. * docs(ts-sdk): rework custom-filters example and teach the mock server custom filters Restructure examples/custom-filters.ts into three scenarios: happy path (mock server, mixed default/registered/ad-hoc filters with a typed custom field), authoring errors (default-name collision, unknown filterType, misspelled resource/method caught at compile time and at runtime), and consumer errors (registered wrong value family, ad-hoc operator/value mismatch). Upgrade the mock server to apply the standard status filter, echo received filters (including customFilters) into filterInfo, and carry a programCode custom field. Export createMockServer() with a guarded auto-start so `pnpm example:server` still runs it standalone. --- .changeset/ts-typed-custom-filters.md | 2 +- .../extensions/custom-filters.spec.ts | 45 ++- lib/ts-sdk/examples/custom-filters.ts | 379 +++++++++--------- lib/ts-sdk/examples/mock-api-server.ts | 233 ++++++----- lib/ts-sdk/src/extensions/custom-filters.ts | 46 ++- 5 files changed, 402 insertions(+), 303 deletions(-) diff --git a/.changeset/ts-typed-custom-filters.md b/.changeset/ts-typed-custom-filters.md index 41b0a00f5..c193c48ae 100644 --- a/.changeset/ts-typed-custom-filters.md +++ b/.changeset/ts-typed-custom-filters.md @@ -6,7 +6,7 @@ Typed custom-filters authoring surface and plugin-scoped client for the TypeScri - `definePlugin()` route keys are closed unions: a misspelled resource or method is a compile error, and an invalid registration throws `FilterError` at definition time. - `plugin.getClient(config)` returns a client pre-bound to the plugin's schemas and registered filters; responses parse with the plugin schema by default (per-call `schema` override still wins). -- Filter validation is fail-fast: an invalid value on any filter — standard, registered custom, or ad-hoc — throws `FilterError` before the request is sent (compile error where the type is known). Well-formed ad-hoc filters still pass through; `filterInfo.errors` now carries server-returned errors only. +- Filter validation is fail-fast: an invalid value on any filter (standard, registered custom, or ad-hoc) throws `FilterError` before the request is sent (a compile error where the type is known). Well-formed ad-hoc filters still pass through, but an ad-hoc filter whose operator and value structure disagree (e.g. `in` with a non-array value) also throws; `filterInfo.errors` only includes errors returned by the server. - `search()`/`list()` return `{ items, errors }`: valid rows in `items`, per-row `ParseFailure` (`index`, `raw`, `error`) in `errors`; `onParseError: "throw"` opts into fail-hard, and `get()` remains fail-hard. - Breaking: `new Client(...)` no longer accepts `routes` (bind them via `plugin.getClient()`); `classifyFilters` now throws `FilterError` instead of returning collected errors, and `ClassifyResult` is removed. - Breaking: the `Opportunities` constructor is now `(client, boundSchema?, routes?)` — `routes` moved from the second to the third parameter (a compile error at existing two-argument call sites). diff --git a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts index dac615fbf..3e63542f6 100644 --- a/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts +++ b/lib/ts-sdk/__tests__/extensions/custom-filters.spec.ts @@ -205,27 +205,55 @@ describe("route and filter-call validation", () => { ).toBeUndefined(); }); - it("returns undefined for an ad-hoc filter with a valid shape (no operator enforcement)", () => { - // Ad-hoc (spec=undefined) — any valid DefaultFilter shape passes + it("returns undefined for an ad-hoc filter with a coherent operator/value pair", () => { + // Ad-hoc (spec=undefined) — operator pairs with the right value structure expect( validateFilterCall(undefined, "legacyTag", { operator: "eq", value: "legacy-2024" }) ).toBeUndefined(); + expect( + validateFilterCall(undefined, "regions", { operator: "in", value: ["US-CA", "US-NY"] }) + ).toBeUndefined(); + expect( + validateFilterCall(undefined, "amountRange", { + operator: "between", + value: { min: 100, max: 500 }, + }) + ).toBeUndefined(); }); it("returns a FilterError for an ad-hoc filter with an invalid shape", () => { - // Missing `operator` key — fails DefaultFilterSchema shape check (operator is required/enum) + // Missing `operator` key — no branch of AdHocFilterSchema matches const err = validateFilterCall(undefined, "badFilter", { value: "something" }); expect(err).toBeInstanceOf(FilterError); }); it("returns a FilterError for an ad-hoc filter with an unknown operator", () => { - // `superCustomOp` is not in AllOperatorsEnum — fails DefaultFilterSchema + // `superCustomOp` is not a known operator — no branch matches const err = validateFilterCall(undefined, "badFilter", { operator: "superCustomOp", value: "x", }); expect(err).toBeInstanceOf(FilterError); }); + + it("returns a FilterError for an ad-hoc operator/value mismatch", () => { + // `in` needs an array value + expect( + validateFilterCall(undefined, "regions", { operator: "in", value: "US-CA" }) + ).toBeInstanceOf(FilterError); + // `between` needs a { min, max } object, not a scalar + expect( + validateFilterCall(undefined, "amountRange", { operator: "between", value: 500 }) + ).toBeInstanceOf(FilterError); + // `like` needs a string + expect( + validateFilterCall(undefined, "title", { operator: "like", value: 42 }) + ).toBeInstanceOf(FilterError); + // `eq` needs a scalar, not an array + expect( + validateFilterCall(undefined, "code", { operator: "eq", value: ["A", "B"] }) + ).toBeInstanceOf(FilterError); + }); }); }); @@ -345,6 +373,15 @@ describe("classifyFilters", () => { expect(result.customFilters?.adHocKey).toBeDefined(); }); + it("throws FilterError on an ad-hoc operator/value mismatch", () => { + expect(() => + classifyFilters(grantsGovRoutes, "opportunities", "search", { + // `in` requires an array value + adHocKey: { operator: "in", value: "not-an-array" }, + }) + ).toThrow(FilterError); + }); + it("omits customFilters entirely when only defaults are present", () => { const result = classifyFilters(grantsGovRoutes, "opportunities", "search", { status: { operator: "in", value: ["open"] }, diff --git a/lib/ts-sdk/examples/custom-filters.ts b/lib/ts-sdk/examples/custom-filters.ts index 4119f3ef9..fffdb0e8a 100644 --- a/lib/ts-sdk/examples/custom-filters.ts +++ b/lib/ts-sdk/examples/custom-filters.ts @@ -1,255 +1,236 @@ /** - * Example script demonstrating custom-filters registration and classification. Shows: - * 1. Defining a grants.gov plugin with route-keyed custom filter specs via - * `definePlugin({ routes: { ... } } as const)`. - * 2. Building a unified consumer `filters` object mixing default fields - * (status, closeDateRange), pre-registered custom filters (agency, - * fundingProgram), and an ad-hoc filter (legacyTag) — using `F.*` helpers. - * Control fields (query, maxResults, signal, schema) stay OUTSIDE `filters`. - * 3. Calling `classifyFilters` to produce the ADR-0012 request body: - * default fields at top-level, custom + ad-hoc under `customFilters` — - * an invalid value on any key throws `FilterError` before a body is built. - * 4. Building a plugin-bound client via `plugin.getClient()` and showing the - * fail-fast guard: an invalid registered filter value rejects BEFORE any - * HTTP request (so this step needs no server). - * 5. A COMMENT block (not executed) demonstrating the `as const` widening - * trap — see custom-filters-types.ts for the compile-time narrowing assertions. + * Example: the custom-filters authoring and consumer surface, end to end. + * + * Three scenarios, each isolated in its own function: + * + * 1. Happy path (mock server): define a plugin with a custom field and a + * registered custom filter, build a client with `plugin.getClient()`, and + * search with a mix of default, registered, and ad-hoc filters. + * 2. Authoring error: `definePlugin()` rejects an invalid registration + * (a filter named after a default field; an unknown filterType). + * 3. Consumer error: `search()` rejects a bad filter value with `FilterError` + * before any HTTP request, for both a registered filter (wrong value + * family) and an ad-hoc filter (operator/value structure mismatch). * * Run with: `pnpm example:custom-filters` * - * @remarks - * The three-bucket classification rule (ADR-0012): - * - Default filters (status, closeDateRange, ...) → named top-level fields on the request body. - * - Pre-registered custom filters (agency, fundingProgram) → `customFilters` record. - * - Ad-hoc filters (legacyTag) → `customFilters` passthrough (shape-only validated). + * Scenario 1 talks to the mock server; start it first in another terminal with + * `pnpm example:server`. If it is not running, scenario 1 is skipped and the two + * offline scenarios still run. * - * No network I/O: this example BUILDS the request body and prints it; it does NOT - * send it over the wire — transport is not handled here. + * Filter buckets (ADR-0012): + * - Default filters (status, closeDateRange, ...) -> named top-level fields. + * - Registered custom filters (region) -> `customFilters` record, validated + * against their declared filterType. + * - Ad-hoc filters (any other key) -> `customFilters` passthrough, validated + * for operator/value structure only (no element-type check). */ -import { classifyFilters, definePlugin, F, FilterError } from "../src/extensions"; +import { definePlugin, F, FilterError } from "../src/extensions"; -// ############################################################################ -// Step 1 — Define the grants.gov plugin with route-keyed custom filters -// ############################################################################ +const BASE_URL = "http://localhost:8000"; -// `as const` is load-bearing: it preserves literal `filterType` values so -// that TypeScript can narrow call-site filter keys, operators, and value shapes. -// Without `as const`, `filterType` widens to `string` and the typed guard is lost. -// See custom-filters-types.ts for the compile-time proof. +// `as const` is load-bearing: it preserves the literal `filterType` values so +// `search({ filters })` can narrow filter names, operators, and value shapes. const grantsGovPlugin = definePlugin({ meta: { name: "grants.gov", version: "0.1.0", sourceSystem: "grants.gov", - capabilities: ["customFilters"], + capabilities: ["customFields", "customFilters"], + }, + schemas: { + Opportunity: { + customFields: { + programCode: { fieldType: "string", description: "Funding program code" }, + }, + }, }, routes: { opportunities: { search: { filters: { - agency: { + region: { filterType: "stringArray", - description: "Filter by funding agency code (e.g. 'HHS', 'DOE')", - }, - fundingProgram: { - filterType: "stringComparison", - description: "Filter by funding program name", + description: "Filter by region code (e.g. 'US-CA')", }, - // NOTE: `as const` is what makes these `filterType` values - // literal strings ("stringArray", "stringComparison") instead of - // widened `string`. Without it, the typed narrowing layer collapses. }, }, }, }, } as const); -console.log("=== Step 1: Plugin registered ==="); -console.log(`Plugin name: ${grantsGovPlugin.meta?.name}`); -console.log( - `Registered filters: ${Object.keys(grantsGovPlugin.routes?.opportunities?.search?.filters ?? {}).join(", ")}` -); +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + console.error(`ASSERTION FAILED: ${message}`); + process.exit(1); + } +} + +/** Collapse a (possibly multi-line) error message to a single readable line. */ +function oneLine(message: string, max = 140): string { + const collapsed = message.replace(/\s+/g, " ").trim(); + return collapsed.length > max ? `${collapsed.slice(0, max)}...` : collapsed; +} // ############################################################################ -// Step 2 — Build a unified consumer filters object +// Scenario 1 — Happy path (mock server) // ############################################################################ -// The consumer-facing `filters` object is flat — it mixes all three filter types -// without the caller needing to know which bucket each belongs to. -// -// IMPORTANT: control fields (query, maxResults, signal, schema) stay OUTSIDE -// the `filters` object. The classifier handles only the filter predicates. -const searchParams = { - // Control fields — outside `filters` - query: "conservation research", - maxResults: 25, - - // The flat consumer filters object - filters: { - // Default filters — will land as named top-level fields on the request body - status: F.in(["open", "forecasted"]), - closeDateRange: F.between("2025-01-01", "2025-12-31"), - - // Pre-registered custom filters — will land in customFilters record - agency: F.in(["HHS", "DOE", "NSF"]), - fundingProgram: F.like("*Conservation*"), - - // Ad-hoc filter (not registered in the plugin) — flows to customFilters verbatim - // (shape-only validated; operator/filterType not enforced for ad-hoc keys) - legacyTag: F.eq("conservation-2024"), - }, -}; +async function happyPath(): Promise<void> { + console.log("=== Scenario 1: happy path (mock server) ==="); -console.log("\n=== Step 2: Consumer filters built ==="); -console.log("Consumer-facing filters (flat):"); -console.log(JSON.stringify(searchParams.filters, null, 2)); + const client = grantsGovPlugin.getClient({ baseUrl: BASE_URL }); -// ############################################################################ -// Step 3 — Classify into the ADR-0012 OppFilters request body -// ############################################################################ + // A flat filters bag mixing all three buckets. The registered `region` key + // autocompletes and type-checks its value; `status` is a default field; + // `fundingType` is an ad-hoc passthrough. + const filters = { + status: F.in(["open"]), // default filter -> top-level field + region: F.in(["US-CA", "US-NY"]), // registered custom filter -> customFilters + fundingType: F.eq("grant"), // ad-hoc, well-formed -> customFilters passthrough + }; -// `classifyFilters` runs the three-bucket classification: -// 1. status, closeDateRange → top-level named request-body fields -// 2. agency, fundingProgram (registered) → customFilters record -// 3. legacyTag (ad-hoc) → customFilters passthrough -// `grantsGovPlugin.routes` is non-null here — we declared it above as a -// non-optional literal object. The `!` assertion removes the `undefined` from -// the union type that `Plugin.routes?:` introduces (routes is optional in the -// interface to support plugins that don't declare filters). -// `classifyFilters` is fail-fast: an invalid value on any key — standard, -// registered, or ad-hoc — throws `FilterError` before a request body exists. -const requestBody = classifyFilters( - grantsGovPlugin.routes!, - "opportunities", - "search", - searchParams.filters -); - -console.log("\n=== Step 3: Classified request body (OppFilters) ==="); -console.log(JSON.stringify(requestBody, null, 2)); + try { + const result = await client.opportunities.search({ filters }); + + console.log(` items returned: ${result.items.length}`); + console.log(` per-row parse failures: ${result.errors.length}`); + if (result.items[0]) { + // The plugin schema is bound by default, so customFields are typed without + // passing a per-call schema. + const programCode = result.items[0].customFields?.programCode?.value; + console.log(` first opportunity: ${result.items[0].title}`); + console.log(` first programCode custom field: ${programCode ?? "(none)"}`); + } + // The mock server echoes the received filters, so we can confirm the + // registered (region) and ad-hoc (fundingType) filters reached the wire. + console.log(` filters received by server: ${JSON.stringify(result.filterInfo?.filters)}`); + } catch (e) { + // A missing server should not fail the example; the offline scenarios below + // assert behavior deterministically without a server. + console.log(` (skipped: could not reach ${BASE_URL}; run \`pnpm example:server\` first)`); + console.log(` ${(e as Error).message}`); + } +} // ############################################################################ -// Assertions — verify the three-bucket classification +// Scenario 2 — Authoring error (definePlugin rejects a bad registration) // ############################################################################ -function fail(message: string): never { - console.error(`ASSERTION FAILED: ${message}`); - process.exit(1); -} +function authoringErrors(): void { + console.log("\n=== Scenario 2: authoring errors (definePlugin) ==="); -// Default filters must appear as named top-level fields -if (!requestBody.status) fail("status should be a top-level field (default filter bucket)"); -if (!requestBody.closeDateRange) - fail("closeDateRange should be a top-level field (default filter bucket)"); - -// Custom + ad-hoc filters must land in customFilters record -if (!requestBody.customFilters) fail("customFilters should exist for registered + ad-hoc filters"); -if (!requestBody.customFilters.agency) - fail("agency should be in customFilters (registered custom filter)"); -if (!requestBody.customFilters.fundingProgram) - fail("fundingProgram should be in customFilters (registered custom filter)"); -if (!requestBody.customFilters.legacyTag) - fail("legacyTag should be in customFilters (ad-hoc passthrough)"); - -// Default filter keys must NOT appear under customFilters -if ((requestBody.customFilters as Record<string, unknown>).status) - fail("status must NOT be in customFilters — it is a default filter"); - -// Fail-fast demo: a wrong value family on a registered filter throws -// FilterError before any request body is produced. -try { - classifyFilters(grantsGovPlugin.routes!, "opportunities", "search", { - agency: { operator: "eq", value: 42 }, // agency is a stringArray filter - }); - fail("expected classifyFilters to throw FilterError for an invalid registered value"); -} catch (e) { - if (!(e instanceof FilterError)) throw e; - console.log(`\n=== Fail-fast demo ===`); - console.log(`FilterError (expected): ${e.message.split("\n")[0]}`); -} + // 2a. A custom filter named after a default field (`status`) is rejected: + // it would otherwise shadow the default bucket at classification time. + try { + definePlugin({ + routes: { + opportunities: { search: { filters: { status: { filterType: "stringArray" } } } }, + }, + } as const); + assert(false, "expected definePlugin to reject a default-name collision"); + } catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log(` default-name collision rejected: ${oneLine(e.message)}`); + } -console.log("\n=== Assertions passed ==="); -console.log(" status → top-level (default filter bucket)"); -console.log(" closeDateRange → top-level (default filter bucket)"); -console.log(" agency → customFilters (registered custom filter)"); -console.log(" fundingProgram → customFilters (registered custom filter)"); -console.log(" legacyTag → customFilters (ad-hoc passthrough)"); + // 2b. An unknown filterType is rejected. A typed author gets a compile error + // here; the cast simulates a plain-JS author hitting the runtime backstop. + try { + definePlugin({ + routes: { + opportunities: { + search: { filters: { region: { filterType: "strin" as never } } }, + }, + }, + } as const); + assert(false, "expected definePlugin to reject an unknown filterType"); + } catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log(` unknown filterType rejected: ${oneLine(e.message)}`); + } + + // 2c. A misspelled resource name. Route keys are a closed union, so this is a + // compile error (the `@ts-expect-error` below proves the type checker flags + // it); an untyped caller hits the runtime backstop and throws the same way. + try { + definePlugin({ + routes: { + // @ts-expect-error - "opportunties" is not a known resource name + opportunties: { search: { filters: { region: { filterType: "stringArray" } } } }, + }, + } as const); + assert(false, "expected definePlugin to reject a misspelled resource"); + } catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log(` misspelled resource rejected: ${oneLine(e.message)}`); + } + + // 2d. A misspelled method name is caught the same way: a compile error, with + // a runtime backstop for untyped callers. + try { + definePlugin({ + routes: { + opportunities: { + // @ts-expect-error - "serach" is not a known route method + serach: { filters: { region: { filterType: "stringArray" } } }, + }, + }, + } as const); + assert(false, "expected definePlugin to reject a misspelled method"); + } catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log(` misspelled method rejected: ${oneLine(e.message)}`); + } +} // ############################################################################ -// Step 5 — plugin.getClient(): the consumer path +// Scenario 3 — Consumer error (search rejects a bad value before any request) // ############################################################################ -// The client is pre-bound to the plugin: responses parse with the plugin's -// compiled schema by default, and `search({ filters })` types the registered -// filter names. Against a live API (e.g. `pnpm example:server`), the consumer -// flow looks like: -// -// const result = await client.opportunities.search({ -// filters: { agency: F.in(["HHS"]) }, -// }); -// for (const opp of result.items) console.log(opp.title); // valid rows -// // ParseFailure rows — err.raw may carry PII; log a redacted projection -// for (const err of result.errors) console.log(err.index, err.error.message); -// -// (Executed versions live in __tests__/extensions/get-client.spec.ts.) -const client = grantsGovPlugin.getClient({ baseUrl: "http://localhost:8000" }); - -// Fail-fast without a server: an invalid registered value throws BEFORE any -// HTTP request is made, so this rejects even though nothing is listening. -// (Async so this file compiles under CommonJS module settings — no top-level -// await; the final logs chain off it below so output order matches source order.) -async function step5FailFastDemo(): Promise<void> { +async function consumerErrors(): Promise<void> { + console.log("\n=== Scenario 3: consumer errors (search, no request sent) ==="); + + const client = grantsGovPlugin.getClient({ baseUrl: BASE_URL }); + + // 3a. A registered filter with the wrong value family. `region` is a + // stringArray, so `F.eq()` (a scalar) is a compile error; the cast simulates + // a plain-JS caller hitting the runtime backstop. + try { + await client.opportunities.search({ + filters: { region: F.eq("US-CA") } as never, + }); + assert(false, "expected a FilterError for a wrong registered value family"); + } catch (e) { + if (!(e instanceof FilterError)) throw e; + console.log(` registered wrong value family rejected: ${oneLine(e.message)}`); + } + + // 3b. An ad-hoc filter with an incoherent operator/value pair. `in` requires + // an array value; a scalar is rejected. Ad-hoc keys are accepted at the type + // level (they support arbitrary escape-hatch filters), so this is caught at + // runtime rather than compile time. try { await client.opportunities.search({ - // Wrong value family for a stringArray filter — also a compile error; - // the cast simulates a plain-JS caller hitting the runtime backstop. - filters: { agency: { operator: "eq", value: 42 } } as never, + filters: { fundingType: { operator: "in", value: "grant" } }, }); - fail("expected search() to reject with FilterError before any request"); + assert(false, "expected a FilterError for an ad-hoc operator/value mismatch"); } catch (e) { if (!(e instanceof FilterError)) throw e; - console.log("\n=== Step 5: getClient fail-fast demo ==="); - console.log(`search() rejected before any request (expected): ${e.message.split("\n")[0]}`); - console.log("\n✓ getClient consumer path complete"); + console.log(` ad-hoc operator/value mismatch rejected: ${oneLine(e.message)}`); } } // ############################################################################ -// Step 6 — The `as const` widening trap (comment block — not executed) +// Run all scenarios in source order // ############################################################################ -// If you forget `as const` on the definePlugin call, TypeScript widens the -// `filterType` values from literal strings to the broad `string` type. The -// TypedConsumerFilters narrowing layer then cannot distinguish filter keys or -// validate operator/value shapes at the call site — unknown keys and wrong -// value types silently pass the type checker. -// -// The compile-time narrowing assertions (the errors that fire WITH `as const`) -// live in: lib/ts-sdk/__tests__/extensions/custom-filters-types.ts -// -// Example of what NOT to do: -// -// const badPlugin = definePlugin({ -// routes: { -// opportunities: { -// search: { filters: { agency: { filterType: "stringArray" } } }, -// }, -// }, -// }); // ← MISSING `as const` -// -// // When `badPlugin` is stored as `Plugin` (no TRoutes generic), -// // or when `as const` is omitted and the function's const-generic cannot -// // infer the literal, the typed guard collapses and the following would NOT -// // be caught at compile time: -// // badPlugin.routes?.opportunities?.search?.filters // type: Record<string, CustomFilterSpec> -// // // → literal filterTypes are lost, so per-key narrowing is impossible -// // // → unknown keys, wrong operators, and wrong value shapes silently accepted - -// Chained after Step 5 settles: the completion banner provably prints last. -void step5FailFastDemo().then(() => { +async function main(): Promise<void> { + await happyPath(); + authoringErrors(); + await consumerErrors(); console.log("\n✓ custom-filters example complete"); - console.log( - " See __tests__/extensions/custom-filters-types.ts for compile-time narrowing proof" - ); -}); +} + +void main(); diff --git a/lib/ts-sdk/examples/mock-api-server.ts b/lib/ts-sdk/examples/mock-api-server.ts index d45f7f7e8..a5a683a87 100644 --- a/lib/ts-sdk/examples/mock-api-server.ts +++ b/lib/ts-sdk/examples/mock-api-server.ts @@ -5,9 +5,14 @@ * pnpm example:server * * Then run any example (get, list, search, get-custom-fields) against http://localhost:8000. + * + * The search route understands `filters`: it applies the standard `status` + * filter, and echoes every received filter (including `customFilters`) back in + * `filterInfo.filters`, so examples can show that registered and ad-hoc custom + * filters reached the server. Import `createMockServer()` to run it in-process. */ -import { createServer } from "http"; +import { createServer, type Server } from "http"; const PORT = parseInt(process.env.PORT ?? "8000", 10); @@ -30,6 +35,12 @@ const MOCK_OPPORTUNITIES = [ value: 12345, description: "Legacy system opportunity ID", }, + programCode: { + name: "programCode", + fieldType: "string", + value: "STEM-ED", + description: "Funding program code", + }, }, }, { @@ -46,6 +57,12 @@ const MOCK_OPPORTUNITIES = [ value: 12346, description: "Legacy system opportunity ID", }, + programCode: { + name: "programCode", + fieldType: "string", + value: "COMM-DEV", + description: "Funding program code", + }, }, }, { @@ -62,6 +79,12 @@ const MOCK_OPPORTUNITIES = [ value: 12347, description: "Legacy system opportunity ID", }, + programCode: { + name: "programCode", + fieldType: "string", + value: "EDU-INIT", + description: "Funding program code", + }, }, }, ]; @@ -86,110 +109,134 @@ function parseQuery(url: string): Record<string, string> { return out; } +/** Status values carried by an opportunity, for the standard `status` filter. */ +function statusOf(opp: (typeof MOCK_OPPORTUNITIES)[number]): string { + return opp.status.value; +} + // ============================================================================= // Mock API Server // ============================================================================= -const server = createServer((req, res) => { - const url = req.url ?? ""; - const method = req.method ?? "GET"; - const path = url.split("?")[0]; - - // ============================================================================= - // GET /common-grants/opportunities - // ============================================================================= - - if (method === "GET" && path === "/common-grants/opportunities") { - const q = parseQuery(url); - const page = Math.max(1, parseInt(q.page ?? "1", 10)); - const pageSize = Math.min(100, Math.max(1, parseInt(q.pageSize ?? "25", 10))); - const start = (page - 1) * pageSize; - const items = MOCK_OPPORTUNITIES.slice(start, start + pageSize); - send(res, 200, { - status: 200, - message: "Success", - items, - paginationInfo: { - page, - pageSize, - totalItems: MOCK_OPPORTUNITIES.length, - totalPages: Math.ceil(MOCK_OPPORTUNITIES.length / pageSize), - }, - }); - return; - } - - // ============================================================================= - // GET /common-grants/opportunities/:id - // ============================================================================= - - if (method === "GET" && path.startsWith("/common-grants/opportunities/")) { - const id = path.slice("/common-grants/opportunities/".length); - const opp = MOCK_OPPORTUNITIES.find(o => o.id === id) ?? { - id, - title: "Mock Opportunity", - description: "Mock opportunity for examples.", - status: { value: "open" }, - createdAt: "2025-01-01T00:00:00Z", - lastModifiedAt: "2025-01-01T00:00:00Z", - }; - send(res, 200, { - status: 200, - message: "Success", - data: opp, - }); - return; - } - // ============================================================================= - // POST /common-grants/opportunities/search - // ============================================================================= - - if (method === "POST" && path === "/common-grants/opportunities/search") { - let body = ""; - req.on("data", chunk => { - body += chunk; - }); - req.on("end", () => { - let items = [...MOCK_OPPORTUNITIES]; - try { - const parsed = body ? (JSON.parse(body) as { search?: string }) : {}; - if (parsed.search && typeof parsed.search === "string") { - const q = parsed.search.toLowerCase(); - items = items.filter(o => o.title.toLowerCase().includes(q)); - } - } catch { - // ignore - } +/** Builds the mock server without starting it, so examples can run it in-process. */ +export function createMockServer(): Server { + return createServer((req, res) => { + const url = req.url ?? ""; + const method = req.method ?? "GET"; + const path = url.split("?")[0]; + + // ========================================================================= + // GET /common-grants/opportunities + // ========================================================================= + + if (method === "GET" && path === "/common-grants/opportunities") { + const q = parseQuery(url); + const page = Math.max(1, parseInt(q.page ?? "1", 10)); + const pageSize = Math.min(100, Math.max(1, parseInt(q.pageSize ?? "25", 10))); + const start = (page - 1) * pageSize; + const items = MOCK_OPPORTUNITIES.slice(start, start + pageSize); send(res, 200, { status: 200, message: "Success", items, paginationInfo: { - page: 1, - pageSize: items.length > 0 ? items.length : 1, - totalItems: items.length, - totalPages: 1, + page, + pageSize, + totalItems: MOCK_OPPORTUNITIES.length, + totalPages: Math.ceil(MOCK_OPPORTUNITIES.length / pageSize), }, - sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, - filterInfo: { filters: {} }, }); - }); - return; - } + return; + } + + // ========================================================================= + // GET /common-grants/opportunities/:id + // ========================================================================= + + if (method === "GET" && path.startsWith("/common-grants/opportunities/")) { + const id = path.slice("/common-grants/opportunities/".length); + const opp = MOCK_OPPORTUNITIES.find(o => o.id === id) ?? { + id, + title: "Mock Opportunity", + description: "Mock opportunity for examples.", + status: { value: "open" }, + createdAt: "2025-01-01T00:00:00Z", + lastModifiedAt: "2025-01-01T00:00:00Z", + }; + send(res, 200, { + status: 200, + message: "Success", + data: opp, + }); + return; + } - send(res, 404, { status: 404, message: "Not found" }); -}); + // ========================================================================= + // POST /common-grants/opportunities/search + // ========================================================================= + + if (method === "POST" && path === "/common-grants/opportunities/search") { + let body = ""; + req.on("data", chunk => { + body += chunk; + }); + req.on("end", () => { + let items = [...MOCK_OPPORTUNITIES]; + let filters: Record<string, unknown> = {}; + try { + const parsed = body + ? (JSON.parse(body) as { search?: string; filters?: Record<string, unknown> }) + : {}; + if (parsed.search && typeof parsed.search === "string") { + const q = parsed.search.toLowerCase(); + items = items.filter(o => o.title.toLowerCase().includes(q)); + } + filters = parsed.filters ?? {}; + // Apply the standard `status` filter; custom filters are echoed but + // not applied by this mock. + const status = filters.status as { value?: unknown } | undefined; + if (Array.isArray(status?.value)) { + const allowed = new Set(status.value as string[]); + items = items.filter(o => allowed.has(statusOf(o))); + } + } catch { + // ignore malformed bodies + } + send(res, 200, { + status: 200, + message: "Success", + items, + paginationInfo: { + page: 1, + pageSize: items.length > 0 ? items.length : 1, + totalItems: items.length, + totalPages: 1, + }, + sortInfo: { sortBy: "lastModifiedAt", sortOrder: "desc" }, + // Echo the received filters so a consumer can confirm registered and + // ad-hoc custom filters reached the server. + filterInfo: { filters }, + }); + }); + return; + } + + send(res, 404, { status: 404, message: "Not found" }); + }); +} // ============================================================================= -// Start Server +// Start Server (only when run directly, e.g. `pnpm example:server`) // ============================================================================= -server.listen(PORT, () => { - console.log(`Mock CommonGrants API listening on http://localhost:${PORT}`); - console.log(""); - console.log("Run examples in another terminal:"); - console.log(" pnpm example:list"); - console.log(" pnpm example:get 573525f2-8e15-4405-83fb-e6523511d893"); - console.log(" pnpm example:search education"); - console.log(" pnpm example:get-custom-fields 573525f2-8e15-4405-83fb-e6523511d893"); - console.log(""); -}); +if (process.argv[1]?.includes("mock-api-server")) { + createMockServer().listen(PORT, () => { + console.log(`Mock CommonGrants API listening on http://localhost:${PORT}`); + console.log(""); + console.log("Run examples in another terminal:"); + console.log(" pnpm example:list"); + console.log(" pnpm example:get 573525f2-8e15-4405-83fb-e6523511d893"); + console.log(" pnpm example:search education"); + console.log(" pnpm example:get-custom-fields 573525f2-8e15-4405-83fb-e6523511d893"); + console.log(""); + }); +} diff --git a/lib/ts-sdk/src/extensions/custom-filters.ts b/lib/ts-sdk/src/extensions/custom-filters.ts index a0821d384..2a5a22e61 100644 --- a/lib/ts-sdk/src/extensions/custom-filters.ts +++ b/lib/ts-sdk/src/extensions/custom-filters.ts @@ -215,6 +215,22 @@ export function validateRoutes(routes: PluginRoutes): void { } } +/** The value shape each operator expects, used to explain an ad-hoc mismatch. */ +const AD_HOC_VALUE_EXPECTATION: Record<string, string> = { + eq: "a scalar value", + neq: "a scalar value", + gt: "a scalar value", + gte: "a scalar value", + lt: "a scalar value", + lte: "a scalar value", + in: "an array value", + notIn: "an array value", + like: "a string value", + notLike: "a string value", + between: "a { min, max } object", + outside: "a { min, max } object", +}; + // ############################################################################ // Public — validateFilterCall (call-time validation) // ############################################################################ @@ -225,8 +241,10 @@ export function validateRoutes(routes: PluginRoutes): void { * - For REGISTERED filters (spec provided): validates the `{operator, value}` * pair against the filterType's Zod schema — each schema constrains both the * allowed operator enum and the value shape, so one parse covers both checks. - * - For AD-HOC filters (spec is undefined): shape-only check against - * `DefaultFilterSchema` (no operator/filterType enforcement — accepted trade-off). + * - For AD-HOC filters (spec is undefined): the `{operator, value}` pair must be + * well-formed for some known filterType (checked against `FILTER_TYPE_SCHEMAS`, + * the same map registered filters use). The element type is not pinned to one + * type, since ad-hoc filters carry no `filterType`. * * Fail-soft: returns a `FilterError` describing the problem, or `undefined` * when the value is valid. The caller (`classifyFilters`) throws returned @@ -245,11 +263,27 @@ export function validateFilterCall( const path = `filters.${filterName}`; if (spec === undefined) { - // Ad-hoc filter — shape-only check against DefaultFilterSchema - const result = DefaultFilterSchema.safeParse(filterValue); - if (!result.success) { + // Ad-hoc filter — no declared filterType, so accept it when its + // {operator, value} is well-formed for SOME known filterType. This reuses + // FILTER_TYPE_SCHEMAS (the single source of truth) rather than a parallel + // schema, and picks up new filter types automatically. + const matchesKnownType = Object.values(FILTER_TYPE_SCHEMAS).some( + schema => schema.safeParse(filterValue).success + ); + if (!matchesKnownType) { + // Derive a targeted message from the operator rather than surfacing the + // combined Zod error across every candidate schema. + const operator = (filterValue as { operator?: unknown } | null)?.operator; + let detail: string; + if (typeof operator !== "string") { + detail = "expected a { operator, value } object"; + } else if (!(operator in AD_HOC_VALUE_EXPECTATION)) { + detail = `operator "${operator}" is not a known filter operator`; + } else { + detail = `operator "${operator}" expects ${AD_HOC_VALUE_EXPECTATION[operator]}`; + } return new FilterError( - `Ad-hoc filter "${filterName}" has an invalid shape: ${result.error.message}`, + `Ad-hoc filter "${filterName}" has an invalid operator/value combination: ${detail}`, { path, sourceValue: filterValue } ); } From 037fb61925a25c86d9756ef8cdcc1132c7c7bb02 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <bryan@snowboardtechie.com> Date: Tue, 7 Jul 2026 15:04:58 -0700 Subject: [PATCH 44/51] fix(py-sdk): import GrantsGovOpportunitySchema from cg_grants_gov.models The grants-gov transforms example landed on main under the mypy gate; HOLD-filters' pyright gate flags the top-level import as private (cg_grants_gov.__all__ only exports grants_gov). Import from the models module, which declares it. --- lib/python-sdk/examples/grants_gov_transforms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/python-sdk/examples/grants_gov_transforms.py b/lib/python-sdk/examples/grants_gov_transforms.py index f5b4b1ca5..886028202 100755 --- a/lib/python-sdk/examples/grants_gov_transforms.py +++ b/lib/python-sdk/examples/grants_gov_transforms.py @@ -17,7 +17,8 @@ import sys from typing import Any -from cg_grants_gov import GrantsGovOpportunitySchema, grants_gov +from cg_grants_gov import grants_gov +from cg_grants_gov.models import GrantsGovOpportunitySchema from common_grants_sdk.client import Client From 19fdd8f54830576e2f2c5f5a8401937543a524f4 Mon Sep 17 00:00:00 2001 From: Billy Daly <billy.daly@agile6.com> Date: Thu, 9 Jul 2026 10:54:35 -0400 Subject: [PATCH 45/51] fix(py-sdk): align python and typescript (#988) * feat(py-sdk): add integer comparison filter Plugin authors can register an integer comparison filter, and consumers can filter by it. It accepts the comparison and equivalence operators against an integer value, and rejects a value that is not an integer, such as 3.5 or a boolean. * feat(py-sdk): validate custom filters in define_plugin() An invalid filter registration is now caught when the plugin author defines the plugin, instead of later when a consumer builds a client. The author sees the mistake right away rather than passing it downstream to whoever uses the plugin. * feat(py-sdk)!: fail fast on invalid custom filters An invalid filter value now raises a FilterError before the request is sent, whether it is a standard, registered, or ad-hoc filter. An invalid ad-hoc filter used to be dropped without a word, which quietly widened the consumer's search instead of telling them they made a mistake; now an ad-hoc filter whose value does not fit its operator, for example "in" with a plain string instead of a list, is rejected too. Only real filter types are accepted as custom filters. BREAKING CHANGE: classify_filters returns the request body directly and raises on the first invalid filter. ClassifyResult and FilterError.strict, which only supported the old drop-and-collect behavior, are removed. * docs(py-sdk): update custom filters example Show that an invalid filter raises a FilterError instead of being collected and dropped, so the example matches how the client behaves. * fix(py-sdk): reword the invalid filter error Say "valid filter model" in the message a plugin author sees when a route registers a filter with the wrong type, matching the rest of the wording. * docs(py-sdk): custom filters example end to end Rework the custom filters example into one runnable walk through the happy path and the unhappy paths: a search mixing standard, registered, and ad-hoc filters against a small mock server, an invalid registration caught at define_plugin, a misspelled resource or method caught when the route is built, and an invalid filter value raising before any request. Add the mock server the happy path talks to. The mistakes an IDE also flags are marked so a reader can see what the type checker catches on top of the runtime guard. * ci(py-sdk): ignore grants.gov transforms example The grants.gov example includes a reference to the published grants.gov plugin that's not imported in this project. * docs(py-sdk): fix inconsistent changeset --- .changeset/python-typed-custom-filters.md | 2 +- .../common_grants_sdk/client/opportunities.py | 23 +- .../common_grants_sdk/extensions/__init__.py | 2 - .../common_grants_sdk/extensions/filters.py | 259 +++-- .../common_grants_sdk/extensions/plugin.py | 9 + .../common_grants_sdk/extensions/types.py | 25 - .../schemas/pydantic/filters/__init__.py | 5 + .../schemas/pydantic/filters/integer.py | 45 + .../schemas/pydantic/filters/opportunity.py | 2 + lib/python-sdk/examples/custom_filters.py | 269 +++-- lib/python-sdk/examples/mock_api_server.py | 159 +++ lib/python-sdk/pyrightconfig.json | 1 + lib/python-sdk/tests/client/test_client.py | 35 +- .../tests/client/test_opportunities.py | 37 +- .../tests/extensions/test_filters.py | 933 +++++++++--------- 15 files changed, 1044 insertions(+), 762 deletions(-) create mode 100644 lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/integer.py create mode 100644 lib/python-sdk/examples/mock_api_server.py diff --git a/.changeset/python-typed-custom-filters.md b/.changeset/python-typed-custom-filters.md index dd19473ef..9651d10a4 100644 --- a/.changeset/python-typed-custom-filters.md +++ b/.changeset/python-typed-custom-filters.md @@ -7,4 +7,4 @@ Typed custom-filter authoring and consumption for the Python SDK. - `OpportunityFilters` is now an open `TypedDict` (PEP 728 `extra_items`): standard filter keys are typed to their value models, a `total=False` subclass gives each registered custom filter its own typed key, and unregistered keys still pass through. One source of truth for both registration and the consumer's `search(filters=...)` call site. Clean value aliases (`StringArray`, `NumberComparison`, `DateComparison`, …) read at the call site. - `define_plugin(routes=...)` takes typed route carriers — `PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters))` — so a misspelled route/method is a type error, replacing the stringly-typed route dict. - `Plugin.get_client(config)` returns a client already scoped with the plugin's routes and schemas: `opportunities.search()` / `.list()` parse responses with the plugin's custom fields by default and return `SearchResult` / `ListResult` that partition successfully parsed `items` from per-row parse `errors`, so one malformed row no longer fails the batch. -- `opportunities.search()` raises `FilterError` on an invalid value for a standard or registered filter (or a conflicting `status` shorthand) before sending the request; ad-hoc (unregistered) keys stay best-effort and are dropped if unusable. `filter_info.errors` now carries server-returned errors only. +- `opportunities.search()` raises `FilterError` on an invalid value for both standard and custom filters (e.g. `{ "operator": "in", "value": "open" }` since `in` should be paired with an array rather than a scalar value). diff --git a/lib/python-sdk/common_grants_sdk/client/opportunities.py b/lib/python-sdk/common_grants_sdk/client/opportunities.py index 223e88bb7..cde3833f2 100644 --- a/lib/python-sdk/common_grants_sdk/client/opportunities.py +++ b/lib/python-sdk/common_grants_sdk/client/opportunities.py @@ -126,10 +126,9 @@ def search( schema: Per-call parse-schema override; defaults to the plugin's Opportunity schema (or ``OpportunityBase`` when unbound). filters: Typed filter dict — registered keys are validated against the - plugin's route filters and standard keys against their models. An - invalid value on a standard or registered filter raises - ``FilterError`` before any request; ad-hoc (unregistered) keys are - best-effort and dropped if unusable. + plugin's route filters, standard keys against their models, and + ad-hoc (unregistered) keys against the known-filter-model union. Any + invalid value raises ``FilterError`` before a request is sent. Returns: ``SearchResult`` — parsed ``items``, per-row parse ``errors``, and @@ -137,23 +136,21 @@ def search( Raises: APIError: If the API request fails. - FilterError: If a standard or registered filter — or the ``status`` - shorthand — is given an invalid or conflicting value. + FilterError: If any filter value — or the ``status`` shorthand — is + invalid or conflicting. """ resolved = self._schema(schema) - # Raise on an invalid standard/registered filter value rather than - # collecting it: filter_info.errors is reserved for server errors. The - # classifier stays fail-soft; ad-hoc keys are best-effort (dropped). + # classify_filters raises FilterError on the first invalid filter value, + # whether it is a standard, registered, or ad-hoc filter, before any request + # is sent. The filter_info.errors field is reserved for errors the server + # reports about the filters it received. filters_body: dict[str, Any] = {} if filters: classified = classify_filters( self.client._routes, "opportunities", "search", filters ) - strict_error = next((e for e in classified.errors if e.strict), None) - if strict_error is not None: - raise strict_error - filters_body = classified.result.model_dump( + filters_body = classified.model_dump( by_alias=True, exclude_none=True, mode="json" ) diff --git a/lib/python-sdk/common_grants_sdk/extensions/__init__.py b/lib/python-sdk/common_grants_sdk/extensions/__init__.py index 012be99fe..ccdff1515 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/__init__.py +++ b/lib/python-sdk/common_grants_sdk/extensions/__init__.py @@ -26,7 +26,6 @@ ) from .transforms import build_transforms from .types import ( - ClassifyResult, FilterError, Handler, PassthroughModel, @@ -62,7 +61,6 @@ "schema", "validate_into", # Custom filters - "ClassifyResult", "FilterError", "PluginRoutes", "ResourceRoutes", diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index 50e534a49..7c8bc8da1 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -2,16 +2,19 @@ Provides: - ``f`` helper singleton for building filter value models. +- ``VALID_FILTER_MODELS`` — the filter models that can be used as registered or + ad-hoc custom filters (single source of truth for route validation and ad-hoc + classification). - ``DEFAULT_FILTER_NAMES`` — frozenset of all core default-filter field names (snake + alias). - ``validate_routes(routes)`` — registration-time validator; raises FilterError. -- ``validate_filter_call(model_cls, filter_name, value)`` — call-time validator; fail-soft. - Validates against ``model_cls`` (a registered filter's value model) or ``DefaultFilter`` - when ``model_cls`` is ``None``; returns ``(wire-ready DefaultFilter, None)`` on success - or ``(None, FilterError)``. -- ``classify_filters(routes, resource, method, consumer_filters)`` — fail-soft classifier - producing a ``ClassifyResult`` (the valid-only ``OppFilters`` search request body plus - the collected ``FilterError``s; never raises on a bad call-time filter value). Registered - custom filters are recovered from the route's TypedDict via ``get_type_hints``. +- ``validate_filter_call(model_cls, filter_name, value)`` — call-time validator. + Validates against ``model_cls`` (a registered filter's value model) or the + ``VALID_FILTER_MODELS`` union when ``model_cls`` is ``None``; returns + ``(wire-ready DefaultFilter, None)`` on success or ``(None, FilterError)``. +- ``classify_filters(routes, resource, method, consumer_filters)`` — fail-fast classifier + producing the ``OppFilters`` search request body; raises ``FilterError`` on the first + invalid filter value (standard, registered, or ad-hoc). Registered custom filters are + recovered from the route's TypedDict via ``get_type_hints``. No generate.py / codegen dependency. Correctness is enforced at runtime by Pydantic v2. """ @@ -34,6 +37,15 @@ StringOperator, ) from common_grants_sdk.schemas.pydantic.filters.boolean import BooleanComparisonFilter +from common_grants_sdk.schemas.pydantic.filters.date import ( + DateComparisonFilter, + DateRangeFilter, +) +from common_grants_sdk.schemas.pydantic.filters.integer import IntegerComparisonFilter +from common_grants_sdk.schemas.pydantic.filters.money import ( + MoneyComparisonFilter, + MoneyRangeFilter, +) from common_grants_sdk.schemas.pydantic.filters.numeric import ( NumberArrayFilter, NumberComparisonFilter, @@ -50,7 +62,38 @@ StringComparisonFilter, ) -from .types import ClassifyResult, FilterError, PluginRoutes +from .types import FilterError, PluginRoutes + +# --------------------------------------------------------------------------- +# Valid filter models (single source of truth) +# --------------------------------------------------------------------------- + +#: The set of filter models that can be used as registered or ad-hoc custom +#: filters, one for each core filter family. This is the single source of truth +#: for two checks: +#: +#: - ``validate_routes`` requires that every registered filter is annotated with +#: one of these models, so a route that registers a filter with any other type +#: is rejected when the plugin is defined. +#: - The classifier requires that an unregistered (ad-hoc) filter validates +#: against one of these models. This is what lets the SDK reject a filter whose +#: value does not match its operator, for example ``{"operator": "in", "value": +#: "x"}``, where "in" expects a list but the value is a plain string. +#: +#: When the API adds a new filter family, add its model here. +VALID_FILTER_MODELS: tuple[type[CommonGrantsBaseModel], ...] = ( + StringComparisonFilter, + StringArrayFilter, + NumberComparisonFilter, + NumberArrayFilter, + NumberRangeFilter, + IntegerComparisonFilter, + BooleanComparisonFilter, + DateComparisonFilter, + DateRangeFilter, + MoneyComparisonFilter, + MoneyRangeFilter, +) # --------------------------------------------------------------------------- # f.* helpers @@ -291,16 +334,21 @@ def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: # --------------------------------------------------------------------------- +def _is_filter_model(ann: Any) -> bool: + """True if ``ann`` is one of the valid filter models (or a subclass).""" + return isinstance(ann, type) and issubclass(ann, VALID_FILTER_MODELS) + + def _registered_filter_models(route_td: Any) -> dict[str, type[BaseModel]]: """Recover a route's registered custom filters from its filter TypedDict. ``route_td`` is the TypedDict class an author put in the route slot (e.g. ``OppSearchFilters``), or ``None``. Returns ``{filterName: value model}`` for every key the author declared beyond the standard ``OpportunityFilters`` keys. - Non-model annotations are skipped here; ``validate_routes`` (run when - ``plugin.get_client`` binds routes) rejects them, so a caller who invokes - ``classify_filters`` directly on unvalidated routes gets silent skipping rather - than a raise. + Non-filter annotations are skipped here; ``validate_routes`` (run at + ``define_plugin`` and when ``plugin.get_client`` binds routes) rejects them, so + a caller who invokes ``classify_filters`` directly on unvalidated routes gets + silent skipping rather than a raise. """ if route_td is None: return {} @@ -308,7 +356,7 @@ def _registered_filter_models(route_td: Any) -> dict[str, type[BaseModel]]: return { name: ann for name, ann in get_type_hints(route_td).items() - if name not in standard and isinstance(ann, type) and issubclass(ann, BaseModel) + if name not in standard and _is_filter_model(ann) } @@ -318,15 +366,20 @@ def validate_routes(routes: PluginRoutes[Any]) -> None: The typed carriers make a misspelled resource/method a *static* error. Two runtime checks remain: - 1. A registered custom filter's value type must be a filter value model (a - ``CommonGrantsBaseModel`` subclass) — a mistyped TypedDict value (e.g. - ``region: int``) would otherwise fail opaquely at call time. + 1. A registered custom filter's value type must be one of the valid filter + models (``VALID_FILTER_MODELS``) — a mistyped TypedDict value (e.g. + ``region: int`` or a non-filter model like ``region: OpportunityBase``) + would otherwise fail opaquely at call time. 2. A route TypedDict must not redeclare a standard ``OpportunityFilters`` key with a different value type. Authors get the standard keys for free; a re-typed standard key (e.g. ``status: StringComparisonFilter``) creates a static/runtime mismatch — the call site sees the override, but classification validates against the real standard field type. + Runs at ``define_plugin`` (so the author catches a bad registration at + definition) and again when ``plugin.get_client`` binds routes (backstop for + hand-built clients). + Args: routes: Typed route registration (``PluginRoutes``). @@ -352,13 +405,11 @@ def validate_routes(routes: PluginRoutes[Any]) -> None: source_value=ann, ) continue - if not ( - isinstance(ann, type) and issubclass(ann, CommonGrantsBaseModel) - ): + if not _is_filter_model(ann): raise FilterError( f'Registered filter "{name}" on ' - f"{resource_field.name}.{method_field.name} must be a filter " - "value model (a CommonGrantsBaseModel subclass)", + f"{resource_field.name}.{method_field.name} must be a valid " + "filter model (e.g. StringArray, NumberComparison)", path=f"{path_prefix}.{name}", source_value=ann, ) @@ -381,20 +432,51 @@ def _first_error_detail(exc: ValidationError) -> str: # --------------------------------------------------------------------------- +#: Describes the value each operator expects. Used to build a readable message +#: when an unregistered (ad-hoc) filter does not match any valid filter model. +AD_HOC_VALUE_EXPECTATION: dict[str, str] = { + "eq": "a scalar value", + "neq": "a scalar value", + "gt": "a scalar value", + "gte": "a scalar value", + "lt": "a scalar value", + "lte": "a scalar value", + "in": "an array value", + "notIn": "an array value", + "like": "a string value", + "notLike": "a string value", + "between": "a { min, max } object", + "outside": "a { min, max } object", +} + + +def _adhoc_detail(payload: Any) -> str: + """Explain why an ad-hoc {operator, value} matched no valid filter model.""" + operator = payload.get("operator") if isinstance(payload, dict) else None + if not isinstance(operator, str): + return "expected a { operator, value } object" + expectation = AD_HOC_VALUE_EXPECTATION.get(operator) + if expectation is None: + return f'operator "{operator}" is not a known filter operator' + return f'operator "{operator}" expects {expectation}' + + def validate_filter_call( model_cls: Optional[type[BaseModel]], filter_name: str, value: Any, ) -> tuple[Optional[DefaultFilter], Optional[FilterError]]: - """Call-time validator for a single filter value (fail-soft). + """Call-time validator for a single filter value. Validates ``value`` against ``model_cls`` when provided (a registered filter's - value model, recovered from the route's TypedDict), or against ``DefaultFilter`` - when ``model_cls`` is ``None`` (ad-hoc / escape-hatch filter). + value model, recovered from the route's TypedDict), or against the + ``VALID_FILTER_MODELS`` union when ``model_cls`` is ``None`` (ad-hoc / + escape-hatch filter): the ``{operator, value}`` pair must be well-formed for + some valid filter model. - Never raises: returns ``(validated_filter, None)`` on success or - ``(None, FilterError)`` on failure. The caller collects the error and skips - the key rather than aborting the whole call. + Never raises itself: returns ``(validated_filter, None)`` on success or + ``(None, FilterError)`` on failure. The caller (``classify_filters``) raises + the returned error (fail-fast) before any request is sent. On success the returned filter carries the coerced operator/value, never the raw input — lax coercion can differ from the input (e.g. ``"42"`` → ``42``). @@ -427,17 +509,22 @@ def validate_filter_call( # operator/value. DefaultFilter.value is Any per the core spec, so # nothing the typed model accepted can fail here. return DefaultFilter.model_validate(validated.model_dump()), None - # Ad-hoc / escape-hatch: validate against DefaultFilter shape only - try: - return DefaultFilter.model_validate(payload), None - except ValidationError as exc: - return None, FilterError( - f'Ad-hoc filter "{filter_name}" has an invalid DefaultFilter shape: ' - f"{exc.error_count()} error(s); first: {_first_error_detail(exc)}", - path=f"filters.{filter_name}", - source_value=value, - cause=exc, - ) + # Ad-hoc filter: there is no registered model for this key, so accept it only + # if its operator and value together match one of the valid filter models. This + # rejects a value that does not fit its operator, for example "in" with a plain + # string instead of a list, instead of letting it pass through unchecked. + for model in VALID_FILTER_MODELS: + try: + validated = model.model_validate(payload) + except ValidationError: + continue + return DefaultFilter.model_validate(validated.model_dump()), None + return None, FilterError( + f'Ad-hoc filter "{filter_name}" has an invalid operator/value combination: ' + f"{_adhoc_detail(payload)}", + path=f"filters.{filter_name}", + source_value=value, + ) # --------------------------------------------------------------------------- @@ -489,7 +576,7 @@ def classify_filters( resource: str, method: str, consumer_filters: Mapping[str, Union[BaseModel, dict[str, Any]]], -) -> ClassifyResult[OppFilters]: +) -> OppFilters: """Classify consumer filter dict into the ``OppFilters`` search request body. Three-bucket classification: @@ -499,19 +586,19 @@ def classify_filters( TypedDict for the given resource/method → land in ``custom_filters``. - Bucket 3 (ad-hoc): any other key → land in ``custom_filters`` passthrough. - Validation is **fail-soft**: a key that fails its call-time validation is - dropped from the result and its ``FilterError`` collected into - ``ClassifyResult.errors``; the call never raises on a bad filter value. - (Registration-time validation in ``validate_routes`` still raises.) + Validation is **fail-fast**: the first key that fails its call-time validation + raises ``FilterError`` before a request body is produced — a standard, + registered, or ad-hoc value alike. A malformed value never silently widens the + consumer's search. (Registration-time validation in ``validate_routes`` also + raises.) Opportunity-bound — see the limitation note above the function. Registered specs are looked up by the exact ``(resource, method)`` strings declared in ``routes``. A non-matching pair (e.g. a pluralization typo in ``resource``) yields no registered bucket at all: every non-default filter is - then validated only against the permissive ``DefaultFilter`` shape, exactly - like ad-hoc input. Call sites must pass the same resource/method strings the - plugin declared. + then validated against the known-model union, exactly like ad-hoc input. Call + sites must pass the same resource/method strings the plugin declared. Construction normalizes all default consumer keys to the form that ``OppFilters(**kwargs)`` accepts. Because ``OppDefaultFilters`` does NOT set @@ -531,13 +618,14 @@ def classify_filters( call site (raw ``{"operator": ..., "value": ...}`` dicts also accepted). Returns: - ``ClassifyResult`` — ``.result`` is the valid-only ``OppFilters`` request - body; ``.errors`` is the collected list of ``FilterError``s for keys that - failed (empty on full success). Call - ``.result.model_dump(by_alias=True, exclude_none=True, mode="json")`` for - the JSON body of the search request — ``mode="json"`` is required because - coerced ``date`` objects are not JSON-serializable in the default python - mode (operator enums are ``StrEnum`` and serialize fine either way). + The ``OppFilters`` request body. Call + ``.model_dump(by_alias=True, exclude_none=True, mode="json")`` for the JSON + body of the search request — ``mode="json"`` is required because coerced + ``date`` objects are not JSON-serializable in the default python mode + (operator enums are ``StrEnum`` and serialize fine either way). + + Raises: + FilterError: On the first invalid filter value (standard, registered, or ad-hoc). """ route = getattr(routes, resource, None) route_td = getattr(route, method, None) if route is not None else None @@ -545,67 +633,48 @@ def classify_filters( default_fields: dict[str, Any] = {} custom_buckets: dict[str, DefaultFilter] = {} - errors: list[FilterError] = [] for key, value in consumer_filters.items(): if key in DEFAULT_FILTER_NAMES: - # Bucket 1: core default filter. - # OppFilters requires the alias form (e.g. "closeDateRange") when constructing - # via **kwargs because populate_by_name is not set on OppDefaultFilters. - # Normalize: camelCase aliases stay as-is; snake_case keys are mapped to their - # alias; keys with no alias (e.g. "status") are passed through unchanged. - # Validate per-field against the named field's actual type (e.g. - # status → StringArrayFilter) — stricter than the permissive DefaultFilter - # check. Fail-soft: an invalid value is collected and skipped. + # Bucket 1: a standard filter. Validate it against the type declared for + # that field (for example, "status" is validated as a StringArrayFilter). + # OppFilters is constructed with keyword arguments and does not set + # populate_by_name, so a snake_case key is first converted to its alias. + # Keys that are already in alias form, and keys that have no alias, pass + # through unchanged. alias_key = _SNAKE_TO_ALIAS.get(key, key) if alias_key in default_fields: - # Snake and camel forms of the same field normalize to one key; - # without this guard, plain dict assignment would silently drop - # whichever form the consumer's dict ordered first. Fail-soft: - # keep the first-seen value, drop the duplicate, collect a warning. - errors.append( - FilterError( - f'Default filter "{alias_key}" was supplied more than once ' - "(snake_case and camelCase forms of the same filter)", - path=f"filters.{alias_key}", - source_value=value, - strict=True, - ) + # Snake and camel forms of the same field normalize to one key. + raise FilterError( + f'Default filter "{alias_key}" was supplied more than once ' + "(snake_case and camelCase forms of the same filter)", + path=f"filters.{alias_key}", + source_value=value, ) - continue validated, error = _validate_default_field(alias_key, value) if error is not None: - # Standard filter — the SDK owns its contract; mark strict so the - # resource client raises rather than fail-softing. - error.strict = True - errors.append(error) - continue + raise error default_fields[alias_key] = validated elif key in registered: - # Bucket 2: registered custom filter — validate against the value - # model recovered from the route's TypedDict; ship the validated - # value, never the raw input. Fail-soft here; strict for the client. + # Bucket 2: a filter the plugin registered for this route. Validate it + # against the model the plugin declared for this key, and keep the + # validated value. validated, error = validate_filter_call(registered[key], key, value) if error is not None: - error.strict = True - errors.append(error) - continue + raise error custom_buckets[key] = validated # type: ignore[assignment] else: - # Bucket 3: ad-hoc / escape-hatch passthrough — ship the validated value. - # Fail-soft: collect the (non-strict) error and skip the key. + # Bucket 3: a filter the plugin did not register. Validate it against the + # valid filter models so that a value that does not fit its operator is + # rejected instead of quietly passing through to the request. validated, error = validate_filter_call(None, key, value) if error is not None: - errors.append(error) - continue + raise error custom_buckets[key] = validated # type: ignore[assignment] # OppFilters requires the alias form for construction (populate_by_name is not set). # Use "customFilters" (the alias) rather than "custom_filters" (the field name). - # Every value in default_fields / custom_buckets already passed validation - # above, so this construction cannot fail on a value-shape error. - result = OppFilters( + return OppFilters( **default_fields, customFilters=custom_buckets if custom_buckets else None, ) - return ClassifyResult(result=result, errors=errors) diff --git a/lib/python-sdk/common_grants_sdk/extensions/plugin.py b/lib/python-sdk/common_grants_sdk/extensions/plugin.py index 19733f1fd..44628c599 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/plugin.py +++ b/lib/python-sdk/common_grants_sdk/extensions/plugin.py @@ -19,6 +19,7 @@ import typing_extensions as te from ..schemas.pydantic.models import OpportunityBase +from .filters import validate_routes from .schema import ( PluginDefinitionError, SchemaOnly, @@ -145,6 +146,9 @@ def define_plugin( Raises: PluginDefinitionError: If any slot does not hold a schema extension, or holds one whose ``schema_name`` does not match its attribute name. + FilterError: If ``routes`` registers a filter whose value type is not a known + filter value model (validated here so the author catches it at definition, + not the consumer at ``get_client``). """ errors: list[str] = [] for fld in fields(cast(Any, schemas)): @@ -161,4 +165,9 @@ def define_plugin( resolved_routes: PluginRoutes[Any] = ( routes if routes is not None else PluginRoutes(opportunities=ResourceRoutes()) ) + # Validate the routes here, when the plugin is defined, so that a plugin author + # sees an invalid filter registration right away instead of a consumer running + # into it later when they build a client. The client validates the routes again + # when it binds them, which covers clients built without define_plugin. + validate_routes(resolved_routes) return Plugin(schemas=schemas, routes=resolved_routes, meta=meta) diff --git a/lib/python-sdk/common_grants_sdk/extensions/types.py b/lib/python-sdk/common_grants_sdk/extensions/types.py index 76d26abad..9ee92a85f 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/types.py +++ b/lib/python-sdk/common_grants_sdk/extensions/types.py @@ -104,10 +104,6 @@ class FilterError(Exception): Note: source_value may contain PII. Adopters are responsible for redacting it before logging or re-raising. The SDK does not redact by default. - - ``strict`` marks a failure on a typed filter — a standard or registered - custom filter, whose contract the SDK owns. The resource client raises on - strict failures; non-strict (ad-hoc / passthrough) failures stay fail-soft. """ def __init__( @@ -118,33 +114,12 @@ def __init__( handler: str | None = None, source_value: Any = None, cause: BaseException | None = None, - strict: bool = False, ) -> None: super().__init__(message) self.path = path self.handler = handler self.source_value = source_value self.cause = cause - self.strict = strict - - -@dataclass -class ClassifyResult(Generic[T]): - """Fail-soft return shape for ``classify_filters``. - - ``classify_filters`` never raises on a bad call-time filter value: each - failing key is dropped from ``result`` and its :class:`FilterError` collected - into ``errors``. - - result: classified filters, containing only the keys that passed validation. - errors: aggregated ``FilterError``s for keys that failed; empty on success. - - Registration-time validation (``validate_routes``) still raises — a malformed - plugin declaration has no result to return. - """ - - result: T - errors: list[FilterError] @dataclass diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py index 2f535ec77..57bc04b17 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/__init__.py @@ -20,6 +20,8 @@ "MoneyRangeFilter", # Boolean Filters "BooleanComparisonFilter", + # Integer Filters + "IntegerComparisonFilter", # Numeric Filters "NumberArrayFilter", "NumberComparisonFilter", @@ -37,6 +39,7 @@ # so range fields are typed with the explicit ``*RangeFilter`` names instead. "BooleanComparison", "DateComparison", + "IntegerComparison", "MoneyComparison", "NumberArray", "NumberComparison", @@ -58,6 +61,7 @@ DateRange, DateRangeFilter, ) +from .integer import IntegerComparisonFilter from .money import ( MoneyComparisonFilter, MoneyRange, @@ -73,6 +77,7 @@ from .opportunity import ( BooleanComparison, DateComparison, + IntegerComparison, MoneyComparison, NumberArray, NumberComparison, diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/integer.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/integer.py new file mode 100644 index 000000000..5c2ea756c --- /dev/null +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/integer.py @@ -0,0 +1,45 @@ +"""Integer filter schemas.""" + +from pydantic import Field, field_validator + +from ..base import CommonGrantsBaseModel +from .base import ComparisonOperator, EquivalenceOperator + +# ############################################################ +# Models +# ############################################################ + + +class IntegerComparisonFilter(CommonGrantsBaseModel): + """Filter that matches an integer against a value. + + Accepts comparison operators (``gt``/``gte``/``lt``/``lte``) and equivalence + operators (``eq``/``neq``). The value must be an integer: a non-integral value + (e.g. 3.5) or a bool is rejected. Implements the core spec's + IntegerComparisonFilter (filters/numeric.tsp). + """ + + operator: ComparisonOperator | EquivalenceOperator = Field( + ..., + description="The comparison operator to apply to the filter value", + ) + value: int = Field(..., description="The integer value to compare against") + + @field_validator("operator", mode="before") + @classmethod + def validate_operator(cls, v): + """Convert string to enum if needed.""" + if isinstance(v, str): + if v in [op.value for op in ComparisonOperator]: + return ComparisonOperator(v) + elif v in [op.value for op in EquivalenceOperator]: + return EquivalenceOperator(v) + return v + + @field_validator("value", mode="before") + @classmethod + def reject_bool(cls, v): + """Reject bool (bool subclasses int, so True would ship as 1).""" + if isinstance(v, bool): + raise ValueError("value must be an integer, not a bool") + return v diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py index dc2a04050..cf21979d7 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py @@ -8,6 +8,7 @@ from .base import DefaultFilter from .boolean import BooleanComparisonFilter from .date import DateComparisonFilter, DateRangeFilter +from .integer import IntegerComparisonFilter from .money import MoneyComparisonFilter, MoneyRangeFilter from .numeric import NumberArrayFilter, NumberComparisonFilter from .string import StringArrayFilter, StringComparisonFilter @@ -20,6 +21,7 @@ StringComparison = StringComparisonFilter NumberArray = NumberArrayFilter NumberComparison = NumberComparisonFilter +IntegerComparison = IntegerComparisonFilter DateComparison = DateComparisonFilter MoneyComparison = MoneyComparisonFilter BooleanComparison = BooleanComparisonFilter diff --git a/lib/python-sdk/examples/custom_filters.py b/lib/python-sdk/examples/custom_filters.py index d6412b58c..a3344b962 100644 --- a/lib/python-sdk/examples/custom_filters.py +++ b/lib/python-sdk/examples/custom_filters.py @@ -1,145 +1,210 @@ #!/usr/bin/env python3 -"""Custom filters example — codegen-free request-body demo. +"""Custom filters — the plugin author and consumer experience, end to end. -Demonstrates typed route registration, flat call-site classification, and the -three-bucket ``OppFilters`` request body (default named fields + customFilters -record) using the grants.gov canonical example from #646/#869. +Three scenarios, each in its own function: -No code generation (custom filters are a pure-runtime classifier). Routes are a -typed ``PluginRoutes`` carrier — the author extends ``OpportunityFilters`` with -one typed key per custom filter, so ``classify_filters`` recovers each key's -value model from the TypedDict. + 1. Happy path (mock server): define a plugin with a custom field and a + registered custom filter, build a client with plugin.get_client(), and search + with a mix of standard, registered, and ad-hoc filters. + 2. Authoring errors: define_plugin rejects a filter registered with a type that + is not a filter, and a misspelled resource or method is caught when the route + carrier is built. + 3. Consumer errors: search rejects a bad filter value before any request is sent + — the wrong kind of value for a registered filter, and an ad-hoc value that + does not fit its operator. + +Some of these mistakes are also flagged by your IDE (pyright). The lines marked +``# type: ignore`` are the ones the type checker flags; the example suppresses the +type error only so it can also show the runtime guard firing on the same mistake. +examples/typed_custom_filters_failures.py asserts the type errors on their own — +run pyright against that file to see them fire. + +Scenario 1 talks to the mock server. Start it first in another terminal: + + poetry run python examples/mock_api_server.py + +If it is not running, scenario 1 is skipped and the two offline scenarios still +run: -Run (from lib/python-sdk/): poetry run python examples/custom_filters.py + +Filter buckets (ADR-0012): + - Standard filters (status, closeDateRange, ...) become named top-level fields. + - Registered custom filters (region) go under customFilters, validated against + the type the plugin declared for them. + - Ad-hoc filters (any other key) go under customFilters, validated against the + filter models the SDK knows. """ from __future__ import annotations import json +from typing import Optional +import httpx +from pydantic import Field + +from common_grants_sdk.client import Config +from common_grants_sdk.client.exceptions import APIError from common_grants_sdk.extensions import ( + CustomField, + CustomFieldSet, + FilterError, + PluginMeta, PluginRoutes, + PluginSchemas, ResourceRoutes, - classify_filters, + define_plugin, f, + schema, ) -from common_grants_sdk.extensions.types import PluginMeta from common_grants_sdk.schemas.pydantic.filters.opportunity import ( OpportunityFilters, StringArray, - StringComparison, ) +from common_grants_sdk.schemas.pydantic.models import OpportunityBase -# --------------------------------------------------------------------------- -# Typed route registration + plugin metadata -# --------------------------------------------------------------------------- +BASE_URL = "http://localhost:8000" -meta = PluginMeta( - name="grants-gov", - version="0.1.0", - source_system="grants.gov", - capabilities=["customFilters"], -) +class OppCustomFields(CustomFieldSet): + """Custom fields the plugin adds to every opportunity it returns.""" -class OppSearchFilters(OpportunityFilters, total=False): - """The custom filters this plugin accepts on opportunities.search. + program_code: Optional[CustomField[str]] = Field( + default=None, description="Funding program code carried from the source system." + ) - Each key's annotation *is* its Pydantic value model, so the call-site value - validates against it and the classifier recovers it from the TypedDict. - """ - agency: StringArray - fundingProgram: StringComparison +class OppSearchFilters(OpportunityFilters, total=False): + """The custom filters this plugin registers on opportunities.search.""" + region: StringArray -routes: PluginRoutes[OppSearchFilters] = PluginRoutes( - opportunities=ResourceRoutes(search=OppSearchFilters) + +opportunity_plugin = define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase[OppCustomFields])), + routes=PluginRoutes(opportunities=ResourceRoutes(search=OppSearchFilters)), + meta=PluginMeta(name="grants-gov", source_system="grants.gov"), ) -def _section(title: str) -> None: - print(f"\n{'=' * 60}") - print(title) - print("=" * 60) +# --------------------------------------------------------------------------- +# Scenario 1 — happy path (mock server) +# --------------------------------------------------------------------------- -def main() -> None: - # --- Declared routes --- - _section("PLUGIN ROUTES (declared)") - print(" opportunities.search custom filters:") - print(" agency: StringArray") - print(" fundingProgram: StringComparison — Program name filter") - - # --- Classify: default + registered custom + ad-hoc --- - _section("CLASSIFY FILTERS — default + custom + ad-hoc (canonical grants.gov demo)") - - # Mixing three kinds of filters in a single flat dict: - # "status" — default core filter (snake_case key, no alias) - # "close_date_range" — default core filter, snake_case key for an ALIASED field: - # the classifier must normalize it to "closeDateRange" - # (an unnormalized snake key is silently dropped — - # the alias landmine) - # "agency" — registered custom filter (opportunities.search.agency) - # "fundingProgram" — registered custom filter (opportunities.search.fundingProgram) - # "legacyTag" — ad-hoc passthrough (not registered, not a core default) - consumer_filters = { - "status": f.in_(["open", "forecasted"]), - "close_date_range": f.between("2026-01-01", "2026-12-31"), - "agency": f.in_(["NSF", "NIH"]), - "fundingProgram": f.eq("research-grants"), - "legacyTag": f.eq("priority"), +def happy_path() -> None: + print("=== Scenario 1: happy path (mock server) ===") + client = opportunity_plugin.get_client(Config(base_url=BASE_URL, api_key="unused")) + + # One flat filter dict mixing all three buckets: "status" is a standard + # filter, "region" is the registered custom filter, and "fundingType" is an + # ad-hoc filter the plugin did not register. + filters: OppSearchFilters = { + "status": f.in_(["open"]), + "region": f.in_(["US-CA", "US-NY"]), + "fundingType": f.eq("grant"), } - classified = classify_filters(routes, "opportunities", "search", consumer_filters) + try: + result = client.opportunities.search(filters=filters) + except (APIError, httpx.HTTPError): + print( + f" (skipped: could not reach {BASE_URL}; start it with " + "`poetry run python examples/mock_api_server.py`)" + ) + return + + print(f" items returned: {len(result.items)}") + print(f" per-row parse failures: {len(result.errors)}") + if result.items: + opp = result.items[0] + program_code = None + if opp.custom_fields is not None and opp.custom_fields.program_code is not None: + program_code = opp.custom_fields.program_code.value + print(f" first opportunity: {opp.title}") + print(f" first program code: {program_code or '(none)'}") + # The client reports the filters it classified and sent, so all three buckets + # are visible: status at the top level, region and fundingType under + # customFilters. + print(f" filters sent to the server: {json.dumps(result.filter_info.filters)}") - print("\nRequest body (by_alias=True, exclude_none=True, mode='json'):") - print( - json.dumps( - classified.result.model_dump(by_alias=True, exclude_none=True, mode="json"), - indent=2, + +# --------------------------------------------------------------------------- +# Scenario 2 — authoring errors (define_plugin / the route carrier) +# --------------------------------------------------------------------------- + + +def authoring_errors() -> None: + print("\n=== Scenario 2: authoring errors ===") + + # A filter registered with a type that is not a filter is rejected when the + # plugin author defines the plugin, rather than later when a consumer builds a + # client. + class BadRegionFilters(OpportunityFilters, total=False): + region: int # not a filter value model + + try: + define_plugin( + PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), + routes=PluginRoutes(opportunities=ResourceRoutes(search=BadRegionFilters)), + meta=PluginMeta(name="grants-gov", source_system="grants.gov"), ) - ) + raise AssertionError("expected define_plugin to reject a non-filter type") + except FilterError as e: + print(f" non-filter registration rejected: {e.path} - {e}") + + # A misspelled resource. PluginRoutes has no "opportunites" field, so your IDE + # flags the line below, and the typed carrier rejects it at runtime too. + try: + PluginRoutes(opportunites=ResourceRoutes(search=OppSearchFilters)) # type: ignore + raise AssertionError("expected a misspelled resource to be rejected") + except TypeError as e: + print(f" misspelled resource rejected: {e}") + + # A misspelled method. ResourceRoutes has no "serach" field — flagged by your + # IDE, rejected at runtime. + try: + ResourceRoutes(serach=OppSearchFilters) # type: ignore + raise AssertionError("expected a misspelled method to be rejected") + except TypeError as e: + print(f" misspelled method rejected: {e}") - # --- FilterError demo — bad call is fail-soft (collected, not raised) --- - _section("VALIDATION — bad operator collected, valid filters still applied") - - # agency is registered as StringArray (expects ArrayOperator: in/notIn). - # Passing f.eq(...) (EquivalenceOperator.EQUAL) fails call-time validation. - # Fail-soft: classify_filters never raises on a bad - # filter value — the bad key is dropped from the result body and a FilterError - # is collected on .errors. A valid sibling filter still classifies normally. - bad_filters = { - "agency": f.eq( - "wrong-operator-for-array-type" - ), # eq is not a valid StringArray op - "legacyTag": f.eq("priority"), # valid ad-hoc filter — survives - } - classified_bad = classify_filters(routes, "opportunities", "search", bad_filters) - for err in classified_bad.errors: - # str(err) summarizes the first failure; the structured fields carry - # the full detail — err.path names the failing filter, err.cause is - # the underlying pydantic ValidationError for programmatic access. - print(f"FilterError collected: {err}") - print(f" path: {err.path}") - print(f" cause: {type(err.cause).__name__}") - # The valid filter still made it into the request body. - print( - "\nValid-only request body: " - + json.dumps( - classified_bad.result.model_dump( - by_alias=True, exclude_none=True, mode="json" - ) + +# --------------------------------------------------------------------------- +# Scenario 3 — consumer errors (search rejects a bad value before any request) +# --------------------------------------------------------------------------- + + +def consumer_errors() -> None: + print("\n=== Scenario 3: consumer errors (search, no request sent) ===") + client = opportunity_plugin.get_client(Config(base_url=BASE_URL, api_key="unused")) + + # A registered filter given the wrong kind of value. "region" is a string + # array filter, so f.eq (a scalar comparison) is flagged by your IDE; the + # runtime guard rejects it before any request too. + try: + client.opportunities.search(filters={"region": f.eq("US-CA")}) # type: ignore + raise AssertionError("expected a FilterError for a wrong region value") + except FilterError as e: + print(f" registered filter, wrong kind of value: {e.path} - {e}") + + # An ad-hoc filter whose value does not fit its operator. "in" expects a list, + # so the plain string is flagged by your IDE and rejected at runtime. + try: + client.opportunities.search( + filters={"fundingType": {"operator": "in", "value": "grant"}} # type: ignore ) - ) + raise AssertionError("expected a FilterError for an ad-hoc value mismatch") + except FilterError as e: + print(f" ad-hoc filter, value does not fit operator: {e.path} - {e}") - # --- Plugin metadata --- - _section("PLUGIN METADATA") - print(f"name: {meta.name}") - print(f"version: {meta.version}") - print(f"sourceSystem: {meta.source_system}") - print(f"capabilities: {meta.capabilities}") + +def main() -> None: + happy_path() + authoring_errors() + consumer_errors() + print("\n✓ custom filters example complete") if __name__ == "__main__": diff --git a/lib/python-sdk/examples/mock_api_server.py b/lib/python-sdk/examples/mock_api_server.py new file mode 100644 index 000000000..fe6600eb6 --- /dev/null +++ b/lib/python-sdk/examples/mock_api_server.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Minimal mock CommonGrants API for running the examples without a real server. + +Serves the GET and POST routes the SDK client calls. The search route applies the +standard ``status`` filter so the happy path returns a realistic subset. Start it +in one terminal: + + poetry run python examples/mock_api_server.py + +Then run an example against http://localhost:8000 in another terminal, and stop +the server with Ctrl-C when you are done. +""" + +from __future__ import annotations + +import json +import os +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any + +PORT = int(os.environ.get("PORT", "8000")) + +# Two opportunities, each carrying a "programCode" custom field so an example can +# show a typed custom field on a parsed row. +MOCK_OPPORTUNITIES: list[dict[str, Any]] = [ + { + "id": "573525f2-8e15-4405-83fb-e6523511d893", + "title": "STEM Education Grant Program", + "description": "A grant program focused on improving STEM education.", + "status": {"value": "open", "description": "Open"}, + "createdAt": "2025-01-01T00:00:00Z", + "lastModifiedAt": "2025-01-15T00:00:00Z", + "customFields": { + "programCode": { + "name": "programCode", + "fieldType": "string", + "value": "STEM-ED", + }, + }, + }, + { + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "title": "Community Development Grant", + "description": "Funding for community development projects.", + "status": {"value": "forecasted", "description": "Forecasted"}, + "createdAt": "2025-01-02T00:00:00Z", + "lastModifiedAt": "2025-01-16T00:00:00Z", + "customFields": { + "programCode": { + "name": "programCode", + "fieldType": "string", + "value": "COMM-DEV", + }, + }, + }, +] + + +def _requested_statuses(filters: dict[str, Any]) -> list[str] | None: + """The status values a search asked for, if it included a status filter.""" + status = filters.get("status") + if isinstance(status, dict) and isinstance(status.get("value"), list): + return [str(v) for v in status["value"]] + return None + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, format: str, *args: Any) -> None: + """Silence the default per-request logging.""" + + def _send(self, status: int, body: dict[str, Any]) -> None: + payload = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: + path = self.path.split("?")[0] + if path == "/common-grants/opportunities": + self._send( + 200, + { + "status": 200, + "message": "Success", + "items": MOCK_OPPORTUNITIES, + "paginationInfo": { + "page": 1, + "pageSize": 25, + "totalItems": len(MOCK_OPPORTUNITIES), + "totalPages": 1, + }, + }, + ) + return + if path.startswith("/common-grants/opportunities/"): + opp_id = path.rsplit("/", 1)[-1] + opp = next( + (o for o in MOCK_OPPORTUNITIES if o["id"] == opp_id), + MOCK_OPPORTUNITIES[0], + ) + self._send(200, {"status": 200, "message": "Success", "data": opp}) + return + self._send(404, {"status": 404, "message": "Not found"}) + + def do_POST(self) -> None: + if self.path.split("?")[0] != "/common-grants/opportunities/search": + self._send(404, {"status": 404, "message": "Not found"}) + return + + length = int(self.headers.get("Content-Length", "0") or "0") + raw = self.rfile.read(length) if length else b"" + try: + body = json.loads(raw) if raw else {} + except json.JSONDecodeError: + body = {} + + filters = body.get("filters") or {} + items = list(MOCK_OPPORTUNITIES) + wanted = _requested_statuses(filters) if isinstance(filters, dict) else None + if wanted is not None: + items = [o for o in items if o["status"]["value"] in wanted] + + self._send( + 200, + { + "status": 200, + "message": "Success", + "items": items, + "paginationInfo": { + "page": 1, + "pageSize": max(len(items), 1), + "totalItems": len(items), + "totalPages": 1, + }, + "sortInfo": { + "sortBy": "lastModifiedAt", + "sortOrder": "desc", + "customSortBy": None, + "errors": [], + }, + "filterInfo": {"filters": {}, "errors": []}, + }, + ) + + +def main() -> None: + server = HTTPServer(("localhost", PORT), Handler) + print(f"Mock CommonGrants API listening on http://localhost:{PORT}") + print("Run an example in another terminal. Ctrl-C to stop.") + try: + server.serve_forever() + except KeyboardInterrupt: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/lib/python-sdk/pyrightconfig.json b/lib/python-sdk/pyrightconfig.json index 79a5d27f1..ed3cafe67 100644 --- a/lib/python-sdk/pyrightconfig.json +++ b/lib/python-sdk/pyrightconfig.json @@ -5,6 +5,7 @@ "**/__pycache__", "generated", "examples/typed_custom_filters_failures.py", + "examples/grants_gov_transforms.py", "**/node_modules", "**/.*", ".venv" diff --git a/lib/python-sdk/tests/client/test_client.py b/lib/python-sdk/tests/client/test_client.py index 90ae38cbc..ddb417471 100644 --- a/lib/python-sdk/tests/client/test_client.py +++ b/lib/python-sdk/tests/client/test_client.py @@ -42,27 +42,40 @@ def test_client_initialization_with_config_and_auth(self): assert client.auth == auth assert isinstance(client.opportunities, type(client.opportunities)) - def test_get_client_validates_routes(self): - """get_client validates routes when binding and raises on a bad declaration. + def test_define_plugin_validates_routes(self): + """define_plugin validates routes at definition, so a bad registration + fails for the author before a consumer ever builds a client. The typed carrier makes a misspelled resource/method a static error, so the - only remaining runtime check is that a registered custom filter's value type - is a filter value model. A TypedDict whose custom key is annotated with a - non-model type (``region: int``) is the genuinely-invalid registration. + remaining runtime check is that a registered custom filter's value type is a + valid filter model. A custom key annotated with a non-filter type + (``region: int``) is the genuinely-invalid registration. """ - with patch("common_grants_sdk.client.client.httpx.Client"): - config = Config(base_url="https://api.example.com", api_key="test-key") - class BadFilters(OpportunityFilters, total=False): - region: int # not a filter value model + class BadFilters(OpportunityFilters, total=False): + region: int # not a filter value model - plugin = define_plugin( + with pytest.raises(FilterError): + define_plugin( PluginSchemas(Opportunity=schema(common_schema=OpportunityBase)), routes=PluginRoutes(opportunities=ResourceRoutes(search=BadFilters)), meta=PluginMeta(name="t", source_system="t"), ) + + def test_bind_routes_backstops_bad_routes(self): + """_bind_routes re-validates as a backstop for hand-built clients that + bypass define_plugin's definition-time check.""" + with patch("common_grants_sdk.client.client.httpx.Client"): + config = Config(base_url="https://api.example.com", api_key="test-key") + + class BadFilters(OpportunityFilters, total=False): + region: int # not a filter value model + + client = Client(config=config) with pytest.raises(FilterError): - plugin.get_client(config) + client._bind_routes( + PluginRoutes(opportunities=ResourceRoutes(search=BadFilters)) + ) def test_get_client_accepts_valid_routes(self): """A valid typed routes carrier binds through get_client (no raise).""" diff --git a/lib/python-sdk/tests/client/test_opportunities.py b/lib/python-sdk/tests/client/test_opportunities.py index fc6c9a03b..9b6e2cd0c 100644 --- a/lib/python-sdk/tests/client/test_opportunities.py +++ b/lib/python-sdk/tests/client/test_opportunities.py @@ -949,32 +949,25 @@ def test_search_invalid_standard_filter_raises( assert exc_info.value.path == "filters.status" mock_httpx_client.post.assert_not_called() - def test_search_invalid_adhoc_filter_passes_through_no_raise( - self, client, mock_httpx_client, sample_search_response - ): - """A malformed ad-hoc (unregistered) filter is best-effort, not fatal (AC6). + def test_search_invalid_adhoc_filter_raises(self, client, mock_httpx_client): + """An invalid ad-hoc (unregistered) filter raises FilterError before any request. - Ad-hoc keys live in the untyped escape hatch: an unusable value is dropped - rather than raised (only standard/registered filters raise). Results return. + Ad-hoc keys are validated against the valid filter models, so a value that + does not fit its operator (here "in" with a plain string instead of a list) + raises rather than being sent to the server. Silently dropping it would + widen the consumer's search without telling them. """ - mock_response = Mock() - mock_response.status_code = 200 - mock_response.text = json.dumps(sample_search_response) - mock_response.json = Mock(return_value=sample_search_response) - mock_response.raise_for_status = Mock() - mock_httpx_client.post = Mock(return_value=mock_response) + mock_httpx_client.post = Mock() - # No routes: "legacyTag" is neither standard nor registered -> ad-hoc. - response = client.opportunities.search( - search="conservation", - filters={"legacyTag": "not-a-filter"}, - ) + # No routes: "legacyTag" is neither standard nor registered, so it is ad-hoc. + with pytest.raises(FilterError): + client.opportunities.search( + search="conservation", + filters={"legacyTag": {"operator": "in", "value": "NSF"}}, + ) - assert isinstance(response, SearchResult) - assert len(response.items) == 2 - # Dropped from the sent body (never reached the wire). - sent_body = mock_httpx_client.post.call_args[1]["json"] - assert "legacyTag" not in sent_body.get("filters", {}).get("customFilters", {}) + # The request is never sent. + mock_httpx_client.post.assert_not_called() def test_search_filter_info_errors_are_server_only( self, mock_httpx_client, sample_search_response diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py index 7a333c9ad..a409eb88a 100644 --- a/lib/python-sdk/tests/extensions/test_filters.py +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -11,12 +11,14 @@ from common_grants_sdk.extensions import PluginRoutes, ResourceRoutes from common_grants_sdk.extensions.filters import ( + VALID_FILTER_MODELS, classify_filters, f, validate_filter_call, validate_routes, ) -from common_grants_sdk.extensions.types import ClassifyResult, FilterError +from common_grants_sdk.extensions.types import FilterError +from common_grants_sdk.schemas.pydantic.filters.integer import IntegerComparisonFilter from common_grants_sdk.schemas.pydantic.filters.money import ( MoneyComparisonFilter, MoneyRangeFilter, @@ -27,6 +29,7 @@ ) from common_grants_sdk.schemas.pydantic.filters.opportunity import ( BooleanComparison, + IntegerComparison, NumberComparison, OpportunityFilters, OppFilters, @@ -81,9 +84,8 @@ def test_f_helper_wire_values(helper, args, operator, value): Asserts the JSON wire dump, not the in-memory ``.value``: the builders return precise filter models whose ``.value`` may be a sub-model (e.g. ``NumberRange``). - - Includes the reserved-word workarounds: Python f.in_ / f.not_in produce - wire operators "in" / "notIn". + ``f.in_`` and ``f.not_in`` use trailing underscores because ``in`` is a Python + keyword, and produce the wire operators "in" and "notIn". """ flt = getattr(f, helper)(*args) wire = flt.model_dump(by_alias=True, mode="json") @@ -92,79 +94,87 @@ def test_f_helper_wire_values(helper, args, operator, value): # --------------------------------------------------------------------------- -# classify_filters: three-bucket classification +# VALID_FILTER_MODELS registry +# --------------------------------------------------------------------------- + + +def test_valid_filter_models_includes_integer_comparison(): + """IntegerComparisonFilter is registered as one of the valid filter models.""" + assert IntegerComparisonFilter in VALID_FILTER_MODELS + + +# --------------------------------------------------------------------------- +# classify_filters: three-bucket classification (happy paths) # --------------------------------------------------------------------------- def test_classify_default_snake_key_lands_in_named_field(): - """Default snake_case key (e.g. "status") lands in a named OppFilters field, not customFilters.""" - consumer_filters = {"status": f.in_(["open"])} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + """A standard snake_case key (e.g. "status") lands in a named OppFilters field, not customFilters.""" + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.in_(["open"])} ) - assert isinstance(classified, ClassifyResult) - assert classified.errors == [] - result = classified.result assert isinstance(result, OppFilters) assert result.status is not None assert result.custom_filters is None def test_classify_default_camel_alias_lands_in_named_field(): - """THE LANDMINE: camelCase alias "closeDateRange" must land in named field, NOT customFilters.""" - consumer_filters = {"closeDateRange": f.between("2026-01-01", "2026-12-31")} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + """A camelCase alias (e.g. "closeDateRange") lands in its named field, not customFilters.""" + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"closeDateRange": f.between("2026-01-01", "2026-12-31")}, ) - assert classified.errors == [] - result = classified.result assert isinstance(result, OppFilters) - # The camelCase alias must normalize to the snake_case field assert result.close_date_range is not None - # It must NOT appear in customFilters (a conditional check here could - # never fail on the path it guards — assert the bucket is empty outright) assert result.custom_filters is None def test_classify_registered_custom_filter_lands_in_custom_filters(): """A registered custom filter (e.g. "agency") lands in OppFilters.custom_filters.""" - consumer_filters = {"agency": f.in_(["NSF", "NIH"])} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.in_(["NSF", "NIH"])} ) - assert classified.errors == [] - result = classified.result - assert isinstance(result, OppFilters) assert result.custom_filters is not None assert "agency" in result.custom_filters def test_classify_adhoc_unregistered_filter_lands_in_custom_filters(): """An unregistered ad-hoc key (e.g. "legacyTag") passes through to customFilters.""" - consumer_filters = {"legacyTag": f.eq("priority")} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"legacyTag": f.eq("priority")} ) - assert classified.errors == [] - result = classified.result - assert isinstance(result, OppFilters) assert result.custom_filters is not None assert "legacyTag" in result.custom_filters def test_classify_escape_hatch_key_lands_in_custom_filters(): """gov.<system>@<filter> escape-hatch keys pass through to customFilters.""" - consumer_filters = {"gov.someSystem@someFilter": f.eq("test")} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"gov.someSystem@someFilter": f.eq("test")}, ) - assert classified.errors == [] - result = classified.result - assert isinstance(result, OppFilters) assert result.custom_filters is not None assert "gov.someSystem@someFilter" in result.custom_filters +def test_classify_registered_integer_comparison_filter(): + """A registered integerComparison filter validates and lands in customFilters.""" + + class IntFilters(OpportunityFilters, total=False): + awardCount: IntegerComparison + + routes = PluginRoutes(opportunities=ResourceRoutes(search=IntFilters)) + result = classify_filters( + routes, "opportunities", "search", {"awardCount": f.gt(5)} + ) + assert result.custom_filters is not None + assert "awardCount" in result.custom_filters + + # --------------------------------------------------------------------------- # Request-body shape # --------------------------------------------------------------------------- @@ -173,76 +183,364 @@ def test_classify_escape_hatch_key_lands_in_custom_filters(): def test_request_body_has_default_fields_at_top_level_and_custom_filters_nested(): """model_dump(by_alias=True, exclude_none=True) yields the ADR-0012 request-body shape. - Default filters appear at top level; custom/ad-hoc appear under "customFilters". + Default filters appear at top level; custom and ad-hoc filters appear under + "customFilters". """ - consumer_filters = { - "status": f.in_(["open"]), - "agency": f.in_(["NSF"]), - "legacyTag": f.eq("priority"), - } - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "status": f.in_(["open"]), + "agency": f.in_(["NSF"]), + "legacyTag": f.eq("priority"), + }, ) - assert classified.errors == [] - request_body = classified.result.model_dump(by_alias=True, exclude_none=True) + request_body = result.model_dump(by_alias=True, exclude_none=True) - # Default field appears at top level assert "status" in request_body - # Custom and ad-hoc filters are nested under customFilters assert "customFilters" in request_body assert "agency" in request_body["customFilters"] assert "legacyTag" in request_body["customFilters"] - # customFilters is NOT a top-level key for a default filter assert "status" not in request_body.get("customFilters", {}) def test_request_body_no_custom_filters_key_when_all_defaults(): """customFilters key is absent from the request body when all filters are default fields.""" - consumer_filters = {"status": f.in_(["open"])} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + result = classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.in_(["open"])} ) - assert classified.errors == [] - request_body = classified.result.model_dump(by_alias=True, exclude_none=True) + request_body = result.model_dump(by_alias=True, exclude_none=True) assert "customFilters" not in request_body def test_oppfilters_mixed_case_roundtrip(): - """Alias-normalization round-trip proof. + """A mix of snake_case and camelCase default keys normalizes to the correct fields. - A mix of snake_case and camelCase default keys — including 'closeDateRange' as - a camelCase alias — must normalize to the correct snake_case field names and - produce the correct ADR-0012 request JSON via model_dump(by_alias=True, exclude_none=True). + "closeDateRange" is given as a camelCase alias; it must land in its named field + (close_date_range) rather than customFilters, and the request body must use the + camelCase aliases. + """ + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "status": f.in_(["open", "forecasted"]), + "closeDateRange": f.between("2026-01-01", "2026-12-31"), + }, + ) + assert result.close_date_range is not None + request_body = result.model_dump(by_alias=True, exclude_none=True) + assert "status" in request_body + assert "closeDateRange" in request_body + assert "customFilters" not in request_body - Asserts: - - closeDateRange lands in its named default field (close_date_range), NOT in customFilters. - - request-body output uses camelCase aliases (by_alias=True). + +def test_request_body_mode_json_round_trip(): + """model_dump(mode="json") yields a body that json.dumps can serialize. + + Coerced date objects only serialize in json mode, which is the mode the + classifier output is dumped with. """ - consumer_filters = { - "status": f.in_(["open", "forecasted"]), # snake_case default - "closeDateRange": f.between( - "2026-01-01", "2026-12-31" - ), # camelCase alias default - } - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "close_date_range": f.between(date(2026, 1, 1), date(2026, 12, 31)), + "agency": f.in_(["NSF"]), + }, ) - assert classified.errors == [] - result = classified.result - assert isinstance(result, OppFilters) + body = json.loads( + json.dumps(result.model_dump(by_alias=True, exclude_none=True, mode="json")) + ) + assert body["closeDateRange"]["operator"] == "between" + assert body["closeDateRange"]["value"]["min"] == "2026-01-01" + assert body["customFilters"]["agency"]["operator"] == "in" - # camelCase alias must land in named field + +def test_classify_empty_filters_dict_yields_empty_body(): + """An empty consumer dict produces an OppFilters with no customFilters entry.""" + result = classify_filters(SAMPLE_ROUTES, "opportunities", "search", {}) + assert result.custom_filters is None + body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert "customFilters" not in body + + +def test_classify_default_snake_form_of_aliased_key_normalizes_to_alias(): + """A snake_case key for an aliased field lands in the named field, not customFilters. + + Without the alias normalization, OppFilters(close_date_range=...) would be + silently dropped by pydantic (populate_by_name is not set) and the field would + stay None. + """ + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"close_date_range": f.between("2026-01-01", "2026-12-31")}, + ) assert result.close_date_range is not None + assert result.custom_filters is None - request_body = result.model_dump(by_alias=True, exclude_none=True) - # Both named fields appear at top level with their alias names - assert "status" in request_body - assert "closeDateRange" in request_body +# --------------------------------------------------------------------------- +# Wire-body integrity: the value that passed validation is the value shipped +# --------------------------------------------------------------------------- - # Nothing landed in customFilters — assert the bucket is absent outright - # (a conditional check could never fail on the path it guards) - assert "customFilters" not in request_body + +class WireFilters(OpportunityFilters, total=False): + """Registered wire-integrity filters: a boolean and a number comparison.""" + + isOpen: BooleanComparison + awardCount: NumberComparison + + +WIRE_ROUTES = PluginRoutes(opportunities=ResourceRoutes(search=WireFilters)) + + +def test_boolean_filter_value_survives_to_wire_as_json_true(): + """f.eq(True) on a booleanComparison filter serializes as JSON true, not 1. + + DefaultFilter.value is Any (the core spec models it as ``unknown``), which keeps + a boolean from being coerced to a number on its way to the wire. + """ + result = classify_filters( + WIRE_ROUTES, "opportunities", "search", {"isOpen": f.eq(True)} + ) + body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert body["customFilters"]["isOpen"]["value"] is True + + +def test_registered_filter_ships_validated_value_not_raw_input(): + """The classifier keeps the value that passed validation, not the raw input. + + A numberComparison filter coerces the string "42" to the integer 42, so the + value that reaches the wire is the validated 42, not the original string. + """ + result = classify_filters( + WIRE_ROUTES, "opportunities", "search", {"awardCount": f.gt("42")} + ) + body = result.model_dump(by_alias=True, exclude_none=True, mode="json") + assert body["customFilters"]["awardCount"]["value"] == 42 + assert isinstance(body["customFilters"]["awardCount"]["value"], int) + + +# --------------------------------------------------------------------------- +# classify_filters: fail-fast on an invalid filter value +# --------------------------------------------------------------------------- + + +def test_classify_invalid_default_raises(): + """An invalid standard filter raises FilterError before a request body is built. + + "status" is a StringArrayFilter (an array operator with a list of strings), so + f.eq("open") (an equivalence operator with a scalar) does not fit it. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} + ) + assert exc.value.path == "filters.status" + assert isinstance(exc.value.cause, ValidationError) + + +def test_classify_invalid_default_via_alias_raises(): + """An invalid standard filter given by its camelCase alias raises, using the alias in the path. + + "closeDateRange" is a DateRangeFilter; f.eq("2026-01-01") does not fit it. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"closeDateRange": f.eq("2026-01-01")}, + ) + assert exc.value.path == "filters.closeDateRange" + + +def test_classify_invalid_money_range_default_raises(): + """An invalid moneyRange standard filter raises. + + "totalFundingAvailableRange" is a MoneyRangeFilter (a range operator with min + and max Money values); f.eq("100") is a valid permissive DefaultFilter but not a + MoneyRangeFilter. This confirms the money-range defaults are validated against + their real type rather than the permissive DefaultFilter shape. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"totalFundingAvailableRange": f.eq("100")}, + ) + assert exc.value.path == "filters.totalFundingAvailableRange" + + +def test_classify_invalid_registered_raises(): + """An invalid registered custom filter raises before a request body is built. + + "agency" is a stringArray; f.eq("NSF") (a scalar equivalence) does not fit it. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} + ) + assert exc.value.path == "filters.agency" + + +def test_classify_registered_number_filter_rejects_bool(): + """A boolean given to a numberComparison filter raises rather than shipping 1. + + bool is a subclass of int, so without an explicit rejection True would be + coerced to 1 and the request would carry a number where a boolean was passed. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + WIRE_ROUTES, "opportunities", "search", {"awardCount": f.eq(True)} + ) + assert exc.value.path == "filters.awardCount" + + +def test_classify_adhoc_value_incompatible_with_operator_raises(): + """An ad-hoc filter whose value does not fit its operator raises FilterError. + + "in" expects a list, so {"operator": "in", "value": "NSF"} (a plain string) is + rejected instead of passing through unchecked to the request. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"legacyTag": {"operator": "in", "value": "NSF"}}, + ) + assert exc.value.path == "filters.legacyTag" + + +def test_classify_adhoc_unknown_operator_raises(): + """An ad-hoc filter with an unknown operator raises FilterError.""" + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"adhoc": {"operator": "bogus", "value": 1}}, + ) + assert exc.value.path == "filters.adhoc" + + +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_classify_mutated_adhoc_instance_is_revalidated_and_raises(): + """An ad-hoc filter mutated after construction is re-validated and raises. + + The filter models are mutable, so the classifier validates the value at call + time rather than trusting that it was valid when it was built. + """ + flt = f.eq("x") + flt.operator = "bogus" # type: ignore[assignment] + with pytest.raises(FilterError) as exc: + classify_filters(SAMPLE_ROUTES, "opportunities", "search", {"legacy": flt}) + assert exc.value.path == "filters.legacy" + + +def test_classify_both_forms_of_same_default_filter_raises(): + """Supplying the snake_case and camelCase forms of one default filter raises. + + Both keys normalize to "closeDateRange"; supplying both is ambiguous, so it + raises rather than silently dropping one of the two values. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "close_date_range": f.between("2026-01-01", "2026-06-30"), + "closeDateRange": f.between("2026-07-01", "2026-12-31"), + }, + ) + assert exc.value.path == "filters.closeDateRange" + assert "more than once" in str(exc.value) + + +def test_classify_raises_on_the_first_invalid_filter(): + """When several filters are invalid, classify raises on the first one it reaches. + + The whole call fails; no partial request body is produced. + """ + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + {"status": f.eq("open"), "closeDateRange": f.eq("x")}, + ) + assert exc.value.path in {"filters.status", "filters.closeDateRange"} + + +def test_classify_mixed_valid_and_invalid_raises(): + """A mix of valid and invalid filters raises; a valid filter does not rescue the call.""" + with pytest.raises(FilterError): + classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "status": f.in_(["open"]), + "agency": f.in_(["NSF"]), + "fundingProgram": f.in_(["a", "b"]), # stringComparison, not an array + }, + ) + + +def test_classify_all_valid_mixed_buckets_returns_all_keys(): + """A mix of valid standard, registered, and ad-hoc filters classifies all of them.""" + result = classify_filters( + SAMPLE_ROUTES, + "opportunities", + "search", + { + "status": f.in_(["open"]), + "agency": f.in_(["NSF"]), + "legacyTag": f.eq("priority"), + }, + ) + assert result.status is not None + assert result.custom_filters is not None + assert "agency" in result.custom_filters + assert "legacyTag" in result.custom_filters + + +@pytest.mark.parametrize( + ("resource", "method"), + [ + ("opportunities", "list"), # method not declared in routes + ("opportunity", "search"), # resource near-miss (pluralization) + ], +) +def test_classify_unmatched_route_treats_registered_name_as_adhoc(resource, method): + """A (resource, method) pair with no routes entry has no registered bucket. + + "agency" is registered as a stringArray under opportunities.search only. Via any + other pair it is validated as an ad-hoc filter, so f.eq("NSF") (a valid string + comparison) passes through to customFilters. This exercises both levels of the + routes lookup: a mis-keyed lookup would either apply the spec where it should + not, or skip it where it should apply. + """ + result = classify_filters(SAMPLE_ROUTES, resource, method, {"agency": f.eq("NSF")}) + assert result.custom_filters is not None + assert result.custom_filters["agency"].value == "NSF" + + # Via the declared pair, "agency" is validated against its stringArray model and + # f.eq("NSF") raises. + with pytest.raises(FilterError) as exc: + classify_filters( + SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} + ) + assert exc.value.path == "filters.agency" # --------------------------------------------------------------------------- @@ -250,39 +548,40 @@ def test_oppfilters_mixed_case_roundtrip(): # --------------------------------------------------------------------------- -def test_validate_routes_non_model_filter_type_raises(): - """validate_routes raises FilterError when a registered custom filter's value - type is not a filter value model. +def test_validate_routes_non_filter_model_type_raises(): + """validate_routes raises when a registered filter is annotated with a type that + is not a valid filter model. - The typed carrier makes a misspelled resource/method a *static* error, so the - only meaningful runtime check left is that each registered key is annotated - with a ``CommonGrantsBaseModel`` subclass. A ``region: int`` annotation is the - genuinely-invalid registration this surfaces at registration time. + The typed carrier makes a misspelled resource or method a static error, so the + remaining runtime check is that each registered key is annotated with one of the + valid filter models. A ``region: int`` annotation is rejected here. """ class BadFilters(OpportunityFilters, total=False): region: int # not a filter value model - routes = PluginRoutes(opportunities=ResourceRoutes(search=BadFilters)) with pytest.raises(FilterError): - validate_routes(routes) + validate_routes(PluginRoutes(opportunities=ResourceRoutes(search=BadFilters))) def test_validate_routes_redeclared_standard_key_wrong_type_raises(): - """validate_routes raises when a route TypedDict redeclares a standard key with - a different value type — the call site would see the override while classify - validates against the real standard field type (a static/runtime mismatch).""" + """validate_routes raises when a route redeclares a standard key with a different type. + + The call site would see the override while the classifier validates against the + real standard field type, so the two would disagree. + """ class RedeclaresStatus(OpportunityFilters, total=False): status: StringComparison # standard "status" is a StringArray filter - routes = PluginRoutes(opportunities=ResourceRoutes(search=RedeclaresStatus)) with pytest.raises(FilterError): - validate_routes(routes) + validate_routes( + PluginRoutes(opportunities=ResourceRoutes(search=RedeclaresStatus)) + ) def test_validate_routes_redeclared_standard_key_same_type_ok(): - """Redeclaring a standard key with its SAME type is a harmless no-op override.""" + """Redeclaring a standard key with its same type is a harmless override.""" class RedeclaresStatusSame(OpportunityFilters, total=False): status: StringArray # same as the standard type @@ -294,7 +593,6 @@ class RedeclaresStatusSame(OpportunityFilters, total=False): def test_validate_routes_valid_routes_do_not_raise(): """validate_routes does not raise for a fully valid typed routes carrier.""" - # Should not raise validate_routes(SAMPLE_ROUTES) @@ -304,53 +602,69 @@ def test_validate_routes_empty_carrier_does_not_raise(): # --------------------------------------------------------------------------- -# Call-time validation (validate_filter_call) — fail-soft +# Call-time validation (validate_filter_call) — returns (value, error) # --------------------------------------------------------------------------- def test_validate_filter_call_registered_bad_operator_returns_error(): """validate_filter_call returns (None, FilterError) for an operator/value mismatch. - A bad registered filter yields a FilterError, not an exception. agency is - validated against StringArrayFilter — an "eq" with a scalar value is wrong for it. + "agency" is validated against StringArrayFilter, so an equivalence operator with + a scalar value does not fit it. """ - bad_filter = f.eq("not-an-array") - value, error = validate_filter_call(StringArrayFilter, "agency", bad_filter) + value, error = validate_filter_call(StringArrayFilter, "agency", f.eq("not-a-list")) assert value is None assert isinstance(error, FilterError) assert error.path == "filters.agency" -def test_validate_filter_call_adhoc_invalid_shape_returns_error(): - """validate_filter_call returns a FilterError when an ad-hoc filter has an invalid shape.""" +def test_validate_filter_call_valid_registered_returns_value_and_no_error(): + """validate_filter_call returns (DefaultFilter, None) for a valid registered filter call.""" + value, error = validate_filter_call(StringArrayFilter, "agency", f.in_(["NSF"])) + assert error is None + assert value is not None + assert value.operator == "in" + + +def test_validate_filter_call_adhoc_accepts_valid_operator_value(): + """Ad-hoc validation accepts a raw operator/value dict that matches a known model.""" + value, error = validate_filter_call(None, "x", {"operator": "eq", "value": "v"}) + assert error is None + assert value is not None + assert value.operator == "eq" + assert value.value == "v" + - # Pass None as model_cls (ad-hoc), with something that isn't a DefaultFilter - class _BadShape: - operator = "not_a_real_operator" - value = object() # not a valid value type +def test_validate_filter_call_adhoc_incompatible_value_returns_error(): + """Ad-hoc validation returns a FilterError when the value does not fit the operator. - value, error = validate_filter_call(None, "legacyTag", _BadShape()) # type: ignore[arg-type] + "in" expects a list, so a plain string does not match any valid filter model. + """ + value, error = validate_filter_call( + None, "legacyTag", {"operator": "in", "value": "NSF"} + ) assert value is None assert isinstance(error, FilterError) assert error.path == "filters.legacyTag" -def test_validate_filter_call_valid_registered_returns_value_and_no_error(): - """validate_filter_call returns (DefaultFilter, None) for a valid registered filter call.""" - valid_filter = f.in_(["NSF", "NIH"]) - # agency is StringArrayFilter, in_ with list is valid - value, error = validate_filter_call(StringArrayFilter, "agency", valid_filter) +def test_validate_filter_call_integer_comparison_rejects_non_integer(): + """An integerComparison filter accepts an integer and rejects a non-integer value.""" + value, error = validate_filter_call(IntegerComparisonFilter, "awardCount", f.gt(5)) assert error is None assert value is not None - assert value.operator == "in" + + value, error = validate_filter_call( + IntegerComparisonFilter, "awardCount", {"operator": "gt", "value": 3.5} + ) + assert value is None + assert isinstance(error, FilterError) def test_validate_filter_call_money_comparison_passes_valid_money(): - """A moneyComparison filter accepts a comparison operator and Money value. + """A moneyComparison filter accepts a comparison operator and a Money value. - Money.amount is a decimal STRING ("1000000"), not a number — the shape that - drifted in the TS compile-time filter map and was locked there with - compile-error tests; covered here at the runtime layer. + Money.amount is a decimal string ("1000000"), not a number. """ value, error = validate_filter_call( MoneyComparisonFilter, @@ -375,7 +689,7 @@ def test_validate_filter_call_money_comparison_rejects_array_operator(): def test_validate_filter_call_money_comparison_rejects_numeric_amount(): """A moneyComparison filter returns a FilterError for a numeric amount. - Money.amount is a DecimalString — a raw number is the wrong shape. + Money.amount is a decimal string, so a raw number is the wrong shape. """ value, error = validate_filter_call( MoneyComparisonFilter, "awardFloor", f.gt({"amount": 1000.5, "currency": "USD"}) @@ -385,7 +699,7 @@ def test_validate_filter_call_money_comparison_rejects_numeric_amount(): def test_validate_filter_call_money_range_passes_valid_range(): - """A moneyRange filter accepts between with {min, max} Money values.""" + """A moneyRange filter accepts between with min and max Money values.""" value, error = validate_filter_call( MoneyRangeFilter, "awardRange", @@ -408,13 +722,11 @@ def test_validate_filter_call_money_range_rejects_comparison_operator(): def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): - """A numberRange filter round-trips its NumberRange sub-model to wire dict. + """A numberRange filter round-trips its NumberRange sub-model to a wire dict. f.between(int, int) returns a NumberRangeFilter whose ``.value`` is a NumberRange - sub-model (not a plain dict); validation must accept it and model_dump must recurse - the sub-model to ``{min, max}``. A regression that left ``.value`` as an un-dumped - NumberRange object — or that dropped the int payload — would ship a non-JSON body. - The moneyRange analog above is covered; this pins the numeric path. + sub-model rather than a plain dict; validation must accept it and model_dump must + recurse into the sub-model to produce ``{min, max}``. """ value, error = validate_filter_call( NumberRangeFilter, "awardCount", f.between(0, 1000) @@ -425,375 +737,14 @@ def test_validate_filter_call_number_range_value_submodel_survives_to_wire(): assert wire["value"] == {"min": 0, "max": 1000} -def test_classify_default_wrong_shape_collects_error_and_omits_key(): - """A wrong-shaped DEFAULT filter is fail-soft: omitted from result, error collected. - - "status" is a StringArrayFilter (ArrayOperator + list[str]); f.eq("open") is an - equivalence filter. classify_filters no longer raises on a bad call-time filter - value: the bad key is dropped from the result body and a - single FilterError with path "filters.status" is collected. - """ - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} - ) - # No raise; the invalid default is absent from the result body. - assert classified.result.status is None - # Exactly one collected error, pinpointed to filters.status. - assert len(classified.errors) == 1 - assert classified.errors[0].path == "filters.status" - # The underlying pydantic error is preserved as the structured cause. - assert isinstance(classified.errors[0].cause, ValidationError) - - def test_validate_filter_call_number_comparison_validates_as_number(): - """A numberComparison filter validates against NumberComparisonFilter. - - A numeric value passes; a non-numeric value fails. - """ + """A numberComparison filter accepts a number and rejects a non-number value.""" value, error = validate_filter_call(NumberComparisonFilter, "awardCount", f.gt(100)) assert error is None assert value is not None + value, error = validate_filter_call( NumberComparisonFilter, "awardCount", f.gt("not a number") ) assert value is None assert isinstance(error, FilterError) - - -def test_classify_default_camel_alias_wrong_shape_collects_error(): - """A wrong-shaped default filter via its camelCase alias is fail-soft, omitted from result. - - "closeDateRange" is a DateRangeFilter; f.eq("2026-01-01") is an equivalence - filter — the alias normalization path must collect a FilterError (not raise) - and omit the key. The error path uses the alias form. - """ - classified = classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - {"closeDateRange": f.eq("2026-01-01")}, - ) - assert classified.result.close_date_range is None - assert len(classified.errors) == 1 - assert classified.errors[0].path == "filters.closeDateRange" - - -def test_classify_default_money_range_wrong_shape_collects_error(): - """A wrong-shaped MoneyRangeFilter default is dropped and its error collected. - - "totalFundingAvailableRange" is a MoneyRangeFilter (RangeOperator + MoneyRange); - f.eq("100") is an equivalence filter with a scalar value — a valid permissive - DefaultFilter, but not a MoneyRangeFilter. MoneyRangeFilter is the one default - type the sibling tests (status -> StringArrayFilter, closeDateRange -> - DateRangeFilter) do not exercise. The regression this guards: if the money-range - defaults were validated against the permissive DefaultFilter shape instead of - their real type, this f.eq value would pass and a bad body would reach the server. - """ - classified = classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - {"totalFundingAvailableRange": f.eq("100")}, - ) - # Fail-soft: omitted from result, error collected (not raised). - assert classified.result.total_funding_available_range is None - assert len(classified.errors) == 1 - assert classified.errors[0].path == "filters.totalFundingAvailableRange" - # Same uniform error contract as the sibling default tests: the underlying - # pydantic ValidationError is preserved as the structured cause. - assert isinstance(classified.errors[0].cause, ValidationError) - - -# --------------------------------------------------------------------------- -# Wire-body integrity: the value that passed validation is the value shipped -# --------------------------------------------------------------------------- - - -class WireFilters(OpportunityFilters, total=False): - """Registered wire-integrity filters: a boolean and a number comparison.""" - - isOpen: BooleanComparison - awardCount: NumberComparison - - -WIRE_ROUTES = PluginRoutes(opportunities=ResourceRoutes(search=WireFilters)) - - -def test_boolean_filter_value_survives_to_wire_as_json_true(): - """f.eq(True) serializes as JSON true, not 1. - - DefaultFilter.value is Any per the core spec (filters/base.tsp `unknown`); - a narrowed union without bool lax-coerced True -> 1 and corrupted the wire. - """ - classified = classify_filters( - WIRE_ROUTES, "opportunities", "search", {"isOpen": f.eq(True)} - ) - assert classified.errors == [] - body = classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") - assert body["customFilters"]["isOpen"]["value"] is True - - -def test_registered_filter_ships_validated_value_not_raw_input(): - """The classifier keeps the value that passed validation, not the raw input. - - NumberComparisonFilter lax-coerces "42" -> 42 (smart-union resolves the - int|float union to int first); shipping the raw string would mean the - value differs from what validation approved. - """ - classified = classify_filters( - WIRE_ROUTES, "opportunities", "search", {"awardCount": f.gt("42")} - ) - assert classified.errors == [] - body = classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") - assert body["customFilters"]["awardCount"]["value"] == 42 - assert isinstance(body["customFilters"]["awardCount"]["value"], int) - - -def test_number_comparison_registered_filter_rejects_bool(): - """f.eq(True) on a numberComparison-registered filter is dropped, never ships 1. - - bool subclasses int; without an explicit rejection the int|float union - lax-coerces True -> 1 and the wire silently carries a number for a - boolean — the corruption class the DefaultFilter.value widening fixed. - Fail-soft: the bad registered filter is collected as an error and omitted - from customFilters rather than raising. - """ - classified = classify_filters( - WIRE_ROUTES, "opportunities", "search", {"awardCount": f.eq(True)} - ) - assert classified.result.custom_filters is None - assert len(classified.errors) == 1 - assert classified.errors[0].path == "filters.awardCount" - - -@pytest.mark.filterwarnings("ignore::UserWarning") # pydantic warns during the -# model_dump of the mutated instance, before re-validation collects the error -def test_mutated_adhoc_instance_is_revalidated_and_collected(): - """An ad-hoc DefaultFilter mutated after construction is collected, not shipped. - - The filter models are mutable; the ad-hoc branch must re-validate instances - rather than trust isinstance. Fail-soft: the error is collected and the key - omitted from customFilters. - """ - flt = f.eq("x") - flt.operator = "bogus" # type: ignore[assignment] - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"legacy": flt} - ) - assert classified.result.custom_filters is None - assert len(classified.errors) == 1 - assert classified.errors[0].path == "filters.legacy" - - -def test_validate_filter_call_adhoc_accepts_raw_dict(): - """Ad-hoc validation accepts a raw operator/value dict and returns (DefaultFilter, None).""" - validated, error = validate_filter_call(None, "x", {"operator": "eq", "value": "v"}) - assert error is None - assert validated is not None - assert validated.operator == "eq" - assert validated.value == "v" - - -# --------------------------------------------------------------------------- -# Alias normalization (snake form), serialization contract, error paths -# --------------------------------------------------------------------------- - - -def test_classify_default_snake_form_of_aliased_key_normalizes_to_alias(): - """Snake_case key for an ALIASED field lands in the named field, not customFilters. - - Exercises the _SNAKE_TO_ALIAS hit branch: without normalization, - OppFilters(close_date_range=...) is silently dropped by pydantic - (populate_by_name is not set) and the field stays None. - """ - consumer_filters = {"close_date_range": f.between("2026-01-01", "2026-12-31")} - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters - ) - assert classified.errors == [] - assert classified.result.close_date_range is not None - assert classified.result.custom_filters is None - - -def test_classify_both_forms_of_same_default_filter_collects_error(): - """Supplying snake AND camel forms of one default filter is fail-soft. - - Both keys normalize to "closeDateRange". The first form is kept; the - duplicate is dropped and a FilterError collected (a bad call-time filter - never raises). Without the dedup guard, dict assignment - would silently drop whichever range the consumer's dict ordered first. - """ - consumer_filters = { - "close_date_range": f.between("2026-01-01", "2026-06-30"), - "closeDateRange": f.between("2026-07-01", "2026-12-31"), - } - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters - ) - # The first-seen value is kept on the result; the duplicate is dropped. - assert classified.result.close_date_range is not None - assert len(classified.errors) == 1 - assert classified.errors[0].path == "filters.closeDateRange" - assert "more than once" in str(classified.errors[0]) - - -@pytest.mark.parametrize( - ("resource", "method"), - [ - ("opportunities", "list"), # method not declared in routes - ("opportunity", "search"), # resource near-miss (pluralization) - ], -) -def test_classify_unmatched_route_treats_registered_name_as_adhoc(resource, method): - """A (resource, method) pair with no routes entry has NO registered bucket. - - "agency" is registered as a stringArray under opportunities.search only; - via any other route pair it is validated as permissive ad-hoc, so - f.eq("NSF") (invalid for stringArray) passes through to customFilters. - Discriminates both levels of the routes[resource][method] lookup — a - regression that flattens or mis-keys it either wrongly applies the spec - or wrongly skips it. - """ - classified = classify_filters( - SAMPLE_ROUTES, resource, method, {"agency": f.eq("NSF")} - ) - assert classified.errors == [] - assert classified.result.custom_filters is not None - assert classified.result.custom_filters["agency"].value == "NSF" - - # ...and the same filter via the declared pair IS spec-validated and dropped - # fail-soft: omitted from result, error collected (no raise). - declared = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} - ) - assert declared.result.custom_filters is None - assert len(declared.errors) == 1 - assert declared.errors[0].path == "filters.agency" - - -def test_request_body_mode_json_round_trip(): - """The documented model_dump(mode="json") call yields a json.dumps-able body. - - Coerced date objects only serialize in json mode, which is what the - classifier dumps. - """ - consumer_filters = { - "close_date_range": f.between(date(2026, 1, 1), date(2026, 12, 31)), - "agency": f.in_(["NSF"]), - } - classified = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", consumer_filters - ) - assert classified.errors == [] - body = json.loads( - json.dumps( - classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") - ) - ) - assert body["closeDateRange"]["operator"] == "between" - assert body["closeDateRange"]["value"]["min"] == "2026-01-01" - assert body["customFilters"]["agency"]["operator"] == "in" - - -def test_classify_empty_filters_dict_yields_empty_body(): - """An empty consumer dict produces an OppFilters with no customFilters entry.""" - classified = classify_filters(SAMPLE_ROUTES, "opportunities", "search", {}) - assert classified.errors == [] - assert classified.result.custom_filters is None - body = classified.result.model_dump(by_alias=True, exclude_none=True, mode="json") - assert "customFilters" not in body - - -def test_collected_error_path_is_uniform_across_buckets(): - """All three buckets collect a FilterError with a filters.<name> path, none raise.""" - c1 = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"status": f.eq("open")} - ) - assert len(c1.errors) == 1 - assert c1.errors[0].path == "filters.status" - - c2 = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"agency": f.eq("NSF")} - ) - assert len(c2.errors) == 1 - assert c2.errors[0].path == "filters.agency" - - c3 = classify_filters( - SAMPLE_ROUTES, "opportunities", "search", {"adhoc": {"operator": "bogus"}} - ) - assert len(c3.errors) == 1 - assert c3.errors[0].path == "filters.adhoc" - - -def test_multiple_failing_defaults_collect_one_error_per_key(): - """Two failing default filters each collect their own FilterError, none raise. - - Per-key fail-soft: each invalid default is dropped and pinpointed - individually (no single collective "filters" path), so a consumer sees - exactly which keys failed. - """ - classified = classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - {"status": f.eq("open"), "closeDateRange": f.eq("x")}, - ) - assert classified.result.status is None - assert classified.result.close_date_range is None - paths = sorted(e.path for e in classified.errors) - assert paths == ["filters.closeDateRange", "filters.status"] - - -def test_classify_invalid_registered_and_invalid_adhoc_both_collected(): - """An invalid registered filter AND an invalid ad-hoc filter are both collected. - - Neither is shipped: both keys are omitted from customFilters and each yields - its own FilterError. "agency" is a stringArray (f.eq scalar is wrong); the - ad-hoc "adhoc" key has a bogus operator. - """ - classified = classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - {"agency": f.eq("NSF"), "adhoc": {"operator": "bogus", "value": 1}}, - ) - assert classified.result.custom_filters is None - paths = sorted(e.path for e in classified.errors) - assert paths == ["filters.adhoc", "filters.agency"] - - -def test_classify_mixed_valid_and_invalid_keeps_valid_drops_invalid(): - """Mixed valid + invalid filters: valid keys present in result, invalid absent. - - Discriminating across all three buckets at once — a valid default (status), - a valid registered custom (agency), a valid ad-hoc (legacyTag), plus an - invalid default (closeDateRange) and an invalid registered (fundingProgram, - a stringComparison given an array value). The three valid keys must survive; - the two invalid keys must be omitted and exactly two errors collected. - """ - classified = classify_filters( - SAMPLE_ROUTES, - "opportunities", - "search", - { - "status": f.in_(["open"]), # valid default - "agency": f.in_(["NSF"]), # valid registered (stringArray) - "legacyTag": f.eq("priority"), # valid ad-hoc - "closeDateRange": f.eq("2026-01-01"), # invalid default (not a range) - "fundingProgram": f.in_( - ["a", "b"] - ), # invalid registered (stringComparison) - }, - ) - result = classified.result - # Valid keys present - assert result.status is not None - assert result.custom_filters is not None - assert "agency" in result.custom_filters - assert "legacyTag" in result.custom_filters - # Invalid keys absent - assert result.close_date_range is None - assert "fundingProgram" not in result.custom_filters - # Exactly the two invalid keys collected (count discriminates). - paths = sorted(e.path for e in classified.errors) - assert paths == ["filters.closeDateRange", "filters.fundingProgram"] From 20795f0302bebae9859b6ed4244395796ffb574f Mon Sep 17 00:00:00 2001 From: Bryan Thompson <18094023+SnowboardTechie@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:23:34 -0700 Subject: [PATCH 46/51] fix(py-sdk): ship PEP 561 py.typed marker (#1002) * fix(py-sdk): ship PEP 561 py.typed marker The package is fully type-annotated but shipped no py.typed marker, so downstream type checkers treated every common_grants_sdk import as untyped. Adding the marker lets consumers pick up the SDK's inline annotations, matching the TypeScript SDK which already ships its .d.ts. * fix(ci): write multiline py_changelog via heredoc in version job The version job greps all changesets for "common-grants-sdk" and wrote the result to $GITHUB_OUTPUT as key=value. When more than one changeset bumps the package (a feature + this py.typed patch), the grep result is multiline and GitHub Actions rejects the key=value form. Use the heredoc delimiter form so any number of changesets is handled. --- .changeset/python-sdk-py-typed.md | 7 +++++++ .github/workflows/ci-bump-version.yml | 9 ++++++++- lib/python-sdk/common_grants_sdk/py.typed | 0 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .changeset/python-sdk-py-typed.md create mode 100644 lib/python-sdk/common_grants_sdk/py.typed diff --git a/.changeset/python-sdk-py-typed.md b/.changeset/python-sdk-py-typed.md new file mode 100644 index 000000000..4e70f2561 --- /dev/null +++ b/.changeset/python-sdk-py-typed.md @@ -0,0 +1,7 @@ +--- +"common-grants-sdk": patch +--- + +Ship a PEP 561 `py.typed` marker so downstream type checkers use the SDK's inline type annotations. + +The package is fully type-annotated but shipped no marker, so `mypy` treated every `common_grants_sdk` import as untyped (`import-untyped`, "missing library stubs or py.typed marker") and consumers got no type-checking from the SDK. Adding the marker brings the Python SDK to parity with the TypeScript SDK, which already ships its `.d.ts` types. diff --git a/.github/workflows/ci-bump-version.yml b/.github/workflows/ci-bump-version.yml index 01757256c..899994f0e 100644 --- a/.github/workflows/ci-bump-version.yml +++ b/.github/workflows/ci-bump-version.yml @@ -71,7 +71,14 @@ jobs: exit 0 else echo "found_py=true" >> $GITHUB_OUTPUT - echo "py_changelog=$PY_CHANGELOG" >> $GITHUB_OUTPUT + # Multiple changesets can bump common-grants-sdk (e.g. a feature + + # a fix landing in the same release), so PY_CHANGELOG may be + # multiline — write it via the heredoc form, not key=value. + { + echo "py_changelog<<EOF" + echo "$PY_CHANGELOG" + echo "EOF" + } >> $GITHUB_OUTPUT fi - name: Bump version for Python package diff --git a/lib/python-sdk/common_grants_sdk/py.typed b/lib/python-sdk/common_grants_sdk/py.typed new file mode 100644 index 000000000..e69de29bb From af1f759696a49fd815fe995f0ffd3ae2c8d9ef4d Mon Sep 17 00:00:00 2001 From: widal001 <billy.daly@agile6.com> Date: Wed, 15 Jul 2026 00:49:00 -0400 Subject: [PATCH 47/51] Revert "feat(core): add boolean and integer comparison filter models (#953)" This reverts commit c861b407cbc92c025cc147c639b03f7ed141a0b4. We want to group these changes with other changes to the core library that will be released with v0.4.0 rather than releasing them with the next SDK version. --- .../boolean-integer-comparison-filters.md | 8 -------- lib/core/lib/core/filters/boolean.tsp | 18 ------------------ lib/core/lib/core/filters/index.tsp | 1 - lib/core/lib/core/filters/numeric.tsp | 15 --------------- lib/core/lib/main.tsp | 1 - lib/python-sdk/.gitignore | 4 ---- 6 files changed, 47 deletions(-) delete mode 100644 .changeset/boolean-integer-comparison-filters.md delete mode 100644 lib/core/lib/core/filters/boolean.tsp diff --git a/.changeset/boolean-integer-comparison-filters.md b/.changeset/boolean-integer-comparison-filters.md deleted file mode 100644 index af4de730a..000000000 --- a/.changeset/boolean-integer-comparison-filters.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -"@common-grants/core": minor ---- - -Add `BooleanComparisonFilter` and `IntegerComparisonFilter` to the core filter catalog (#895). -`BooleanComparisonFilter` compares a boolean value with `eq`/`neq`; `IntegerComparisonFilter` -compares an integer value with the full comparison + equivalence operator set. Both are added at -protocol version 0.4.0 and emitted to JSON Schema output. diff --git a/lib/core/lib/core/filters/boolean.tsp b/lib/core/lib/core/filters/boolean.tsp deleted file mode 100644 index ee884f5e8..000000000 --- a/lib/core/lib/core/filters/boolean.tsp +++ /dev/null @@ -1,18 +0,0 @@ -import "./base.tsp"; - -namespace CommonGrants.Filters; - -// ############################################################################ -// Boolean comparison filter -// ############################################################################ - -/** A filter that applies a comparison to a boolean value */ -@Versioning.added(CommonGrants.Versions.v0_4) -model BooleanComparisonFilter { - /** The operator to apply to the filter value */ - operator: EquivalenceOperators; - - /** The value to use for the filter operation */ - @example(true) - value: boolean; -} diff --git a/lib/core/lib/core/filters/index.tsp b/lib/core/lib/core/filters/index.tsp index c85dd2a17..c111b1844 100644 --- a/lib/core/lib/core/filters/index.tsp +++ b/lib/core/lib/core/filters/index.tsp @@ -1,5 +1,4 @@ import "./base.tsp"; -import "./boolean.tsp"; import "./date.tsp"; import "./numeric.tsp"; import "./money.tsp"; diff --git a/lib/core/lib/core/filters/numeric.tsp b/lib/core/lib/core/filters/numeric.tsp index 6cfbf743f..b01933e23 100644 --- a/lib/core/lib/core/filters/numeric.tsp +++ b/lib/core/lib/core/filters/numeric.tsp @@ -20,21 +20,6 @@ model NumberComparisonFilter { value: numeric; } -// ############################################################################ -// Integer comparison filter -// ############################################################################ - -/** Filters by comparing a field to an integer value */ -@Versioning.added(CommonGrants.Versions.v0_4) -model IntegerComparisonFilter { - /** The comparison operator to apply to the filter value */ - operator: ComparisonOperators | EquivalenceOperators; - - /** The value to use for the filter operation */ - @example(100) - value: integer; -} - // ############################################################################ // Number range filter // ############################################################################ diff --git a/lib/core/lib/main.tsp b/lib/core/lib/main.tsp index 80c27f8da..8d270ac86 100644 --- a/lib/core/lib/main.tsp +++ b/lib/core/lib/main.tsp @@ -19,5 +19,4 @@ enum Versions { v0_1: "0.1.0", v0_2: "0.2.0", v0_3: "0.3.0", - v0_4: "0.4.0", } diff --git a/lib/python-sdk/.gitignore b/lib/python-sdk/.gitignore index c2ace7c4e..a4db3a2c8 100644 --- a/lib/python-sdk/.gitignore +++ b/lib/python-sdk/.gitignore @@ -11,9 +11,5 @@ __pycache__/ .pytest_cache/ .coverage -# Virtual environments -.venv/ -venv/ - #Generated Schema objects generated/ \ No newline at end of file From 7f56f447b2fb695b04e4bcf841afc1b8444977f9 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <bryan@snowboardtechie.com> Date: Wed, 15 Jul 2026 09:46:42 -0700 Subject: [PATCH 48/51] fix(py-sdk): let snake_case construction of aliased models type-check Convert the Event models (eventType, startDate, startTime, endDate, endTime) and the customValue field on OppStatus / ApplicantType from bare alias= to validation_alias + serialization_alias, matching the pattern CustomField and PluginMeta already use, and set populate_by_name=True on CommonGrantsBaseModel. Bare alias= makes pyright type the constructor as alias-only, so idiomatic snake_case calls like SingleDateEvent(event_type=...) were rejected statically and ignored by validation at runtime. Wire I/O is unchanged: parsing still accepts camelCase and model_dump(by_alias=True) still emits it. Internal readers of field_info.alias (default-filter classification, transform mapping validation) now consult all three alias fields so models on either alias style resolve. --- .changeset/python-sdk-populate-by-name.md | 5 ++++ .../common_grants_sdk/extensions/filters.py | 30 ++++++++++--------- .../common_grants_sdk/extensions/schema.py | 5 ++-- .../extensions/transforms.py | 9 ++++-- .../schemas/pydantic/base.py | 3 ++ .../schemas/pydantic/fields/event.py | 24 ++++++++++----- .../pydantic/models/opp_applicant_type.py | 3 +- .../schemas/pydantic/models/opp_status.py | 3 +- 8 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 .changeset/python-sdk-populate-by-name.md diff --git a/.changeset/python-sdk-populate-by-name.md b/.changeset/python-sdk-populate-by-name.md new file mode 100644 index 000000000..9b0388194 --- /dev/null +++ b/.changeset/python-sdk-populate-by-name.md @@ -0,0 +1,5 @@ +--- +"common-grants-sdk": patch +--- + +Make snake_case construction of aliased models type-check and validate: the Event models (`eventType`, `startDate`, `startTime`, `endDate`, `endTime`) and the `customValue` field on `OppStatus` / `ApplicantType` now use `validation_alias` + `serialization_alias` (the pattern `CustomField` and `PluginMeta` already use) instead of bare `alias`, and `CommonGrantsBaseModel` sets `populate_by_name=True`. Static type checkers previously rejected idiomatic calls like `SingleDateEvent(event_type=...)`, and validation ignored the snake_case name. Wire I/O is unchanged: parsing still accepts camelCase and `model_dump(by_alias=True)` still emits it. Internal alias readers (filter classification, transform mapping validation) now consult all three alias fields, so both alias styles resolve. diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index 7c8bc8da1..b65e24126 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -301,32 +301,34 @@ def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: #: field (an alias-only set would silently drop snake_case keys into ``customFilters``). DEFAULT_FILTER_NAMES: frozenset[str] = frozenset( list(OppDefaultFilters.model_fields.keys()) - + [v.alias for v in OppDefaultFilters.model_fields.values() if v.alias] + + [ + name + for v in OppDefaultFilters.model_fields.values() + for name in (v.alias, v.validation_alias, v.serialization_alias) + if isinstance(name, str) + ] ) # --------------------------------------------------------------------------- # Alias-normalization maps for classify_filters # -# OppDefaultFilters uses snake_case field names with camelCase aliases but does NOT -# set populate_by_name=True. Pydantic v2 therefore requires the alias form when -# constructing OppFilters via **kwargs — passing the snake_case field name silently -# results in None (the alias is the required construction key). -# -# classify_filters normalizes consumer keys to the alias (or field-name for fields -# without an alias) before passing them to OppFilters(**...): +# Consumers may pass either the snake_case field name or the camelCase wire +# alias. classify_filters normalizes keys to the wire alias (or field-name for +# fields without one) before passing them to OppFilters(**...): # - snake_case keys with a camelCase alias → converted to the alias (closeDateRange) # - camelCase alias keys → kept as-is (already the alias) # - keys with no alias (e.g. "status") → kept as-is (snake == request key) # --------------------------------------------------------------------------- -# Map from snake_case field name → camelCase alias (used for OppFilters construction). -# Only fields that declare an alias are included; alias-form keys and fields without -# aliases fall through ``_SNAKE_TO_ALIAS.get(key, key)`` unchanged — one lookup -# normalizes all three key classes. +# Map from snake_case field name → camelCase wire alias (used for OppFilters +# construction). Reads ``serialization_alias`` before bare ``alias`` so fields on +# either alias style resolve; fields without an alias and alias-form keys fall +# through ``_SNAKE_TO_ALIAS.get(key, key)`` unchanged — one lookup normalizes all +# three key classes. _SNAKE_TO_ALIAS: dict[str, str] = { - field_name: field_info.alias + field_name: wire_name for field_name, field_info in OppDefaultFilters.model_fields.items() - if field_info.alias + if (wire_name := field_info.serialization_alias or field_info.alias) } # --------------------------------------------------------------------------- diff --git a/lib/python-sdk/common_grants_sdk/extensions/schema.py b/lib/python-sdk/common_grants_sdk/extensions/schema.py index 6e848e2c9..22746d926 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/schema.py +++ b/lib/python-sdk/common_grants_sdk/extensions/schema.py @@ -217,8 +217,9 @@ def _output_field_names(model: type[BaseModel]) -> set[str]: """Valid top-level output keys for a model: field names plus their aliases.""" names: set[str] = set(model.model_fields.keys()) for info in model.model_fields.values(): - if info.alias: - names.add(info.alias) + for alias in (info.alias, info.validation_alias, info.serialization_alias): + if isinstance(alias, str): + names.add(alias) return names diff --git a/lib/python-sdk/common_grants_sdk/extensions/transforms.py b/lib/python-sdk/common_grants_sdk/extensions/transforms.py index 4f3c0dbe9..61fc0cbab 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/transforms.py +++ b/lib/python-sdk/common_grants_sdk/extensions/transforms.py @@ -48,8 +48,13 @@ def _validate_output_paths( valid_names: set[str] = set(model.model_fields.keys()) for field_info in model.model_fields.values(): - if field_info.alias: - valid_names.add(field_info.alias) + for alias in ( + field_info.alias, + field_info.validation_alias, + field_info.serialization_alias, + ): + if isinstance(alias, str): + valid_names.add(alias) # Top-level handler invocations (rare but structurally valid) are not output keys output_keys = {k for k in mapping if k not in known_handlers} diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py index f3a5edc1c..f12defab6 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py @@ -12,6 +12,9 @@ class CommonGrantsBaseModel(BaseModel): model_config = ConfigDict( from_attributes=True, strict=False, # Coerces strings to enums, datetimes, etc. + # Accept snake_case field names alongside their camelCase wire aliases + # in constructors and validation; serialization still uses the aliases. + populate_by_name=True, ) def dump(self) -> dict: diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py index c231db931..395857e71 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py @@ -29,7 +29,8 @@ class EventBase(CommonGrantsBaseModel): ) event_type: EventType = Field( ..., - alias="eventType", + validation_alias="eventType", + serialization_alias="eventType", description="Type of event", ) description: Optional[str] = Field( @@ -44,7 +45,8 @@ class SingleDateEvent(EventBase): event_type: Literal[EventType.SINGLE_DATE] = Field( EventType.SINGLE_DATE, - alias="eventType", + validation_alias="eventType", + serialization_alias="eventType", ) date: ISODate = Field( ..., @@ -62,26 +64,31 @@ class DateRangeEvent(EventBase): event_type: Literal[EventType.DATE_RANGE] = Field( EventType.DATE_RANGE, - alias="eventType", + validation_alias="eventType", + serialization_alias="eventType", ) start_date: ISODate = Field( ..., - alias="startDate", + validation_alias="startDate", + serialization_alias="startDate", description="Start date of the event in ISO 8601 format: YYYY-MM-DD", ) start_time: Optional[ISOTime] = Field( default=None, - alias="startTime", + validation_alias="startTime", + serialization_alias="startTime", description="Start time of the event in ISO 8601 format: HH:MM:SS", ) end_date: ISODate = Field( ..., - alias="endDate", + validation_alias="endDate", + serialization_alias="endDate", description="End date of the event in ISO 8601 format: YYYY-MM-DD", ) end_time: Optional[ISOTime] = Field( default=None, - alias="endTime", + validation_alias="endTime", + serialization_alias="endTime", description="End time of the event in ISO 8601 format: HH:MM:SS", ) @@ -92,7 +99,8 @@ class OtherEvent(EventBase): event_type: Literal[EventType.OTHER] = Field( EventType.OTHER, - alias="eventType", + validation_alias="eventType", + serialization_alias="eventType", ) details: Optional[str] = Field( default=None, diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py index f2941ac20..8e3e58ae0 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py @@ -41,7 +41,8 @@ class ApplicantType(CommonGrantsBaseModel): ) custom_value: Optional[str] = Field( default=None, - alias="customValue", + validation_alias="customValue", + serialization_alias="customValue", description="A custom applicant type value, only meaningful when value='custom'", ) description: Optional[str] = Field( diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py index 6abb482b8..d612f4f96 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py @@ -36,7 +36,8 @@ class OppStatus(CommonGrantsBaseModel): ) custom_value: Optional[str] = Field( default=None, - alias="customValue", + validation_alias="customValue", + serialization_alias="customValue", description="A custom status value", ) description: Optional[str] = Field( From abc366eda436f3ab78513c11894f01a7b6361207 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <bryan@snowboardtechie.com> Date: Wed, 15 Jul 2026 15:38:50 -0700 Subject: [PATCH 49/51] fix(py-sdk): derive camelCase wire naming from a shared alias generator Rework of the per-field validation_alias/serialization_alias approach after review: a to_camel AliasGenerator plus populate_by_name on the model config gives every field its camelCase wire alias while static checkers accept the snake_case field names. Applied to CommonGrantsBaseModel and to the wire models that do not extend it (sorting, pagination, filters), replacing ~60 per-field alias declarations. Every removed declaration matched its field's to_camel form exactly, so wire I/O is unchanged; the one irregular wire name (CustomField.schema_url -> "schema") keeps its explicit field-level alias, which overrides the generator. Internal camelCase constructor calls (pagination aggregation, filter classification, default sorting) move to snake_case field names accordingly. --- .changeset/python-sdk-populate-by-name.md | 2 +- .../common_grants_sdk/client/pagination.py | 8 +++---- .../common_grants_sdk/extensions/filters.py | 3 +-- .../schemas/pydantic/base.py | 21 ++++++++++++++++--- .../schemas/pydantic/fields/custom.py | 2 -- .../schemas/pydantic/fields/event.py | 16 -------------- .../schemas/pydantic/fields/metadata.py | 2 -- .../schemas/pydantic/filters/opportunity.py | 9 +++----- .../pydantic/models/opp_applicant_type.py | 2 -- .../schemas/pydantic/models/opp_base.py | 3 --- .../schemas/pydantic/models/opp_funding.py | 6 ------ .../schemas/pydantic/models/opp_status.py | 2 -- .../schemas/pydantic/models/opp_timeline.py | 3 --- .../schemas/pydantic/pagination.py | 9 ++++---- .../schemas/pydantic/requests/opportunity.py | 2 +- .../schemas/pydantic/responses/opportunity.py | 4 ---- .../schemas/pydantic/responses/success.py | 3 --- .../schemas/pydantic/sorting.py | 14 ++++--------- .../common_grants_sdk/utils/custom_fields.py | 6 +++--- 19 files changed, 39 insertions(+), 78 deletions(-) diff --git a/.changeset/python-sdk-populate-by-name.md b/.changeset/python-sdk-populate-by-name.md index 9b0388194..5256540c5 100644 --- a/.changeset/python-sdk-populate-by-name.md +++ b/.changeset/python-sdk-populate-by-name.md @@ -2,4 +2,4 @@ "common-grants-sdk": patch --- -Make snake_case construction of aliased models type-check and validate: the Event models (`eventType`, `startDate`, `startTime`, `endDate`, `endTime`) and the `customValue` field on `OppStatus` / `ApplicantType` now use `validation_alias` + `serialization_alias` (the pattern `CustomField` and `PluginMeta` already use) instead of bare `alias`, and `CommonGrantsBaseModel` sets `populate_by_name=True`. Static type checkers previously rejected idiomatic calls like `SingleDateEvent(event_type=...)`, and validation ignored the snake_case name. Wire I/O is unchanged: parsing still accepts camelCase and `model_dump(by_alias=True)` still emits it. Internal alias readers (filter classification, transform mapping validation) now consult all three alias fields, so both alias styles resolve. +camelCase wire naming now comes from a shared alias generator on the model config instead of per-field `alias=` declarations. Validation and serialization use each field's `to_camel` alias, `populate_by_name` keeps snake_case construction working, and static type checkers now accept snake_case constructor calls like `SingleDateEvent(event_type=...)`, which bare `alias=` declarations made them reject. The config applies to `CommonGrantsBaseModel` and to the wire models that don't extend it (sorting, pagination, filters). Wire I/O is unchanged: every removed declaration matched its field's `to_camel` form exactly, and irregular wire names (`CustomField.schema_url` -> `"schema"`) keep explicit field-level aliases. diff --git a/lib/python-sdk/common_grants_sdk/client/pagination.py b/lib/python-sdk/common_grants_sdk/client/pagination.py index 7ebdc2567..f88862512 100644 --- a/lib/python-sdk/common_grants_sdk/client/pagination.py +++ b/lib/python-sdk/common_grants_sdk/client/pagination.py @@ -107,9 +107,9 @@ def wrapper( aggregated_pagination_info = PaginatedResultsInfo( page=1, - pageSize=len(items) or page_size, - totalItems=len(items), - totalPages=1, + page_size=len(items) or page_size, + total_items=len(items), + total_pages=1, ) # Build aggregated response. Copy the first page's response so any extra @@ -131,7 +131,7 @@ def wrapper( status=latest_response.status if latest_response else 200, message=latest_response.message if latest_response else "Success", items=cast(list[ItemsT], items), - paginationInfo=aggregated_pagination_info, + pagination_info=aggregated_pagination_info, ) return wrapper diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index b65e24126..8299e1d35 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -675,8 +675,7 @@ def classify_filters( custom_buckets[key] = validated # type: ignore[assignment] # OppFilters requires the alias form for construction (populate_by_name is not set). - # Use "customFilters" (the alias) rather than "custom_filters" (the field name). return OppFilters( **default_fields, - customFilters=custom_buckets if custom_buckets else None, + custom_filters=custom_buckets if custom_buckets else None, ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py index f12defab6..42d51fc22 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py @@ -1,10 +1,26 @@ import json from typing import Self -from pydantic import BaseModel, ConfigDict +from pydantic import AliasGenerator, BaseModel, ConfigDict +from pydantic.alias_generators import to_camel from common_grants_sdk.utils.transformation import transform_from_mapping +# camelCase wire naming as a reusable config: validation and serialization use +# the to_camel alias while populate_by_name keeps snake_case construction +# working (and type-checking, since static checkers fall back to field names +# when aliases come from a generator). Field-level alias settings override the +# generator for irregular wire names. Used by CommonGrantsBaseModel and by the +# wire models that do not extend it (sorting, pagination, filters). +CAMEL_ALIASES = AliasGenerator( + validation_alias=to_camel, + serialization_alias=to_camel, +) +CAMEL_WIRE_CONFIG = ConfigDict( + populate_by_name=True, + alias_generator=CAMEL_ALIASES, +) + class CommonGrantsBaseModel(BaseModel): """Base model with common configuration and methods for CommonGrants models.""" @@ -12,9 +28,8 @@ class CommonGrantsBaseModel(BaseModel): model_config = ConfigDict( from_attributes=True, strict=False, # Coerces strings to enums, datetimes, etc. - # Accept snake_case field names alongside their camelCase wire aliases - # in constructors and validation; serialization still uses the aliases. populate_by_name=True, + alias_generator=CAMEL_ALIASES, ) def dump(self) -> dict: diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py index 907a20edc..d5f1a61ff 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py @@ -44,8 +44,6 @@ class CustomField(CommonGrantsBaseModel, Generic[V]): ) field_type: CustomFieldType = Field( ..., - validation_alias="fieldType", - serialization_alias="fieldType", description="The JSON schema type to use when de-serializing the `value` field", ) schema_url: Optional[HttpUrl] = Field( diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py index 395857e71..2c7fd53fc 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/event.py @@ -29,8 +29,6 @@ class EventBase(CommonGrantsBaseModel): ) event_type: EventType = Field( ..., - validation_alias="eventType", - serialization_alias="eventType", description="Type of event", ) description: Optional[str] = Field( @@ -45,8 +43,6 @@ class SingleDateEvent(EventBase): event_type: Literal[EventType.SINGLE_DATE] = Field( EventType.SINGLE_DATE, - validation_alias="eventType", - serialization_alias="eventType", ) date: ISODate = Field( ..., @@ -64,31 +60,21 @@ class DateRangeEvent(EventBase): event_type: Literal[EventType.DATE_RANGE] = Field( EventType.DATE_RANGE, - validation_alias="eventType", - serialization_alias="eventType", ) start_date: ISODate = Field( ..., - validation_alias="startDate", - serialization_alias="startDate", description="Start date of the event in ISO 8601 format: YYYY-MM-DD", ) start_time: Optional[ISOTime] = Field( default=None, - validation_alias="startTime", - serialization_alias="startTime", description="Start time of the event in ISO 8601 format: HH:MM:SS", ) end_date: ISODate = Field( ..., - validation_alias="endDate", - serialization_alias="endDate", description="End date of the event in ISO 8601 format: YYYY-MM-DD", ) end_time: Optional[ISOTime] = Field( default=None, - validation_alias="endTime", - serialization_alias="endTime", description="End time of the event in ISO 8601 format: HH:MM:SS", ) @@ -99,8 +85,6 @@ class OtherEvent(EventBase): event_type: Literal[EventType.OTHER] = Field( EventType.OTHER, - validation_alias="eventType", - serialization_alias="eventType", ) details: Optional[str] = Field( default=None, diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/metadata.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/metadata.py index 39230e400..6c67f29ba 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/metadata.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/metadata.py @@ -12,12 +12,10 @@ class SystemMetadata(CommonGrantsBaseModel): created_at: UTCDateTime = Field( ..., - alias="createdAt", description="The timestamp (in UTC) at which the record was created.", ) last_modified_at: UTCDateTime = Field( ..., - alias="lastModifiedAt", description="The timestamp (in UTC) at which the record was last modified.", ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py index cf21979d7..16c178479 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/filters/opportunity.py @@ -4,7 +4,7 @@ from typing_extensions import TypedDict -from ..base import CommonGrantsBaseModel +from ..base import CAMEL_WIRE_CONFIG, CommonGrantsBaseModel from .base import DefaultFilter from .boolean import BooleanComparisonFilter from .date import DateComparisonFilter, DateRangeFilter @@ -34,28 +34,26 @@ class OppDefaultFilters(BaseModel): """Standard filters available for searching opportunities.""" + model_config = CAMEL_WIRE_CONFIG + status: Optional[StringArrayFilter] = Field( default=None, description="`status.value` matches one of the following values", ) close_date_range: Optional[DateRangeFilter] = Field( default=None, - alias="closeDateRange", description="`keyDates.closeDate` is between the given range", ) total_funding_available_range: Optional[MoneyRangeFilter] = Field( default=None, - alias="totalFundingAvailableRange", description="`funding.totalAmountAvailable` is between the given range", ) min_award_amount_range: Optional[MoneyRangeFilter] = Field( default=None, - alias="minAwardAmountRange", description="`funding.minAwardAmount` is between the given range", ) max_award_amount_range: Optional[MoneyRangeFilter] = Field( default=None, - alias="maxAwardAmountRange", description="`funding.maxAwardAmount` is between the given range", ) @@ -66,7 +64,6 @@ class OppFilters(OppDefaultFilters): custom_filters: Optional[dict[str, DefaultFilter]] = Field( default=None, description="Additional custom filters to apply to the search", - alias="customFilters", ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py index 8e3e58ae0..5ed782287 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_applicant_type.py @@ -41,8 +41,6 @@ class ApplicantType(CommonGrantsBaseModel): ) custom_value: Optional[str] = Field( default=None, - validation_alias="customValue", - serialization_alias="customValue", description="A custom applicant type value, only meaningful when value='custom'", ) description: Optional[str] = Field( diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py index 407adc47d..caea500b3 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_base.py @@ -49,7 +49,6 @@ class OpportunityBase(SystemMetadata, CommonGrantsBaseModel, Generic[CF]): ) key_dates: Optional[OppTimeline] = Field( default=None, - alias="keyDates", description="Key dates for the opportunity, such as when the application opens and closes", ) source: Optional[HttpUrl] = Field( @@ -58,12 +57,10 @@ class OpportunityBase(SystemMetadata, CommonGrantsBaseModel, Generic[CF]): ) custom_fields: Optional[CF] = Field( default=None, - alias="customFields", description="Additional custom fields specific to this opportunity", ) accepted_applicant_types: Optional[list[ApplicantType]] = Field( default=None, - alias="acceptedApplicantTypes", description="The type of applicant for the opportunity", ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_funding.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_funding.py index 5e237e64d..cdf5272a0 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_funding.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_funding.py @@ -17,31 +17,25 @@ class OppFunding(CommonGrantsBaseModel): ) total_amount_available: Optional[Money] = Field( default=None, - alias="totalAmountAvailable", description="Total amount of funding available for this opportunity", ) min_award_amount: Optional[Money] = Field( default=None, - alias="minAwardAmount", description="Minimum amount of funding granted per award", ) max_award_amount: Optional[Money] = Field( default=None, - alias="maxAwardAmount", description="Maximum amount of funding granted per award", ) min_award_count: Optional[int] = Field( default=None, - alias="minAwardCount", description="Minimum number of awards granted", ) max_award_count: Optional[int] = Field( default=None, - alias="maxAwardCount", description="Maximum number of awards granted", ) estimated_award_count: Optional[int] = Field( default=None, - alias="estimatedAwardCount", description="Estimated number of awards that will be granted", ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py index d612f4f96..fe4de9357 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_status.py @@ -36,8 +36,6 @@ class OppStatus(CommonGrantsBaseModel): ) custom_value: Optional[str] = Field( default=None, - validation_alias="customValue", - serialization_alias="customValue", description="A custom status value", ) description: Optional[str] = Field( diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_timeline.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_timeline.py index 682dfb082..17d5ff2fc 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_timeline.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/models/opp_timeline.py @@ -13,16 +13,13 @@ class OppTimeline(CommonGrantsBaseModel): post_date: Optional[Event] = Field( default=None, - alias="postDate", description="The date (and time) at which the opportunity is posted", ) close_date: Optional[Event] = Field( default=None, - alias="closeDate", description="The date (and time) at which the opportunity closes", ) other_dates: Optional[dict[str, Event]] = Field( default=None, - alias="otherDates", description="An optional map of other key dates or events in the opportunity timeline", ) diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/pagination.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/pagination.py index 372c8b66b..c48c4fd74 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/pagination.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/pagination.py @@ -2,6 +2,8 @@ from pydantic import BaseModel, Field +from .base import CAMEL_WIRE_CONFIG + class PaginatedBase(BaseModel): """Parameters for pagination.""" @@ -13,12 +15,11 @@ class PaginatedBase(BaseModel): ) page_size: int = Field( default=10, - alias="pageSize", description="The number of items per page", ge=1, ) - model_config = {"populate_by_name": True} + model_config = CAMEL_WIRE_CONFIG class PaginatedBodyParams(PaginatedBase): @@ -34,13 +35,11 @@ class PaginatedResultsInfo(PaginatedBase): total_items: int = Field( ..., - alias="totalItems", description="The total number of items", ) total_pages: int = Field( ..., - alias="totalPages", description="The total number of pages", ) - model_config = {"populate_by_name": True} + model_config = CAMEL_WIRE_CONFIG diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/requests/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/requests/opportunity.py index 36705a168..e71e3ea03 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/requests/opportunity.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/requests/opportunity.py @@ -21,5 +21,5 @@ class OpportunitySearchRequest(BaseModel): default_factory=OppFilters, description="Filters to apply to the opportunity search", ) - sorting: OppSorting = OppSorting(sortBy=OppSortBy.LAST_MODIFIED_AT) + sorting: OppSorting = OppSorting(sort_by=OppSortBy.LAST_MODIFIED_AT) pagination: PaginatedBodyParams = PaginatedBodyParams() diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/opportunity.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/opportunity.py index 4e7a391a6..886d4ac7f 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/opportunity.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/opportunity.py @@ -16,7 +16,6 @@ class OpportunitiesListResponse(DefaultResponse): pagination_info: PaginatedResultsInfo = Field( ..., description="The pagination details", - alias="paginationInfo", ) model_config = {"populate_by_name": True} @@ -29,17 +28,14 @@ class OpportunitiesSearchResponse(DefaultResponse): pagination_info: PaginatedResultsInfo = Field( ..., description="The pagination details", - alias="paginationInfo", ) sort_info: SortedResultsInfo = Field( ..., description="The sorting details", - alias="sortInfo", ) filter_info: FilterInfo[dict] = Field( ..., description="The filter details", - alias="filterInfo", ) model_config = {"populate_by_name": True} diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/success.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/success.py index 45bf2a02c..d8b9b8259 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/success.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/responses/success.py @@ -36,7 +36,6 @@ class Paginated(Success, Generic[ItemsT]): pagination_info: PaginatedResultsInfo = Field( ..., description="Details about the paginated results", - alias="paginationInfo", ) model_config = {"populate_by_name": True} @@ -48,7 +47,6 @@ class Sorted(Paginated[ItemsT], Generic[ItemsT]): sort_info: SortedResultsInfo = Field( ..., description="The sort order of the items", - alias="sortInfo", ) model_config = {"populate_by_name": True} @@ -72,7 +70,6 @@ class Filtered(Sorted[ItemsT], Generic[ItemsT, FilterT]): filter_info: FilterInfo[FilterT] = Field( ..., description="The filters applied to the response items", - alias="filterInfo", ) model_config = {"populate_by_name": True} diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/sorting.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/sorting.py index 14a61f40c..a639bf928 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/sorting.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/sorting.py @@ -5,6 +5,8 @@ from pydantic import BaseModel, Field, model_validator +from .base import CAMEL_WIRE_CONFIG + class SortOrder(str, Enum): """Sort order enumeration.""" @@ -18,18 +20,16 @@ class SortBase(BaseModel): sort_by: Union[str, None] = Field( ..., - alias="sortBy", description="The field to sort by", examples=["lastModifiedAt"], ) custom_sort_by: Optional[str] = Field( default=None, - alias="customSortBy", description="Implementation-defined sort key", examples=["customField"], ) - model_config = {"populate_by_name": True} + model_config = CAMEL_WIRE_CONFIG class SortQueryParams(SortBase): @@ -37,7 +37,6 @@ class SortQueryParams(SortBase): sort_order: Optional[SortOrder] = Field( default=None, - alias="sortOrder", description="The order to sort by", examples=[SortOrder.ASC], ) @@ -48,7 +47,6 @@ class SortBodyParams(SortBase): sort_order: Optional[SortOrder] = Field( default=None, - alias="sortOrder", description="The order to sort by", examples=[SortOrder.ASC], ) @@ -59,7 +57,6 @@ class SortedResultsInfo(SortBase): sort_order: str = Field( ..., - alias="sortOrder", description="The order in which the results are sorted", ) errors: Optional[list[str]] = Field( @@ -90,17 +87,14 @@ class OppSorting(BaseModel): sort_by: OppSortBy = Field( ..., description="The field to sort by", - alias="sortBy", ) sort_order: str = Field( default="desc", description="The sort order (asc or desc)", - alias="sortOrder", ) custom_sort_by: Optional[str] = Field( default=None, description="The custom field to sort by when sortBy is 'custom'", - alias="customSortBy", ) @model_validator(mode="after") @@ -111,4 +105,4 @@ def validate_custom_sort_by(self) -> "OppSorting": raise ValueError(e) return self - model_config = {"populate_by_name": True} + model_config = CAMEL_WIRE_CONFIG diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index c98db17ef..694ea6cc3 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -80,7 +80,7 @@ def _parse_extra_as_custom_fields(cls, values: Any) -> Any: __base__=cls, custom_fields=( Optional[CustomFieldsContainer], - Field(default=None, alias="customFields"), + Field(default=None), ), ) @@ -160,10 +160,10 @@ class OpportunityLegacyIdField(CustomField): CustomFieldForAttr = create_model( _create_model_name(name=name, key=key), __base__=CustomField, - # pin expected type (still accepts wire key "fieldType" via alias) + # pin expected type (wire key "fieldType" comes from the base alias generator) field_type=( CustomFieldType, - Field(default=field.field_type, alias="fieldType"), + Field(default=field.field_type), ), # pin name and description from spec name=(str, Field(default=field.name or key)), From 82c98b4820dc31e97cb4850db77c86cf49a4a007 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <bryan@snowboardtechie.com> Date: Wed, 15 Jul 2026 15:53:28 -0700 Subject: [PATCH 50/51] chore: trim mechanism comments to intent Comments on the alias generator, the snake-to-alias map, and the classify normalization banner explained mechanics readable from the code; keep only the non-obvious intent (generated aliases keep field names type-checkable, field-level settings override). --- .../common_grants_sdk/extensions/filters.py | 17 +---------------- .../common_grants_sdk/schemas/pydantic/base.py | 9 +++------ .../common_grants_sdk/utils/custom_fields.py | 2 +- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index 8299e1d35..ed0573fe6 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -309,22 +309,7 @@ def _range(self, operator: RangeOperator, min: Any, max: Any) -> BaseModel: ] ) -# --------------------------------------------------------------------------- -# Alias-normalization maps for classify_filters -# -# Consumers may pass either the snake_case field name or the camelCase wire -# alias. classify_filters normalizes keys to the wire alias (or field-name for -# fields without one) before passing them to OppFilters(**...): -# - snake_case keys with a camelCase alias → converted to the alias (closeDateRange) -# - camelCase alias keys → kept as-is (already the alias) -# - keys with no alias (e.g. "status") → kept as-is (snake == request key) -# --------------------------------------------------------------------------- - -# Map from snake_case field name → camelCase wire alias (used for OppFilters -# construction). Reads ``serialization_alias`` before bare ``alias`` so fields on -# either alias style resolve; fields without an alias and alias-form keys fall -# through ``_SNAKE_TO_ALIAS.get(key, key)`` unchanged — one lookup normalizes all -# three key classes. +# snake_case field name → camelCase wire alias, for OppFilters construction. _SNAKE_TO_ALIAS: dict[str, str] = { field_name: wire_name for field_name, field_info in OppDefaultFilters.model_fields.items() diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py index 42d51fc22..5b7a876bc 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/base.py @@ -6,12 +6,9 @@ from common_grants_sdk.utils.transformation import transform_from_mapping -# camelCase wire naming as a reusable config: validation and serialization use -# the to_camel alias while populate_by_name keeps snake_case construction -# working (and type-checking, since static checkers fall back to field names -# when aliases come from a generator). Field-level alias settings override the -# generator for irregular wire names. Used by CommonGrantsBaseModel and by the -# wire models that do not extend it (sorting, pagination, filters). +# Generated (not field-level) aliases keep snake_case construction type-checking: +# static checkers fall back to field names when the alias comes from a generator. +# Field-level alias settings still override for irregular wire names. CAMEL_ALIASES = AliasGenerator( validation_alias=to_camel, serialization_alias=to_camel, diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index 694ea6cc3..9f351347a 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -160,7 +160,7 @@ class OpportunityLegacyIdField(CustomField): CustomFieldForAttr = create_model( _create_model_name(name=name, key=key), __base__=CustomField, - # pin expected type (wire key "fieldType" comes from the base alias generator) + # pin expected type field_type=( CustomFieldType, Field(default=field.field_type), From cfe907c60bba0bfc70528d7698b622d92e2e5e77 Mon Sep 17 00:00:00 2001 From: Bryan Thompson <bryan@snowboardtechie.com> Date: Wed, 15 Jul 2026 16:13:38 -0700 Subject: [PATCH 51/51] docs: correct stale alias claims classify_filters' docstring, two inline comments, a test docstring, and the CustomField docstring still described per-field aliases and an OppFilters that rejects snake_case construction; both are superseded by the shared alias generator. The key normalization in classify_filters stays because it collapses a snake_case key and its camelCase alias onto one kwarg. Also align the custom-fields known-field check with the other alias readers (consult all three alias fields). --- .../common_grants_sdk/extensions/filters.py | 18 ++++-------------- .../schemas/pydantic/fields/custom.py | 7 ++++--- .../common_grants_sdk/utils/custom_fields.py | 10 +++++++--- .../tests/extensions/test_filters.py | 7 +------ 4 files changed, 16 insertions(+), 26 deletions(-) diff --git a/lib/python-sdk/common_grants_sdk/extensions/filters.py b/lib/python-sdk/common_grants_sdk/extensions/filters.py index ed0573fe6..8eb3597b9 100644 --- a/lib/python-sdk/common_grants_sdk/extensions/filters.py +++ b/lib/python-sdk/common_grants_sdk/extensions/filters.py @@ -587,15 +587,10 @@ def classify_filters( then validated against the known-model union, exactly like ad-hoc input. Call sites must pass the same resource/method strings the plugin declared. - Construction normalizes all default consumer keys to the form that - ``OppFilters(**kwargs)`` accepts. Because ``OppDefaultFilters`` does NOT set - ``populate_by_name=True``, Pydantic v2 requires the alias form (e.g. - ``closeDateRange``) for aliased fields. Snake_case keys (e.g. - ``close_date_range``) are therefore mapped to their alias via ``_SNAKE_TO_ALIAS`` - before construction. Fields without aliases (e.g. ``status``) pass through - unchanged. The alternative — enabling ``populate_by_name=True`` on - ``OppFilters.model_config`` — is deliberately avoided: the classifier must not - modify core schema model config. + Default consumer keys are normalized to the wire alias via ``_SNAKE_TO_ALIAS`` + before ``OppFilters(**kwargs)`` construction. ``populate_by_name`` would accept + either form directly, but normalizing first collapses a snake_case key and its + camelCase alias onto one kwarg instead of passing pydantic both. Args: routes: Plugin route declarations (used to identify registered custom filters). @@ -625,10 +620,6 @@ def classify_filters( if key in DEFAULT_FILTER_NAMES: # Bucket 1: a standard filter. Validate it against the type declared for # that field (for example, "status" is validated as a StringArrayFilter). - # OppFilters is constructed with keyword arguments and does not set - # populate_by_name, so a snake_case key is first converted to its alias. - # Keys that are already in alias form, and keys that have no alias, pass - # through unchanged. alias_key = _SNAKE_TO_ALIAS.get(key, key) if alias_key in default_fields: # Snake and camel forms of the same field normalize to one key. @@ -659,7 +650,6 @@ def classify_filters( raise error custom_buckets[key] = validated # type: ignore[assignment] - # OppFilters requires the alias form for construction (populate_by_name is not set). return OppFilters( **default_fields, custom_filters=custom_buckets if custom_buckets else None, diff --git a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py index d5f1a61ff..335936d81 100644 --- a/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py +++ b/lib/python-sdk/common_grants_sdk/schemas/pydantic/fields/custom.py @@ -30,9 +30,10 @@ class CustomField(CommonGrantsBaseModel, Generic[V]): Generic over its value type ``V`` (default ``Any``): the bare ``CustomField`` keeps the protocol's untyped-value behavior, while ``CustomField[int]`` (or a Pydantic model) gives plugin authors and consumers a concrete, inspectable - ``value`` type. ``populate_by_name`` plus ``validation_alias``/ - ``serialization_alias`` keep JSON I/O camelCase (``fieldType``) while - snake_case field-name construction (``CustomField(field_type=...)``) type-checks. + ``value`` type. Wire naming comes from the base alias generator (camelCase + ``fieldType`` on the wire, ``CustomField(field_type=...)`` in code); the one + irregular wire name, ``schema_url`` -> ``"schema"``, keeps an explicit + field-level alias. """ model_config = ConfigDict(populate_by_name=True) diff --git a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py index 9f351347a..8378572b5 100644 --- a/lib/python-sdk/common_grants_sdk/utils/custom_fields.py +++ b/lib/python-sdk/common_grants_sdk/utils/custom_fields.py @@ -55,9 +55,13 @@ def _parse_extra_as_custom_fields(cls, values: Any) -> Any: if hasattr(cls, "model_fields"): for attr, field_info in cls.model_fields.items(): known.add(attr) - alias = getattr(field_info, "alias", None) - if alias: - known.add(alias) + for alias in ( + getattr(field_info, "alias", None), + getattr(field_info, "validation_alias", None), + getattr(field_info, "serialization_alias", None), + ): + if isinstance(alias, str): + known.add(alias) for key, val in list(values.items()): if key not in known and isinstance(val, dict) and "fieldType" in val: # Inject the key as the name if not already present diff --git a/lib/python-sdk/tests/extensions/test_filters.py b/lib/python-sdk/tests/extensions/test_filters.py index a409eb88a..93e9c1903 100644 --- a/lib/python-sdk/tests/extensions/test_filters.py +++ b/lib/python-sdk/tests/extensions/test_filters.py @@ -269,12 +269,7 @@ def test_classify_empty_filters_dict_yields_empty_body(): def test_classify_default_snake_form_of_aliased_key_normalizes_to_alias(): - """A snake_case key for an aliased field lands in the named field, not customFilters. - - Without the alias normalization, OppFilters(close_date_range=...) would be - silently dropped by pydantic (populate_by_name is not set) and the field would - stay None. - """ + """A snake_case key for an aliased field lands in the named field, not customFilters.""" result = classify_filters( SAMPLE_ROUTES, "opportunities",