From ed48b06d728b41d2f17a5dec884ad46129c6b914 Mon Sep 17 00:00:00 2001 From: Pavel Makarchuk Date: Fri, 12 Jun 2026 01:06:35 -0400 Subject: [PATCH 1/4] populace.data.contract: the release artifact contract, enforced at the producer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The releases already on policyengine/populace-us disagree with each other: 1abddeb has no build_manifest.json and an unversioned release_manifest schema, while later releases carry schema_version 1. Every consumer was left to implement its own defensive filter. validate_release_dir() is the single gate: required files, manifest schema version, and build-id agreement between both manifests and the directory name — raising ReleaseContractError that names every violation at once. Verified against the real 9f1260b release (passes) and the real 1abddeb release (rejected with six named failures). Fixes #11. Co-Authored-By: Claude Fable 5 --- .../src/populace/data/__init__.py | 10 + .../src/populace/data/contract.py | 196 ++++++++++++++++++ packages/populace-data/tests/test_contract.py | 159 ++++++++++++++ 3 files changed, 365 insertions(+) create mode 100644 packages/populace-data/src/populace/data/contract.py create mode 100644 packages/populace-data/tests/test_contract.py diff --git a/packages/populace-data/src/populace/data/__init__.py b/packages/populace-data/src/populace/data/__init__.py index ed94607..3422288 100644 --- a/packages/populace-data/src/populace/data/__init__.py +++ b/packages/populace-data/src/populace/data/__init__.py @@ -21,6 +21,12 @@ there is no kernel in its dependency closure to gate against. """ +from populace.data.contract import ( + RELEASE_MANIFEST_SCHEMA_VERSION, + REQUIRED_RELEASE_FILES, + ReleaseContractError, + validate_release_dir, +) from populace.data.loader import ( available, download, @@ -39,6 +45,10 @@ "DatasetSpec", "REGISTRY", "register", + "RELEASE_MANIFEST_SCHEMA_VERSION", + "REQUIRED_RELEASE_FILES", + "ReleaseContractError", + "validate_release_dir", ] __version__ = "0.1.0" diff --git a/packages/populace-data/src/populace/data/contract.py b/packages/populace-data/src/populace/data/contract.py new file mode 100644 index 0000000..b11f837 --- /dev/null +++ b/packages/populace-data/src/populace/data/contract.py @@ -0,0 +1,196 @@ +"""The release artifact contract: what a published release MUST contain. + +The releases already on the Hub disagree with each other — one carries no +``build_manifest.json`` at all, and two different ``release_manifest.json`` +schemas coexist (an unversioned early shape next to ``schema_version: 1``). +A consumer iterating ``releases/`` therefore cannot trust the listing, and +every consumer ends up re-implementing its own defensive filter. The charter +makes "stage manifests are load-bearing" a binding process rule; the release +directory is the most public manifest of all, so its contract lives here, +with the producer — not in every consumer. + +:func:`validate_release_dir` is the single gate: it checks a local release +directory against the contract and raises :class:`ReleaseContractError` +naming **every** failure at once (a publisher should see the full repair +list, not play whack-a-mole one failure per run). Publishing code calls it +before any byte reaches the Hub. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +__all__ = [ + "RELEASE_MANIFEST_SCHEMA_VERSION", + "REQUIRED_RELEASE_FILES", + "ReleaseContractError", + "validate_release_dir", +] + +#: The release-manifest schema this library reads and writes. Bump it with the +#: schema, and keep :func:`validate_release_dir` rejecting drift loudly — the +#: unversioned 1abddeb-era manifest is exactly the silence this guards against. +RELEASE_MANIFEST_SCHEMA_VERSION = 1 + +#: Files a release directory must contain to count as published. A release +#: missing any of these is invisible to :func:`validate_release_dir`-respecting +#: publishers, by design. +REQUIRED_RELEASE_FILES = ( + "build_manifest.json", + "release_manifest.json", + "sound_ecps_replacement_comparison.json", +) + + +class ReleaseContractError(ValueError): + """A release directory violates the release contract. + + Attributes: + failures: Every contract violation found, each a self-contained + human-readable sentence naming the file and field at fault. + """ + + def __init__(self, release_dir: Path, failures: list[str]) -> None: + self.failures = list(failures) + bullet_list = "\n".join(f" - {failure}" for failure in self.failures) + super().__init__( + f"Release directory {release_dir} violates the release contract " + f"({len(self.failures)} failure(s)):\n{bullet_list}" + ) + + +def _load_json(path: Path, failures: list[str]) -> Mapping | None: + try: + loaded = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + failures.append(f"{path.name} is not valid JSON: {exc}.") + return None + if not isinstance(loaded, Mapping): + failures.append( + f"{path.name} must be a JSON object, got {type(loaded).__name__}." + ) + return None + return loaded + + +def _check_build_manifest( + manifest: Mapping, release_id: str, failures: list[str] +) -> None: + build_id = manifest.get("build_id") + if not build_id: + failures.append("build_manifest.json is missing 'build_id'.") + elif build_id != release_id: + failures.append( + f"build_manifest.json 'build_id' is {build_id!r} but the release " + f"directory is named {release_id!r}; the directory name IS the " + f"build id." + ) + dataset = manifest.get("dataset") + if not isinstance(dataset, Mapping): + failures.append("build_manifest.json is missing the 'dataset' object.") + else: + for key in ("filename", "sha256"): + if not dataset.get(key): + failures.append( + f"build_manifest.json 'dataset' is missing {key!r}." + ) + if not isinstance(manifest.get("gates"), Mapping): + failures.append( + "build_manifest.json is missing the 'gates' object (the " + "acceptance-gate verdicts are the point of the manifest)." + ) + + +def _check_release_manifest( + manifest: Mapping, release_id: str, failures: list[str] +) -> None: + schema_version = manifest.get("schema_version") + if schema_version is None: + failures.append( + "release_manifest.json has no 'schema_version'; unversioned " + "manifests (the 1abddeb-era shape) are not publishable." + ) + elif schema_version != RELEASE_MANIFEST_SCHEMA_VERSION: + failures.append( + f"release_manifest.json 'schema_version' is {schema_version!r}; " + f"this library publishes version " + f"{RELEASE_MANIFEST_SCHEMA_VERSION}." + ) + build = manifest.get("build") + if not isinstance(build, Mapping) or not build.get("build_id"): + failures.append( + "release_manifest.json is missing 'build.build_id'." + ) + elif build["build_id"] != release_id: + failures.append( + f"release_manifest.json 'build.build_id' is " + f"{build['build_id']!r} but the release directory is named " + f"{release_id!r}." + ) + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping) or not artifacts: + failures.append( + "release_manifest.json must declare a non-empty 'artifacts' " + "mapping." + ) + else: + for key, entry in artifacts.items(): + if not isinstance(entry, Mapping): + failures.append( + f"release_manifest.json artifact {key!r} must be an " + f"object." + ) + continue + for field in ("path", "repo_id", "sha256"): + if not entry.get(field): + failures.append( + f"release_manifest.json artifact {key!r} is missing " + f"{field!r}." + ) + + +def validate_release_dir(release_dir: Path | str) -> None: + """Check a local release directory against the release contract. + + The directory name is the build id (``populace-us-2024--``); + its files are what :data:`REQUIRED_RELEASE_FILES` names; and both + manifests must agree with the directory about which build this is. + + Args: + release_dir: The local ``releases/`` directory about to be + published. + + Raises: + ReleaseContractError: Naming every violation found — missing files, + unparseable or unversioned manifests, schema drift, and build-id + mismatches between the manifests and the directory name. + """ + release_dir = Path(release_dir) + release_id = release_dir.name + failures: list[str] = [] + + if not release_dir.is_dir(): + raise ReleaseContractError( + release_dir, [f"{release_dir} is not a directory."] + ) + + for filename in REQUIRED_RELEASE_FILES: + if not (release_dir / filename).is_file(): + failures.append(f"required file {filename!r} is missing.") + + build_manifest_path = release_dir / "build_manifest.json" + if build_manifest_path.is_file(): + manifest = _load_json(build_manifest_path, failures) + if manifest is not None: + _check_build_manifest(manifest, release_id, failures) + + release_manifest_path = release_dir / "release_manifest.json" + if release_manifest_path.is_file(): + manifest = _load_json(release_manifest_path, failures) + if manifest is not None: + _check_release_manifest(manifest, release_id, failures) + + if failures: + raise ReleaseContractError(release_dir, failures) diff --git a/packages/populace-data/tests/test_contract.py b/packages/populace-data/tests/test_contract.py new file mode 100644 index 0000000..133d725 --- /dev/null +++ b/packages/populace-data/tests/test_contract.py @@ -0,0 +1,159 @@ +"""The release contract: every published release looks the same, loudly. + +These are behavioral tests against the failure modes already observed on the +Hub: a release with no build manifest at all (1abddeb), and two coexisting +release-manifest schemas (an unversioned early shape next to +``schema_version: 1``). A valid release passes silently; every broken release +fails with each violation named. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from populace.data import ( + RELEASE_MANIFEST_SCHEMA_VERSION, + REQUIRED_RELEASE_FILES, + ReleaseContractError, + validate_release_dir, +) + +RELEASE_ID = "populace-us-2024-9f1260b-20260611" + + +def _build_manifest(release_id: str = RELEASE_ID) -> dict: + return { + "build_id": release_id, + "builder": "populace", + "dataset": {"filename": "populace_us_2024.h5", "sha256": "dc75c0"}, + "calibration": { + "filename": "populace_us_2024_calibration.npz", + "sha256": "a3da2f", + }, + "gates": {"parity_gaps": 0}, + "score_vs_enhanced_cps": {"per_target_wins": {}}, + } + + +def _release_manifest(release_id: str = RELEASE_ID) -> dict: + return { + "schema_version": RELEASE_MANIFEST_SCHEMA_VERSION, + "data_package": {"name": "populace-data", "version": "0.1.0"}, + "build": {"build_id": release_id}, + "artifacts": { + "populace_us_2024": { + "kind": "microdata", + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "sha256": "dc75c0", + } + }, + } + + +@pytest.fixture +def release_dir(tmp_path: Path) -> Path: + """A complete, contract-valid release directory.""" + directory = tmp_path / "releases" / RELEASE_ID + directory.mkdir(parents=True) + (directory / "build_manifest.json").write_text(json.dumps(_build_manifest())) + (directory / "release_manifest.json").write_text( + json.dumps(_release_manifest()) + ) + (directory / "sound_ecps_replacement_comparison.json").write_text( + json.dumps({"schema_version": 1, "target_diagnostics": {}}) + ) + return directory + + +def test_a_complete_release_passes(release_dir: Path) -> None: + validate_release_dir(release_dir) + + +@pytest.mark.parametrize("filename", REQUIRED_RELEASE_FILES) +def test_each_required_file_is_named_when_missing( + release_dir: Path, filename: str +) -> None: + (release_dir / filename).unlink() + with pytest.raises(ReleaseContractError, match=filename): + validate_release_dir(release_dir) + + +def test_the_1abddeb_shape_is_rejected(release_dir: Path) -> None: + """The regression: a release with only an unversioned release manifest.""" + (release_dir / "build_manifest.json").unlink() + (release_dir / "sound_ecps_replacement_comparison.json").unlink() + (release_dir / "release_manifest.json").write_text( + json.dumps( + { + "release_id": RELEASE_ID, + "country_id": "us", + "artifacts": {}, + "validation": {}, + } + ) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + failures = "\n".join(excinfo.value.failures) + assert "build_manifest.json" in failures + assert "schema_version" in failures + + +def test_schema_drift_is_rejected_by_version(release_dir: Path) -> None: + manifest = _release_manifest() + manifest["schema_version"] = RELEASE_MANIFEST_SCHEMA_VERSION + 1 + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError, match="schema_version"): + validate_release_dir(release_dir) + + +def test_build_id_mismatch_names_both_ids(release_dir: Path) -> None: + (release_dir / "build_manifest.json").write_text( + json.dumps(_build_manifest("populace-us-2024-other-20260101")) + ) + with pytest.raises(ReleaseContractError, match="populace-us-2024-other"): + validate_release_dir(release_dir) + + +def test_release_manifest_build_id_must_match_directory( + release_dir: Path, +) -> None: + manifest = _release_manifest("populace-us-2024-other-20260101") + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError, match="build.build_id"): + validate_release_dir(release_dir) + + +def test_artifact_entries_must_carry_provenance(release_dir: Path) -> None: + manifest = _release_manifest() + manifest["artifacts"]["populace_us_2024"].pop("sha256") + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError, match="sha256"): + validate_release_dir(release_dir) + + +def test_unparseable_manifest_is_a_named_failure(release_dir: Path) -> None: + (release_dir / "build_manifest.json").write_text("{not json") + with pytest.raises(ReleaseContractError, match="not valid JSON"): + validate_release_dir(release_dir) + + +def test_all_failures_reported_at_once(release_dir: Path) -> None: + """A publisher sees the full repair list, not one failure per run.""" + (release_dir / "sound_ecps_replacement_comparison.json").unlink() + manifest = _release_manifest() + del manifest["schema_version"] + manifest["artifacts"] = {} + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + assert len(excinfo.value.failures) >= 3 + + +def test_a_missing_directory_is_a_contract_error(tmp_path: Path) -> None: + with pytest.raises(ReleaseContractError, match="is not a directory"): + validate_release_dir(tmp_path / "releases" / "nope") From 18598a2c43b14ec070ce2d0d8b1bb74953909eb2 Mon Sep 17 00:00:00 2001 From: Pavel Makarchuk Date: Fri, 12 Jun 2026 01:08:32 -0400 Subject: [PATCH 2/4] populace.data.release: latest.json pointer + contract-gated publishing The Hub repo had no way to say which release is current: consumers listed the releases/ tree and guessed, and same-day builds (the ids end in a date) have no defined ordering. publish_release() validates the release directory against the contract (an invalid release uploads nothing), uploads its files, and writes the latest.json pointer LAST so a reader never sees a pointer to files that are not there yet. latest_release() is the consumer side: one call that returns the typed pointer. The Hub client is injected so the suite tests the real ordering against a fake, network-free. Fixes #9. Stacked on #11's contract branch. Co-Authored-By: Claude Fable 5 --- .../src/populace/data/__init__.py | 14 ++ .../src/populace/data/release.py | 212 ++++++++++++++++++ packages/populace-data/tests/test_release.py | 191 ++++++++++++++++ 3 files changed, 417 insertions(+) create mode 100644 packages/populace-data/src/populace/data/release.py create mode 100644 packages/populace-data/tests/test_release.py diff --git a/packages/populace-data/src/populace/data/__init__.py b/packages/populace-data/src/populace/data/__init__.py index 3422288..855ab7d 100644 --- a/packages/populace-data/src/populace/data/__init__.py +++ b/packages/populace-data/src/populace/data/__init__.py @@ -35,6 +35,14 @@ resolve, ) from populace.data.registry import REGISTRY, DatasetSpec, register +from populace.data.release import ( + LATEST_POINTER_PATH, + LATEST_POINTER_SCHEMA_VERSION, + LatestPointer, + latest_pointer_payload, + latest_release, + publish_release, +) __all__ = [ "load", @@ -49,6 +57,12 @@ "REQUIRED_RELEASE_FILES", "ReleaseContractError", "validate_release_dir", + "LATEST_POINTER_PATH", + "LATEST_POINTER_SCHEMA_VERSION", + "LatestPointer", + "latest_pointer_payload", + "latest_release", + "publish_release", ] __version__ = "0.1.0" diff --git a/packages/populace-data/src/populace/data/release.py b/packages/populace-data/src/populace/data/release.py new file mode 100644 index 0000000..4aa4b30 --- /dev/null +++ b/packages/populace-data/src/populace/data/release.py @@ -0,0 +1,212 @@ +"""Publish a release and point ``latest.json`` at it. + +The Hub repo publishes builds under ``releases//``, but nothing +identified which release is *current*: a consumer had to list the tree and +guess, and because build ids end in a date (``populace-us-2024-9f1260b- +20260611``), two builds published the same day have no defined ordering. +``latest.json`` at the repo root is that missing pointer — a tiny manifest +naming the current release and the path of each of its contract files. + +Two sides of the pointer live here: + +- :func:`publish_release` is the producer: it validates the local release + directory against the :mod:`release contract ` + (a release that fails the contract refuses to publish), uploads its + files, and uploads ``latest.json`` **last** — so a reader never sees the + pointer before the release it points at. +- :func:`latest_release` is the consumer: it downloads ``latest.json`` and + returns the typed pointer, the one-call answer to "which release is + current?" for dashboards and scorers. + +The Hub client is injected (``api=``) everywhere it is used, so the suite +exercises the real ordering and payloads against a fake — no network, no +mocking of our own internals. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from populace.data.contract import ( + REQUIRED_RELEASE_FILES, + validate_release_dir, +) + +__all__ = [ + "LATEST_POINTER_PATH", + "LATEST_POINTER_SCHEMA_VERSION", + "LatestPointer", + "latest_pointer_payload", + "publish_release", + "latest_release", +] + +#: Where the pointer lives in the dataset repo. The root, not a release +#: directory: the pointer is repo state, not release state. +LATEST_POINTER_PATH = "latest.json" + +#: Version of the pointer payload itself, so the pointer can evolve without +#: consumers guessing (the same discipline the release manifest learned). +LATEST_POINTER_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class LatestPointer: + """The parsed ``latest.json``: which release is current, and where. + + Attributes: + release_id: The current build id (the ``releases/`` directory name). + updated_at: ISO-8601 UTC timestamp of when the pointer was written. + paths: Repo-relative path of each contract file, keyed by its stem + (``"build_manifest"``, ``"release_manifest"``, + ``"sound_ecps_replacement_comparison"``). + """ + + release_id: str + updated_at: str + paths: dict[str, str] + + +def latest_pointer_payload( + release_id: str, *, updated_at: str | None = None +) -> dict: + """The ``latest.json`` payload for ``release_id``. + + Paths are derived from the release contract — the pointer names exactly + the files :data:`~populace.data.contract.REQUIRED_RELEASE_FILES` + guarantees exist. + + Args: + release_id: The build id being made current. + updated_at: ISO-8601 UTC timestamp; defaults to now. + """ + if updated_at is None: + updated_at = ( + datetime.now(UTC).replace(microsecond=0).isoformat() + ) + return { + "schema_version": LATEST_POINTER_SCHEMA_VERSION, + "release_id": release_id, + "updated_at": updated_at, + "paths": { + filename.removesuffix(".json"): f"releases/{release_id}/{filename}" + for filename in REQUIRED_RELEASE_FILES + }, + } + + +def _hf_api(): + try: + from huggingface_hub import HfApi + except ImportError as exc: # pragma: no cover - declared dependency + raise ImportError( + "populace-data needs huggingface_hub to publish releases; " + "reinstall populace-data with its dependencies." + ) from exc + return HfApi() + + +def publish_release( + release_dir: Path | str, + repo_id: str, + *, + api=None, + extra_files: tuple[str, ...] = (), + updated_at: str | None = None, +) -> dict: + """Upload a release directory and point ``latest.json`` at it. + + The order is the guarantee: the release contract is validated first (an + invalid release never reaches the Hub), every release file is uploaded + next, and the pointer goes up **last**, so a consumer that reads + ``latest.json`` always finds the files it names. + + Args: + release_dir: Local ``releases/`` directory. + repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. + api: A ``huggingface_hub.HfApi``-shaped object (anything with + ``upload_file(path_or_fileobj=, path_in_repo=, repo_id=, + repo_type=)``); constructed lazily when omitted. + extra_files: Additional filenames in ``release_dir`` to upload + beyond the contract files (e.g. a diagnostics artifact). + updated_at: Pointer timestamp; defaults to now (UTC). + + Returns: + The ``latest.json`` payload that was published. + + Raises: + ReleaseContractError: If the release directory violates the + contract. Nothing is uploaded in that case. + FileNotFoundError: If an ``extra_files`` entry does not exist. + """ + release_dir = Path(release_dir) + validate_release_dir(release_dir) + release_id = release_dir.name + if api is None: + api = _hf_api() + + filenames = list(REQUIRED_RELEASE_FILES) + [ + name for name in extra_files if name not in REQUIRED_RELEASE_FILES + ] + for filename in filenames: + local = release_dir / filename + if not local.is_file(): + raise FileNotFoundError( + f"extra release file {filename!r} not found in {release_dir}." + ) + api.upload_file( + path_or_fileobj=str(local), + path_in_repo=f"releases/{release_id}/{filename}", + repo_id=repo_id, + repo_type="dataset", + ) + + payload = latest_pointer_payload(release_id, updated_at=updated_at) + api.upload_file( + path_or_fileobj=json.dumps(payload, indent=1).encode(), + path_in_repo=LATEST_POINTER_PATH, + repo_id=repo_id, + repo_type="dataset", + ) + return payload + + +def latest_release(repo_id: str, *, api=None) -> LatestPointer: + """Read ``latest.json`` from a dataset repo: which release is current. + + Args: + repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. + api: A ``huggingface_hub.HfApi``-shaped object (anything with + ``hf_hub_download(repo_id=, filename=, repo_type=)``); + constructed lazily when omitted. + + Raises: + ValueError: If the pointer is malformed or its schema version is + newer than this library understands. + """ + if api is None: + api = _hf_api() + local = api.hf_hub_download( + repo_id=repo_id, filename=LATEST_POINTER_PATH, repo_type="dataset" + ) + payload = json.loads(Path(local).read_text()) + schema_version = payload.get("schema_version") + if schema_version != LATEST_POINTER_SCHEMA_VERSION: + raise ValueError( + f"{LATEST_POINTER_PATH} in {repo_id} has schema_version " + f"{schema_version!r}; this populace-data reads version " + f"{LATEST_POINTER_SCHEMA_VERSION}. Upgrade populace-data." + ) + release_id = payload.get("release_id") + if not release_id: + raise ValueError( + f"{LATEST_POINTER_PATH} in {repo_id} has no 'release_id'." + ) + return LatestPointer( + release_id=str(release_id), + updated_at=str(payload.get("updated_at", "")), + paths={str(k): str(v) for k, v in (payload.get("paths") or {}).items()}, + ) diff --git a/packages/populace-data/tests/test_release.py b/packages/populace-data/tests/test_release.py new file mode 100644 index 0000000..3dda64e --- /dev/null +++ b/packages/populace-data/tests/test_release.py @@ -0,0 +1,191 @@ +"""Publishing behavior: contract-gated uploads and a last-written pointer. + +The fake Hub client records every upload in order, so the suite asserts the +real guarantees — an invalid release uploads nothing, and ``latest.json`` +lands strictly after the files it points at — rather than implementation +details. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from populace.data import ReleaseContractError +from populace.data.contract import REQUIRED_RELEASE_FILES +from populace.data.release import ( + LATEST_POINTER_PATH, + LATEST_POINTER_SCHEMA_VERSION, + latest_pointer_payload, + latest_release, + publish_release, +) + +RELEASE_ID = "populace-us-2024-9f1260b-20260611" + + +class FakeHub: + """Records uploads in order; serves downloads from what was uploaded.""" + + def __init__(self) -> None: + self.uploads: list[tuple[str, bytes]] = [] + + def upload_file( + self, *, path_or_fileobj, path_in_repo, repo_id, repo_type + ) -> None: + assert repo_type == "dataset" + assert repo_id == "policyengine/populace-us" + if isinstance(path_or_fileobj, bytes): + content = path_or_fileobj + else: + content = Path(path_or_fileobj).read_bytes() + self.uploads.append((path_in_repo, content)) + + def hf_hub_download(self, *, repo_id, filename, repo_type) -> str: + assert repo_type == "dataset" + for path_in_repo, content in reversed(self.uploads): + if path_in_repo == filename: + local = self._download_dir / filename + local.parent.mkdir(parents=True, exist_ok=True) + local.write_bytes(content) + return str(local) + raise FileNotFoundError(filename) + + +@pytest.fixture +def hub(tmp_path: Path) -> FakeHub: + fake = FakeHub() + fake._download_dir = tmp_path / "hub-cache" + return fake + + +@pytest.fixture +def release_dir(tmp_path: Path) -> Path: + directory = tmp_path / "releases" / RELEASE_ID + directory.mkdir(parents=True) + (directory / "build_manifest.json").write_text( + json.dumps( + { + "build_id": RELEASE_ID, + "dataset": {"filename": "populace_us_2024.h5", "sha256": "dc"}, + "gates": {"parity_gaps": 0}, + } + ) + ) + (directory / "release_manifest.json").write_text( + json.dumps( + { + "schema_version": 1, + "build": {"build_id": RELEASE_ID}, + "artifacts": { + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "sha256": "dc", + } + }, + } + ) + ) + (directory / "sound_ecps_replacement_comparison.json").write_text("{}") + return directory + + +def test_pointer_payload_names_every_contract_file() -> None: + payload = latest_pointer_payload(RELEASE_ID, updated_at="2026-06-11T13:53:15+00:00") + assert payload["schema_version"] == LATEST_POINTER_SCHEMA_VERSION + assert payload["release_id"] == RELEASE_ID + assert set(payload["paths"]) == { + name.removesuffix(".json") for name in REQUIRED_RELEASE_FILES + } + assert ( + payload["paths"]["build_manifest"] + == f"releases/{RELEASE_ID}/build_manifest.json" + ) + + +def test_publish_uploads_pointer_last(hub: FakeHub, release_dir: Path) -> None: + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + updated_at="2026-06-11T13:53:15+00:00", + ) + uploaded_paths = [path for path, _ in hub.uploads] + assert uploaded_paths[-1] == LATEST_POINTER_PATH + for filename in REQUIRED_RELEASE_FILES: + assert f"releases/{RELEASE_ID}/{filename}" in uploaded_paths[:-1] + + +def test_invalid_release_uploads_nothing(hub: FakeHub, release_dir: Path) -> None: + (release_dir / "build_manifest.json").unlink() + with pytest.raises(ReleaseContractError): + publish_release(release_dir, "policyengine/populace-us", api=hub) + assert hub.uploads == [] + + +def test_extra_files_ride_along_before_the_pointer( + hub: FakeHub, release_dir: Path +) -> None: + (release_dir / "calibration_diagnostics.json").write_text("{}") + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + extra_files=("calibration_diagnostics.json",), + ) + uploaded_paths = [path for path, _ in hub.uploads] + extra = f"releases/{RELEASE_ID}/calibration_diagnostics.json" + assert extra in uploaded_paths + assert uploaded_paths.index(extra) < uploaded_paths.index(LATEST_POINTER_PATH) + + +def test_missing_extra_file_fails_loudly(hub: FakeHub, release_dir: Path) -> None: + with pytest.raises(FileNotFoundError, match="calibration_diagnostics"): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + extra_files=("calibration_diagnostics.json",), + ) + + +def test_publish_then_resolve_round_trips( + hub: FakeHub, release_dir: Path +) -> None: + published = publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + updated_at="2026-06-11T13:53:15+00:00", + ) + pointer = latest_release("policyengine/populace-us", api=hub) + assert pointer.release_id == RELEASE_ID + assert pointer.updated_at == "2026-06-11T13:53:15+00:00" + assert pointer.paths == published["paths"] + + +def test_future_pointer_schema_is_refused(hub: FakeHub) -> None: + hub.uploads.append( + ( + LATEST_POINTER_PATH, + json.dumps( + {"schema_version": LATEST_POINTER_SCHEMA_VERSION + 1} + ).encode(), + ) + ) + with pytest.raises(ValueError, match="Upgrade populace-data"): + latest_release("policyengine/populace-us", api=hub) + + +def test_pointer_without_release_id_is_refused(hub: FakeHub) -> None: + hub.uploads.append( + ( + LATEST_POINTER_PATH, + json.dumps({"schema_version": LATEST_POINTER_SCHEMA_VERSION}).encode(), + ) + ) + with pytest.raises(ValueError, match="release_id"): + latest_release("policyengine/populace-us", api=hub) From 2b089694b71ba819a69d3d50d8cb43e2e0affb9c Mon Sep 17 00:00:00 2001 From: Pavel Makarchuk Date: Fri, 12 Jun 2026 01:11:22 -0400 Subject: [PATCH 3/4] populace.calibrate.diagnostics: the calibration evidence ships with the artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CalibrationResult already carries everything an auditor needs — per-target initial/final estimates and tolerance verdicts, the per-epoch loss trajectory, every skipped target with its reason, and the solver options actually used — but none of it left the build machine: the build pushed diagnostics to telemetry and dropped them, and the published .npz kept only closing scalars. "Skipped and reported, never dropped silently" is only true if the report ships. diagnostics_payload() renders the result as strict JSON (non-finite floats become null; the writer passes allow_nan=False so anything that escapes the scrub fails loudly), and write_calibration_diagnostics() writes the calibration_diagnostics.json a release directory publishes next to its manifests. Fixes #10. Co-Authored-By: Claude Fable 5 --- .../src/populace/calibrate/__init__.py | 8 ++ .../src/populace/calibrate/diagnostics.py | 116 ++++++++++++++++++ .../tests/test_diagnostics.py | 106 ++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100644 packages/populace-calibrate/src/populace/calibrate/diagnostics.py create mode 100644 packages/populace-calibrate/tests/test_diagnostics.py diff --git a/packages/populace-calibrate/src/populace/calibrate/__init__.py b/packages/populace-calibrate/src/populace/calibrate/__init__.py index d8b190a..0d36917 100644 --- a/packages/populace-calibrate/src/populace/calibrate/__init__.py +++ b/packages/populace-calibrate/src/populace/calibrate/__init__.py @@ -74,6 +74,11 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: _assert_frame_compatible(_frame_version, _REQUIRED_FRAME_SERIES) +from populace.calibrate.diagnostics import ( # noqa: E402 - after the compat gate + CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + diagnostics_payload, + write_calibration_diagnostics, +) from populace.calibrate.matrix import ( # noqa: E402 - after the compat gate CalibrationProblem, SkippedTarget, @@ -102,6 +107,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: __all__ = [ "AGGREGATIONS", + "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", "CONSERVE_MASS", "FREE_MASS", "CalibrationProblem", @@ -114,7 +120,9 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "TargetSpec", "build_constraint_matrix", "calibrate", + "diagnostics_payload", "relative_error_loss", "specs_from_pe_surface", + "write_calibration_diagnostics", "__version__", ] diff --git a/packages/populace-calibrate/src/populace/calibrate/diagnostics.py b/packages/populace-calibrate/src/populace/calibrate/diagnostics.py new file mode 100644 index 0000000..13e0701 --- /dev/null +++ b/packages/populace-calibrate/src/populace/calibrate/diagnostics.py @@ -0,0 +1,116 @@ +"""Serialize a calibration's diagnostics so they travel with the artifact. + +A :class:`~populace.calibrate.solve.CalibrationResult` carries everything a +reviewer needs to audit what calibration did — per-target estimates before +and after, the per-epoch loss trajectory, the targets that failed to compile +*and why*, and the solver options actually used. Until now none of it left +the build machine: the build pushed the diagnostics to telemetry and dropped +them, and the published ``.npz`` kept only closing scalars. "Skipped and +reported, never dropped silently" is only true if the report ships. + +:func:`diagnostics_payload` renders the result as a JSON-stable dict, and +:func:`write_calibration_diagnostics` writes it as +``calibration_diagnostics.json`` — the artifact a release publishes next to +its manifests (charter rule: artifacts carry their environment; a published +dataset's calibration evidence belongs with the dataset, not in telemetry). +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +from populace.calibrate.solve import CalibrationResult + +__all__ = [ + "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", + "diagnostics_payload", + "write_calibration_diagnostics", +] + +#: Version of the diagnostics payload. Consumers (dashboards, scorers) key +#: their readers on it; bump it with any shape change. +CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 1 + + +def _finite(value: float) -> float | None: + """JSON has no NaN/inf; a non-finite diagnostic serializes as null.""" + value = float(value) + return value if math.isfinite(value) else None + + +def _jsonable(value: object) -> object: + """An option value as strict JSON: non-finite floats become null.""" + if isinstance(value, float): + return _finite(value) + return value + + +def diagnostics_payload(result: CalibrationResult) -> dict: + """Render a calibration result as a JSON-stable diagnostics payload. + + The payload carries the full evidence, not summaries: every per-target + row, the whole loss trajectory, and every skipped target with its + reason. Summary scalars (``final_loss``, ``fraction_within_10pct``) are + included so a consumer need not recompute them, but they are derived + from the rows, never a substitute. + + Args: + result: The :func:`~populace.calibrate.solve.calibrate` output. + + Returns: + A dict that round-trips through ``json`` unchanged (non-finite + floats become ``null``). + """ + return { + "schema_version": CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + "weight_entity": result.weight_entity, + "options": {key: _jsonable(value) for key, value in result.options.items()}, + "l0_lambda": _finite(result.l0_lambda), + "n_nonzero": int(result.n_nonzero), + "n_records": int(result.weights.shape[0]), + "initial_loss": _finite(result.initial_loss), + "final_loss": _finite(result.final_loss), + "fraction_within_10pct": _finite(result.fraction_within_10pct), + "loss_trajectory": [_finite(loss) for loss in result.loss_trajectory], + "skipped": [ + {"name": skip.target.name, "reason": skip.reason} + for skip in result.skipped + ], + "targets": [ + { + "name": diagnostic.name, + "target": _finite(diagnostic.target), + "initial_estimate": _finite(diagnostic.initial_estimate), + "final_estimate": _finite(diagnostic.final_estimate), + "relative_error": _finite(diagnostic.relative_error), + "within_tolerance": diagnostic.within_tolerance, + } + for diagnostic in result.diagnostics + ], + } + + +def write_calibration_diagnostics( + result: CalibrationResult, path: Path | str +) -> Path: + """Write the diagnostics payload to ``path`` as JSON. + + The conventional filename is ``calibration_diagnostics.json`` inside a + release directory, alongside ``build_manifest.json``. + + Args: + result: The :func:`~populace.calibrate.solve.calibrate` output. + path: Destination file path; parent directories must exist. + + Returns: + The path written. + """ + path = Path(path) + # allow_nan=False is the guard: a non-finite value that escaped the + # scrub is a bug here, not something to smuggle out as invalid JSON. + path.write_text( + json.dumps(diagnostics_payload(result), indent=1, allow_nan=False) + ) + return path diff --git a/packages/populace-calibrate/tests/test_diagnostics.py b/packages/populace-calibrate/tests/test_diagnostics.py new file mode 100644 index 0000000..7e523ff --- /dev/null +++ b/packages/populace-calibrate/tests/test_diagnostics.py @@ -0,0 +1,106 @@ +"""The diagnostics artifact: a calibration's evidence ships with it. + +Behavioral contracts: the payload carries every per-target row, the whole +loss trajectory, and every skipped target with its reason; it round-trips +through strict JSON (no NaN/Infinity tokens); and the file writer produces +the artifact a release directory publishes. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from populace.calibrate import ( + CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + Target, + TargetSet, + calibrate, + diagnostics_payload, + write_calibration_diagnostics, +) + + +def _result(feasible_frame, *, with_skip: bool = False, epochs: int = 120): + frame, truths = feasible_frame() + targets = [ + Target( + name="population", + entity="household", + aggregation="count", + value=truths["population"] * 1.2, + ), + Target( + name="income", + entity="household", + aggregation="sum", + value=truths["income"] * 1.2, + measure="income", + tolerance=truths["income"], + ), + ] + if with_skip: + targets.append( + Target( + name="ghost", + entity="household", + aggregation="sum", + value=1.0, + measure="no_such_column", + ) + ) + return calibrate(frame, TargetSet(tuple(targets)), epochs=epochs, seed=0) + + +def test_payload_carries_full_evidence(feasible_frame) -> None: + result = _result(feasible_frame, epochs=120) + payload = diagnostics_payload(result) + + assert payload["schema_version"] == CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION + assert payload["weight_entity"] == "household" + assert len(payload["loss_trajectory"]) == 120 + assert len(payload["targets"]) == len(result.diagnostics) + assert payload["n_records"] == result.weights.shape[0] + assert payload["final_loss"] == result.final_loss + assert payload["fraction_within_10pct"] == result.fraction_within_10pct + assert payload["options"]["epochs"] == 120 + assert payload["options"]["seed"] == 0 + + income = next( + row for row in payload["targets"] if row["name"].startswith("income") + ) + assert income["initial_estimate"] is not None + assert income["final_estimate"] is not None + assert income["within_tolerance"] is True # tolerance was a whole truth wide + population = next( + row for row in payload["targets"] if row["name"].startswith("population") + ) + assert population["within_tolerance"] is None # no tolerance declared + + +def test_skipped_targets_ship_with_their_reason(feasible_frame) -> None: + result = _result(feasible_frame, with_skip=True) + payload = diagnostics_payload(result) + assert len(payload["skipped"]) == 1 + skip = payload["skipped"][0] + assert skip["name"] == "ghost" + assert "no_such_column" in skip["reason"] + # The skip never leaks into the compiled target rows. + assert all(not row["name"].startswith("ghost") for row in payload["targets"]) + + +def test_payload_is_strict_json(feasible_frame) -> None: + result = _result(feasible_frame) + payload = diagnostics_payload(result) + encoded = json.dumps(payload, allow_nan=False) # raises on NaN/inf + assert json.loads(encoded) == payload + + +def test_writer_round_trips(feasible_frame, tmp_path: Path) -> None: + result = _result(feasible_frame) + path = write_calibration_diagnostics( + result, tmp_path / "calibration_diagnostics.json" + ) + loaded = json.loads(path.read_text()) + assert loaded == diagnostics_payload(result) + assert loaded["schema_version"] == CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION From 5e24b56c5d0e40cf06c8100243847f232c58d986 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 14 Jun 2026 15:03:25 -0400 Subject: [PATCH 4/4] Fix release contract and export gates --- packages/populace-build/pyproject.toml | 9 +- .../src/populace/build/__init__.py | 6 + .../src/populace/build/gates.py | 242 ++++++++++++++++- .../src/populace/build/trace.py | 108 +++++--- .../src/populace/build/us/poverty.py | 132 +++++++++ .../src/populace/build/us/source_coverage.py | 251 ++++++++++++++++++ packages/populace-build/tests/test_gates.py | 121 ++++++++- packages/populace-build/tests/test_trace.py | 87 +++++- .../populace-build/tests/test_us_poverty.py | 79 ++++++ .../tests/test_us_source_coverage.py | 35 +++ .../src/populace/calibrate/diagnostics.py | 11 +- .../tests/test_diagnostics.py | 4 +- packages/populace-data/pyproject.toml | 2 +- .../src/populace/data/contract.py | 83 ++++-- .../src/populace/data/release.py | 38 ++- packages/populace-data/tests/test_contract.py | 48 +++- packages/populace-data/tests/test_release.py | 91 ++++++- packages/populace-frame/pyproject.toml | 2 +- .../frame/adapters/policyengine_us.py | 154 +++++++++-- .../src/populace/frame/rules.py | 17 +- .../populace-frame/tests/test_adapters.py | 3 + .../tests/test_policyengine_us_adapter.py | 239 +++++++++++++++++ uv.lock | 14 +- 23 files changed, 1621 insertions(+), 155 deletions(-) create mode 100644 packages/populace-build/src/populace/build/us/poverty.py create mode 100644 packages/populace-build/src/populace/build/us/source_coverage.py create mode 100644 packages/populace-build/tests/test_us_poverty.py create mode 100644 packages/populace-build/tests/test_us_source_coverage.py diff --git a/packages/populace-build/pyproject.toml b/packages/populace-build/pyproject.toml index 431f66d..b013c0e 100644 --- a/packages/populace-build/pyproject.toml +++ b/packages/populace-build/pyproject.toml @@ -18,7 +18,14 @@ dependencies = [ # The US build pulls survey loaders and the rules engine; the base install # (stage machinery + gates) stays light so scorers can consume the gates # without an engine. -us = ["policyengine-us>=1.723,<2", "policyengine-us-data>=1.69,<2", "h5py>=3"] +us = [ + "policyengine-us>=1.729,<2", + # 1.115.4 pins policyengine-us==1.693.5, which conflicts with the + # current engine surface needed for formula-owned export checks. Lift the + # cap once policyengine-us-data publishes compatible metadata again. + "policyengine-us-data>=1.115.2,<1.115.4", + "h5py>=3", +] [project.urls] Homepage = "https://populace.dev" diff --git a/packages/populace-build/src/populace/build/__init__.py b/packages/populace-build/src/populace/build/__init__.py index f59fdf2..1670262 100644 --- a/packages/populace-build/src/populace/build/__init__.py +++ b/packages/populace-build/src/populace/build/__init__.py @@ -53,10 +53,13 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: GateReport, GateResult, aggregate_admin_gate, + enum_domain_gate, exported_nonzero_gate, + formula_owned_export_gate, parity_gate, per_family_fit_gate, relative_error_loss, + source_coverage_gate, support_gate, ) from populace.build.holdout import ( # noqa: E402 - after the compat gate @@ -80,11 +83,14 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "StagePlan", "StageRecord", "aggregate_admin_gate", + "enum_domain_gate", "exported_nonzero_gate", + "formula_owned_export_gate", "parity_gate", "per_family_fit_gate", "relative_error_loss", "rotated_folds", + "source_coverage_gate", "summarize_rotations", "support_gate", "__version__", diff --git a/packages/populace-build/src/populace/build/gates.py b/packages/populace-build/src/populace/build/gates.py index 3bbbbf2..e3ac390 100644 --- a/packages/populace-build/src/populace/build/gates.py +++ b/packages/populace-build/src/populace/build/gates.py @@ -4,7 +4,7 @@ mutates anything, every failure names the exact variable/target involved, and a :class:`GateReport` aggregates the suite into one publishable verdict. -The four gates encode the build lessons of 2026: +The gates encode the build lessons of 2026: - :func:`parity_gate` — the incumbent-replacement contract: every variable layer the reference populates, the candidate populates. An all-zero layer @@ -17,6 +17,11 @@ gains at −$3.9T and investment-interest at $33.5T. - :func:`per_family_fit_gate` — calibration fit reported per source family, so a collapsed family cannot hide inside a good global average. +- :func:`source_coverage_gate` — hard-target source families must be active or + explicitly excluded, while validation-only families must stay out of hard + calibration. +- :func:`enum_domain_gate` — enum-typed engine inputs must carry engine enum + member names, not raw source-system codes. Scoring uses :func:`relative_error_loss` — the calibrator's own objective — so there is no calibrator-vs-scorer objective mismatch: what the solver @@ -36,11 +41,14 @@ __all__ = [ "GateResult", "GateReport", + "enum_domain_gate", + "formula_owned_export_gate", "exported_nonzero_gate", "parity_gate", "support_gate", "aggregate_admin_gate", "per_family_fit_gate", + "source_coverage_gate", "relative_error_loss", ] @@ -125,7 +133,7 @@ def exported_nonzero_gate( all-zero stored column is either a pipeline bug (real values lost on the way to export — the v3 head-carry incident) or dead scaffolding that masks the engine's own defaults/formulas; the fix is to populate - it or drop it, never to ship zeros. + it or remove it upstream, never to ship zeros. Args: column_shares: Stored column -> share of records with a non-zero @@ -153,7 +161,7 @@ def exported_nonzero_gate( if share > 0.0 or name in exemptions: continue failures.append( - f"{name}: stored but all-zero — populate it or drop it " + f"{name}: stored but all-zero — populate it or remove it upstream " "(zeros mask engine defaults/formulas)." ) unused = sorted(set(exemptions) - set(column_shares)) @@ -173,6 +181,234 @@ def exported_nonzero_gate( ) +def formula_owned_export_gate( + exported_columns: Iterable[str], + formula_owned_columns: Iterable[str], + *, + structural_columns: Iterable[str] = (), +) -> GateResult: + """Formula-owned engine outputs must not be persisted as inputs. + + A PolicyEngine-native HDF5 file turns every persisted variable column into + a simulation input. Persisting a formula-owned variable such as ``ssi`` + therefore pins the baseline value and masks reforms; the artifact must + arrive at export without it, so the engine computes it. Entity ids and + memberships can be exempted via ``structural_columns`` because those are + reconstruction scaffolding, not policy inputs. + + Args: + exported_columns: Columns the artifact will persist. + formula_owned_columns: Variables owned by engine formulas. + structural_columns: Non-input structural columns allowed through even + when their names overlap the engine's variable registry. + + Returns: + Pass iff no non-structural exported column is formula-owned. + """ + exported = set(exported_columns) + structural = set(structural_columns) + formula_owned = set(formula_owned_columns) + offenders = sorted((exported & formula_owned) - structural) + return GateResult( + name="formula_owned_export", + passed=not offenders, + failures=tuple( + f"{name}: formula-owned engine output exported as an input; " + "remove it upstream before export." + for name in offenders + ), + details={ + "columns_checked": len(exported), + "formula_owned_columns": len(formula_owned), + "structural_exemptions": sorted(structural & exported & formula_owned), + "offenders": offenders, + }, + ) + + +def _enum_member_name(value: object) -> str: + if isinstance(value, bytes): + return value.decode() + name = getattr(value, "name", None) + if isinstance(name, str): + return name + return str(value) + + +def _enum_domain_names(domain: Iterable[object] | object) -> tuple[str, ...]: + members = getattr(domain, "__members__", None) + if isinstance(members, Mapping): + return tuple(str(name) for name in members) + return tuple(_enum_member_name(value) for value in domain) # type: ignore[arg-type] + + +def enum_domain_gate( + column_values: Mapping[str, Iterable[object]], + enum_domains: Mapping[str, Iterable[object] | object], +) -> GateResult: + """Validate exported enum inputs against their engine enum domains. + + A non-zero raw source code can pass parity and nonzero checks while still + being impossible for the rules engine to interpret. This gate operates on + exported columns whose corresponding engine variable declares enum + ``possible_values`` and requires stored values to be enum member names + such as ``"WHITE"`` rather than source codes such as ``"10"``. + + Args: + column_values: Exported enum column -> stored values. + enum_domains: Exported enum column -> valid enum members, member + names, or an enum class exposing ``__members__``. + + Returns: + Pass iff every provided enum column's non-missing values are inside + its declared domain. Missing values are treated as invalid because a + present enum input column should be fully interpretable by the engine; + omit the column to let the engine default it. + """ + failures: list[str] = [] + invalid_counts: dict[str, int] = {} + invalid_examples: dict[str, list[str]] = {} + allowed_values: dict[str, list[str]] = {} + columns_checked = 0 + + for column, values in sorted(column_values.items()): + if column not in enum_domains: + continue + allowed = set(_enum_domain_names(enum_domains[column])) + allowed_values[column] = sorted(allowed) + columns_checked += 1 + invalid: list[str] = [] + total = 0 + for value in values: + total += 1 + if value is None or ( + isinstance(value, (float, np.floating)) and np.isnan(value) + ): + invalid.append("") + continue + name = _enum_member_name(value) + if name not in allowed: + invalid.append(name) + if not invalid: + continue + examples = sorted(set(invalid))[:8] + invalid_counts[column] = len(invalid) + invalid_examples[column] = examples + failures.append( + f"{column}: {len(invalid)}/{total} value(s) outside enum domain; " + f"invalid examples {examples}; allowed values {sorted(allowed)[:8]}." + ) + + return GateResult( + name="enum_domain", + passed=not failures, + failures=tuple(failures), + details={ + "columns_checked": columns_checked, + "invalid_counts": invalid_counts, + "invalid_examples": invalid_examples, + "allowed_values": allowed_values, + }, + ) + + +def _coverage_field(entry: object, name: str, default: object = None) -> object: + if isinstance(entry, Mapping): + return entry.get(name, default) + return getattr(entry, name, default) + + +def source_coverage_gate( + coverage_entries: Iterable[object], + *, + active_target_aliases: Iterable[str] = (), + active_target_families: Iterable[str] = (), + reviewed_exclusions: Mapping[str, str] | Iterable[str] = (), +) -> GateResult: + """Gate source-family coverage for a release target profile. + + Hard-target source package aliases must either appear in the active target + inventory or have an explicit reviewed exclusion. Validation-only families + can appear in diagnostics, but fail the gate if activated as hard targets. + Source gaps are reported in details without failing; they are facts about + source availability, not evidence that the build covered the family. + + ``coverage_entries`` intentionally accepts either dict-like entries or the + ``SourceCoverageEntry`` dataclass from ``populace.build.us.source_coverage`` + so callers can also pass a live Arch coverage contract. + """ + active_aliases = set(active_target_aliases) + active_families = set(active_target_families) + if isinstance(reviewed_exclusions, Mapping): + exclusion_reasons = { + str(alias): str(reason) for alias, reason in reviewed_exclusions.items() + } + else: + exclusion_reasons = { + str(alias): "reviewed exclusion" for alias in reviewed_exclusions + } + + failures: list[str] = [] + missing_hard_targets: list[str] = [] + reviewed: dict[str, str] = {} + validation_misuse: list[str] = [] + source_gaps: dict[str, tuple[str, ...]] = {} + + for entry in coverage_entries: + family = str(_coverage_field(entry, "family_id", "")) + role = str(_coverage_field(entry, "role", "")) + aliases = tuple( + str(a) for a in (_coverage_field(entry, "package_aliases", ()) or ()) + ) + if role == "hard_target": + for alias in aliases: + if alias in active_aliases: + continue + if alias in exclusion_reasons: + reviewed[alias] = exclusion_reasons[alias] + continue + missing_hard_targets.append(alias) + failures.append( + f"{family}/{alias}: hard-target source alias is not active " + "and has no reviewed exclusion." + ) + elif role == "validation_only": + if family in active_families or any( + alias in active_aliases for alias in aliases + ): + validation_misuse.append(family) + failures.append( + f"{family}: validation-only source family activated as a hard target." + ) + elif role == "source_gap": + source_gaps[family] = tuple( + str(item) + for item in ( + _coverage_field(entry, "missing_source_packages", ()) or () + ) + ) + + unused_exclusions = sorted(set(exclusion_reasons) - set(reviewed)) + if unused_exclusions: + failures.append( + f"Reviewed exclusions not in coverage contract: {unused_exclusions}." + ) + + return GateResult( + name="source_coverage", + passed=not failures, + failures=tuple(failures), + details={ + "active_target_aliases": sorted(active_aliases), + "active_target_families": sorted(active_families), + "missing_hard_targets": sorted(missing_hard_targets), + "reviewed_exclusions": reviewed, + "validation_only_activated": sorted(validation_misuse), + "source_gaps": source_gaps, + }, + ) + + def parity_gate( candidate_nonzero: Mapping[str, float], reference_nonzero: Mapping[str, float], diff --git a/packages/populace-build/src/populace/build/trace.py b/packages/populace-build/src/populace/build/trace.py index cae335a..f04cf87 100644 --- a/packages/populace-build/src/populace/build/trace.py +++ b/packages/populace-build/src/populace/build/trace.py @@ -525,20 +525,15 @@ def tro_from_release_manifest( ) -> dict: """Convert a populace release manifest into a TRACE TRO. - Maps the real field names of - ``packages/populace-data/build/us/release_manifest.json``: + Maps the release manifest fields: - - ``build_id`` / ``builder`` / ``build_sha`` / ``build_date`` -> - identity on the build performance; - - ``dataset`` and ``calibration`` (each ``{filename, sha256}``) -> the two - output artifacts, located at their canonical Hugging Face URLs; + - schema-v1 ``build`` / ``artifacts`` fields, or the earlier top-level + ``build_id`` / ``dataset`` / ``calibration`` smoke-manifest fields; + - output artifacts, located at their canonical Hugging Face URLs; - the whole ``gates`` block -> the ``gate_report`` payload, with ``pop:gatesPassed``/``pop:gateNames`` derived from its keys; - - ``construction`` -> the build-config payload (the manifest's only - configuration record); - - ``score_vs_enhanced_cps`` -> the stage-records payload slot (the - manifest has no per-stage records, so the scoring summary stands in as - the build's recorded evidence trail). + - ``construction`` -> the build-config payload; + - optional ``stage_records`` -> the stage-records payload slot. Restricted inputs (the IRS PUF, Fed SCF, Census SIPP) are not in the release manifest — pass them via ``input_artifacts`` to bind them too. @@ -551,41 +546,72 @@ def tro_from_release_manifest( Raises: ValueError: If a declared output artifact lacks a ``sha256``. """ + build_info = manifest.get("build") + if isinstance(build_info, Mapping): + build_id = str(build_info.get("build_id", "")) + builder = str(build_info.get("builder", manifest.get("builder", "populace"))) + build_sha = str(build_info.get("build_sha", manifest.get("build_sha", ""))) + build_date = str(build_info.get("build_date", manifest.get("build_date", ""))) + else: + build_id = str(manifest["build_id"]) + builder = str(manifest.get("builder", "populace")) + build_sha = str(manifest.get("build_sha", "")) + build_date = str(manifest.get("build_date", "")) + outputs: list[dict[str, Any]] = [] - dataset = manifest.get("dataset") - if isinstance(dataset, Mapping): - outputs.append( - _output_from_manifest_artifact( - dataset, spec_id="dataset", hf_repo=hf_repo, hf_revision=hf_revision + artifacts = manifest.get("artifacts") + if isinstance(artifacts, Mapping): + for artifact_id, artifact in artifacts.items(): + if not isinstance(artifact, Mapping): + continue + outputs.append( + _output_from_manifest_artifact( + artifact, + spec_id=str(artifact_id), + hf_repo=hf_repo, + hf_revision=hf_revision, + ) ) - ) - calibration = manifest.get("calibration") - if isinstance(calibration, Mapping): - outputs.append( - _output_from_manifest_artifact( - calibration, - spec_id="calibration", - hf_repo=hf_repo, - hf_revision=hf_revision, + else: + dataset = manifest.get("dataset") + if isinstance(dataset, Mapping): + outputs.append( + _output_from_manifest_artifact( + dataset, + spec_id="dataset", + hf_repo=hf_repo, + hf_revision=hf_revision, + ) + ) + calibration = manifest.get("calibration") + if isinstance(calibration, Mapping): + outputs.append( + _output_from_manifest_artifact( + calibration, + spec_id="calibration", + hf_repo=hf_repo, + hf_revision=hf_revision, + ) ) - ) config_manifest = ( {"construction": manifest["construction"]} if "construction" in manifest else None ) + stage_records_value = manifest.get("stage_records") stage_records = ( - [{"score_vs_enhanced_cps": manifest["score_vs_enhanced_cps"]}] - if "score_vs_enhanced_cps" in manifest + list(stage_records_value) + if isinstance(stage_records_value, Sequence) + and not isinstance(stage_records_value, (str, bytes)) else None ) return build_build_tro( - build_id=str(manifest["build_id"]), - builder=str(manifest.get("builder", "populace")), - build_sha=str(manifest.get("build_sha", "")), - build_date=str(manifest.get("build_date", "")), + build_id=build_id, + builder=builder, + build_sha=build_sha, + build_date=build_date, config_manifest=config_manifest, stage_records=stage_records, gate_manifest=manifest.get("gates"), @@ -602,7 +628,12 @@ def _created_at_from_manifest(manifest: Mapping[str, Any]) -> str | None: Uses the build date as-is (it is a calendar date, not an instant — the manifest records no finer timestamp). """ - build_date = manifest.get("build_date") + build_info = manifest.get("build") + build_date = ( + build_info.get("build_date") + if isinstance(build_info, Mapping) + else manifest.get("build_date") + ) return str(build_date) if build_date else None @@ -613,19 +644,24 @@ def _output_from_manifest_artifact( hf_repo: str, hf_revision: str, ) -> dict[str, Any]: - """Build an output-artifact mapping from a manifest ``{filename, sha256}``. + """Build an output-artifact mapping from a manifest artifact entry. Locates the file at its canonical Hugging Face dataset URL. The sha256 is carried through as-is; :func:`build_build_tro` raises when it is missing so the failure is legible. """ - filename = artifact.get("filename") + filename = artifact.get("filename") or artifact.get("path") + artifact_repo = artifact.get("repo_id") + if not isinstance(artifact_repo, str) or not artifact_repo: + artifact_repo = hf_repo return { "id": spec_id, "name": filename, "sha256": artifact.get("sha256"), "location": ( - _huggingface_location(filename, repo=hf_repo, revision=hf_revision) + _huggingface_location( + filename, repo=artifact_repo, revision=hf_revision + ) if filename else None ), diff --git a/packages/populace-build/src/populace/build/us/poverty.py b/packages/populace-build/src/populace/build/us/poverty.py new file mode 100644 index 0000000..3d7cc1a --- /dev/null +++ b/packages/populace-build/src/populace/build/us/poverty.py @@ -0,0 +1,132 @@ +"""US poverty/SPM release diagnostics. + +These helpers summarize modeled SPM resources against thresholds. They are +validation diagnostics, not calibration targets: official CPS SPM references can +travel in the output metadata, but callers must not use this module to turn CPS +SPM rates into hard target rows. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +import numpy as np +import pandas as pd + +__all__ = ["spm_resource_diagnostics", "write_spm_resource_diagnostics"] + + +def _weighted_sum(values: np.ndarray, weights: np.ndarray) -> float: + return float(np.sum(values * weights)) + + +def _weighted_quantile( + values: np.ndarray, weights: np.ndarray, quantiles: tuple[float, ...] +) -> dict[str, float | None]: + valid = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(valid): + return {str(q): None for q in quantiles} + values = values[valid] + weights = weights[valid] + order = np.argsort(values) + values = values[order] + weights = weights[order] + cumulative = np.cumsum(weights) + total = cumulative[-1] + return { + str(q): float(values[np.searchsorted(cumulative, q * total, side="left")]) + for q in quantiles + } + + +def spm_resource_diagnostics( + frame: pd.DataFrame, + *, + resource_column: str, + threshold_column: str, + weight_column: str, + group_columns: tuple[str, ...] = (), + validation_references: Mapping[str, object] | None = None, +) -> dict: + """Return SPM poverty and negative-resource diagnostics for a frame. + + Args: + frame: One row per SPM unit or person, already aligned to resources, + thresholds, and weights. + resource_column: Modeled SPM resources. + threshold_column: SPM poverty threshold. + weight_column: Person or SPM-unit weight. + group_columns: Optional subgroup columns for poverty-rate breakdowns. + validation_references: Metadata about official/public validation + references. These are copied under ``validation_references`` and + marked validation-only. + """ + required = (resource_column, threshold_column, weight_column) + missing = [column for column in required if column not in frame.columns] + if missing: + raise ValueError(f"SPM diagnostic frame missing required columns {missing}.") + + resources = frame[resource_column].to_numpy(dtype=float) + thresholds = frame[threshold_column].to_numpy(dtype=float) + weights = frame[weight_column].to_numpy(dtype=float) + valid = np.isfinite(resources) & np.isfinite(thresholds) & np.isfinite(weights) + valid &= weights > 0 + if not np.any(valid): + raise ValueError("SPM diagnostic frame has no positive-weight finite rows.") + + poverty = (resources < thresholds) & valid + negative = (resources < 0) & valid + population = float(weights[valid].sum()) + poverty_count = _weighted_sum(poverty[valid].astype(float), weights[valid]) + negative_count = _weighted_sum(negative[valid].astype(float), weights[valid]) + + groups: dict[str, dict[str, dict[str, float | int | str]]] = {} + for column in group_columns: + if column not in frame.columns: + raise ValueError(f"SPM diagnostic group column {column!r} is missing.") + groups[column] = {} + for value, subframe in frame.loc[valid].groupby(column, dropna=False): + sub = spm_resource_diagnostics( + subframe, + resource_column=resource_column, + threshold_column=threshold_column, + weight_column=weight_column, + ) + groups[column][str(value)] = { + "population": sub["population"], + "poverty_count": sub["poverty_count"], + "poverty_rate": sub["poverty_rate"], + } + + payload = { + "schema_version": 1, + "classification": "validation_only", + "population": population, + "poverty_count": poverty_count, + "poverty_rate": poverty_count / population, + "resource_quantiles": _weighted_quantile( + resources[valid], weights[valid], (0.1, 0.25, 0.5, 0.75, 0.9) + ), + "negative_resources": { + "count": negative_count, + "share": negative_count / population, + "minimum": float(np.min(resources[valid])), + "total_negative_mass": float( + np.sum(resources[negative] * weights[negative]) + ), + }, + "groups": groups, + "validation_references": dict(validation_references or {}), + } + return payload + + +def write_spm_resource_diagnostics( + payload: Mapping[str, object], path: Path | str +) -> Path: + """Write an SPM diagnostics payload as strict JSON.""" + path = Path(path) + path.write_text(json.dumps(payload, indent=1, allow_nan=False)) + return path diff --git a/packages/populace-build/src/populace/build/us/source_coverage.py b/packages/populace-build/src/populace/build/us/source_coverage.py new file mode 100644 index 0000000..fa8cb51 --- /dev/null +++ b/packages/populace-build/src/populace/build/us/source_coverage.py @@ -0,0 +1,251 @@ +"""US source-family coverage for poverty and nonfiler release gates. + +This is a pinned Populace-side copy of the Arch source coverage contract +merged in ``PolicyEngine/arch-data`` PR #53 +(``5fa48f07436a806ad75ff76fd22cfb8613bddbe0``). Arch owns source packages; +Populace owns whether source families are active hard targets, validation-only +diagnostics, or explicit source gaps in a release profile. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +__all__ = [ + "ARCH_US_POVERTY_CONTRACT_COMMIT", + "CoverageRole", + "SourceCoverageEntry", + "US_POVERTY_NONFILER_SOURCE_COVERAGE", + "hard_target_package_aliases", + "source_gap_family_ids", + "validation_only_family_ids", +] + +ARCH_US_POVERTY_CONTRACT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" + +CoverageRole = Literal["hard_target", "validation_only", "source_gap"] + + +@dataclass(frozen=True) +class SourceCoverageEntry: + """Coverage status for one source family relevant to US poverty work.""" + + family_id: str + label: str + role: CoverageRole + package_aliases: tuple[str, ...] = () + missing_source_packages: tuple[str, ...] = () + notes: str = "" + + +US_POVERTY_NONFILER_SOURCE_COVERAGE: tuple[SourceCoverageEntry, ...] = ( + SourceCoverageEntry( + "population_age_sex", + "Population by age, sex, state, and congressional district", + "hard_target", + ( + "census-pep-2024-national-age-sex", + "census-pep-2024-state-age-sex", + "census-acs-s0101-national-age-2024", + "census-acs-s0101-state-age-2024", + "census-acs-s0101-congressional-district-age-2024", + ), + ), + SourceCoverageEntry( + "nipa_personal_income", + "NIPA personal income, transfers, taxes, and pensions", + "hard_target", + ( + "bea-nipa-total-wages-salaries", + "bea-nipa-personal-income-components", + "bea-nipa-personal-income-disposition", + "bea-nipa-pension-contributions", + ), + ), + SourceCoverageEntry( + "irs_soi_filer_income_tax_credits", + "SOI filer income, taxes, deductions, and credits", + "hard_target", + ( + "soi-table-1-1", + "soi-table-1-2", + "soi-table-1-4", + "soi-table-2-1", + "soi-table-2-5", + "soi-table-2-5-eitc-agi-children-2022", + "soi-table-4-3", + "soi-state-2022", + "soi-historic-table-2", + "soi-historic-table-2-state-agi-2022", + "soi-historic-table-2-state-broad-2022", + "soi-historic-table-2-state-eitc-2022", + "soi-w2-statistics-2020", + ), + ), + SourceCoverageEntry( + "social_security_ssi", + "Social Security and SSI payments", + "hard_target", + ("ssa-annual-statistical-supplement-2025", "ssa-ssi-table-7b1-2024"), + ), + SourceCoverageEntry( + "snap_admin", + "SNAP participation and benefit cost", + "hard_target", + ("usda-snap-fy69-to-current",), + ), + SourceCoverageEntry( + "tanf_admin", + "TANF caseload and financial data", + "hard_target", + ("hhs-acf-tanf-caseload-2024", "hhs-acf-tanf-financial-2024"), + ), + SourceCoverageEntry( + "liheap_admin", + "LIHEAP households and benefits", + "hard_target", + ( + "hhs-acf-liheap-fy2023-national-profile", + "hhs-acf-liheap-fy2024-national-profile", + ), + ), + SourceCoverageEntry( + "health_programs", + "Medicaid, CHIP, ACA, Medicare, and NHE controls", + "hard_target", + ( + "cms-medicaid-chip-monthly-enrollment-dataset", + "cms-medicaid-chip-monthly-enrollment-december-2024", + "cms-nhe-historical-service-source", + "cms-aca-oep-state-level", + "cms-aca-oep-state-level-2022", + "cms-aca-oep-state-level-2025", + "cms-aca-effectuated-enrollment-2022", + "cms-medicare-trustees-report-2025-part-b-premium-income", + ), + ), + SourceCoverageEntry( + "state_income_tax_collections", + "State individual income tax collections", + "hard_target", + ("census-stc-individual-income-tax",), + ), + SourceCoverageEntry( + "snap_local_proxy", + "SNAP congressional district household estimates", + "validation_only", + ("census-acs-s2201-congressional-district-snap-2024",), + ), + SourceCoverageEntry( + "cbo_income_revenue_projection", + "CBO income and revenue projections", + "validation_only", + ("cbo-revenue-projections-income-by-source-2026-02",), + ), + SourceCoverageEntry( + "wealth_balance_sheet", + "Household net worth balance-sheet checks", + "validation_only", + ("federal-reserve-z1-household-net-worth",), + ), + SourceCoverageEntry( + "census_cps_spm", + "Census CPS ASEC SPM poverty, resources, and thresholds", + "validation_only", + notes="Validation only; not a hard calibration target.", + ), + SourceCoverageEntry( + "dina_distributional_accounts", + "Distributional national accounts", + "validation_only", + notes="Validation only; not a hard calibration target.", + ), + SourceCoverageEntry( + "acs_poverty_income_distribution", + "ACS poverty and income distributions", + "validation_only", + notes="Validation only; not an SPM hard target.", + ), + SourceCoverageEntry( + "hud_assisted_housing", + "Housing assistance and subsidy controls", + "source_gap", + missing_source_packages=( + "HUD Picture of Subsidized Households", + "HUD assisted-housing expenditure or unit-count tables", + ), + ), + SourceCoverageEntry( + "usda_wic", + "WIC participation and benefits", + "source_gap", + missing_source_packages=("USDA FNS WIC program data",), + ), + SourceCoverageEntry( + "usda_school_meals", + "School lunch and breakfast benefits", + "source_gap", + missing_source_packages=( + "USDA FNS National School Lunch Program data", + "USDA FNS School Breakfast Program data", + ), + ), + SourceCoverageEntry( + "ocse_child_support", + "Child support received and paid", + "source_gap", + missing_source_packages=("HHS OCSE child support annual report tables",), + ), + SourceCoverageEntry( + "dol_workers_compensation", + "Workers' compensation benefits", + "source_gap", + missing_source_packages=( + "DOL or NASI workers' compensation benefit totals", + "State workers' compensation benefit totals", + ), + ), + SourceCoverageEntry( + "moop_work_childcare_costs", + "MOOP, work expenses, and childcare expense validation", + "source_gap", + missing_source_packages=( + "MEPS out-of-pocket medical spending tables", + "BLS Consumer Expenditure work-related expense tables", + "Childcare expense validation source", + ), + ), +) + + +def hard_target_package_aliases() -> tuple[str, ...]: + """Arch package aliases required for hard-target source coverage.""" + return tuple( + sorted( + { + alias + for entry in US_POVERTY_NONFILER_SOURCE_COVERAGE + if entry.role == "hard_target" + for alias in entry.package_aliases + } + ) + ) + + +def validation_only_family_ids() -> tuple[str, ...]: + """Source families that can be diagnostics but must not be hard targets.""" + return tuple( + entry.family_id + for entry in US_POVERTY_NONFILER_SOURCE_COVERAGE + if entry.role == "validation_only" + ) + + +def source_gap_family_ids() -> tuple[str, ...]: + """Source families the release must report as currently unsupported.""" + return tuple( + entry.family_id + for entry in US_POVERTY_NONFILER_SOURCE_COVERAGE + if entry.role == "source_gap" + ) diff --git a/packages/populace-build/tests/test_gates.py b/packages/populace-build/tests/test_gates.py index ff10166..ba46934 100644 --- a/packages/populace-build/tests/test_gates.py +++ b/packages/populace-build/tests/test_gates.py @@ -13,9 +13,12 @@ GateReport, GateResult, aggregate_admin_gate, + enum_domain_gate, + formula_owned_export_gate, parity_gate, per_family_fit_gate, relative_error_loss, + source_coverage_gate, support_gate, ) from populace.calibrate import TargetSpec @@ -218,7 +221,7 @@ def test_all_zero_column_fails_with_remedy_named(self) -> None: result = exported_nonzero_gate({"snap": 0.1, "net_worth": 0.0}) assert not result.passed assert "net_worth" in result.failures[0] - assert "populate it or drop it" in result.failures[0] + assert "populate it or remove it upstream" in result.failures[0] def test_fully_populated_export_passes(self) -> None: from populace.build import exported_nonzero_gate @@ -251,3 +254,119 @@ def test_unused_exemptions_are_surfaced(self) -> None: ) assert result.passed assert result.details["unused_exemptions"] == ["gone_var"] + + +class TestFormulaOwnedExportGate: + def test_formula_owned_column_fails_with_remedy_named(self) -> None: + result = formula_owned_export_gate( + ["person_id", "employment_income", "ssi"], + ["ssi", "income_tax"], + structural_columns=["person_id"], + ) + assert not result.passed + assert result.failures == ( + "ssi: formula-owned engine output exported as an input; " + "remove it upstream before export.", + ) + assert result.details["offenders"] == ["ssi"] + + def test_structural_overlap_is_exempted(self) -> None: + result = formula_owned_export_gate( + ["person_id", "employment_income"], + ["person_id", "income_tax"], + structural_columns=["person_id"], + ) + assert result.passed + assert result.details["structural_exemptions"] == ["person_id"] + + +class TestEnumDomainGate: + def test_valid_enum_names_pass(self) -> None: + result = enum_domain_gate( + {"race": ["WHITE", "BLACK", "HISPANIC", "OTHER"]}, + {"race": ("WHITE", "BLACK", "HISPANIC", "OTHER")}, + ) + assert result.passed + assert result.details["columns_checked"] == 1 + + def test_raw_source_codes_fail_with_examples(self) -> None: + result = enum_domain_gate( + {"race": [0, 1, 10, 11]}, + {"race": ("WHITE", "BLACK", "HISPANIC", "OTHER")}, + ) + assert not result.passed + assert result.failures and "race" in result.failures[0] + assert result.details["invalid_counts"] == {"race": 4} + assert "10" in result.details["invalid_examples"]["race"] + + def test_enum_class_domains_are_supported(self) -> None: + import enum + + class Race(enum.Enum): + WHITE = "white" + BLACK = "black" + + result = enum_domain_gate({"race": [Race.WHITE, "BLACK"]}, {"race": Race}) + assert result.passed + + +class TestSourceCoverageGate: + def test_hard_targets_must_be_active_or_reviewed(self) -> None: + coverage = ( + { + "family_id": "population_age_sex", + "role": "hard_target", + "package_aliases": ("census-pep-state-age-sex",), + }, + ) + result = source_coverage_gate(coverage) + assert not result.passed + assert "census-pep-state-age-sex" in result.failures[0] + + def test_reviewed_hard_target_exclusion_passes_and_is_recorded(self) -> None: + coverage = ( + { + "family_id": "population_age_sex", + "role": "hard_target", + "package_aliases": ("census-pep-state-age-sex",), + }, + ) + result = source_coverage_gate( + coverage, + reviewed_exclusions={ + "census-pep-state-age-sex": "state-age targets not in this smoke build" + }, + ) + assert result.passed + assert result.details["reviewed_exclusions"] == { + "census-pep-state-age-sex": "state-age targets not in this smoke build" + } + + def test_validation_only_family_cannot_be_a_hard_target(self) -> None: + coverage = ( + { + "family_id": "census_cps_spm", + "role": "validation_only", + "package_aliases": ("census-cps-spm-2024",), + }, + ) + result = source_coverage_gate( + coverage, + active_target_families=("census_cps_spm",), + ) + assert not result.passed + assert "validation-only" in result.failures[0] + + def test_source_gaps_are_reported_without_failing(self) -> None: + coverage = ( + { + "family_id": "usda_wic", + "role": "source_gap", + "missing_source_packages": ("USDA FNS WIC program data",), + }, + ) + result = source_coverage_gate(coverage) + assert result.passed + assert result.details["source_gaps"] == { + "usda_wic": ("USDA FNS WIC program data",) + } diff --git a/packages/populace-build/tests/test_trace.py b/packages/populace-build/tests/test_trace.py index fecdb0c..0744030 100644 --- a/packages/populace-build/tests/test_trace.py +++ b/packages/populace-build/tests/test_trace.py @@ -41,9 +41,9 @@ "sha256": ("a3da2f59085c45f0e16b06337818e3513c2635911dc0d16fa7deb5006263c12a"), }, "construction": ( - "eCPS-free: every layer from primary sources (CPS ASEC, IRS PUF 2015 " - "uprated, Fed SCF 2022, Census SIPP, CPS-ORG, MEPS-IC parameters, " - "Census ACS 2022); enhanced CPS used only as the scoring benchmark" + "Every layer from primary sources (CPS ASEC, IRS PUF 2015 uprated, " + "Fed SCF 2022, Census SIPP, CPS-ORG, MEPS-IC parameters, Census ACS " + "2022); incumbent comparisons live in the external benchmark repo" ), "gates": { "parity_gaps": 0, @@ -62,16 +62,43 @@ "net_stcg_b": -77.5, }, }, - "score_vs_enhanced_cps": { - "protocol": ( - "matched 41,314 households, symmetric refit, 739-target holdout " - "(seed 20260529)" - ), - "train_loss": {"populace": 0.18957, "enhanced_cps": 1.08879}, - "holdout_loss": {"populace": 0.03837, "enhanced_cps": 0.3167}, - "full_loss": {"populace": 0.22794, "enhanced_cps": 1.40549}, - "per_target_wins": {"populace": 1040, "enhanced_cps": 2613, "ties": 51}, + "stage_records": [ + { + "stage": "support_export", + "artifact": "populace_us_2024.h5", + "passed": True, + } + ], +} + +SCHEMA_V1_RELEASE_MANIFEST_FIXTURE: dict = { + "schema_version": 1, + "data_package": {"name": "populace-data", "version": "0.1.0"}, + "build": { + "build_id": "populace-us-2024-9f1260b-20260611", + "builder": "populace", + "build_sha": "9f1260b", + "build_date": "2026-06-11", + }, + "artifacts": { + "populace_us_2024": { + "kind": "microdata", + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "sha256": ( + "dc75c0d4fdedd57946db84a8d838dbc5b61a284365c3ce6eb6508b8e81111a4b" + ), + }, + "calibration_diagnostics": { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": "policyengine/populace-diagnostics", + "sha256": ( + "a3da2f59085c45f0e16b06337818e3513c2635911dc0d16fa7deb5006263c12a" + ), + }, }, + "gates": {"exported_nonzero": {"passed": True}}, } # The IRS PUF: a restricted input the build consumes but a re-runner cannot @@ -454,6 +481,42 @@ def test_hf_revision_pins_artifact_locations(self) -> None: "resolve/4a8e7d39eb9e/populace_us_2024.h5" ] + def test_schema_v1_release_manifest_outputs_artifacts(self) -> None: + tro = trace.tro_from_release_manifest( + SCHEMA_V1_RELEASE_MANIFEST_FIXTURE, + hf_revision="abc123", + ) + node = _graph_node(tro) + perf = node["trov:hasPerformance"] + assert perf["pop:buildId"] == "populace-us-2024-9f1260b-20260611" + assert perf["pop:buildSha"] == "9f1260b" + hashes = _artifact_hashes(node) + assert ( + SCHEMA_V1_RELEASE_MANIFEST_FIXTURE["artifacts"]["populace_us_2024"][ + "sha256" + ] + in hashes + ) + assert ( + SCHEMA_V1_RELEASE_MANIFEST_FIXTURE["artifacts"]["calibration_diagnostics"][ + "sha256" + ] + in hashes + ) + locations = { + loc["trov:hasLocation"] + for loc in _locations(node) + if "trov:hasLocation" in loc + } + assert ( + "https://huggingface.co/datasets/policyengine/populace-us/resolve/" + "abc123/populace_us_2024.h5" + ) in locations + assert ( + "https://huggingface.co/datasets/policyengine/populace-diagnostics/" + "resolve/abc123/calibration_diagnostics.json" + ) in locations + class TestGatesPassedDerivation: def _tro_with_gates(self, gates: dict) -> dict: diff --git a/packages/populace-build/tests/test_us_poverty.py b/packages/populace-build/tests/test_us_poverty.py new file mode 100644 index 0000000..ee45aa0 --- /dev/null +++ b/packages/populace-build/tests/test_us_poverty.py @@ -0,0 +1,79 @@ +import json + +import pandas as pd +import pytest + +from populace.build.us.poverty import ( + spm_resource_diagnostics, + write_spm_resource_diagnostics, +) + + +def test_spm_resource_diagnostics_reports_validation_only_payload() -> None: + frame = pd.DataFrame( + { + "resources": [9_000.0, 40_000.0, -500.0], + "threshold": [15_000.0, 30_000.0, 10_000.0], + "weight": [2.0, 3.0, 1.0], + "state": ["CA", "CA", "NY"], + } + ) + + diagnostics = spm_resource_diagnostics( + frame, + resource_column="resources", + threshold_column="threshold", + weight_column="weight", + group_columns=("state",), + validation_references={"census_cps_spm": {"classification": "validation_only"}}, + ) + + assert diagnostics["classification"] == "validation_only" + assert diagnostics["population"] == 6.0 + assert diagnostics["poverty_count"] == 3.0 + assert diagnostics["poverty_rate"] == pytest.approx(0.5) + assert diagnostics["negative_resources"]["count"] == 1.0 + assert diagnostics["negative_resources"]["minimum"] == -500.0 + assert diagnostics["groups"]["state"]["CA"]["poverty_count"] == 2.0 + assert diagnostics["validation_references"]["census_cps_spm"] == { + "classification": "validation_only" + } + + +def test_spm_resource_diagnostics_requires_positive_finite_rows() -> None: + frame = pd.DataFrame({"resources": [1.0], "threshold": [2.0], "weight": [0.0]}) + with pytest.raises(ValueError, match="positive-weight finite rows"): + spm_resource_diagnostics( + frame, + resource_column="resources", + threshold_column="threshold", + weight_column="weight", + ) + + +def test_spm_resource_diagnostics_ignores_invalid_rows_in_counts() -> None: + frame = pd.DataFrame( + { + "resources": [9_000.0, 40_000.0, 1_000.0, 2_000.0], + "threshold": [15_000.0, 30_000.0, 10_000.0, float("nan")], + "weight": [2.0, 3.0, float("nan"), 5.0], + } + ) + + diagnostics = spm_resource_diagnostics( + frame, + resource_column="resources", + threshold_column="threshold", + weight_column="weight", + ) + + assert diagnostics["population"] == 5.0 + assert diagnostics["poverty_count"] == 2.0 + assert diagnostics["poverty_rate"] == pytest.approx(0.4) + assert diagnostics["negative_resources"]["count"] == 0.0 + + +def test_write_spm_resource_diagnostics_writes_strict_json(tmp_path) -> None: + payload = {"schema_version": 1, "classification": "validation_only"} + path = write_spm_resource_diagnostics(payload, tmp_path / "spm.json") + assert json.loads(path.read_text()) == payload diff --git a/packages/populace-build/tests/test_us_source_coverage.py b/packages/populace-build/tests/test_us_source_coverage.py new file mode 100644 index 0000000..52b4a3a --- /dev/null +++ b/packages/populace-build/tests/test_us_source_coverage.py @@ -0,0 +1,35 @@ +from populace.build.gates import source_coverage_gate +from populace.build.us.source_coverage import ( + ARCH_US_POVERTY_CONTRACT_COMMIT, + US_POVERTY_NONFILER_SOURCE_COVERAGE, + hard_target_package_aliases, + source_gap_family_ids, + validation_only_family_ids, +) + + +def test_us_poverty_source_coverage_snapshot_has_expected_roles() -> None: + assert len(ARCH_US_POVERTY_CONTRACT_COMMIT) == 40 + assert "ssa-ssi-table-7b1-2024" in hard_target_package_aliases() + assert "cms-aca-oep-state-level-2025" in hard_target_package_aliases() + assert "census_cps_spm" in validation_only_family_ids() + assert "usda_wic" in source_gap_family_ids() + + +def test_us_poverty_source_coverage_gate_passes_when_hard_aliases_are_active() -> None: + result = source_coverage_gate( + US_POVERTY_NONFILER_SOURCE_COVERAGE, + active_target_aliases=hard_target_package_aliases(), + ) + assert result.passed + assert result.details["source_gaps"]["hud_assisted_housing"] + + +def test_us_poverty_source_coverage_gate_blocks_cps_spm_hard_target() -> None: + result = source_coverage_gate( + US_POVERTY_NONFILER_SOURCE_COVERAGE, + active_target_aliases=hard_target_package_aliases(), + active_target_families=("census_cps_spm",), + ) + assert not result.passed + assert any("census_cps_spm" in failure for failure in result.failures) diff --git a/packages/populace-calibrate/src/populace/calibrate/diagnostics.py b/packages/populace-calibrate/src/populace/calibrate/diagnostics.py index 13e0701..60edca3 100644 --- a/packages/populace-calibrate/src/populace/calibrate/diagnostics.py +++ b/packages/populace-calibrate/src/populace/calibrate/diagnostics.py @@ -75,8 +75,7 @@ def diagnostics_payload(result: CalibrationResult) -> dict: "fraction_within_10pct": _finite(result.fraction_within_10pct), "loss_trajectory": [_finite(loss) for loss in result.loss_trajectory], "skipped": [ - {"name": skip.target.name, "reason": skip.reason} - for skip in result.skipped + {"name": skip.target.name, "reason": skip.reason} for skip in result.skipped ], "targets": [ { @@ -92,9 +91,7 @@ def diagnostics_payload(result: CalibrationResult) -> dict: } -def write_calibration_diagnostics( - result: CalibrationResult, path: Path | str -) -> Path: +def write_calibration_diagnostics(result: CalibrationResult, path: Path | str) -> Path: """Write the diagnostics payload to ``path`` as JSON. The conventional filename is ``calibration_diagnostics.json`` inside a @@ -110,7 +107,5 @@ def write_calibration_diagnostics( path = Path(path) # allow_nan=False is the guard: a non-finite value that escaped the # scrub is a bug here, not something to smuggle out as invalid JSON. - path.write_text( - json.dumps(diagnostics_payload(result), indent=1, allow_nan=False) - ) + path.write_text(json.dumps(diagnostics_payload(result), indent=1, allow_nan=False)) return path diff --git a/packages/populace-calibrate/tests/test_diagnostics.py b/packages/populace-calibrate/tests/test_diagnostics.py index 7e523ff..83e9357 100644 --- a/packages/populace-calibrate/tests/test_diagnostics.py +++ b/packages/populace-calibrate/tests/test_diagnostics.py @@ -66,9 +66,7 @@ def test_payload_carries_full_evidence(feasible_frame) -> None: assert payload["options"]["epochs"] == 120 assert payload["options"]["seed"] == 0 - income = next( - row for row in payload["targets"] if row["name"].startswith("income") - ) + income = next(row for row in payload["targets"] if row["name"].startswith("income")) assert income["initial_estimate"] is not None assert income["final_estimate"] is not None assert income["within_tolerance"] is True # tolerance was a whole truth wide diff --git a/packages/populace-data/pyproject.toml b/packages/populace-data/pyproject.toml index f0403e9..c7dc7a7 100644 --- a/packages/populace-data/pyproject.toml +++ b/packages/populace-data/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ # Floor at the engine version verified to load the published artifact (the # USSingleYearDataset loader + entity-flatten behavior the file is built # against are recent); cap below 2 to guard a breaking dataset-schema major. -us = ["policyengine-us>=1.723,<2"] +us = ["policyengine-us>=1.729,<2"] # The UK artifact lives in a PRIVATE repo (UK Data Service licence): loads # require an authenticated HF token with access. The loader surfaces the # 401 with that explanation rather than retrying. diff --git a/packages/populace-data/src/populace/data/contract.py b/packages/populace-data/src/populace/data/contract.py index b11f837..ff9efab 100644 --- a/packages/populace-data/src/populace/data/contract.py +++ b/packages/populace-data/src/populace/data/contract.py @@ -40,9 +40,11 @@ REQUIRED_RELEASE_FILES = ( "build_manifest.json", "release_manifest.json", - "sound_ecps_replacement_comparison.json", + "calibration_diagnostics.json", ) +CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 1 + class ReleaseContractError(ValueError): """A release directory violates the release contract. @@ -63,8 +65,11 @@ def __init__(self, release_dir: Path, failures: list[str]) -> None: def _load_json(path: Path, failures: list[str]) -> Mapping | None: try: - loaded = json.loads(path.read_text()) - except json.JSONDecodeError as exc: + loaded = json.loads( + path.read_text(), + parse_constant=_reject_json_constant, + ) + except (json.JSONDecodeError, ValueError) as exc: failures.append(f"{path.name} is not valid JSON: {exc}.") return None if not isinstance(loaded, Mapping): @@ -75,6 +80,10 @@ def _load_json(path: Path, failures: list[str]) -> Mapping | None: return loaded +def _reject_json_constant(token: str) -> None: + raise ValueError(f"non-standard JSON constant {token}") + + def _check_build_manifest( manifest: Mapping, release_id: str, failures: list[str] ) -> None: @@ -93,9 +102,7 @@ def _check_build_manifest( else: for key in ("filename", "sha256"): if not dataset.get(key): - failures.append( - f"build_manifest.json 'dataset' is missing {key!r}." - ) + failures.append(f"build_manifest.json 'dataset' is missing {key!r}.") if not isinstance(manifest.get("gates"), Mapping): failures.append( "build_manifest.json is missing the 'gates' object (the " @@ -120,9 +127,7 @@ def _check_release_manifest( ) build = manifest.get("build") if not isinstance(build, Mapping) or not build.get("build_id"): - failures.append( - "release_manifest.json is missing 'build.build_id'." - ) + failures.append("release_manifest.json is missing 'build.build_id'.") elif build["build_id"] != release_id: failures.append( f"release_manifest.json 'build.build_id' is " @@ -132,22 +137,60 @@ def _check_release_manifest( artifacts = manifest.get("artifacts") if not isinstance(artifacts, Mapping) or not artifacts: failures.append( - "release_manifest.json must declare a non-empty 'artifacts' " - "mapping." + "release_manifest.json must declare a non-empty 'artifacts' mapping." ) else: for key, entry in artifacts.items(): if not isinstance(entry, Mapping): failures.append( - f"release_manifest.json artifact {key!r} must be an " - f"object." + f"release_manifest.json artifact {key!r} must be an object." ) continue for field in ("path", "repo_id", "sha256"): if not entry.get(field): failures.append( - f"release_manifest.json artifact {key!r} is missing " - f"{field!r}." + f"release_manifest.json artifact {key!r} is missing {field!r}." + ) + + +def _check_calibration_diagnostics(diagnostics: Mapping, failures: list[str]) -> None: + schema_version = diagnostics.get("schema_version") + if schema_version is None: + failures.append("calibration_diagnostics.json is missing 'schema_version'.") + elif schema_version != CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION: + failures.append( + f"calibration_diagnostics.json 'schema_version' is {schema_version!r}; " + f"this library publishes version " + f"{CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION}." + ) + + expected_sections = { + "targets": list, + "loss_trajectory": list, + "skipped": list, + "options": Mapping, + } + for section, expected_type in expected_sections.items(): + value = diagnostics.get(section) + if not isinstance(value, expected_type): + failures.append( + f"calibration_diagnostics.json is missing a {section!r} " + f"{expected_type.__name__}." + ) + + targets = diagnostics.get("targets") + if isinstance(targets, list): + for index, target in enumerate(targets): + if not isinstance(target, Mapping): + failures.append( + f"calibration_diagnostics.json target row {index} must be an object." + ) + continue + for field in ("name", "target", "initial_estimate", "final_estimate"): + if field not in target: + failures.append( + "calibration_diagnostics.json target row " + f"{index} is missing {field!r}." ) @@ -172,9 +215,7 @@ def validate_release_dir(release_dir: Path | str) -> None: failures: list[str] = [] if not release_dir.is_dir(): - raise ReleaseContractError( - release_dir, [f"{release_dir} is not a directory."] - ) + raise ReleaseContractError(release_dir, [f"{release_dir} is not a directory."]) for filename in REQUIRED_RELEASE_FILES: if not (release_dir / filename).is_file(): @@ -192,5 +233,11 @@ def validate_release_dir(release_dir: Path | str) -> None: if manifest is not None: _check_release_manifest(manifest, release_id, failures) + calibration_diagnostics_path = release_dir / "calibration_diagnostics.json" + if calibration_diagnostics_path.is_file(): + diagnostics = _load_json(calibration_diagnostics_path, failures) + if diagnostics is not None: + _check_calibration_diagnostics(diagnostics, failures) + if failures: raise ReleaseContractError(release_dir, failures) diff --git a/packages/populace-data/src/populace/data/release.py b/packages/populace-data/src/populace/data/release.py index 4aa4b30..db5d1f3 100644 --- a/packages/populace-data/src/populace/data/release.py +++ b/packages/populace-data/src/populace/data/release.py @@ -62,7 +62,7 @@ class LatestPointer: updated_at: ISO-8601 UTC timestamp of when the pointer was written. paths: Repo-relative path of each contract file, keyed by its stem (``"build_manifest"``, ``"release_manifest"``, - ``"sound_ecps_replacement_comparison"``). + ``"calibration_diagnostics"``). """ release_id: str @@ -70,9 +70,7 @@ class LatestPointer: paths: dict[str, str] -def latest_pointer_payload( - release_id: str, *, updated_at: str | None = None -) -> dict: +def latest_pointer_payload(release_id: str, *, updated_at: str | None = None) -> dict: """The ``latest.json`` payload for ``release_id``. Paths are derived from the release contract — the pointer names exactly @@ -84,9 +82,7 @@ def latest_pointer_payload( updated_at: ISO-8601 UTC timestamp; defaults to now. """ if updated_at is None: - updated_at = ( - datetime.now(UTC).replace(microsecond=0).isoformat() - ) + updated_at = datetime.now(UTC).replace(microsecond=0).isoformat() return { "schema_version": LATEST_POINTER_SCHEMA_VERSION, "release_id": release_id, @@ -145,8 +141,6 @@ def publish_release( release_dir = Path(release_dir) validate_release_dir(release_dir) release_id = release_dir.name - if api is None: - api = _hf_api() filenames = list(REQUIRED_RELEASE_FILES) + [ name for name in extra_files if name not in REQUIRED_RELEASE_FILES @@ -157,6 +151,12 @@ def publish_release( raise FileNotFoundError( f"extra release file {filename!r} not found in {release_dir}." ) + + if api is None: + api = _hf_api() + + for filename in filenames: + local = release_dir / filename api.upload_file( path_or_fileobj=str(local), path_in_repo=f"releases/{release_id}/{filename}", @@ -202,11 +202,27 @@ def latest_release(repo_id: str, *, api=None) -> LatestPointer: ) release_id = payload.get("release_id") if not release_id: + raise ValueError(f"{LATEST_POINTER_PATH} in {repo_id} has no 'release_id'.") + paths = payload.get("paths") + if not isinstance(paths, dict): + raise ValueError(f"{LATEST_POINTER_PATH} in {repo_id} has no 'paths' object.") + expected_paths = latest_pointer_payload(str(release_id), updated_at="")["paths"] + observed_paths = {str(key): value for key, value in paths.items()} + missing_paths = sorted(set(expected_paths) - set(observed_paths)) + unexpected_paths = sorted(set(observed_paths) - set(expected_paths)) + malformed_paths = sorted( + key + for key in set(expected_paths) & set(observed_paths) + if observed_paths[key] != expected_paths[key] + ) + if missing_paths or unexpected_paths or malformed_paths: raise ValueError( - f"{LATEST_POINTER_PATH} in {repo_id} has no 'release_id'." + f"{LATEST_POINTER_PATH} in {repo_id} has incomplete paths: " + f"missing={missing_paths}, unexpected={unexpected_paths}, " + f"malformed={malformed_paths}." ) return LatestPointer( release_id=str(release_id), updated_at=str(payload.get("updated_at", "")), - paths={str(k): str(v) for k, v in (payload.get("paths") or {}).items()}, + paths={str(k): str(v) for k, v in paths.items()}, ) diff --git a/packages/populace-data/tests/test_contract.py b/packages/populace-data/tests/test_contract.py index 133d725..a891f44 100644 --- a/packages/populace-data/tests/test_contract.py +++ b/packages/populace-data/tests/test_contract.py @@ -33,8 +33,7 @@ def _build_manifest(release_id: str = RELEASE_ID) -> dict: "filename": "populace_us_2024_calibration.npz", "sha256": "a3da2f", }, - "gates": {"parity_gaps": 0}, - "score_vs_enhanced_cps": {"per_target_wins": {}}, + "gates": {"exported_nonzero": {"passed": True}}, } @@ -54,17 +53,35 @@ def _release_manifest(release_id: str = RELEASE_ID) -> dict: } +def _calibration_diagnostics() -> dict: + return { + "schema_version": 1, + "weight_entity": "household", + "options": {"epochs": 120}, + "loss_trajectory": [1.0, 0.5], + "skipped": [], + "targets": [ + { + "name": "population", + "target": 1.0, + "initial_estimate": 0.8, + "final_estimate": 1.0, + "relative_error": 0.0, + "within_tolerance": True, + } + ], + } + + @pytest.fixture def release_dir(tmp_path: Path) -> Path: """A complete, contract-valid release directory.""" directory = tmp_path / "releases" / RELEASE_ID directory.mkdir(parents=True) (directory / "build_manifest.json").write_text(json.dumps(_build_manifest())) - (directory / "release_manifest.json").write_text( - json.dumps(_release_manifest()) - ) - (directory / "sound_ecps_replacement_comparison.json").write_text( - json.dumps({"schema_version": 1, "target_diagnostics": {}}) + (directory / "release_manifest.json").write_text(json.dumps(_release_manifest())) + (directory / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics()) ) return directory @@ -85,7 +102,7 @@ def test_each_required_file_is_named_when_missing( def test_the_1abddeb_shape_is_rejected(release_dir: Path) -> None: """The regression: a release with only an unversioned release manifest.""" (release_dir / "build_manifest.json").unlink() - (release_dir / "sound_ecps_replacement_comparison.json").unlink() + (release_dir / "calibration_diagnostics.json").unlink() (release_dir / "release_manifest.json").write_text( json.dumps( { @@ -142,9 +159,22 @@ def test_unparseable_manifest_is_a_named_failure(release_dir: Path) -> None: validate_release_dir(release_dir) +def test_malformed_calibration_diagnostics_is_rejected( + release_dir: Path, +) -> None: + (release_dir / "calibration_diagnostics.json").write_text( + json.dumps({"schema_version": 1}) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + failures = "\n".join(excinfo.value.failures) + assert "calibration_diagnostics.json" in failures + assert "targets" in failures + + def test_all_failures_reported_at_once(release_dir: Path) -> None: """A publisher sees the full repair list, not one failure per run.""" - (release_dir / "sound_ecps_replacement_comparison.json").unlink() + (release_dir / "calibration_diagnostics.json").unlink() manifest = _release_manifest() del manifest["schema_version"] manifest["artifacts"] = {} diff --git a/packages/populace-data/tests/test_release.py b/packages/populace-data/tests/test_release.py index 3dda64e..278a45e 100644 --- a/packages/populace-data/tests/test_release.py +++ b/packages/populace-data/tests/test_release.py @@ -26,15 +26,33 @@ RELEASE_ID = "populace-us-2024-9f1260b-20260611" +def _calibration_diagnostics() -> dict: + return { + "schema_version": 1, + "weight_entity": "household", + "options": {"epochs": 120}, + "loss_trajectory": [1.0, 0.5], + "skipped": [], + "targets": [ + { + "name": "population", + "target": 1.0, + "initial_estimate": 0.8, + "final_estimate": 1.0, + "relative_error": 0.0, + "within_tolerance": True, + } + ], + } + + class FakeHub: """Records uploads in order; serves downloads from what was uploaded.""" def __init__(self) -> None: self.uploads: list[tuple[str, bytes]] = [] - def upload_file( - self, *, path_or_fileobj, path_in_repo, repo_id, repo_type - ) -> None: + def upload_file(self, *, path_or_fileobj, path_in_repo, repo_id, repo_type) -> None: assert repo_type == "dataset" assert repo_id == "policyengine/populace-us" if isinstance(path_or_fileobj, bytes): @@ -89,7 +107,9 @@ def release_dir(tmp_path: Path) -> Path: } ) ) - (directory / "sound_ecps_replacement_comparison.json").write_text("{}") + (directory / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics()) + ) return directory @@ -126,10 +146,30 @@ def test_invalid_release_uploads_nothing(hub: FakeHub, release_dir: Path) -> Non assert hub.uploads == [] -def test_extra_files_ride_along_before_the_pointer( +def test_invalid_calibration_diagnostics_uploads_nothing( hub: FakeHub, release_dir: Path ) -> None: (release_dir / "calibration_diagnostics.json").write_text("{}") + with pytest.raises(ReleaseContractError, match="calibration_diagnostics"): + publish_release(release_dir, "policyengine/populace-us", api=hub) + assert hub.uploads == [] + + +def test_nonstandard_nan_calibration_diagnostics_uploads_nothing( + hub: FakeHub, release_dir: Path +) -> None: + (release_dir / "calibration_diagnostics.json").write_text( + '{"schema_version": 1, "targets": [], "loss_trajectory": [NaN], ' + '"skipped": [], "options": {}}' + ) + with pytest.raises(ReleaseContractError, match="calibration_diagnostics"): + publish_release(release_dir, "policyengine/populace-us", api=hub) + assert hub.uploads == [] + + +def test_extra_files_ride_along_before_the_pointer( + hub: FakeHub, release_dir: Path +) -> None: publish_release( release_dir, "policyengine/populace-us", @@ -143,18 +183,17 @@ def test_extra_files_ride_along_before_the_pointer( def test_missing_extra_file_fails_loudly(hub: FakeHub, release_dir: Path) -> None: - with pytest.raises(FileNotFoundError, match="calibration_diagnostics"): + with pytest.raises(FileNotFoundError, match="support_audit"): publish_release( release_dir, "policyengine/populace-us", api=hub, - extra_files=("calibration_diagnostics.json",), + extra_files=("support_audit.json",), ) + assert hub.uploads == [] -def test_publish_then_resolve_round_trips( - hub: FakeHub, release_dir: Path -) -> None: +def test_publish_then_resolve_round_trips(hub: FakeHub, release_dir: Path) -> None: published = publish_release( release_dir, "policyengine/populace-us", @@ -171,9 +210,7 @@ def test_future_pointer_schema_is_refused(hub: FakeHub) -> None: hub.uploads.append( ( LATEST_POINTER_PATH, - json.dumps( - {"schema_version": LATEST_POINTER_SCHEMA_VERSION + 1} - ).encode(), + json.dumps({"schema_version": LATEST_POINTER_SCHEMA_VERSION + 1}).encode(), ) ) with pytest.raises(ValueError, match="Upgrade populace-data"): @@ -189,3 +226,31 @@ def test_pointer_without_release_id_is_refused(hub: FakeHub) -> None: ) with pytest.raises(ValueError, match="release_id"): latest_release("policyengine/populace-us", api=hub) + + +def test_pointer_without_contract_paths_is_refused(hub: FakeHub) -> None: + hub.uploads.append( + ( + LATEST_POINTER_PATH, + json.dumps( + { + "schema_version": LATEST_POINTER_SCHEMA_VERSION, + "release_id": RELEASE_ID, + "paths": {"build_manifest": "releases/x/build_manifest.json"}, + } + ).encode(), + ) + ) + with pytest.raises(ValueError, match="paths"): + latest_release("policyengine/populace-us", api=hub) + + +def test_pointer_with_swapped_contract_path_is_refused(hub: FakeHub) -> None: + payload = latest_pointer_payload(RELEASE_ID) + payload["paths"]["build_manifest"] = ( + f"releases/{RELEASE_ID}/calibration_diagnostics.json" + ) + hub.uploads.append((LATEST_POINTER_PATH, json.dumps(payload).encode())) + + with pytest.raises(ValueError, match="malformed=\\['build_manifest'\\]"): + latest_release("policyengine/populace-us", api=hub) diff --git a/packages/populace-frame/pyproject.toml b/packages/populace-frame/pyproject.toml index 01850b3..a69e0f0 100644 --- a/packages/populace-frame/pyproject.toml +++ b/packages/populace-frame/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] us = ["microunit>=0.1.0"] -policyengine = ["policyengine-us>=1.0", "microunit>=0.1.0"] +policyengine = ["policyengine-us>=1.729,<2", "microunit>=0.1.0"] [dependency-groups] dev = [ diff --git a/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py b/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py index 1b488c4..3f34e7b 100644 --- a/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py +++ b/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py @@ -54,25 +54,49 @@ _PERIOD_BY_DEFINITION: dict[str, str] = {"year": "year", "month": "month"} -def _has_formula(variable: Any) -> bool: +def _is_engine_computed(variable: Any, period: int | str | None = None) -> bool: """Return whether a PolicyEngine variable is computed by a formula. - Input variables (read from data, what a pool must produce) have no - formula; formula-owned outputs do. PolicyEngine exposes formulas either as - a ``formula`` attribute or a ``formulas`` mapping keyed by start date. + Input variables (read from data, what a pool must produce) are plain source + variables. Formula-owned outputs may be backed by a direct formula or a + formula mapping keyed by start date. """ + if period is not None: + return variable.get_formula(str(period)) is not None if getattr(variable, "formula", None) is not None: return True formulas = getattr(variable, "formulas", None) return bool(formulas) +def _enum_domain(variable: Any) -> tuple[str, ...]: + possible_values = getattr(variable, "possible_values", None) + members = getattr(possible_values, "__members__", None) + if isinstance(members, Mapping): + return tuple(str(name) for name in members) + return () + + +def _stored_enum_name(value: object) -> str | None: + if value is None: + return None + if isinstance(value, (float, np.floating)) and np.isnan(value): + return None + if isinstance(value, bytes): + return value.decode() + name = getattr(value, "name", None) + if isinstance(name, str): + return name + return str(value) + + class PolicyEngineUSEngine: """RulesEngine adapter backed by ``policyengine_us``. Args: contract: Column-parity contract for :meth:`write_dataset` exports. - ``None`` means an empty contract (no required/forbidden checks). + ``None`` means an empty contract (no required/forbidden/closed + surface checks). defaults: Scalar defaults broadcast onto the owning entity table for contract-required columns no bundle table provides. @@ -129,7 +153,7 @@ def variables(self) -> list[str]: return sorted( name for name, variable in system_variables.items() - if not _has_formula(variable) + if not _is_engine_computed(variable) ) def _entity_of(self, name: str) -> str: @@ -204,11 +228,12 @@ def write_dataset( ) -> None: """Write the bundle as a ``USSingleYearDataset`` HDF5 file. - Applies the export gate: forbidden and formula-owned columns are - dropped, defaults are broadcast onto the owning entity table for - required columns no table provides, and a dataset with missing - required columns is never written. After writing, the dataset is - reloaded and every persisted column verified (round-trip check). + Applies the export gate: forbidden and formula-owned columns block the + export, defaults are broadcast onto the owning entity table for + required columns no table provides, closed contracts reject unexpected + non-structural columns, and a dataset with violations is never + written. After writing, the dataset is reloaded and every persisted + column verified (round-trip check). Args: bundle: A US-schema bundle. @@ -218,8 +243,8 @@ def write_dataset( Raises: ImportError: If ``policyengine_us`` is not installed. ValueError: If ``path`` does not end in ``.h5``, the contract is - violated (the message lists the missing/forbidden columns), - or the round-trip verification fails. + violated (the message lists missing/forbidden/formula-owned/ + unexpected columns), or the round-trip verification fails. """ output_path = Path(path) if output_path.suffix != ".h5": @@ -227,17 +252,6 @@ def write_dataset( contract = self._contract tables = self._engine_tables(bundle) - forbidden = set(contract.forbidden) - drop_on_sight = forbidden | set(contract.formula_owned_excluded) - dropped: set[str] = set() - forbidden_present: set[str] = set() - for name, frame in tables.items(): - present_drops = drop_on_sight.intersection(frame.columns) - if present_drops: - tables[name] = frame.drop(columns=sorted(present_drops)) - dropped.update(present_drops) - forbidden_present.update(forbidden.intersection(present_drops)) - present_columns: set[str] = set() for frame in tables.values(): present_columns.update(frame.columns) @@ -256,11 +270,36 @@ def write_dataset( continue missing_required.append(column) - if forbidden_present or missing_required: + forbidden_present = set(contract.forbidden).intersection(present_columns) + formula_owned_present = self._engine_computed_columns( + tables, period=period + ) | set(contract.formula_owned_excluded).intersection(present_columns) + unexpected: set[str] = set() + if contract.closed: + allowed = ( + set(contract.required) + | set(contract.optional) + | self._structural_columns() + | {_HOUSEHOLD_WEIGHT_COLUMN} + ) + unexpected = present_columns - allowed + + enum_domain_failures = self._enum_domain_failures(tables) + + if ( + forbidden_present + or missing_required + or formula_owned_present + or unexpected + or enum_domain_failures + ): raise ValueError( "Export contract violated; nothing was written. Missing " f"required column(s): {sorted(missing_required)}; forbidden " - f"column(s) present: {sorted(forbidden_present)}." + f"column(s) present: {sorted(forbidden_present)}; formula-owned " + f"column(s) present: {sorted(formula_owned_present)}; unexpected column(s) " + f"present: {sorted(unexpected)}; enum-domain violation(s): " + f"{enum_domain_failures}." ) self._write_and_verify(tables, period=int(period), output_path=output_path) @@ -337,6 +376,69 @@ def _default_entity(self, column: str) -> str: return variables[column].entity.key return _PERSON_TABLE + def _engine_computed_columns( + self, + tables: Mapping[str, pd.DataFrame], + *, + period: int | str, + ) -> set[str]: + """PolicyEngine-computed columns present in the pending export. + + Formula-owned columns cannot be allowed through implicitly: if a + source table carries a PolicyEngine output name such as ``ssi``, + keeping it in the HDF5 file turns that formula output into an input + and masks reforms. Such columns must be removed upstream before the + writer is called, after checking aggregate deltas. + """ + variables = self._tax_benefit_system().variables + present = {column for frame in tables.values() for column in frame.columns} + structural = self._structural_columns() + return { + column + for column in present + if column not in structural + and column in variables + and _is_engine_computed(variables[column], period=period) + } + + def _enum_domain_failures( + self, + tables: Mapping[str, pd.DataFrame], + ) -> list[str]: + """Return enum input columns carrying values outside engine domains.""" + variables = self._tax_benefit_system().variables + structural = self._structural_columns() + failures: list[str] = [] + for entity, frame in tables.items(): + for column in frame.columns: + if column in structural or column not in variables: + continue + allowed = set(_enum_domain(variables[column])) + if not allowed: + continue + invalid: list[str] = [] + for value in frame[column].to_numpy(dtype=object): + name = _stored_enum_name(value) + if name not in allowed: + invalid.append("" if name is None else name) + if invalid: + failures.append( + f"{entity}.{column}: {len(invalid)}/{len(frame)} value(s) " + "outside enum domain; invalid examples " + f"{sorted(set(invalid))[:8]}; allowed values " + f"{sorted(allowed)[:8]}" + ) + return failures + + def _structural_columns(self) -> set[str]: + """Entity ids and memberships required to reconstruct the frame.""" + schema = self.entity_schema() + return {schema.person_id_column} | { + column + for group in schema.group_entities + for column in (schema.id_column(group), schema.membership_column(group)) + } + def _write_and_verify( self, tables: Mapping[str, pd.DataFrame], diff --git a/packages/populace-frame/src/populace/frame/rules.py b/packages/populace-frame/src/populace/frame/rules.py index 133039b..d8f35ee 100644 --- a/packages/populace-frame/src/populace/frame/rules.py +++ b/packages/populace-frame/src/populace/frame/rules.py @@ -7,7 +7,8 @@ :class:`ExportContract` is the frozen column-parity contract a dataset export is gated against: which columns it must contain, must not contain, may carry, -and must leave to the engine's own formulas. +must leave to the engine's own formulas, and whether the surface is closed to +unexpected extras. """ import json @@ -105,19 +106,24 @@ class ExportContract: Attributes: required: Columns the export MUST contain. A missing required column fails the export gate. - forbidden: Columns the export MUST NOT contain. They are dropped on - sight and their presence fails the gate. + forbidden: Columns the export MUST NOT contain. Their presence fails + the export gate. optional: Bookkeeping columns that are neither required nor forbidden; an export passes them through untouched if present. formula_owned_excluded: Variables the engine owns through formulas - and the baseline does not persist as inputs. They are silently - dropped if present so the engine computes them itself. + and the baseline must not persist as inputs. Their presence fails + the export gate; upstream build stages must drop them before + calling the engine writer. + closed: If true, exports may only contain required/optional columns + plus adapter-defined structural columns. Unexpected extras fail + the export gate before anything is written. """ required: tuple[str, ...] forbidden: tuple[str, ...] optional: tuple[str, ...] formula_owned_excluded: tuple[str, ...] + closed: bool = False @classmethod def empty(cls) -> "ExportContract": @@ -150,6 +156,7 @@ def from_path(cls, path: str | Path) -> "ExportContract": formula_owned_excluded=_as_str_tuple( sections.get("formula_owned_excluded", ()) ), + closed=bool(sections.get("closed", False)), ) diff --git a/packages/populace-frame/tests/test_adapters.py b/packages/populace-frame/tests/test_adapters.py index c625c4e..daf9f41 100644 --- a/packages/populace-frame/tests/test_adapters.py +++ b/packages/populace-frame/tests/test_adapters.py @@ -80,6 +80,7 @@ def test_empty_contract_has_no_constraints(self) -> None: assert contract.forbidden == () assert contract.optional == () assert contract.formula_owned_excluded == () + assert contract.closed is False def test_from_path_parses_sections_and_ignores_metadata(self, tmp_path) -> None: manifest = { @@ -88,6 +89,7 @@ def test_from_path_parses_sections_and_ignores_metadata(self, tmp_path) -> None: "forbidden": ["spm_unit_net_income"], "optional": ["engine_bookkeeping"], "formula_owned_excluded": ["eitc"], + "closed": True, } path = tmp_path / "contract.json" path.write_text(json.dumps(manifest), encoding="utf-8") @@ -96,6 +98,7 @@ def test_from_path_parses_sections_and_ignores_metadata(self, tmp_path) -> None: assert contract.forbidden == ("spm_unit_net_income",) assert contract.optional == ("engine_bookkeeping",) assert contract.formula_owned_excluded == ("eitc",) + assert contract.closed is True def test_from_path_ignores_unknown_sections(self, tmp_path) -> None: manifest = {"legacy_internal_optional": ["spm_unit_pre_subsidy_childcare"]} diff --git a/packages/populace-frame/tests/test_policyengine_us_adapter.py b/packages/populace-frame/tests/test_policyengine_us_adapter.py index 483263b..9ac2b6c 100644 --- a/packages/populace-frame/tests/test_policyengine_us_adapter.py +++ b/packages/populace-frame/tests/test_policyengine_us_adapter.py @@ -75,6 +75,12 @@ def test_unknown_variable_is_named(self, adapter) -> None: def test_variables_lists_inputs_not_outputs(self, adapter) -> None: names = adapter.variables() assert "employment_income" in names # input + assert "partnership_income" in names + assert "s_corp_income" in names + assert "partnership_s_corp_income" in names + assert "in_nyc" not in names + assert "ssi" not in names # formula-owned output + assert "spm_unit_capped_work_childcare_expenses" not in names assert "income_tax" not in names # formula-owned output @@ -136,6 +142,239 @@ def test_forbidden_column_blocks_the_write(self, us_bundle, tmp_path) -> None: gated.write_dataset(us_bundle, path, period=2024) assert not path.exists() + def test_formula_owned_columns_block_the_write( + self, adapter, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["ssi"] = [12_000.0, 0.0, 0.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="ssi"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_partnership_s_corp_inputs_round_trip( + self, adapter, us_bundle, tmp_path + ) -> None: + from policyengine_us.data import USSingleYearDataset + + person = us_bundle.person.copy() + person["partnership_income"] = [1_500.0, 0.0, 400.0] + person["s_corp_income"] = [500.0, 0.0, 100.0] + person["partnership_s_corp_income"] = [2_000.0, 0.0, 500.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "partnership_s_corp_inputs.h5" + adapter.write_dataset(rebuilt, path, period=2024) + reloaded = USSingleYearDataset(file_path=str(path)) + assert reloaded.person["partnership_income"].tolist() == [1_500.0, 0.0, 400.0] + assert reloaded.person["s_corp_income"].tolist() == [500.0, 0.0, 100.0] + assert reloaded.person["partnership_s_corp_income"].tolist() == [ + 2_000.0, + 0.0, + 500.0, + ] + + def test_enum_domain_column_blocks_raw_source_codes( + self, adapter, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["race"] = [0, 1, 10] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "enum_blocked.h5" + with pytest.raises(ValueError, match="enum-domain.*race"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_enum_domain_names_round_trip_for_true_input( + self, adapter, us_bundle, tmp_path + ) -> None: + from policyengine_us.data import USSingleYearDataset + + household = us_bundle.table("household").copy() + household["tenure_type"] = ["RENTED", "OWNED_WITH_MORTGAGE"] + rebuilt = Frame( + { + name: (household if name == "household" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "enum_valid.h5" + adapter.write_dataset(rebuilt, path, period=2024) + reloaded = USSingleYearDataset(file_path=str(path)) + assert reloaded.household["tenure_type"].tolist() == [ + "RENTED", + "OWNED_WITH_MORTGAGE", + ] + + def test_in_nyc_formula_owned_column_blocks_the_write( + self, adapter, us_bundle, tmp_path + ) -> None: + household = us_bundle.table("household").copy() + household["in_nyc"] = [False, True] + rebuilt = Frame( + { + name: (household if name == "household" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="in_nyc"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_spm_unit_formula_owned_column_blocks_the_write( + self, adapter, us_bundle, tmp_path + ) -> None: + spm_unit = us_bundle.table("spm_unit").copy() + spm_unit["spm_unit_capped_work_childcare_expenses"] = [3_000.0, 0.0] + rebuilt = Frame( + { + name: (spm_unit if name == "spm_unit" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="spm_unit_capped_work_childcare_expenses"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_contract_listed_formula_owned_columns_still_block_the_write( + self, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["ssi"] = [12_000.0, 0.0, 0.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + adapter = PolicyEngineUSEngine( + contract=ExportContract( + required=(), + forbidden=(), + optional=(), + formula_owned_excluded=("ssi",), + ) + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="ssi"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_contract_formula_owned_exclusion_blocks_non_engine_column( + self, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["legacy_formula_output"] = [1.0, 0.0, 1.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + adapter = PolicyEngineUSEngine( + contract=ExportContract( + required=(), + forbidden=(), + optional=(), + formula_owned_excluded=("legacy_formula_output",), + ) + ) + + path = tmp_path / "contract_formula_blocked.h5" + with pytest.raises(ValueError, match="legacy_formula_output"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_future_formula_does_not_block_current_period_input( + self, adapter, us_bundle, tmp_path + ) -> None: + from policyengine_us.data import USSingleYearDataset + + person = us_bundle.person.copy() + person["weeks_worked"] = [52, 26, 40] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "future_formula_input.h5" + adapter.write_dataset(rebuilt, path, period=2024) + + reloaded = USSingleYearDataset(file_path=str(path)) + assert reloaded.person["weeks_worked"].tolist() == [52, 26, 40] + + def test_closed_contract_blocks_unexpected_columns( + self, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["internal_bookkeeping"] = [1.0, 0.0, 1.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + gated = PolicyEngineUSEngine( + contract=ExportContract( + required=(), + forbidden=(), + optional=("age", "employment_income", "state_fips"), + formula_owned_excluded=(), + closed=True, + ) + ) + + path = tmp_path / "closed.h5" + with pytest.raises(ValueError, match="internal_bookkeeping"): + gated.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + def test_defaults_broadcast_onto_owning_entity(self, us_bundle, tmp_path) -> None: from policyengine_us.data import USSingleYearDataset diff --git a/uv.lock b/uv.lock index 9c38538..6998a2e 100644 --- a/uv.lock +++ b/uv.lock @@ -1499,7 +1499,7 @@ wheels = [ [[package]] name = "policyengine-us" -version = "1.724.0" +version = "1.729.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1509,9 +1509,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/9c/8835e12d62415794144fbb14d14244f00f4fd50a95b55e47128f6fbef9bb/policyengine_us-1.724.0.tar.gz", hash = "sha256:9cb67f2537d3613ae2cef20d91b8b82ae6b5673c8360befffad82ad9efebb369", size = 10136935, upload-time = "2026-06-10T17:43:12.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/cb/b2efba2094a708cd71890d98d72b99394fabc5894a4cceec14381e03fa35/policyengine_us-1.729.0.tar.gz", hash = "sha256:ac05c4d621c7f848b0806effc14e913160d5d47d777eadced6bc18edf392d75c", size = 10373862, upload-time = "2026-06-14T18:05:25.747Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/a2/792ab362ea7c5f08a3931644fe1f36df2d167097a675781c1790c4951e37/policyengine_us-1.724.0-py3-none-any.whl", hash = "sha256:6cc45f6f17f21b1dc62800f0bd1988c833c3e0975c39c926dcdca904bf0942dd", size = 11327533, upload-time = "2026-06-10T17:43:08.875Z" }, + { url = "https://files.pythonhosted.org/packages/b9/7d/778f92ae94997b00c3c9ac34b345f6c9333435f905670ee4eeb2f5e19809/policyengine_us-1.729.0-py3-none-any.whl", hash = "sha256:8d21d3f7c0e82a9415edffe8ea53939330a63d9c8f6bd334299bddb697cf2c00", size = 11905076, upload-time = "2026-06-14T18:05:21.806Z" }, ] [[package]] @@ -1575,8 +1575,8 @@ requires-dist = [ { name = "h5py", marker = "extra == 'us'", specifier = ">=3" }, { name = "numpy", specifier = ">=1.26" }, { name = "pandas", specifier = ">=2" }, - { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.723,<2" }, - { name = "policyengine-us-data", marker = "extra == 'us'", specifier = ">=1.69,<2" }, + { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.729,<2" }, + { name = "policyengine-us-data", marker = "extra == 'us'", specifier = ">=1.115.2,<1.115.4" }, { name = "populace-calibrate", editable = "packages/populace-calibrate" }, { name = "populace-fit", editable = "packages/populace-fit" }, { name = "populace-frame", editable = "packages/populace-frame" }, @@ -1642,7 +1642,7 @@ requires-dist = [ { name = "h5py", specifier = ">=3" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, - { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.723,<2" }, + { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.729,<2" }, ] provides-extras = ["us", "uk"] @@ -1708,7 +1708,7 @@ requires-dist = [ { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=2" }, { name = "pandas", specifier = ">=2.3" }, - { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.0" }, + { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.729,<2" }, ] provides-extras = ["us", "policyengine"]