diff --git a/packages/populace-build/pyproject.toml b/packages/populace-build/pyproject.toml index 431f66d..b013c0e 100644 --- a/packages/populace-build/pyproject.toml +++ b/packages/populace-build/pyproject.toml @@ -18,7 +18,14 @@ dependencies = [ # The US build pulls survey loaders and the rules engine; the base install # (stage machinery + gates) stays light so scorers can consume the gates # without an engine. -us = ["policyengine-us>=1.723,<2", "policyengine-us-data>=1.69,<2", "h5py>=3"] +us = [ + "policyengine-us>=1.729,<2", + # 1.115.4 pins policyengine-us==1.693.5, which conflicts with the + # current engine surface needed for formula-owned export checks. Lift the + # cap once policyengine-us-data publishes compatible metadata again. + "policyengine-us-data>=1.115.2,<1.115.4", + "h5py>=3", +] [project.urls] Homepage = "https://populace.dev" diff --git a/packages/populace-build/src/populace/build/__init__.py b/packages/populace-build/src/populace/build/__init__.py index f59fdf2..1670262 100644 --- a/packages/populace-build/src/populace/build/__init__.py +++ b/packages/populace-build/src/populace/build/__init__.py @@ -53,10 +53,13 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: GateReport, GateResult, aggregate_admin_gate, + enum_domain_gate, exported_nonzero_gate, + formula_owned_export_gate, parity_gate, per_family_fit_gate, relative_error_loss, + source_coverage_gate, support_gate, ) from populace.build.holdout import ( # noqa: E402 - after the compat gate @@ -80,11 +83,14 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "StagePlan", "StageRecord", "aggregate_admin_gate", + "enum_domain_gate", "exported_nonzero_gate", + "formula_owned_export_gate", "parity_gate", "per_family_fit_gate", "relative_error_loss", "rotated_folds", + "source_coverage_gate", "summarize_rotations", "support_gate", "__version__", diff --git a/packages/populace-build/src/populace/build/gates.py b/packages/populace-build/src/populace/build/gates.py index 3bbbbf2..e3ac390 100644 --- a/packages/populace-build/src/populace/build/gates.py +++ b/packages/populace-build/src/populace/build/gates.py @@ -4,7 +4,7 @@ mutates anything, every failure names the exact variable/target involved, and a :class:`GateReport` aggregates the suite into one publishable verdict. -The four gates encode the build lessons of 2026: +The gates encode the build lessons of 2026: - :func:`parity_gate` — the incumbent-replacement contract: every variable layer the reference populates, the candidate populates. An all-zero layer @@ -17,6 +17,11 @@ gains at −$3.9T and investment-interest at $33.5T. - :func:`per_family_fit_gate` — calibration fit reported per source family, so a collapsed family cannot hide inside a good global average. +- :func:`source_coverage_gate` — hard-target source families must be active or + explicitly excluded, while validation-only families must stay out of hard + calibration. +- :func:`enum_domain_gate` — enum-typed engine inputs must carry engine enum + member names, not raw source-system codes. Scoring uses :func:`relative_error_loss` — the calibrator's own objective — so there is no calibrator-vs-scorer objective mismatch: what the solver @@ -36,11 +41,14 @@ __all__ = [ "GateResult", "GateReport", + "enum_domain_gate", + "formula_owned_export_gate", "exported_nonzero_gate", "parity_gate", "support_gate", "aggregate_admin_gate", "per_family_fit_gate", + "source_coverage_gate", "relative_error_loss", ] @@ -125,7 +133,7 @@ def exported_nonzero_gate( all-zero stored column is either a pipeline bug (real values lost on the way to export — the v3 head-carry incident) or dead scaffolding that masks the engine's own defaults/formulas; the fix is to populate - it or drop it, never to ship zeros. + it or remove it upstream, never to ship zeros. Args: column_shares: Stored column -> share of records with a non-zero @@ -153,7 +161,7 @@ def exported_nonzero_gate( if share > 0.0 or name in exemptions: continue failures.append( - f"{name}: stored but all-zero — populate it or drop it " + f"{name}: stored but all-zero — populate it or remove it upstream " "(zeros mask engine defaults/formulas)." ) unused = sorted(set(exemptions) - set(column_shares)) @@ -173,6 +181,234 @@ def exported_nonzero_gate( ) +def formula_owned_export_gate( + exported_columns: Iterable[str], + formula_owned_columns: Iterable[str], + *, + structural_columns: Iterable[str] = (), +) -> GateResult: + """Formula-owned engine outputs must not be persisted as inputs. + + A PolicyEngine-native HDF5 file turns every persisted variable column into + a simulation input. Persisting a formula-owned variable such as ``ssi`` + therefore pins the baseline value and masks reforms; the artifact must + arrive at export without it, so the engine computes it. Entity ids and + memberships can be exempted via ``structural_columns`` because those are + reconstruction scaffolding, not policy inputs. + + Args: + exported_columns: Columns the artifact will persist. + formula_owned_columns: Variables owned by engine formulas. + structural_columns: Non-input structural columns allowed through even + when their names overlap the engine's variable registry. + + Returns: + Pass iff no non-structural exported column is formula-owned. + """ + exported = set(exported_columns) + structural = set(structural_columns) + formula_owned = set(formula_owned_columns) + offenders = sorted((exported & formula_owned) - structural) + return GateResult( + name="formula_owned_export", + passed=not offenders, + failures=tuple( + f"{name}: formula-owned engine output exported as an input; " + "remove it upstream before export." + for name in offenders + ), + details={ + "columns_checked": len(exported), + "formula_owned_columns": len(formula_owned), + "structural_exemptions": sorted(structural & exported & formula_owned), + "offenders": offenders, + }, + ) + + +def _enum_member_name(value: object) -> str: + if isinstance(value, bytes): + return value.decode() + name = getattr(value, "name", None) + if isinstance(name, str): + return name + return str(value) + + +def _enum_domain_names(domain: Iterable[object] | object) -> tuple[str, ...]: + members = getattr(domain, "__members__", None) + if isinstance(members, Mapping): + return tuple(str(name) for name in members) + return tuple(_enum_member_name(value) for value in domain) # type: ignore[arg-type] + + +def enum_domain_gate( + column_values: Mapping[str, Iterable[object]], + enum_domains: Mapping[str, Iterable[object] | object], +) -> GateResult: + """Validate exported enum inputs against their engine enum domains. + + A non-zero raw source code can pass parity and nonzero checks while still + being impossible for the rules engine to interpret. This gate operates on + exported columns whose corresponding engine variable declares enum + ``possible_values`` and requires stored values to be enum member names + such as ``"WHITE"`` rather than source codes such as ``"10"``. + + Args: + column_values: Exported enum column -> stored values. + enum_domains: Exported enum column -> valid enum members, member + names, or an enum class exposing ``__members__``. + + Returns: + Pass iff every provided enum column's non-missing values are inside + its declared domain. Missing values are treated as invalid because a + present enum input column should be fully interpretable by the engine; + omit the column to let the engine default it. + """ + failures: list[str] = [] + invalid_counts: dict[str, int] = {} + invalid_examples: dict[str, list[str]] = {} + allowed_values: dict[str, list[str]] = {} + columns_checked = 0 + + for column, values in sorted(column_values.items()): + if column not in enum_domains: + continue + allowed = set(_enum_domain_names(enum_domains[column])) + allowed_values[column] = sorted(allowed) + columns_checked += 1 + invalid: list[str] = [] + total = 0 + for value in values: + total += 1 + if value is None or ( + isinstance(value, (float, np.floating)) and np.isnan(value) + ): + invalid.append("") + continue + name = _enum_member_name(value) + if name not in allowed: + invalid.append(name) + if not invalid: + continue + examples = sorted(set(invalid))[:8] + invalid_counts[column] = len(invalid) + invalid_examples[column] = examples + failures.append( + f"{column}: {len(invalid)}/{total} value(s) outside enum domain; " + f"invalid examples {examples}; allowed values {sorted(allowed)[:8]}." + ) + + return GateResult( + name="enum_domain", + passed=not failures, + failures=tuple(failures), + details={ + "columns_checked": columns_checked, + "invalid_counts": invalid_counts, + "invalid_examples": invalid_examples, + "allowed_values": allowed_values, + }, + ) + + +def _coverage_field(entry: object, name: str, default: object = None) -> object: + if isinstance(entry, Mapping): + return entry.get(name, default) + return getattr(entry, name, default) + + +def source_coverage_gate( + coverage_entries: Iterable[object], + *, + active_target_aliases: Iterable[str] = (), + active_target_families: Iterable[str] = (), + reviewed_exclusions: Mapping[str, str] | Iterable[str] = (), +) -> GateResult: + """Gate source-family coverage for a release target profile. + + Hard-target source package aliases must either appear in the active target + inventory or have an explicit reviewed exclusion. Validation-only families + can appear in diagnostics, but fail the gate if activated as hard targets. + Source gaps are reported in details without failing; they are facts about + source availability, not evidence that the build covered the family. + + ``coverage_entries`` intentionally accepts either dict-like entries or the + ``SourceCoverageEntry`` dataclass from ``populace.build.us.source_coverage`` + so callers can also pass a live Arch coverage contract. + """ + active_aliases = set(active_target_aliases) + active_families = set(active_target_families) + if isinstance(reviewed_exclusions, Mapping): + exclusion_reasons = { + str(alias): str(reason) for alias, reason in reviewed_exclusions.items() + } + else: + exclusion_reasons = { + str(alias): "reviewed exclusion" for alias in reviewed_exclusions + } + + failures: list[str] = [] + missing_hard_targets: list[str] = [] + reviewed: dict[str, str] = {} + validation_misuse: list[str] = [] + source_gaps: dict[str, tuple[str, ...]] = {} + + for entry in coverage_entries: + family = str(_coverage_field(entry, "family_id", "")) + role = str(_coverage_field(entry, "role", "")) + aliases = tuple( + str(a) for a in (_coverage_field(entry, "package_aliases", ()) or ()) + ) + if role == "hard_target": + for alias in aliases: + if alias in active_aliases: + continue + if alias in exclusion_reasons: + reviewed[alias] = exclusion_reasons[alias] + continue + missing_hard_targets.append(alias) + failures.append( + f"{family}/{alias}: hard-target source alias is not active " + "and has no reviewed exclusion." + ) + elif role == "validation_only": + if family in active_families or any( + alias in active_aliases for alias in aliases + ): + validation_misuse.append(family) + failures.append( + f"{family}: validation-only source family activated as a hard target." + ) + elif role == "source_gap": + source_gaps[family] = tuple( + str(item) + for item in ( + _coverage_field(entry, "missing_source_packages", ()) or () + ) + ) + + unused_exclusions = sorted(set(exclusion_reasons) - set(reviewed)) + if unused_exclusions: + failures.append( + f"Reviewed exclusions not in coverage contract: {unused_exclusions}." + ) + + return GateResult( + name="source_coverage", + passed=not failures, + failures=tuple(failures), + details={ + "active_target_aliases": sorted(active_aliases), + "active_target_families": sorted(active_families), + "missing_hard_targets": sorted(missing_hard_targets), + "reviewed_exclusions": reviewed, + "validation_only_activated": sorted(validation_misuse), + "source_gaps": source_gaps, + }, + ) + + def parity_gate( candidate_nonzero: Mapping[str, float], reference_nonzero: Mapping[str, float], diff --git a/packages/populace-build/src/populace/build/trace.py b/packages/populace-build/src/populace/build/trace.py index cae335a..f04cf87 100644 --- a/packages/populace-build/src/populace/build/trace.py +++ b/packages/populace-build/src/populace/build/trace.py @@ -525,20 +525,15 @@ def tro_from_release_manifest( ) -> dict: """Convert a populace release manifest into a TRACE TRO. - Maps the real field names of - ``packages/populace-data/build/us/release_manifest.json``: + Maps the release manifest fields: - - ``build_id`` / ``builder`` / ``build_sha`` / ``build_date`` -> - identity on the build performance; - - ``dataset`` and ``calibration`` (each ``{filename, sha256}``) -> the two - output artifacts, located at their canonical Hugging Face URLs; + - schema-v1 ``build`` / ``artifacts`` fields, or the earlier top-level + ``build_id`` / ``dataset`` / ``calibration`` smoke-manifest fields; + - output artifacts, located at their canonical Hugging Face URLs; - the whole ``gates`` block -> the ``gate_report`` payload, with ``pop:gatesPassed``/``pop:gateNames`` derived from its keys; - - ``construction`` -> the build-config payload (the manifest's only - configuration record); - - ``score_vs_enhanced_cps`` -> the stage-records payload slot (the - manifest has no per-stage records, so the scoring summary stands in as - the build's recorded evidence trail). + - ``construction`` -> the build-config payload; + - optional ``stage_records`` -> the stage-records payload slot. Restricted inputs (the IRS PUF, Fed SCF, Census SIPP) are not in the release manifest — pass them via ``input_artifacts`` to bind them too. @@ -551,41 +546,72 @@ def tro_from_release_manifest( Raises: ValueError: If a declared output artifact lacks a ``sha256``. """ + build_info = manifest.get("build") + if isinstance(build_info, Mapping): + build_id = str(build_info.get("build_id", "")) + builder = str(build_info.get("builder", manifest.get("builder", "populace"))) + build_sha = str(build_info.get("build_sha", manifest.get("build_sha", ""))) + build_date = str(build_info.get("build_date", manifest.get("build_date", ""))) + else: + build_id = str(manifest["build_id"]) + builder = str(manifest.get("builder", "populace")) + build_sha = str(manifest.get("build_sha", "")) + build_date = str(manifest.get("build_date", "")) + outputs: list[dict[str, Any]] = [] - dataset = manifest.get("dataset") - if isinstance(dataset, Mapping): - outputs.append( - _output_from_manifest_artifact( - dataset, spec_id="dataset", hf_repo=hf_repo, hf_revision=hf_revision + artifacts = manifest.get("artifacts") + if isinstance(artifacts, Mapping): + for artifact_id, artifact in artifacts.items(): + if not isinstance(artifact, Mapping): + continue + outputs.append( + _output_from_manifest_artifact( + artifact, + spec_id=str(artifact_id), + hf_repo=hf_repo, + hf_revision=hf_revision, + ) ) - ) - calibration = manifest.get("calibration") - if isinstance(calibration, Mapping): - outputs.append( - _output_from_manifest_artifact( - calibration, - spec_id="calibration", - hf_repo=hf_repo, - hf_revision=hf_revision, + else: + dataset = manifest.get("dataset") + if isinstance(dataset, Mapping): + outputs.append( + _output_from_manifest_artifact( + dataset, + spec_id="dataset", + hf_repo=hf_repo, + hf_revision=hf_revision, + ) + ) + calibration = manifest.get("calibration") + if isinstance(calibration, Mapping): + outputs.append( + _output_from_manifest_artifact( + calibration, + spec_id="calibration", + hf_repo=hf_repo, + hf_revision=hf_revision, + ) ) - ) config_manifest = ( {"construction": manifest["construction"]} if "construction" in manifest else None ) + stage_records_value = manifest.get("stage_records") stage_records = ( - [{"score_vs_enhanced_cps": manifest["score_vs_enhanced_cps"]}] - if "score_vs_enhanced_cps" in manifest + list(stage_records_value) + if isinstance(stage_records_value, Sequence) + and not isinstance(stage_records_value, (str, bytes)) else None ) return build_build_tro( - build_id=str(manifest["build_id"]), - builder=str(manifest.get("builder", "populace")), - build_sha=str(manifest.get("build_sha", "")), - build_date=str(manifest.get("build_date", "")), + build_id=build_id, + builder=builder, + build_sha=build_sha, + build_date=build_date, config_manifest=config_manifest, stage_records=stage_records, gate_manifest=manifest.get("gates"), @@ -602,7 +628,12 @@ def _created_at_from_manifest(manifest: Mapping[str, Any]) -> str | None: Uses the build date as-is (it is a calendar date, not an instant — the manifest records no finer timestamp). """ - build_date = manifest.get("build_date") + build_info = manifest.get("build") + build_date = ( + build_info.get("build_date") + if isinstance(build_info, Mapping) + else manifest.get("build_date") + ) return str(build_date) if build_date else None @@ -613,19 +644,24 @@ def _output_from_manifest_artifact( hf_repo: str, hf_revision: str, ) -> dict[str, Any]: - """Build an output-artifact mapping from a manifest ``{filename, sha256}``. + """Build an output-artifact mapping from a manifest artifact entry. Locates the file at its canonical Hugging Face dataset URL. The sha256 is carried through as-is; :func:`build_build_tro` raises when it is missing so the failure is legible. """ - filename = artifact.get("filename") + filename = artifact.get("filename") or artifact.get("path") + artifact_repo = artifact.get("repo_id") + if not isinstance(artifact_repo, str) or not artifact_repo: + artifact_repo = hf_repo return { "id": spec_id, "name": filename, "sha256": artifact.get("sha256"), "location": ( - _huggingface_location(filename, repo=hf_repo, revision=hf_revision) + _huggingface_location( + filename, repo=artifact_repo, revision=hf_revision + ) if filename else None ), diff --git a/packages/populace-build/src/populace/build/us/poverty.py b/packages/populace-build/src/populace/build/us/poverty.py new file mode 100644 index 0000000..3d7cc1a --- /dev/null +++ b/packages/populace-build/src/populace/build/us/poverty.py @@ -0,0 +1,132 @@ +"""US poverty/SPM release diagnostics. + +These helpers summarize modeled SPM resources against thresholds. They are +validation diagnostics, not calibration targets: official CPS SPM references can +travel in the output metadata, but callers must not use this module to turn CPS +SPM rates into hard target rows. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +import numpy as np +import pandas as pd + +__all__ = ["spm_resource_diagnostics", "write_spm_resource_diagnostics"] + + +def _weighted_sum(values: np.ndarray, weights: np.ndarray) -> float: + return float(np.sum(values * weights)) + + +def _weighted_quantile( + values: np.ndarray, weights: np.ndarray, quantiles: tuple[float, ...] +) -> dict[str, float | None]: + valid = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(valid): + return {str(q): None for q in quantiles} + values = values[valid] + weights = weights[valid] + order = np.argsort(values) + values = values[order] + weights = weights[order] + cumulative = np.cumsum(weights) + total = cumulative[-1] + return { + str(q): float(values[np.searchsorted(cumulative, q * total, side="left")]) + for q in quantiles + } + + +def spm_resource_diagnostics( + frame: pd.DataFrame, + *, + resource_column: str, + threshold_column: str, + weight_column: str, + group_columns: tuple[str, ...] = (), + validation_references: Mapping[str, object] | None = None, +) -> dict: + """Return SPM poverty and negative-resource diagnostics for a frame. + + Args: + frame: One row per SPM unit or person, already aligned to resources, + thresholds, and weights. + resource_column: Modeled SPM resources. + threshold_column: SPM poverty threshold. + weight_column: Person or SPM-unit weight. + group_columns: Optional subgroup columns for poverty-rate breakdowns. + validation_references: Metadata about official/public validation + references. These are copied under ``validation_references`` and + marked validation-only. + """ + required = (resource_column, threshold_column, weight_column) + missing = [column for column in required if column not in frame.columns] + if missing: + raise ValueError(f"SPM diagnostic frame missing required columns {missing}.") + + resources = frame[resource_column].to_numpy(dtype=float) + thresholds = frame[threshold_column].to_numpy(dtype=float) + weights = frame[weight_column].to_numpy(dtype=float) + valid = np.isfinite(resources) & np.isfinite(thresholds) & np.isfinite(weights) + valid &= weights > 0 + if not np.any(valid): + raise ValueError("SPM diagnostic frame has no positive-weight finite rows.") + + poverty = (resources < thresholds) & valid + negative = (resources < 0) & valid + population = float(weights[valid].sum()) + poverty_count = _weighted_sum(poverty[valid].astype(float), weights[valid]) + negative_count = _weighted_sum(negative[valid].astype(float), weights[valid]) + + groups: dict[str, dict[str, dict[str, float | int | str]]] = {} + for column in group_columns: + if column not in frame.columns: + raise ValueError(f"SPM diagnostic group column {column!r} is missing.") + groups[column] = {} + for value, subframe in frame.loc[valid].groupby(column, dropna=False): + sub = spm_resource_diagnostics( + subframe, + resource_column=resource_column, + threshold_column=threshold_column, + weight_column=weight_column, + ) + groups[column][str(value)] = { + "population": sub["population"], + "poverty_count": sub["poverty_count"], + "poverty_rate": sub["poverty_rate"], + } + + payload = { + "schema_version": 1, + "classification": "validation_only", + "population": population, + "poverty_count": poverty_count, + "poverty_rate": poverty_count / population, + "resource_quantiles": _weighted_quantile( + resources[valid], weights[valid], (0.1, 0.25, 0.5, 0.75, 0.9) + ), + "negative_resources": { + "count": negative_count, + "share": negative_count / population, + "minimum": float(np.min(resources[valid])), + "total_negative_mass": float( + np.sum(resources[negative] * weights[negative]) + ), + }, + "groups": groups, + "validation_references": dict(validation_references or {}), + } + return payload + + +def write_spm_resource_diagnostics( + payload: Mapping[str, object], path: Path | str +) -> Path: + """Write an SPM diagnostics payload as strict JSON.""" + path = Path(path) + path.write_text(json.dumps(payload, indent=1, allow_nan=False)) + return path diff --git a/packages/populace-build/src/populace/build/us/source_coverage.py b/packages/populace-build/src/populace/build/us/source_coverage.py new file mode 100644 index 0000000..fa8cb51 --- /dev/null +++ b/packages/populace-build/src/populace/build/us/source_coverage.py @@ -0,0 +1,251 @@ +"""US source-family coverage for poverty and nonfiler release gates. + +This is a pinned Populace-side copy of the Arch source coverage contract +merged in ``PolicyEngine/arch-data`` PR #53 +(``5fa48f07436a806ad75ff76fd22cfb8613bddbe0``). Arch owns source packages; +Populace owns whether source families are active hard targets, validation-only +diagnostics, or explicit source gaps in a release profile. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +__all__ = [ + "ARCH_US_POVERTY_CONTRACT_COMMIT", + "CoverageRole", + "SourceCoverageEntry", + "US_POVERTY_NONFILER_SOURCE_COVERAGE", + "hard_target_package_aliases", + "source_gap_family_ids", + "validation_only_family_ids", +] + +ARCH_US_POVERTY_CONTRACT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" + +CoverageRole = Literal["hard_target", "validation_only", "source_gap"] + + +@dataclass(frozen=True) +class SourceCoverageEntry: + """Coverage status for one source family relevant to US poverty work.""" + + family_id: str + label: str + role: CoverageRole + package_aliases: tuple[str, ...] = () + missing_source_packages: tuple[str, ...] = () + notes: str = "" + + +US_POVERTY_NONFILER_SOURCE_COVERAGE: tuple[SourceCoverageEntry, ...] = ( + SourceCoverageEntry( + "population_age_sex", + "Population by age, sex, state, and congressional district", + "hard_target", + ( + "census-pep-2024-national-age-sex", + "census-pep-2024-state-age-sex", + "census-acs-s0101-national-age-2024", + "census-acs-s0101-state-age-2024", + "census-acs-s0101-congressional-district-age-2024", + ), + ), + SourceCoverageEntry( + "nipa_personal_income", + "NIPA personal income, transfers, taxes, and pensions", + "hard_target", + ( + "bea-nipa-total-wages-salaries", + "bea-nipa-personal-income-components", + "bea-nipa-personal-income-disposition", + "bea-nipa-pension-contributions", + ), + ), + SourceCoverageEntry( + "irs_soi_filer_income_tax_credits", + "SOI filer income, taxes, deductions, and credits", + "hard_target", + ( + "soi-table-1-1", + "soi-table-1-2", + "soi-table-1-4", + "soi-table-2-1", + "soi-table-2-5", + "soi-table-2-5-eitc-agi-children-2022", + "soi-table-4-3", + "soi-state-2022", + "soi-historic-table-2", + "soi-historic-table-2-state-agi-2022", + "soi-historic-table-2-state-broad-2022", + "soi-historic-table-2-state-eitc-2022", + "soi-w2-statistics-2020", + ), + ), + SourceCoverageEntry( + "social_security_ssi", + "Social Security and SSI payments", + "hard_target", + ("ssa-annual-statistical-supplement-2025", "ssa-ssi-table-7b1-2024"), + ), + SourceCoverageEntry( + "snap_admin", + "SNAP participation and benefit cost", + "hard_target", + ("usda-snap-fy69-to-current",), + ), + SourceCoverageEntry( + "tanf_admin", + "TANF caseload and financial data", + "hard_target", + ("hhs-acf-tanf-caseload-2024", "hhs-acf-tanf-financial-2024"), + ), + SourceCoverageEntry( + "liheap_admin", + "LIHEAP households and benefits", + "hard_target", + ( + "hhs-acf-liheap-fy2023-national-profile", + "hhs-acf-liheap-fy2024-national-profile", + ), + ), + SourceCoverageEntry( + "health_programs", + "Medicaid, CHIP, ACA, Medicare, and NHE controls", + "hard_target", + ( + "cms-medicaid-chip-monthly-enrollment-dataset", + "cms-medicaid-chip-monthly-enrollment-december-2024", + "cms-nhe-historical-service-source", + "cms-aca-oep-state-level", + "cms-aca-oep-state-level-2022", + "cms-aca-oep-state-level-2025", + "cms-aca-effectuated-enrollment-2022", + "cms-medicare-trustees-report-2025-part-b-premium-income", + ), + ), + SourceCoverageEntry( + "state_income_tax_collections", + "State individual income tax collections", + "hard_target", + ("census-stc-individual-income-tax",), + ), + SourceCoverageEntry( + "snap_local_proxy", + "SNAP congressional district household estimates", + "validation_only", + ("census-acs-s2201-congressional-district-snap-2024",), + ), + SourceCoverageEntry( + "cbo_income_revenue_projection", + "CBO income and revenue projections", + "validation_only", + ("cbo-revenue-projections-income-by-source-2026-02",), + ), + SourceCoverageEntry( + "wealth_balance_sheet", + "Household net worth balance-sheet checks", + "validation_only", + ("federal-reserve-z1-household-net-worth",), + ), + SourceCoverageEntry( + "census_cps_spm", + "Census CPS ASEC SPM poverty, resources, and thresholds", + "validation_only", + notes="Validation only; not a hard calibration target.", + ), + SourceCoverageEntry( + "dina_distributional_accounts", + "Distributional national accounts", + "validation_only", + notes="Validation only; not a hard calibration target.", + ), + SourceCoverageEntry( + "acs_poverty_income_distribution", + "ACS poverty and income distributions", + "validation_only", + notes="Validation only; not an SPM hard target.", + ), + SourceCoverageEntry( + "hud_assisted_housing", + "Housing assistance and subsidy controls", + "source_gap", + missing_source_packages=( + "HUD Picture of Subsidized Households", + "HUD assisted-housing expenditure or unit-count tables", + ), + ), + SourceCoverageEntry( + "usda_wic", + "WIC participation and benefits", + "source_gap", + missing_source_packages=("USDA FNS WIC program data",), + ), + SourceCoverageEntry( + "usda_school_meals", + "School lunch and breakfast benefits", + "source_gap", + missing_source_packages=( + "USDA FNS National School Lunch Program data", + "USDA FNS School Breakfast Program data", + ), + ), + SourceCoverageEntry( + "ocse_child_support", + "Child support received and paid", + "source_gap", + missing_source_packages=("HHS OCSE child support annual report tables",), + ), + SourceCoverageEntry( + "dol_workers_compensation", + "Workers' compensation benefits", + "source_gap", + missing_source_packages=( + "DOL or NASI workers' compensation benefit totals", + "State workers' compensation benefit totals", + ), + ), + SourceCoverageEntry( + "moop_work_childcare_costs", + "MOOP, work expenses, and childcare expense validation", + "source_gap", + missing_source_packages=( + "MEPS out-of-pocket medical spending tables", + "BLS Consumer Expenditure work-related expense tables", + "Childcare expense validation source", + ), + ), +) + + +def hard_target_package_aliases() -> tuple[str, ...]: + """Arch package aliases required for hard-target source coverage.""" + return tuple( + sorted( + { + alias + for entry in US_POVERTY_NONFILER_SOURCE_COVERAGE + if entry.role == "hard_target" + for alias in entry.package_aliases + } + ) + ) + + +def validation_only_family_ids() -> tuple[str, ...]: + """Source families that can be diagnostics but must not be hard targets.""" + return tuple( + entry.family_id + for entry in US_POVERTY_NONFILER_SOURCE_COVERAGE + if entry.role == "validation_only" + ) + + +def source_gap_family_ids() -> tuple[str, ...]: + """Source families the release must report as currently unsupported.""" + return tuple( + entry.family_id + for entry in US_POVERTY_NONFILER_SOURCE_COVERAGE + if entry.role == "source_gap" + ) diff --git a/packages/populace-build/tests/test_gates.py b/packages/populace-build/tests/test_gates.py index ff10166..ba46934 100644 --- a/packages/populace-build/tests/test_gates.py +++ b/packages/populace-build/tests/test_gates.py @@ -13,9 +13,12 @@ GateReport, GateResult, aggregate_admin_gate, + enum_domain_gate, + formula_owned_export_gate, parity_gate, per_family_fit_gate, relative_error_loss, + source_coverage_gate, support_gate, ) from populace.calibrate import TargetSpec @@ -218,7 +221,7 @@ def test_all_zero_column_fails_with_remedy_named(self) -> None: result = exported_nonzero_gate({"snap": 0.1, "net_worth": 0.0}) assert not result.passed assert "net_worth" in result.failures[0] - assert "populate it or drop it" in result.failures[0] + assert "populate it or remove it upstream" in result.failures[0] def test_fully_populated_export_passes(self) -> None: from populace.build import exported_nonzero_gate @@ -251,3 +254,119 @@ def test_unused_exemptions_are_surfaced(self) -> None: ) assert result.passed assert result.details["unused_exemptions"] == ["gone_var"] + + +class TestFormulaOwnedExportGate: + def test_formula_owned_column_fails_with_remedy_named(self) -> None: + result = formula_owned_export_gate( + ["person_id", "employment_income", "ssi"], + ["ssi", "income_tax"], + structural_columns=["person_id"], + ) + assert not result.passed + assert result.failures == ( + "ssi: formula-owned engine output exported as an input; " + "remove it upstream before export.", + ) + assert result.details["offenders"] == ["ssi"] + + def test_structural_overlap_is_exempted(self) -> None: + result = formula_owned_export_gate( + ["person_id", "employment_income"], + ["person_id", "income_tax"], + structural_columns=["person_id"], + ) + assert result.passed + assert result.details["structural_exemptions"] == ["person_id"] + + +class TestEnumDomainGate: + def test_valid_enum_names_pass(self) -> None: + result = enum_domain_gate( + {"race": ["WHITE", "BLACK", "HISPANIC", "OTHER"]}, + {"race": ("WHITE", "BLACK", "HISPANIC", "OTHER")}, + ) + assert result.passed + assert result.details["columns_checked"] == 1 + + def test_raw_source_codes_fail_with_examples(self) -> None: + result = enum_domain_gate( + {"race": [0, 1, 10, 11]}, + {"race": ("WHITE", "BLACK", "HISPANIC", "OTHER")}, + ) + assert not result.passed + assert result.failures and "race" in result.failures[0] + assert result.details["invalid_counts"] == {"race": 4} + assert "10" in result.details["invalid_examples"]["race"] + + def test_enum_class_domains_are_supported(self) -> None: + import enum + + class Race(enum.Enum): + WHITE = "white" + BLACK = "black" + + result = enum_domain_gate({"race": [Race.WHITE, "BLACK"]}, {"race": Race}) + assert result.passed + + +class TestSourceCoverageGate: + def test_hard_targets_must_be_active_or_reviewed(self) -> None: + coverage = ( + { + "family_id": "population_age_sex", + "role": "hard_target", + "package_aliases": ("census-pep-state-age-sex",), + }, + ) + result = source_coverage_gate(coverage) + assert not result.passed + assert "census-pep-state-age-sex" in result.failures[0] + + def test_reviewed_hard_target_exclusion_passes_and_is_recorded(self) -> None: + coverage = ( + { + "family_id": "population_age_sex", + "role": "hard_target", + "package_aliases": ("census-pep-state-age-sex",), + }, + ) + result = source_coverage_gate( + coverage, + reviewed_exclusions={ + "census-pep-state-age-sex": "state-age targets not in this smoke build" + }, + ) + assert result.passed + assert result.details["reviewed_exclusions"] == { + "census-pep-state-age-sex": "state-age targets not in this smoke build" + } + + def test_validation_only_family_cannot_be_a_hard_target(self) -> None: + coverage = ( + { + "family_id": "census_cps_spm", + "role": "validation_only", + "package_aliases": ("census-cps-spm-2024",), + }, + ) + result = source_coverage_gate( + coverage, + active_target_families=("census_cps_spm",), + ) + assert not result.passed + assert "validation-only" in result.failures[0] + + def test_source_gaps_are_reported_without_failing(self) -> None: + coverage = ( + { + "family_id": "usda_wic", + "role": "source_gap", + "missing_source_packages": ("USDA FNS WIC program data",), + }, + ) + result = source_coverage_gate(coverage) + assert result.passed + assert result.details["source_gaps"] == { + "usda_wic": ("USDA FNS WIC program data",) + } diff --git a/packages/populace-build/tests/test_trace.py b/packages/populace-build/tests/test_trace.py index fecdb0c..0744030 100644 --- a/packages/populace-build/tests/test_trace.py +++ b/packages/populace-build/tests/test_trace.py @@ -41,9 +41,9 @@ "sha256": ("a3da2f59085c45f0e16b06337818e3513c2635911dc0d16fa7deb5006263c12a"), }, "construction": ( - "eCPS-free: every layer from primary sources (CPS ASEC, IRS PUF 2015 " - "uprated, Fed SCF 2022, Census SIPP, CPS-ORG, MEPS-IC parameters, " - "Census ACS 2022); enhanced CPS used only as the scoring benchmark" + "Every layer from primary sources (CPS ASEC, IRS PUF 2015 uprated, " + "Fed SCF 2022, Census SIPP, CPS-ORG, MEPS-IC parameters, Census ACS " + "2022); incumbent comparisons live in the external benchmark repo" ), "gates": { "parity_gaps": 0, @@ -62,16 +62,43 @@ "net_stcg_b": -77.5, }, }, - "score_vs_enhanced_cps": { - "protocol": ( - "matched 41,314 households, symmetric refit, 739-target holdout " - "(seed 20260529)" - ), - "train_loss": {"populace": 0.18957, "enhanced_cps": 1.08879}, - "holdout_loss": {"populace": 0.03837, "enhanced_cps": 0.3167}, - "full_loss": {"populace": 0.22794, "enhanced_cps": 1.40549}, - "per_target_wins": {"populace": 1040, "enhanced_cps": 2613, "ties": 51}, + "stage_records": [ + { + "stage": "support_export", + "artifact": "populace_us_2024.h5", + "passed": True, + } + ], +} + +SCHEMA_V1_RELEASE_MANIFEST_FIXTURE: dict = { + "schema_version": 1, + "data_package": {"name": "populace-data", "version": "0.1.0"}, + "build": { + "build_id": "populace-us-2024-9f1260b-20260611", + "builder": "populace", + "build_sha": "9f1260b", + "build_date": "2026-06-11", + }, + "artifacts": { + "populace_us_2024": { + "kind": "microdata", + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "sha256": ( + "dc75c0d4fdedd57946db84a8d838dbc5b61a284365c3ce6eb6508b8e81111a4b" + ), + }, + "calibration_diagnostics": { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": "policyengine/populace-diagnostics", + "sha256": ( + "a3da2f59085c45f0e16b06337818e3513c2635911dc0d16fa7deb5006263c12a" + ), + }, }, + "gates": {"exported_nonzero": {"passed": True}}, } # The IRS PUF: a restricted input the build consumes but a re-runner cannot @@ -454,6 +481,42 @@ def test_hf_revision_pins_artifact_locations(self) -> None: "resolve/4a8e7d39eb9e/populace_us_2024.h5" ] + def test_schema_v1_release_manifest_outputs_artifacts(self) -> None: + tro = trace.tro_from_release_manifest( + SCHEMA_V1_RELEASE_MANIFEST_FIXTURE, + hf_revision="abc123", + ) + node = _graph_node(tro) + perf = node["trov:hasPerformance"] + assert perf["pop:buildId"] == "populace-us-2024-9f1260b-20260611" + assert perf["pop:buildSha"] == "9f1260b" + hashes = _artifact_hashes(node) + assert ( + SCHEMA_V1_RELEASE_MANIFEST_FIXTURE["artifacts"]["populace_us_2024"][ + "sha256" + ] + in hashes + ) + assert ( + SCHEMA_V1_RELEASE_MANIFEST_FIXTURE["artifacts"]["calibration_diagnostics"][ + "sha256" + ] + in hashes + ) + locations = { + loc["trov:hasLocation"] + for loc in _locations(node) + if "trov:hasLocation" in loc + } + assert ( + "https://huggingface.co/datasets/policyengine/populace-us/resolve/" + "abc123/populace_us_2024.h5" + ) in locations + assert ( + "https://huggingface.co/datasets/policyengine/populace-diagnostics/" + "resolve/abc123/calibration_diagnostics.json" + ) in locations + class TestGatesPassedDerivation: def _tro_with_gates(self, gates: dict) -> dict: diff --git a/packages/populace-build/tests/test_us_poverty.py b/packages/populace-build/tests/test_us_poverty.py new file mode 100644 index 0000000..ee45aa0 --- /dev/null +++ b/packages/populace-build/tests/test_us_poverty.py @@ -0,0 +1,79 @@ +import json + +import pandas as pd +import pytest + +from populace.build.us.poverty import ( + spm_resource_diagnostics, + write_spm_resource_diagnostics, +) + + +def test_spm_resource_diagnostics_reports_validation_only_payload() -> None: + frame = pd.DataFrame( + { + "resources": [9_000.0, 40_000.0, -500.0], + "threshold": [15_000.0, 30_000.0, 10_000.0], + "weight": [2.0, 3.0, 1.0], + "state": ["CA", "CA", "NY"], + } + ) + + diagnostics = spm_resource_diagnostics( + frame, + resource_column="resources", + threshold_column="threshold", + weight_column="weight", + group_columns=("state",), + validation_references={"census_cps_spm": {"classification": "validation_only"}}, + ) + + assert diagnostics["classification"] == "validation_only" + assert diagnostics["population"] == 6.0 + assert diagnostics["poverty_count"] == 3.0 + assert diagnostics["poverty_rate"] == pytest.approx(0.5) + assert diagnostics["negative_resources"]["count"] == 1.0 + assert diagnostics["negative_resources"]["minimum"] == -500.0 + assert diagnostics["groups"]["state"]["CA"]["poverty_count"] == 2.0 + assert diagnostics["validation_references"]["census_cps_spm"] == { + "classification": "validation_only" + } + + +def test_spm_resource_diagnostics_requires_positive_finite_rows() -> None: + frame = pd.DataFrame({"resources": [1.0], "threshold": [2.0], "weight": [0.0]}) + with pytest.raises(ValueError, match="positive-weight finite rows"): + spm_resource_diagnostics( + frame, + resource_column="resources", + threshold_column="threshold", + weight_column="weight", + ) + + +def test_spm_resource_diagnostics_ignores_invalid_rows_in_counts() -> None: + frame = pd.DataFrame( + { + "resources": [9_000.0, 40_000.0, 1_000.0, 2_000.0], + "threshold": [15_000.0, 30_000.0, 10_000.0, float("nan")], + "weight": [2.0, 3.0, float("nan"), 5.0], + } + ) + + diagnostics = spm_resource_diagnostics( + frame, + resource_column="resources", + threshold_column="threshold", + weight_column="weight", + ) + + assert diagnostics["population"] == 5.0 + assert diagnostics["poverty_count"] == 2.0 + assert diagnostics["poverty_rate"] == pytest.approx(0.4) + assert diagnostics["negative_resources"]["count"] == 0.0 + + +def test_write_spm_resource_diagnostics_writes_strict_json(tmp_path) -> None: + payload = {"schema_version": 1, "classification": "validation_only"} + path = write_spm_resource_diagnostics(payload, tmp_path / "spm.json") + assert json.loads(path.read_text()) == payload diff --git a/packages/populace-build/tests/test_us_source_coverage.py b/packages/populace-build/tests/test_us_source_coverage.py new file mode 100644 index 0000000..52b4a3a --- /dev/null +++ b/packages/populace-build/tests/test_us_source_coverage.py @@ -0,0 +1,35 @@ +from populace.build.gates import source_coverage_gate +from populace.build.us.source_coverage import ( + ARCH_US_POVERTY_CONTRACT_COMMIT, + US_POVERTY_NONFILER_SOURCE_COVERAGE, + hard_target_package_aliases, + source_gap_family_ids, + validation_only_family_ids, +) + + +def test_us_poverty_source_coverage_snapshot_has_expected_roles() -> None: + assert len(ARCH_US_POVERTY_CONTRACT_COMMIT) == 40 + assert "ssa-ssi-table-7b1-2024" in hard_target_package_aliases() + assert "cms-aca-oep-state-level-2025" in hard_target_package_aliases() + assert "census_cps_spm" in validation_only_family_ids() + assert "usda_wic" in source_gap_family_ids() + + +def test_us_poverty_source_coverage_gate_passes_when_hard_aliases_are_active() -> None: + result = source_coverage_gate( + US_POVERTY_NONFILER_SOURCE_COVERAGE, + active_target_aliases=hard_target_package_aliases(), + ) + assert result.passed + assert result.details["source_gaps"]["hud_assisted_housing"] + + +def test_us_poverty_source_coverage_gate_blocks_cps_spm_hard_target() -> None: + result = source_coverage_gate( + US_POVERTY_NONFILER_SOURCE_COVERAGE, + active_target_aliases=hard_target_package_aliases(), + active_target_families=("census_cps_spm",), + ) + assert not result.passed + assert any("census_cps_spm" in failure for failure in result.failures) diff --git a/packages/populace-calibrate/src/populace/calibrate/__init__.py b/packages/populace-calibrate/src/populace/calibrate/__init__.py index d8b190a..0d36917 100644 --- a/packages/populace-calibrate/src/populace/calibrate/__init__.py +++ b/packages/populace-calibrate/src/populace/calibrate/__init__.py @@ -74,6 +74,11 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: _assert_frame_compatible(_frame_version, _REQUIRED_FRAME_SERIES) +from populace.calibrate.diagnostics import ( # noqa: E402 - after the compat gate + CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + diagnostics_payload, + write_calibration_diagnostics, +) from populace.calibrate.matrix import ( # noqa: E402 - after the compat gate CalibrationProblem, SkippedTarget, @@ -102,6 +107,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: __all__ = [ "AGGREGATIONS", + "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", "CONSERVE_MASS", "FREE_MASS", "CalibrationProblem", @@ -114,7 +120,9 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "TargetSpec", "build_constraint_matrix", "calibrate", + "diagnostics_payload", "relative_error_loss", "specs_from_pe_surface", + "write_calibration_diagnostics", "__version__", ] diff --git a/packages/populace-calibrate/src/populace/calibrate/diagnostics.py b/packages/populace-calibrate/src/populace/calibrate/diagnostics.py new file mode 100644 index 0000000..60edca3 --- /dev/null +++ b/packages/populace-calibrate/src/populace/calibrate/diagnostics.py @@ -0,0 +1,111 @@ +"""Serialize a calibration's diagnostics so they travel with the artifact. + +A :class:`~populace.calibrate.solve.CalibrationResult` carries everything a +reviewer needs to audit what calibration did — per-target estimates before +and after, the per-epoch loss trajectory, the targets that failed to compile +*and why*, and the solver options actually used. Until now none of it left +the build machine: the build pushed the diagnostics to telemetry and dropped +them, and the published ``.npz`` kept only closing scalars. "Skipped and +reported, never dropped silently" is only true if the report ships. + +:func:`diagnostics_payload` renders the result as a JSON-stable dict, and +:func:`write_calibration_diagnostics` writes it as +``calibration_diagnostics.json`` — the artifact a release publishes next to +its manifests (charter rule: artifacts carry their environment; a published +dataset's calibration evidence belongs with the dataset, not in telemetry). +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +from populace.calibrate.solve import CalibrationResult + +__all__ = [ + "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", + "diagnostics_payload", + "write_calibration_diagnostics", +] + +#: Version of the diagnostics payload. Consumers (dashboards, scorers) key +#: their readers on it; bump it with any shape change. +CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 1 + + +def _finite(value: float) -> float | None: + """JSON has no NaN/inf; a non-finite diagnostic serializes as null.""" + value = float(value) + return value if math.isfinite(value) else None + + +def _jsonable(value: object) -> object: + """An option value as strict JSON: non-finite floats become null.""" + if isinstance(value, float): + return _finite(value) + return value + + +def diagnostics_payload(result: CalibrationResult) -> dict: + """Render a calibration result as a JSON-stable diagnostics payload. + + The payload carries the full evidence, not summaries: every per-target + row, the whole loss trajectory, and every skipped target with its + reason. Summary scalars (``final_loss``, ``fraction_within_10pct``) are + included so a consumer need not recompute them, but they are derived + from the rows, never a substitute. + + Args: + result: The :func:`~populace.calibrate.solve.calibrate` output. + + Returns: + A dict that round-trips through ``json`` unchanged (non-finite + floats become ``null``). + """ + return { + "schema_version": CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + "weight_entity": result.weight_entity, + "options": {key: _jsonable(value) for key, value in result.options.items()}, + "l0_lambda": _finite(result.l0_lambda), + "n_nonzero": int(result.n_nonzero), + "n_records": int(result.weights.shape[0]), + "initial_loss": _finite(result.initial_loss), + "final_loss": _finite(result.final_loss), + "fraction_within_10pct": _finite(result.fraction_within_10pct), + "loss_trajectory": [_finite(loss) for loss in result.loss_trajectory], + "skipped": [ + {"name": skip.target.name, "reason": skip.reason} for skip in result.skipped + ], + "targets": [ + { + "name": diagnostic.name, + "target": _finite(diagnostic.target), + "initial_estimate": _finite(diagnostic.initial_estimate), + "final_estimate": _finite(diagnostic.final_estimate), + "relative_error": _finite(diagnostic.relative_error), + "within_tolerance": diagnostic.within_tolerance, + } + for diagnostic in result.diagnostics + ], + } + + +def write_calibration_diagnostics(result: CalibrationResult, path: Path | str) -> Path: + """Write the diagnostics payload to ``path`` as JSON. + + The conventional filename is ``calibration_diagnostics.json`` inside a + release directory, alongside ``build_manifest.json``. + + Args: + result: The :func:`~populace.calibrate.solve.calibrate` output. + path: Destination file path; parent directories must exist. + + Returns: + The path written. + """ + path = Path(path) + # allow_nan=False is the guard: a non-finite value that escaped the + # scrub is a bug here, not something to smuggle out as invalid JSON. + path.write_text(json.dumps(diagnostics_payload(result), indent=1, allow_nan=False)) + return path diff --git a/packages/populace-calibrate/tests/test_diagnostics.py b/packages/populace-calibrate/tests/test_diagnostics.py new file mode 100644 index 0000000..83e9357 --- /dev/null +++ b/packages/populace-calibrate/tests/test_diagnostics.py @@ -0,0 +1,104 @@ +"""The diagnostics artifact: a calibration's evidence ships with it. + +Behavioral contracts: the payload carries every per-target row, the whole +loss trajectory, and every skipped target with its reason; it round-trips +through strict JSON (no NaN/Infinity tokens); and the file writer produces +the artifact a release directory publishes. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from populace.calibrate import ( + CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + Target, + TargetSet, + calibrate, + diagnostics_payload, + write_calibration_diagnostics, +) + + +def _result(feasible_frame, *, with_skip: bool = False, epochs: int = 120): + frame, truths = feasible_frame() + targets = [ + Target( + name="population", + entity="household", + aggregation="count", + value=truths["population"] * 1.2, + ), + Target( + name="income", + entity="household", + aggregation="sum", + value=truths["income"] * 1.2, + measure="income", + tolerance=truths["income"], + ), + ] + if with_skip: + targets.append( + Target( + name="ghost", + entity="household", + aggregation="sum", + value=1.0, + measure="no_such_column", + ) + ) + return calibrate(frame, TargetSet(tuple(targets)), epochs=epochs, seed=0) + + +def test_payload_carries_full_evidence(feasible_frame) -> None: + result = _result(feasible_frame, epochs=120) + payload = diagnostics_payload(result) + + assert payload["schema_version"] == CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION + assert payload["weight_entity"] == "household" + assert len(payload["loss_trajectory"]) == 120 + assert len(payload["targets"]) == len(result.diagnostics) + assert payload["n_records"] == result.weights.shape[0] + assert payload["final_loss"] == result.final_loss + assert payload["fraction_within_10pct"] == result.fraction_within_10pct + assert payload["options"]["epochs"] == 120 + assert payload["options"]["seed"] == 0 + + income = next(row for row in payload["targets"] if row["name"].startswith("income")) + assert income["initial_estimate"] is not None + assert income["final_estimate"] is not None + assert income["within_tolerance"] is True # tolerance was a whole truth wide + population = next( + row for row in payload["targets"] if row["name"].startswith("population") + ) + assert population["within_tolerance"] is None # no tolerance declared + + +def test_skipped_targets_ship_with_their_reason(feasible_frame) -> None: + result = _result(feasible_frame, with_skip=True) + payload = diagnostics_payload(result) + assert len(payload["skipped"]) == 1 + skip = payload["skipped"][0] + assert skip["name"] == "ghost" + assert "no_such_column" in skip["reason"] + # The skip never leaks into the compiled target rows. + assert all(not row["name"].startswith("ghost") for row in payload["targets"]) + + +def test_payload_is_strict_json(feasible_frame) -> None: + result = _result(feasible_frame) + payload = diagnostics_payload(result) + encoded = json.dumps(payload, allow_nan=False) # raises on NaN/inf + assert json.loads(encoded) == payload + + +def test_writer_round_trips(feasible_frame, tmp_path: Path) -> None: + result = _result(feasible_frame) + path = write_calibration_diagnostics( + result, tmp_path / "calibration_diagnostics.json" + ) + loaded = json.loads(path.read_text()) + assert loaded == diagnostics_payload(result) + assert loaded["schema_version"] == CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION diff --git a/packages/populace-data/pyproject.toml b/packages/populace-data/pyproject.toml index f0403e9..c7dc7a7 100644 --- a/packages/populace-data/pyproject.toml +++ b/packages/populace-data/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ # Floor at the engine version verified to load the published artifact (the # USSingleYearDataset loader + entity-flatten behavior the file is built # against are recent); cap below 2 to guard a breaking dataset-schema major. -us = ["policyengine-us>=1.723,<2"] +us = ["policyengine-us>=1.729,<2"] # The UK artifact lives in a PRIVATE repo (UK Data Service licence): loads # require an authenticated HF token with access. The loader surfaces the # 401 with that explanation rather than retrying. diff --git a/packages/populace-data/src/populace/data/__init__.py b/packages/populace-data/src/populace/data/__init__.py index ed94607..855ab7d 100644 --- a/packages/populace-data/src/populace/data/__init__.py +++ b/packages/populace-data/src/populace/data/__init__.py @@ -21,6 +21,12 @@ there is no kernel in its dependency closure to gate against. """ +from populace.data.contract import ( + RELEASE_MANIFEST_SCHEMA_VERSION, + REQUIRED_RELEASE_FILES, + ReleaseContractError, + validate_release_dir, +) from populace.data.loader import ( available, download, @@ -29,6 +35,14 @@ resolve, ) from populace.data.registry import REGISTRY, DatasetSpec, register +from populace.data.release import ( + LATEST_POINTER_PATH, + LATEST_POINTER_SCHEMA_VERSION, + LatestPointer, + latest_pointer_payload, + latest_release, + publish_release, +) __all__ = [ "load", @@ -39,6 +53,16 @@ "DatasetSpec", "REGISTRY", "register", + "RELEASE_MANIFEST_SCHEMA_VERSION", + "REQUIRED_RELEASE_FILES", + "ReleaseContractError", + "validate_release_dir", + "LATEST_POINTER_PATH", + "LATEST_POINTER_SCHEMA_VERSION", + "LatestPointer", + "latest_pointer_payload", + "latest_release", + "publish_release", ] __version__ = "0.1.0" diff --git a/packages/populace-data/src/populace/data/contract.py b/packages/populace-data/src/populace/data/contract.py new file mode 100644 index 0000000..ff9efab --- /dev/null +++ b/packages/populace-data/src/populace/data/contract.py @@ -0,0 +1,243 @@ +"""The release artifact contract: what a published release MUST contain. + +The releases already on the Hub disagree with each other — one carries no +``build_manifest.json`` at all, and two different ``release_manifest.json`` +schemas coexist (an unversioned early shape next to ``schema_version: 1``). +A consumer iterating ``releases/`` therefore cannot trust the listing, and +every consumer ends up re-implementing its own defensive filter. The charter +makes "stage manifests are load-bearing" a binding process rule; the release +directory is the most public manifest of all, so its contract lives here, +with the producer — not in every consumer. + +:func:`validate_release_dir` is the single gate: it checks a local release +directory against the contract and raises :class:`ReleaseContractError` +naming **every** failure at once (a publisher should see the full repair +list, not play whack-a-mole one failure per run). Publishing code calls it +before any byte reaches the Hub. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +__all__ = [ + "RELEASE_MANIFEST_SCHEMA_VERSION", + "REQUIRED_RELEASE_FILES", + "ReleaseContractError", + "validate_release_dir", +] + +#: The release-manifest schema this library reads and writes. Bump it with the +#: schema, and keep :func:`validate_release_dir` rejecting drift loudly — the +#: unversioned 1abddeb-era manifest is exactly the silence this guards against. +RELEASE_MANIFEST_SCHEMA_VERSION = 1 + +#: Files a release directory must contain to count as published. A release +#: missing any of these is invisible to :func:`validate_release_dir`-respecting +#: publishers, by design. +REQUIRED_RELEASE_FILES = ( + "build_manifest.json", + "release_manifest.json", + "calibration_diagnostics.json", +) + +CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 1 + + +class ReleaseContractError(ValueError): + """A release directory violates the release contract. + + Attributes: + failures: Every contract violation found, each a self-contained + human-readable sentence naming the file and field at fault. + """ + + def __init__(self, release_dir: Path, failures: list[str]) -> None: + self.failures = list(failures) + bullet_list = "\n".join(f" - {failure}" for failure in self.failures) + super().__init__( + f"Release directory {release_dir} violates the release contract " + f"({len(self.failures)} failure(s)):\n{bullet_list}" + ) + + +def _load_json(path: Path, failures: list[str]) -> Mapping | None: + try: + loaded = json.loads( + path.read_text(), + parse_constant=_reject_json_constant, + ) + except (json.JSONDecodeError, ValueError) as exc: + failures.append(f"{path.name} is not valid JSON: {exc}.") + return None + if not isinstance(loaded, Mapping): + failures.append( + f"{path.name} must be a JSON object, got {type(loaded).__name__}." + ) + return None + return loaded + + +def _reject_json_constant(token: str) -> None: + raise ValueError(f"non-standard JSON constant {token}") + + +def _check_build_manifest( + manifest: Mapping, release_id: str, failures: list[str] +) -> None: + build_id = manifest.get("build_id") + if not build_id: + failures.append("build_manifest.json is missing 'build_id'.") + elif build_id != release_id: + failures.append( + f"build_manifest.json 'build_id' is {build_id!r} but the release " + f"directory is named {release_id!r}; the directory name IS the " + f"build id." + ) + dataset = manifest.get("dataset") + if not isinstance(dataset, Mapping): + failures.append("build_manifest.json is missing the 'dataset' object.") + else: + for key in ("filename", "sha256"): + if not dataset.get(key): + failures.append(f"build_manifest.json 'dataset' is missing {key!r}.") + if not isinstance(manifest.get("gates"), Mapping): + failures.append( + "build_manifest.json is missing the 'gates' object (the " + "acceptance-gate verdicts are the point of the manifest)." + ) + + +def _check_release_manifest( + manifest: Mapping, release_id: str, failures: list[str] +) -> None: + schema_version = manifest.get("schema_version") + if schema_version is None: + failures.append( + "release_manifest.json has no 'schema_version'; unversioned " + "manifests (the 1abddeb-era shape) are not publishable." + ) + elif schema_version != RELEASE_MANIFEST_SCHEMA_VERSION: + failures.append( + f"release_manifest.json 'schema_version' is {schema_version!r}; " + f"this library publishes version " + f"{RELEASE_MANIFEST_SCHEMA_VERSION}." + ) + build = manifest.get("build") + if not isinstance(build, Mapping) or not build.get("build_id"): + failures.append("release_manifest.json is missing 'build.build_id'.") + elif build["build_id"] != release_id: + failures.append( + f"release_manifest.json 'build.build_id' is " + f"{build['build_id']!r} but the release directory is named " + f"{release_id!r}." + ) + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping) or not artifacts: + failures.append( + "release_manifest.json must declare a non-empty 'artifacts' mapping." + ) + else: + for key, entry in artifacts.items(): + if not isinstance(entry, Mapping): + failures.append( + f"release_manifest.json artifact {key!r} must be an object." + ) + continue + for field in ("path", "repo_id", "sha256"): + if not entry.get(field): + failures.append( + f"release_manifest.json artifact {key!r} is missing {field!r}." + ) + + +def _check_calibration_diagnostics(diagnostics: Mapping, failures: list[str]) -> None: + schema_version = diagnostics.get("schema_version") + if schema_version is None: + failures.append("calibration_diagnostics.json is missing 'schema_version'.") + elif schema_version != CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION: + failures.append( + f"calibration_diagnostics.json 'schema_version' is {schema_version!r}; " + f"this library publishes version " + f"{CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION}." + ) + + expected_sections = { + "targets": list, + "loss_trajectory": list, + "skipped": list, + "options": Mapping, + } + for section, expected_type in expected_sections.items(): + value = diagnostics.get(section) + if not isinstance(value, expected_type): + failures.append( + f"calibration_diagnostics.json is missing a {section!r} " + f"{expected_type.__name__}." + ) + + targets = diagnostics.get("targets") + if isinstance(targets, list): + for index, target in enumerate(targets): + if not isinstance(target, Mapping): + failures.append( + f"calibration_diagnostics.json target row {index} must be an object." + ) + continue + for field in ("name", "target", "initial_estimate", "final_estimate"): + if field not in target: + failures.append( + "calibration_diagnostics.json target row " + f"{index} is missing {field!r}." + ) + + +def validate_release_dir(release_dir: Path | str) -> None: + """Check a local release directory against the release contract. + + The directory name is the build id (``populace-us-2024--``); + its files are what :data:`REQUIRED_RELEASE_FILES` names; and both + manifests must agree with the directory about which build this is. + + Args: + release_dir: The local ``releases/`` directory about to be + published. + + Raises: + ReleaseContractError: Naming every violation found — missing files, + unparseable or unversioned manifests, schema drift, and build-id + mismatches between the manifests and the directory name. + """ + release_dir = Path(release_dir) + release_id = release_dir.name + failures: list[str] = [] + + if not release_dir.is_dir(): + raise ReleaseContractError(release_dir, [f"{release_dir} is not a directory."]) + + for filename in REQUIRED_RELEASE_FILES: + if not (release_dir / filename).is_file(): + failures.append(f"required file {filename!r} is missing.") + + build_manifest_path = release_dir / "build_manifest.json" + if build_manifest_path.is_file(): + manifest = _load_json(build_manifest_path, failures) + if manifest is not None: + _check_build_manifest(manifest, release_id, failures) + + release_manifest_path = release_dir / "release_manifest.json" + if release_manifest_path.is_file(): + manifest = _load_json(release_manifest_path, failures) + if manifest is not None: + _check_release_manifest(manifest, release_id, failures) + + calibration_diagnostics_path = release_dir / "calibration_diagnostics.json" + if calibration_diagnostics_path.is_file(): + diagnostics = _load_json(calibration_diagnostics_path, failures) + if diagnostics is not None: + _check_calibration_diagnostics(diagnostics, failures) + + if failures: + raise ReleaseContractError(release_dir, failures) diff --git a/packages/populace-data/src/populace/data/release.py b/packages/populace-data/src/populace/data/release.py new file mode 100644 index 0000000..db5d1f3 --- /dev/null +++ b/packages/populace-data/src/populace/data/release.py @@ -0,0 +1,228 @@ +"""Publish a release and point ``latest.json`` at it. + +The Hub repo publishes builds under ``releases//``, but nothing +identified which release is *current*: a consumer had to list the tree and +guess, and because build ids end in a date (``populace-us-2024-9f1260b- +20260611``), two builds published the same day have no defined ordering. +``latest.json`` at the repo root is that missing pointer — a tiny manifest +naming the current release and the path of each of its contract files. + +Two sides of the pointer live here: + +- :func:`publish_release` is the producer: it validates the local release + directory against the :mod:`release contract ` + (a release that fails the contract refuses to publish), uploads its + files, and uploads ``latest.json`` **last** — so a reader never sees the + pointer before the release it points at. +- :func:`latest_release` is the consumer: it downloads ``latest.json`` and + returns the typed pointer, the one-call answer to "which release is + current?" for dashboards and scorers. + +The Hub client is injected (``api=``) everywhere it is used, so the suite +exercises the real ordering and payloads against a fake — no network, no +mocking of our own internals. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from populace.data.contract import ( + REQUIRED_RELEASE_FILES, + validate_release_dir, +) + +__all__ = [ + "LATEST_POINTER_PATH", + "LATEST_POINTER_SCHEMA_VERSION", + "LatestPointer", + "latest_pointer_payload", + "publish_release", + "latest_release", +] + +#: Where the pointer lives in the dataset repo. The root, not a release +#: directory: the pointer is repo state, not release state. +LATEST_POINTER_PATH = "latest.json" + +#: Version of the pointer payload itself, so the pointer can evolve without +#: consumers guessing (the same discipline the release manifest learned). +LATEST_POINTER_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class LatestPointer: + """The parsed ``latest.json``: which release is current, and where. + + Attributes: + release_id: The current build id (the ``releases/`` directory name). + updated_at: ISO-8601 UTC timestamp of when the pointer was written. + paths: Repo-relative path of each contract file, keyed by its stem + (``"build_manifest"``, ``"release_manifest"``, + ``"calibration_diagnostics"``). + """ + + release_id: str + updated_at: str + paths: dict[str, str] + + +def latest_pointer_payload(release_id: str, *, updated_at: str | None = None) -> dict: + """The ``latest.json`` payload for ``release_id``. + + Paths are derived from the release contract — the pointer names exactly + the files :data:`~populace.data.contract.REQUIRED_RELEASE_FILES` + guarantees exist. + + Args: + release_id: The build id being made current. + updated_at: ISO-8601 UTC timestamp; defaults to now. + """ + if updated_at is None: + updated_at = datetime.now(UTC).replace(microsecond=0).isoformat() + return { + "schema_version": LATEST_POINTER_SCHEMA_VERSION, + "release_id": release_id, + "updated_at": updated_at, + "paths": { + filename.removesuffix(".json"): f"releases/{release_id}/{filename}" + for filename in REQUIRED_RELEASE_FILES + }, + } + + +def _hf_api(): + try: + from huggingface_hub import HfApi + except ImportError as exc: # pragma: no cover - declared dependency + raise ImportError( + "populace-data needs huggingface_hub to publish releases; " + "reinstall populace-data with its dependencies." + ) from exc + return HfApi() + + +def publish_release( + release_dir: Path | str, + repo_id: str, + *, + api=None, + extra_files: tuple[str, ...] = (), + updated_at: str | None = None, +) -> dict: + """Upload a release directory and point ``latest.json`` at it. + + The order is the guarantee: the release contract is validated first (an + invalid release never reaches the Hub), every release file is uploaded + next, and the pointer goes up **last**, so a consumer that reads + ``latest.json`` always finds the files it names. + + Args: + release_dir: Local ``releases/`` directory. + repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. + api: A ``huggingface_hub.HfApi``-shaped object (anything with + ``upload_file(path_or_fileobj=, path_in_repo=, repo_id=, + repo_type=)``); constructed lazily when omitted. + extra_files: Additional filenames in ``release_dir`` to upload + beyond the contract files (e.g. a diagnostics artifact). + updated_at: Pointer timestamp; defaults to now (UTC). + + Returns: + The ``latest.json`` payload that was published. + + Raises: + ReleaseContractError: If the release directory violates the + contract. Nothing is uploaded in that case. + FileNotFoundError: If an ``extra_files`` entry does not exist. + """ + release_dir = Path(release_dir) + validate_release_dir(release_dir) + release_id = release_dir.name + + filenames = list(REQUIRED_RELEASE_FILES) + [ + name for name in extra_files if name not in REQUIRED_RELEASE_FILES + ] + for filename in filenames: + local = release_dir / filename + if not local.is_file(): + raise FileNotFoundError( + f"extra release file {filename!r} not found in {release_dir}." + ) + + if api is None: + api = _hf_api() + + for filename in filenames: + local = release_dir / filename + api.upload_file( + path_or_fileobj=str(local), + path_in_repo=f"releases/{release_id}/{filename}", + repo_id=repo_id, + repo_type="dataset", + ) + + payload = latest_pointer_payload(release_id, updated_at=updated_at) + api.upload_file( + path_or_fileobj=json.dumps(payload, indent=1).encode(), + path_in_repo=LATEST_POINTER_PATH, + repo_id=repo_id, + repo_type="dataset", + ) + return payload + + +def latest_release(repo_id: str, *, api=None) -> LatestPointer: + """Read ``latest.json`` from a dataset repo: which release is current. + + Args: + repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. + api: A ``huggingface_hub.HfApi``-shaped object (anything with + ``hf_hub_download(repo_id=, filename=, repo_type=)``); + constructed lazily when omitted. + + Raises: + ValueError: If the pointer is malformed or its schema version is + newer than this library understands. + """ + if api is None: + api = _hf_api() + local = api.hf_hub_download( + repo_id=repo_id, filename=LATEST_POINTER_PATH, repo_type="dataset" + ) + payload = json.loads(Path(local).read_text()) + schema_version = payload.get("schema_version") + if schema_version != LATEST_POINTER_SCHEMA_VERSION: + raise ValueError( + f"{LATEST_POINTER_PATH} in {repo_id} has schema_version " + f"{schema_version!r}; this populace-data reads version " + f"{LATEST_POINTER_SCHEMA_VERSION}. Upgrade populace-data." + ) + release_id = payload.get("release_id") + if not release_id: + raise ValueError(f"{LATEST_POINTER_PATH} in {repo_id} has no 'release_id'.") + paths = payload.get("paths") + if not isinstance(paths, dict): + raise ValueError(f"{LATEST_POINTER_PATH} in {repo_id} has no 'paths' object.") + expected_paths = latest_pointer_payload(str(release_id), updated_at="")["paths"] + observed_paths = {str(key): value for key, value in paths.items()} + missing_paths = sorted(set(expected_paths) - set(observed_paths)) + unexpected_paths = sorted(set(observed_paths) - set(expected_paths)) + malformed_paths = sorted( + key + for key in set(expected_paths) & set(observed_paths) + if observed_paths[key] != expected_paths[key] + ) + if missing_paths or unexpected_paths or malformed_paths: + raise ValueError( + f"{LATEST_POINTER_PATH} in {repo_id} has incomplete paths: " + f"missing={missing_paths}, unexpected={unexpected_paths}, " + f"malformed={malformed_paths}." + ) + return LatestPointer( + release_id=str(release_id), + updated_at=str(payload.get("updated_at", "")), + paths={str(k): str(v) for k, v in paths.items()}, + ) diff --git a/packages/populace-data/tests/test_contract.py b/packages/populace-data/tests/test_contract.py new file mode 100644 index 0000000..a891f44 --- /dev/null +++ b/packages/populace-data/tests/test_contract.py @@ -0,0 +1,189 @@ +"""The release contract: every published release looks the same, loudly. + +These are behavioral tests against the failure modes already observed on the +Hub: a release with no build manifest at all (1abddeb), and two coexisting +release-manifest schemas (an unversioned early shape next to +``schema_version: 1``). A valid release passes silently; every broken release +fails with each violation named. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from populace.data import ( + RELEASE_MANIFEST_SCHEMA_VERSION, + REQUIRED_RELEASE_FILES, + ReleaseContractError, + validate_release_dir, +) + +RELEASE_ID = "populace-us-2024-9f1260b-20260611" + + +def _build_manifest(release_id: str = RELEASE_ID) -> dict: + return { + "build_id": release_id, + "builder": "populace", + "dataset": {"filename": "populace_us_2024.h5", "sha256": "dc75c0"}, + "calibration": { + "filename": "populace_us_2024_calibration.npz", + "sha256": "a3da2f", + }, + "gates": {"exported_nonzero": {"passed": True}}, + } + + +def _release_manifest(release_id: str = RELEASE_ID) -> dict: + return { + "schema_version": RELEASE_MANIFEST_SCHEMA_VERSION, + "data_package": {"name": "populace-data", "version": "0.1.0"}, + "build": {"build_id": release_id}, + "artifacts": { + "populace_us_2024": { + "kind": "microdata", + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "sha256": "dc75c0", + } + }, + } + + +def _calibration_diagnostics() -> dict: + return { + "schema_version": 1, + "weight_entity": "household", + "options": {"epochs": 120}, + "loss_trajectory": [1.0, 0.5], + "skipped": [], + "targets": [ + { + "name": "population", + "target": 1.0, + "initial_estimate": 0.8, + "final_estimate": 1.0, + "relative_error": 0.0, + "within_tolerance": True, + } + ], + } + + +@pytest.fixture +def release_dir(tmp_path: Path) -> Path: + """A complete, contract-valid release directory.""" + directory = tmp_path / "releases" / RELEASE_ID + directory.mkdir(parents=True) + (directory / "build_manifest.json").write_text(json.dumps(_build_manifest())) + (directory / "release_manifest.json").write_text(json.dumps(_release_manifest())) + (directory / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics()) + ) + return directory + + +def test_a_complete_release_passes(release_dir: Path) -> None: + validate_release_dir(release_dir) + + +@pytest.mark.parametrize("filename", REQUIRED_RELEASE_FILES) +def test_each_required_file_is_named_when_missing( + release_dir: Path, filename: str +) -> None: + (release_dir / filename).unlink() + with pytest.raises(ReleaseContractError, match=filename): + validate_release_dir(release_dir) + + +def test_the_1abddeb_shape_is_rejected(release_dir: Path) -> None: + """The regression: a release with only an unversioned release manifest.""" + (release_dir / "build_manifest.json").unlink() + (release_dir / "calibration_diagnostics.json").unlink() + (release_dir / "release_manifest.json").write_text( + json.dumps( + { + "release_id": RELEASE_ID, + "country_id": "us", + "artifacts": {}, + "validation": {}, + } + ) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + failures = "\n".join(excinfo.value.failures) + assert "build_manifest.json" in failures + assert "schema_version" in failures + + +def test_schema_drift_is_rejected_by_version(release_dir: Path) -> None: + manifest = _release_manifest() + manifest["schema_version"] = RELEASE_MANIFEST_SCHEMA_VERSION + 1 + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError, match="schema_version"): + validate_release_dir(release_dir) + + +def test_build_id_mismatch_names_both_ids(release_dir: Path) -> None: + (release_dir / "build_manifest.json").write_text( + json.dumps(_build_manifest("populace-us-2024-other-20260101")) + ) + with pytest.raises(ReleaseContractError, match="populace-us-2024-other"): + validate_release_dir(release_dir) + + +def test_release_manifest_build_id_must_match_directory( + release_dir: Path, +) -> None: + manifest = _release_manifest("populace-us-2024-other-20260101") + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError, match="build.build_id"): + validate_release_dir(release_dir) + + +def test_artifact_entries_must_carry_provenance(release_dir: Path) -> None: + manifest = _release_manifest() + manifest["artifacts"]["populace_us_2024"].pop("sha256") + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError, match="sha256"): + validate_release_dir(release_dir) + + +def test_unparseable_manifest_is_a_named_failure(release_dir: Path) -> None: + (release_dir / "build_manifest.json").write_text("{not json") + with pytest.raises(ReleaseContractError, match="not valid JSON"): + validate_release_dir(release_dir) + + +def test_malformed_calibration_diagnostics_is_rejected( + release_dir: Path, +) -> None: + (release_dir / "calibration_diagnostics.json").write_text( + json.dumps({"schema_version": 1}) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + failures = "\n".join(excinfo.value.failures) + assert "calibration_diagnostics.json" in failures + assert "targets" in failures + + +def test_all_failures_reported_at_once(release_dir: Path) -> None: + """A publisher sees the full repair list, not one failure per run.""" + (release_dir / "calibration_diagnostics.json").unlink() + manifest = _release_manifest() + del manifest["schema_version"] + manifest["artifacts"] = {} + (release_dir / "release_manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + assert len(excinfo.value.failures) >= 3 + + +def test_a_missing_directory_is_a_contract_error(tmp_path: Path) -> None: + with pytest.raises(ReleaseContractError, match="is not a directory"): + validate_release_dir(tmp_path / "releases" / "nope") diff --git a/packages/populace-data/tests/test_release.py b/packages/populace-data/tests/test_release.py new file mode 100644 index 0000000..278a45e --- /dev/null +++ b/packages/populace-data/tests/test_release.py @@ -0,0 +1,256 @@ +"""Publishing behavior: contract-gated uploads and a last-written pointer. + +The fake Hub client records every upload in order, so the suite asserts the +real guarantees — an invalid release uploads nothing, and ``latest.json`` +lands strictly after the files it points at — rather than implementation +details. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from populace.data import ReleaseContractError +from populace.data.contract import REQUIRED_RELEASE_FILES +from populace.data.release import ( + LATEST_POINTER_PATH, + LATEST_POINTER_SCHEMA_VERSION, + latest_pointer_payload, + latest_release, + publish_release, +) + +RELEASE_ID = "populace-us-2024-9f1260b-20260611" + + +def _calibration_diagnostics() -> dict: + return { + "schema_version": 1, + "weight_entity": "household", + "options": {"epochs": 120}, + "loss_trajectory": [1.0, 0.5], + "skipped": [], + "targets": [ + { + "name": "population", + "target": 1.0, + "initial_estimate": 0.8, + "final_estimate": 1.0, + "relative_error": 0.0, + "within_tolerance": True, + } + ], + } + + +class FakeHub: + """Records uploads in order; serves downloads from what was uploaded.""" + + def __init__(self) -> None: + self.uploads: list[tuple[str, bytes]] = [] + + def upload_file(self, *, path_or_fileobj, path_in_repo, repo_id, repo_type) -> None: + assert repo_type == "dataset" + assert repo_id == "policyengine/populace-us" + if isinstance(path_or_fileobj, bytes): + content = path_or_fileobj + else: + content = Path(path_or_fileobj).read_bytes() + self.uploads.append((path_in_repo, content)) + + def hf_hub_download(self, *, repo_id, filename, repo_type) -> str: + assert repo_type == "dataset" + for path_in_repo, content in reversed(self.uploads): + if path_in_repo == filename: + local = self._download_dir / filename + local.parent.mkdir(parents=True, exist_ok=True) + local.write_bytes(content) + return str(local) + raise FileNotFoundError(filename) + + +@pytest.fixture +def hub(tmp_path: Path) -> FakeHub: + fake = FakeHub() + fake._download_dir = tmp_path / "hub-cache" + return fake + + +@pytest.fixture +def release_dir(tmp_path: Path) -> Path: + directory = tmp_path / "releases" / RELEASE_ID + directory.mkdir(parents=True) + (directory / "build_manifest.json").write_text( + json.dumps( + { + "build_id": RELEASE_ID, + "dataset": {"filename": "populace_us_2024.h5", "sha256": "dc"}, + "gates": {"parity_gaps": 0}, + } + ) + ) + (directory / "release_manifest.json").write_text( + json.dumps( + { + "schema_version": 1, + "build": {"build_id": RELEASE_ID}, + "artifacts": { + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "sha256": "dc", + } + }, + } + ) + ) + (directory / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics()) + ) + return directory + + +def test_pointer_payload_names_every_contract_file() -> None: + payload = latest_pointer_payload(RELEASE_ID, updated_at="2026-06-11T13:53:15+00:00") + assert payload["schema_version"] == LATEST_POINTER_SCHEMA_VERSION + assert payload["release_id"] == RELEASE_ID + assert set(payload["paths"]) == { + name.removesuffix(".json") for name in REQUIRED_RELEASE_FILES + } + assert ( + payload["paths"]["build_manifest"] + == f"releases/{RELEASE_ID}/build_manifest.json" + ) + + +def test_publish_uploads_pointer_last(hub: FakeHub, release_dir: Path) -> None: + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + updated_at="2026-06-11T13:53:15+00:00", + ) + uploaded_paths = [path for path, _ in hub.uploads] + assert uploaded_paths[-1] == LATEST_POINTER_PATH + for filename in REQUIRED_RELEASE_FILES: + assert f"releases/{RELEASE_ID}/{filename}" in uploaded_paths[:-1] + + +def test_invalid_release_uploads_nothing(hub: FakeHub, release_dir: Path) -> None: + (release_dir / "build_manifest.json").unlink() + with pytest.raises(ReleaseContractError): + publish_release(release_dir, "policyengine/populace-us", api=hub) + assert hub.uploads == [] + + +def test_invalid_calibration_diagnostics_uploads_nothing( + hub: FakeHub, release_dir: Path +) -> None: + (release_dir / "calibration_diagnostics.json").write_text("{}") + with pytest.raises(ReleaseContractError, match="calibration_diagnostics"): + publish_release(release_dir, "policyengine/populace-us", api=hub) + assert hub.uploads == [] + + +def test_nonstandard_nan_calibration_diagnostics_uploads_nothing( + hub: FakeHub, release_dir: Path +) -> None: + (release_dir / "calibration_diagnostics.json").write_text( + '{"schema_version": 1, "targets": [], "loss_trajectory": [NaN], ' + '"skipped": [], "options": {}}' + ) + with pytest.raises(ReleaseContractError, match="calibration_diagnostics"): + publish_release(release_dir, "policyengine/populace-us", api=hub) + assert hub.uploads == [] + + +def test_extra_files_ride_along_before_the_pointer( + hub: FakeHub, release_dir: Path +) -> None: + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + extra_files=("calibration_diagnostics.json",), + ) + uploaded_paths = [path for path, _ in hub.uploads] + extra = f"releases/{RELEASE_ID}/calibration_diagnostics.json" + assert extra in uploaded_paths + assert uploaded_paths.index(extra) < uploaded_paths.index(LATEST_POINTER_PATH) + + +def test_missing_extra_file_fails_loudly(hub: FakeHub, release_dir: Path) -> None: + with pytest.raises(FileNotFoundError, match="support_audit"): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + extra_files=("support_audit.json",), + ) + assert hub.uploads == [] + + +def test_publish_then_resolve_round_trips(hub: FakeHub, release_dir: Path) -> None: + published = publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + updated_at="2026-06-11T13:53:15+00:00", + ) + pointer = latest_release("policyengine/populace-us", api=hub) + assert pointer.release_id == RELEASE_ID + assert pointer.updated_at == "2026-06-11T13:53:15+00:00" + assert pointer.paths == published["paths"] + + +def test_future_pointer_schema_is_refused(hub: FakeHub) -> None: + hub.uploads.append( + ( + LATEST_POINTER_PATH, + json.dumps({"schema_version": LATEST_POINTER_SCHEMA_VERSION + 1}).encode(), + ) + ) + with pytest.raises(ValueError, match="Upgrade populace-data"): + latest_release("policyengine/populace-us", api=hub) + + +def test_pointer_without_release_id_is_refused(hub: FakeHub) -> None: + hub.uploads.append( + ( + LATEST_POINTER_PATH, + json.dumps({"schema_version": LATEST_POINTER_SCHEMA_VERSION}).encode(), + ) + ) + with pytest.raises(ValueError, match="release_id"): + latest_release("policyengine/populace-us", api=hub) + + +def test_pointer_without_contract_paths_is_refused(hub: FakeHub) -> None: + hub.uploads.append( + ( + LATEST_POINTER_PATH, + json.dumps( + { + "schema_version": LATEST_POINTER_SCHEMA_VERSION, + "release_id": RELEASE_ID, + "paths": {"build_manifest": "releases/x/build_manifest.json"}, + } + ).encode(), + ) + ) + with pytest.raises(ValueError, match="paths"): + latest_release("policyengine/populace-us", api=hub) + + +def test_pointer_with_swapped_contract_path_is_refused(hub: FakeHub) -> None: + payload = latest_pointer_payload(RELEASE_ID) + payload["paths"]["build_manifest"] = ( + f"releases/{RELEASE_ID}/calibration_diagnostics.json" + ) + hub.uploads.append((LATEST_POINTER_PATH, json.dumps(payload).encode())) + + with pytest.raises(ValueError, match="malformed=\\['build_manifest'\\]"): + latest_release("policyengine/populace-us", api=hub) diff --git a/packages/populace-frame/pyproject.toml b/packages/populace-frame/pyproject.toml index 01850b3..a69e0f0 100644 --- a/packages/populace-frame/pyproject.toml +++ b/packages/populace-frame/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] us = ["microunit>=0.1.0"] -policyengine = ["policyengine-us>=1.0", "microunit>=0.1.0"] +policyengine = ["policyengine-us>=1.729,<2", "microunit>=0.1.0"] [dependency-groups] dev = [ diff --git a/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py b/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py index 1b488c4..3f34e7b 100644 --- a/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py +++ b/packages/populace-frame/src/populace/frame/adapters/policyengine_us.py @@ -54,25 +54,49 @@ _PERIOD_BY_DEFINITION: dict[str, str] = {"year": "year", "month": "month"} -def _has_formula(variable: Any) -> bool: +def _is_engine_computed(variable: Any, period: int | str | None = None) -> bool: """Return whether a PolicyEngine variable is computed by a formula. - Input variables (read from data, what a pool must produce) have no - formula; formula-owned outputs do. PolicyEngine exposes formulas either as - a ``formula`` attribute or a ``formulas`` mapping keyed by start date. + Input variables (read from data, what a pool must produce) are plain source + variables. Formula-owned outputs may be backed by a direct formula or a + formula mapping keyed by start date. """ + if period is not None: + return variable.get_formula(str(period)) is not None if getattr(variable, "formula", None) is not None: return True formulas = getattr(variable, "formulas", None) return bool(formulas) +def _enum_domain(variable: Any) -> tuple[str, ...]: + possible_values = getattr(variable, "possible_values", None) + members = getattr(possible_values, "__members__", None) + if isinstance(members, Mapping): + return tuple(str(name) for name in members) + return () + + +def _stored_enum_name(value: object) -> str | None: + if value is None: + return None + if isinstance(value, (float, np.floating)) and np.isnan(value): + return None + if isinstance(value, bytes): + return value.decode() + name = getattr(value, "name", None) + if isinstance(name, str): + return name + return str(value) + + class PolicyEngineUSEngine: """RulesEngine adapter backed by ``policyengine_us``. Args: contract: Column-parity contract for :meth:`write_dataset` exports. - ``None`` means an empty contract (no required/forbidden checks). + ``None`` means an empty contract (no required/forbidden/closed + surface checks). defaults: Scalar defaults broadcast onto the owning entity table for contract-required columns no bundle table provides. @@ -129,7 +153,7 @@ def variables(self) -> list[str]: return sorted( name for name, variable in system_variables.items() - if not _has_formula(variable) + if not _is_engine_computed(variable) ) def _entity_of(self, name: str) -> str: @@ -204,11 +228,12 @@ def write_dataset( ) -> None: """Write the bundle as a ``USSingleYearDataset`` HDF5 file. - Applies the export gate: forbidden and formula-owned columns are - dropped, defaults are broadcast onto the owning entity table for - required columns no table provides, and a dataset with missing - required columns is never written. After writing, the dataset is - reloaded and every persisted column verified (round-trip check). + Applies the export gate: forbidden and formula-owned columns block the + export, defaults are broadcast onto the owning entity table for + required columns no table provides, closed contracts reject unexpected + non-structural columns, and a dataset with violations is never + written. After writing, the dataset is reloaded and every persisted + column verified (round-trip check). Args: bundle: A US-schema bundle. @@ -218,8 +243,8 @@ def write_dataset( Raises: ImportError: If ``policyengine_us`` is not installed. ValueError: If ``path`` does not end in ``.h5``, the contract is - violated (the message lists the missing/forbidden columns), - or the round-trip verification fails. + violated (the message lists missing/forbidden/formula-owned/ + unexpected columns), or the round-trip verification fails. """ output_path = Path(path) if output_path.suffix != ".h5": @@ -227,17 +252,6 @@ def write_dataset( contract = self._contract tables = self._engine_tables(bundle) - forbidden = set(contract.forbidden) - drop_on_sight = forbidden | set(contract.formula_owned_excluded) - dropped: set[str] = set() - forbidden_present: set[str] = set() - for name, frame in tables.items(): - present_drops = drop_on_sight.intersection(frame.columns) - if present_drops: - tables[name] = frame.drop(columns=sorted(present_drops)) - dropped.update(present_drops) - forbidden_present.update(forbidden.intersection(present_drops)) - present_columns: set[str] = set() for frame in tables.values(): present_columns.update(frame.columns) @@ -256,11 +270,36 @@ def write_dataset( continue missing_required.append(column) - if forbidden_present or missing_required: + forbidden_present = set(contract.forbidden).intersection(present_columns) + formula_owned_present = self._engine_computed_columns( + tables, period=period + ) | set(contract.formula_owned_excluded).intersection(present_columns) + unexpected: set[str] = set() + if contract.closed: + allowed = ( + set(contract.required) + | set(contract.optional) + | self._structural_columns() + | {_HOUSEHOLD_WEIGHT_COLUMN} + ) + unexpected = present_columns - allowed + + enum_domain_failures = self._enum_domain_failures(tables) + + if ( + forbidden_present + or missing_required + or formula_owned_present + or unexpected + or enum_domain_failures + ): raise ValueError( "Export contract violated; nothing was written. Missing " f"required column(s): {sorted(missing_required)}; forbidden " - f"column(s) present: {sorted(forbidden_present)}." + f"column(s) present: {sorted(forbidden_present)}; formula-owned " + f"column(s) present: {sorted(formula_owned_present)}; unexpected column(s) " + f"present: {sorted(unexpected)}; enum-domain violation(s): " + f"{enum_domain_failures}." ) self._write_and_verify(tables, period=int(period), output_path=output_path) @@ -337,6 +376,69 @@ def _default_entity(self, column: str) -> str: return variables[column].entity.key return _PERSON_TABLE + def _engine_computed_columns( + self, + tables: Mapping[str, pd.DataFrame], + *, + period: int | str, + ) -> set[str]: + """PolicyEngine-computed columns present in the pending export. + + Formula-owned columns cannot be allowed through implicitly: if a + source table carries a PolicyEngine output name such as ``ssi``, + keeping it in the HDF5 file turns that formula output into an input + and masks reforms. Such columns must be removed upstream before the + writer is called, after checking aggregate deltas. + """ + variables = self._tax_benefit_system().variables + present = {column for frame in tables.values() for column in frame.columns} + structural = self._structural_columns() + return { + column + for column in present + if column not in structural + and column in variables + and _is_engine_computed(variables[column], period=period) + } + + def _enum_domain_failures( + self, + tables: Mapping[str, pd.DataFrame], + ) -> list[str]: + """Return enum input columns carrying values outside engine domains.""" + variables = self._tax_benefit_system().variables + structural = self._structural_columns() + failures: list[str] = [] + for entity, frame in tables.items(): + for column in frame.columns: + if column in structural or column not in variables: + continue + allowed = set(_enum_domain(variables[column])) + if not allowed: + continue + invalid: list[str] = [] + for value in frame[column].to_numpy(dtype=object): + name = _stored_enum_name(value) + if name not in allowed: + invalid.append("" if name is None else name) + if invalid: + failures.append( + f"{entity}.{column}: {len(invalid)}/{len(frame)} value(s) " + "outside enum domain; invalid examples " + f"{sorted(set(invalid))[:8]}; allowed values " + f"{sorted(allowed)[:8]}" + ) + return failures + + def _structural_columns(self) -> set[str]: + """Entity ids and memberships required to reconstruct the frame.""" + schema = self.entity_schema() + return {schema.person_id_column} | { + column + for group in schema.group_entities + for column in (schema.id_column(group), schema.membership_column(group)) + } + def _write_and_verify( self, tables: Mapping[str, pd.DataFrame], diff --git a/packages/populace-frame/src/populace/frame/rules.py b/packages/populace-frame/src/populace/frame/rules.py index 133039b..d8f35ee 100644 --- a/packages/populace-frame/src/populace/frame/rules.py +++ b/packages/populace-frame/src/populace/frame/rules.py @@ -7,7 +7,8 @@ :class:`ExportContract` is the frozen column-parity contract a dataset export is gated against: which columns it must contain, must not contain, may carry, -and must leave to the engine's own formulas. +must leave to the engine's own formulas, and whether the surface is closed to +unexpected extras. """ import json @@ -105,19 +106,24 @@ class ExportContract: Attributes: required: Columns the export MUST contain. A missing required column fails the export gate. - forbidden: Columns the export MUST NOT contain. They are dropped on - sight and their presence fails the gate. + forbidden: Columns the export MUST NOT contain. Their presence fails + the export gate. optional: Bookkeeping columns that are neither required nor forbidden; an export passes them through untouched if present. formula_owned_excluded: Variables the engine owns through formulas - and the baseline does not persist as inputs. They are silently - dropped if present so the engine computes them itself. + and the baseline must not persist as inputs. Their presence fails + the export gate; upstream build stages must drop them before + calling the engine writer. + closed: If true, exports may only contain required/optional columns + plus adapter-defined structural columns. Unexpected extras fail + the export gate before anything is written. """ required: tuple[str, ...] forbidden: tuple[str, ...] optional: tuple[str, ...] formula_owned_excluded: tuple[str, ...] + closed: bool = False @classmethod def empty(cls) -> "ExportContract": @@ -150,6 +156,7 @@ def from_path(cls, path: str | Path) -> "ExportContract": formula_owned_excluded=_as_str_tuple( sections.get("formula_owned_excluded", ()) ), + closed=bool(sections.get("closed", False)), ) diff --git a/packages/populace-frame/tests/test_adapters.py b/packages/populace-frame/tests/test_adapters.py index c625c4e..daf9f41 100644 --- a/packages/populace-frame/tests/test_adapters.py +++ b/packages/populace-frame/tests/test_adapters.py @@ -80,6 +80,7 @@ def test_empty_contract_has_no_constraints(self) -> None: assert contract.forbidden == () assert contract.optional == () assert contract.formula_owned_excluded == () + assert contract.closed is False def test_from_path_parses_sections_and_ignores_metadata(self, tmp_path) -> None: manifest = { @@ -88,6 +89,7 @@ def test_from_path_parses_sections_and_ignores_metadata(self, tmp_path) -> None: "forbidden": ["spm_unit_net_income"], "optional": ["engine_bookkeeping"], "formula_owned_excluded": ["eitc"], + "closed": True, } path = tmp_path / "contract.json" path.write_text(json.dumps(manifest), encoding="utf-8") @@ -96,6 +98,7 @@ def test_from_path_parses_sections_and_ignores_metadata(self, tmp_path) -> None: assert contract.forbidden == ("spm_unit_net_income",) assert contract.optional == ("engine_bookkeeping",) assert contract.formula_owned_excluded == ("eitc",) + assert contract.closed is True def test_from_path_ignores_unknown_sections(self, tmp_path) -> None: manifest = {"legacy_internal_optional": ["spm_unit_pre_subsidy_childcare"]} diff --git a/packages/populace-frame/tests/test_policyengine_us_adapter.py b/packages/populace-frame/tests/test_policyengine_us_adapter.py index 483263b..9ac2b6c 100644 --- a/packages/populace-frame/tests/test_policyengine_us_adapter.py +++ b/packages/populace-frame/tests/test_policyengine_us_adapter.py @@ -75,6 +75,12 @@ def test_unknown_variable_is_named(self, adapter) -> None: def test_variables_lists_inputs_not_outputs(self, adapter) -> None: names = adapter.variables() assert "employment_income" in names # input + assert "partnership_income" in names + assert "s_corp_income" in names + assert "partnership_s_corp_income" in names + assert "in_nyc" not in names + assert "ssi" not in names # formula-owned output + assert "spm_unit_capped_work_childcare_expenses" not in names assert "income_tax" not in names # formula-owned output @@ -136,6 +142,239 @@ def test_forbidden_column_blocks_the_write(self, us_bundle, tmp_path) -> None: gated.write_dataset(us_bundle, path, period=2024) assert not path.exists() + def test_formula_owned_columns_block_the_write( + self, adapter, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["ssi"] = [12_000.0, 0.0, 0.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="ssi"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_partnership_s_corp_inputs_round_trip( + self, adapter, us_bundle, tmp_path + ) -> None: + from policyengine_us.data import USSingleYearDataset + + person = us_bundle.person.copy() + person["partnership_income"] = [1_500.0, 0.0, 400.0] + person["s_corp_income"] = [500.0, 0.0, 100.0] + person["partnership_s_corp_income"] = [2_000.0, 0.0, 500.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "partnership_s_corp_inputs.h5" + adapter.write_dataset(rebuilt, path, period=2024) + reloaded = USSingleYearDataset(file_path=str(path)) + assert reloaded.person["partnership_income"].tolist() == [1_500.0, 0.0, 400.0] + assert reloaded.person["s_corp_income"].tolist() == [500.0, 0.0, 100.0] + assert reloaded.person["partnership_s_corp_income"].tolist() == [ + 2_000.0, + 0.0, + 500.0, + ] + + def test_enum_domain_column_blocks_raw_source_codes( + self, adapter, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["race"] = [0, 1, 10] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "enum_blocked.h5" + with pytest.raises(ValueError, match="enum-domain.*race"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_enum_domain_names_round_trip_for_true_input( + self, adapter, us_bundle, tmp_path + ) -> None: + from policyengine_us.data import USSingleYearDataset + + household = us_bundle.table("household").copy() + household["tenure_type"] = ["RENTED", "OWNED_WITH_MORTGAGE"] + rebuilt = Frame( + { + name: (household if name == "household" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "enum_valid.h5" + adapter.write_dataset(rebuilt, path, period=2024) + reloaded = USSingleYearDataset(file_path=str(path)) + assert reloaded.household["tenure_type"].tolist() == [ + "RENTED", + "OWNED_WITH_MORTGAGE", + ] + + def test_in_nyc_formula_owned_column_blocks_the_write( + self, adapter, us_bundle, tmp_path + ) -> None: + household = us_bundle.table("household").copy() + household["in_nyc"] = [False, True] + rebuilt = Frame( + { + name: (household if name == "household" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="in_nyc"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_spm_unit_formula_owned_column_blocks_the_write( + self, adapter, us_bundle, tmp_path + ) -> None: + spm_unit = us_bundle.table("spm_unit").copy() + spm_unit["spm_unit_capped_work_childcare_expenses"] = [3_000.0, 0.0] + rebuilt = Frame( + { + name: (spm_unit if name == "spm_unit" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="spm_unit_capped_work_childcare_expenses"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_contract_listed_formula_owned_columns_still_block_the_write( + self, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["ssi"] = [12_000.0, 0.0, 0.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + adapter = PolicyEngineUSEngine( + contract=ExportContract( + required=(), + forbidden=(), + optional=(), + formula_owned_excluded=("ssi",), + ) + ) + + path = tmp_path / "formula_blocked.h5" + with pytest.raises(ValueError, match="ssi"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_contract_formula_owned_exclusion_blocks_non_engine_column( + self, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["legacy_formula_output"] = [1.0, 0.0, 1.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + adapter = PolicyEngineUSEngine( + contract=ExportContract( + required=(), + forbidden=(), + optional=(), + formula_owned_excluded=("legacy_formula_output",), + ) + ) + + path = tmp_path / "contract_formula_blocked.h5" + with pytest.raises(ValueError, match="legacy_formula_output"): + adapter.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + + def test_future_formula_does_not_block_current_period_input( + self, adapter, us_bundle, tmp_path + ) -> None: + from policyengine_us.data import USSingleYearDataset + + person = us_bundle.person.copy() + person["weeks_worked"] = [52, 26, 40] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + + path = tmp_path / "future_formula_input.h5" + adapter.write_dataset(rebuilt, path, period=2024) + + reloaded = USSingleYearDataset(file_path=str(path)) + assert reloaded.person["weeks_worked"].tolist() == [52, 26, 40] + + def test_closed_contract_blocks_unexpected_columns( + self, us_bundle, tmp_path + ) -> None: + person = us_bundle.person.copy() + person["internal_bookkeeping"] = [1.0, 0.0, 1.0] + rebuilt = Frame( + { + name: (person if name == "person" else us_bundle.table(name)) + for name in us_bundle.entities + }, + US_SCHEMA, + {"household": us_bundle.weights_for("household")}, + ) + gated = PolicyEngineUSEngine( + contract=ExportContract( + required=(), + forbidden=(), + optional=("age", "employment_income", "state_fips"), + formula_owned_excluded=(), + closed=True, + ) + ) + + path = tmp_path / "closed.h5" + with pytest.raises(ValueError, match="internal_bookkeeping"): + gated.write_dataset(rebuilt, path, period=2024) + assert not path.exists() + def test_defaults_broadcast_onto_owning_entity(self, us_bundle, tmp_path) -> None: from policyengine_us.data import USSingleYearDataset diff --git a/uv.lock b/uv.lock index 9c38538..6998a2e 100644 --- a/uv.lock +++ b/uv.lock @@ -1499,7 +1499,7 @@ wheels = [ [[package]] name = "policyengine-us" -version = "1.724.0" +version = "1.729.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1509,9 +1509,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/9c/8835e12d62415794144fbb14d14244f00f4fd50a95b55e47128f6fbef9bb/policyengine_us-1.724.0.tar.gz", hash = "sha256:9cb67f2537d3613ae2cef20d91b8b82ae6b5673c8360befffad82ad9efebb369", size = 10136935, upload-time = "2026-06-10T17:43:12.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/cb/b2efba2094a708cd71890d98d72b99394fabc5894a4cceec14381e03fa35/policyengine_us-1.729.0.tar.gz", hash = "sha256:ac05c4d621c7f848b0806effc14e913160d5d47d777eadced6bc18edf392d75c", size = 10373862, upload-time = "2026-06-14T18:05:25.747Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/a2/792ab362ea7c5f08a3931644fe1f36df2d167097a675781c1790c4951e37/policyengine_us-1.724.0-py3-none-any.whl", hash = "sha256:6cc45f6f17f21b1dc62800f0bd1988c833c3e0975c39c926dcdca904bf0942dd", size = 11327533, upload-time = "2026-06-10T17:43:08.875Z" }, + { url = "https://files.pythonhosted.org/packages/b9/7d/778f92ae94997b00c3c9ac34b345f6c9333435f905670ee4eeb2f5e19809/policyengine_us-1.729.0-py3-none-any.whl", hash = "sha256:8d21d3f7c0e82a9415edffe8ea53939330a63d9c8f6bd334299bddb697cf2c00", size = 11905076, upload-time = "2026-06-14T18:05:21.806Z" }, ] [[package]] @@ -1575,8 +1575,8 @@ requires-dist = [ { name = "h5py", marker = "extra == 'us'", specifier = ">=3" }, { name = "numpy", specifier = ">=1.26" }, { name = "pandas", specifier = ">=2" }, - { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.723,<2" }, - { name = "policyengine-us-data", marker = "extra == 'us'", specifier = ">=1.69,<2" }, + { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.729,<2" }, + { name = "policyengine-us-data", marker = "extra == 'us'", specifier = ">=1.115.2,<1.115.4" }, { name = "populace-calibrate", editable = "packages/populace-calibrate" }, { name = "populace-fit", editable = "packages/populace-fit" }, { name = "populace-frame", editable = "packages/populace-frame" }, @@ -1642,7 +1642,7 @@ requires-dist = [ { name = "h5py", specifier = ">=3" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.88" }, - { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.723,<2" }, + { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=1.729,<2" }, ] provides-extras = ["us", "uk"] @@ -1708,7 +1708,7 @@ requires-dist = [ { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=2" }, { name = "pandas", specifier = ">=2.3" }, - { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.0" }, + { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.729,<2" }, ] provides-extras = ["us", "policyengine"]