diff --git a/changelog.d/421.fixed.md b/changelog.d/421.fixed.md new file mode 100644 index 00000000..4c25aae3 --- /dev/null +++ b/changelog.d/421.fixed.md @@ -0,0 +1 @@ +Restore the previous US Populace default dataset and reject Populace releases whose critical fiscal calibration targets miss certification tolerances. diff --git a/src/policyengine/data/release_manifests/us.json b/src/policyengine/data/release_manifests/us.json index ba725e08..017505d1 100644 --- a/src/policyengine/data/release_manifests/us.json +++ b/src/policyengine/data/release_manifests/us.json @@ -5,63 +5,63 @@ "certified_by": "populace-data release manifest", "certified_for_model_version": "1.729.0", "compatibility_basis": "exact_build_model_version", - "data_build_id": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z" + "data_build_id": "populace-us-2024-a912aea-76666318a202-20260616T175345Z" }, "certified_data_artifact": { - "build_id": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", + "build_id": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", "data_package": { "name": "populace-data", "version": "0.1.0" }, "dataset": "populace_us_2024", - "sha256": "bee9a8d104c45094a4505f9b8d8530495302148c3664712ae0bc8859d9c7cf3d", - "uri": "hf://policyengine/populace-us/populace_us_2024.h5@822609aa834528162d530c1c2bfca2cb347a01c2" + "sha256": "9d87c7ff370be524e73aaf68d151b00846eefcae4b00a63760102e2c6f285f92", + "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-a912aea-76666318a202-20260616T175345Z" }, "country_id": "us", "data_package": { "name": "populace-data", - "release_manifest_path": "releases/populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z/release_manifest.json", - "release_manifest_revision": "822609aa834528162d530c1c2bfca2cb347a01c2", + "release_manifest_path": "releases/populace-us-2024-a912aea-76666318a202-20260616T175345Z/release_manifest.json", + "release_manifest_revision": "c4e2fd454ddce0e1889ab77abff178a7bdd72b18", "repo_id": "policyengine/populace-us", "repo_type": "dataset", "version": "0.1.0" }, "datasets": { "calibration_diagnostics": { - "path": "releases/populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z/calibration_diagnostics.json", + "path": "releases/populace-us-2024-a912aea-76666318a202-20260616T175345Z/calibration_diagnostics.json", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", - "sha256": "f92442571fbf33f2188662791902ff26832c3a80ecf3bce2cf3ec31ef715593e" + "revision": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", + "sha256": "154a1b217211d92c50e0fb84750888920cf8a63afcf9437efa85e484a7d501c9" }, "populace_us_2024": { "path": "populace_us_2024.h5", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", - "sha256": "bee9a8d104c45094a4505f9b8d8530495302148c3664712ae0bc8859d9c7cf3d" + "revision": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", + "sha256": "9d87c7ff370be524e73aaf68d151b00846eefcae4b00a63760102e2c6f285f92" }, "populace_us_2024_calibration": { "path": "populace_us_2024_calibration.npz", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", - "sha256": "fe258399d0a76ec81df10195028627364d185b3eae4cfceafeae53e8d4b74121" + "revision": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", + "sha256": "0679dd35dbb198164beee6d56626af2b2fb57d6a3b6ea6511daf908e66296175" }, "us_source_coverage": { - "path": "releases/populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z/us_source_coverage.json", + "path": "releases/populace-us-2024-a912aea-76666318a202-20260616T175345Z/us_source_coverage.json", "repo_id": "policyengine/populace-us", - "revision": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", - "sha256": "813aae6aaf217df6a8bf67e9c4ad168952f73814124cdff748af3cb2325ffae6" + "revision": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", + "sha256": "233a87ccc1c1eb8ed95321b7ebe586cd483e4e5af37686e182803f3b88edc76d" }, "reform_validation": { - "path": "releases/populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z/reform_validation.json", - "revision": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", + "path": "releases/populace-us-2024-a912aea-76666318a202-20260616T175345Z/reform_validation.json", + "revision": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", "repo_id": "policyengine/populace-us", - "sha256": "846af5ea0355ffe8cc8f082e12c64b5135912c685865264db25975f1db1c1d80" + "sha256": "266851a23595eb832fdc3a88453fd60dfc12cd258e557b4ae7fd92b19eeb4f9e" }, "demographics": { - "path": "releases/populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z/demographics.json", - "revision": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", + "path": "releases/populace-us-2024-a912aea-76666318a202-20260616T175345Z/demographics.json", + "revision": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", "repo_id": "policyengine/populace-us", - "sha256": "e0530f7cb82dee2cdd53576b389e75c6d5b66bae610aa04061d0e4d35f0034c2" + "sha256": "073b9edf5a0594ab9e24c8aaae2a981a09ae510869a3f6c6cec3bb5500914897" } }, "default_dataset": "populace_us_2024", diff --git a/src/policyengine/data/release_manifests/us.trace.tro.jsonld b/src/policyengine/data/release_manifests/us.trace.tro.jsonld index 3f556b95..d09f1679 100644 --- a/src/policyengine/data/release_manifests/us.trace.tro.jsonld +++ b/src/policyengine/data/release_manifests/us.trace.tro.jsonld @@ -17,7 +17,7 @@ "schema:name": "PolicyEngine", "schema:url": "https://policyengine.org" }, - "schema:dateCreated": "2026-06-17T03:33:44.404151+00:00", + "schema:dateCreated": "2026-06-16T18:48:19.511357+00:00", "schema:description": "TRACE TRO for certified runtime bundle us-4.17.6 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.", "schema:name": "policyengine us certified bundle TRO", "trov:createdWith": { @@ -45,7 +45,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/data_release_manifest" }, - "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/822609aa834528162d530c1c2bfca2cb347a01c2/releases/populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z/release_manifest.json" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/c4e2fd454ddce0e1889ab77abff178a7bdd72b18/releases/populace-us-2024-a912aea-76666318a202-20260616T175345Z/release_manifest.json" }, { "@id": "arrangement/1/location/dataset", @@ -53,7 +53,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/dataset" }, - "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/822609aa834528162d530c1c2bfca2cb347a01c2/populace_us_2024.h5" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-a912aea-76666318a202-20260616T175345Z/populace_us_2024.h5" }, { "@id": "arrangement/1/location/model_wheel", @@ -75,21 +75,21 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "d827ad68a1de5db65c1493c4460216ca48e88901a818d40d3234f02fa9dacca0" + "trov:sha256": "8050796e60593a976279360f9b4ec214e1d786acf5d7a29c10222817bc6200b3" }, { "@id": "composition/1/artifact/data_release_manifest", "@type": "trov:ResearchArtifact", "schema:name": "populace-data release manifest 0.1.0", "trov:mimeType": "application/json", - "trov:sha256": "0a9faa8ff2818c7bb8251ae510dcf68984ca0b956c2838006bd8f7f3165ed22c" + "trov:sha256": "b50925148419218dec45800baa1c3c683b966a622a8cb281e49410b65c92d099" }, { "@id": "composition/1/artifact/dataset", "@type": "trov:ResearchArtifact", "schema:name": "populace_us_2024", "trov:mimeType": "application/x-hdf5", - "trov:sha256": "bee9a8d104c45094a4505f9b8d8530495302148c3664712ae0bc8859d9c7cf3d" + "trov:sha256": "9d87c7ff370be524e73aaf68d151b00846eefcae4b00a63760102e2c6f285f92" }, { "@id": "composition/1/artifact/model_wheel", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "493989b8237159691fabe2eec599ba661fba425a2e6c8613e06d17b04a577b27" + "trov:sha256": "75c8f49c868118c99b4ff346e6645df56bf520a714ad7353bd97a0c8a131cc22" } }, "trov:hasPerformance": { @@ -111,17 +111,14 @@ "pe:builtWithModelVersion": "1.729.0", "pe:certifiedBy": "populace-data release manifest", "pe:certifiedForModelVersion": "1.729.0", - "pe:ciGitRef": "refs/heads/main", - "pe:ciGitSha": "f0c8436314a4e3edc75440835e155ab9513be63f", - "pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/27664843839", "pe:compatibilityBasis": "exact_build_model_version", - "pe:dataBuildId": "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z", - "pe:emittedIn": "github-actions", - "rdfs:comment": "Certification of build populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z for policyengine-us 1.729.0.", + "pe:dataBuildId": "populace-us-2024-a912aea-76666318a202-20260616T175345Z", + "pe:emittedIn": "local", + "rdfs:comment": "Certification of build populace-us-2024-a912aea-76666318a202-20260616T175345Z for policyengine-us 1.729.0.", "trov:accessedArrangement": { "@id": "arrangement/1" }, - "trov:startedAtTime": "2026-06-17T03:33:44.404151+00:00", + "trov:startedAtTime": "2026-06-16T18:48:19.511357+00:00", "trov:wasConductedBy": { "@id": "trs" } diff --git a/src/policyengine/provenance/bundle.py b/src/policyengine/provenance/bundle.py index b6be5c16..14feb0ee 100644 --- a/src/policyengine/provenance/bundle.py +++ b/src/policyengine/provenance/bundle.py @@ -62,6 +62,28 @@ SEMVER_PATTERN = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") +@dataclass(frozen=True) +class _CriticalCalibrationTarget: + name: str + max_abs_relative_error: float + + +_US_POPULACE_CRITICAL_CALIBRATION_TARGETS = ( + _CriticalCalibrationTarget( + name="irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024", + max_abs_relative_error=0.05, + ), + _CriticalCalibrationTarget( + name="irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_returns@2024", + max_abs_relative_error=0.10, + ), + _CriticalCalibrationTarget( + name="ssa_supplement.cy2024.oasdi_ssi_payments.social_security_benefits.payment_amount@2024", + max_abs_relative_error=0.05, + ), +) + + # --------------------------------------------------------------------------- # policyengine.py bundle identity # --------------------------------------------------------------------------- @@ -233,6 +255,130 @@ def _fetch_data_release_manifest( return None +def _fetch_json_release_artifact( + artifact: dict, + *, + release_manifest_path: str | None, + data_repo_id: str, + data_repo_type: str, + default_revision: str, +) -> dict: + path = _release_scoped_artifact_path( + artifact, + release_manifest_path=release_manifest_path, + data_repo_id=data_repo_id, + ) + repo_id = artifact.get("repo_id") or data_repo_id + revision = artifact.get("revision") or default_revision + repo_type = artifact.get("repo_type") + if repo_type is None: + repo_type = data_repo_type if repo_id == data_repo_id else "model" + url = https_dataset_uri( + repo_id=repo_id, + path_in_repo=path, + revision=revision, + repo_type=repo_type, + ) + headers = {"User-Agent": "policyengine.py"} + token = os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + + with urlopen(Request(url, headers=headers)) as f: + content = f.read() + + expected_sha256 = artifact.get("sha256") + if expected_sha256: + actual_sha256 = hashlib.sha256(content).hexdigest() + if actual_sha256 != expected_sha256: + raise ValueError( + "Data release artifact hash mismatch for " + f"{path!r}: expected {expected_sha256}, got {actual_sha256}." + ) + + try: + return json.loads(content) + except ValueError as exc: + raise ValueError(f"Data release artifact {path!r} is not valid JSON.") from exc + + +def _calibration_relative_error(target: dict) -> float | None: + relative_error = target.get("relative_error") + if relative_error is not None: + return float(relative_error) + + target_value = target.get("target") + final_estimate = target.get("final_estimate", target.get("final")) + if target_value in (None, 0) or final_estimate is None: + return None + return (float(final_estimate) - float(target_value)) / float(target_value) + + +def _validate_populace_critical_calibration_targets( + *, + country: str, + release_manifest_json: dict, + release_manifest_path: str | None, + data_package_name: str, + data_repo_id: str, + data_repo_type: str, + default_revision: str, +) -> None: + if country != "us" or data_package_name != "populace-data": + return + + diagnostics_artifact = release_manifest_json.get("artifacts", {}).get( + "calibration_diagnostics" + ) + if diagnostics_artifact is None: + raise ValueError( + "Populace release manifest is missing calibration_diagnostics; " + "refusing to certify without critical calibration target gates." + ) + + diagnostics = _fetch_json_release_artifact( + diagnostics_artifact, + release_manifest_path=release_manifest_path, + data_repo_id=data_repo_id, + data_repo_type=data_repo_type, + default_revision=default_revision, + ) + targets = diagnostics.get("targets") + if not isinstance(targets, list): + raise ValueError( + "Populace calibration_diagnostics is missing the target list; " + "refusing to certify without critical calibration target gates." + ) + targets_by_name = { + target.get("name"): target + for target in targets + if isinstance(target, dict) and target.get("name") + } + + failures = [] + for critical_target in _US_POPULACE_CRITICAL_CALIBRATION_TARGETS: + target = targets_by_name.get(critical_target.name) + if target is None: + failures.append(f"{critical_target.name}: missing") + continue + + relative_error = _calibration_relative_error(target) + if relative_error is None: + failures.append(f"{critical_target.name}: missing relative error") + continue + + if abs(relative_error) > critical_target.max_abs_relative_error: + failures.append( + f"{critical_target.name}: relative_error={relative_error:.6g} " + f"exceeds {critical_target.max_abs_relative_error:.6g}" + ) + + if failures: + raise ValueError( + "Populace critical calibration target gate failed: " + "; ".join(failures) + ) + + def _updated_release_manifest_path( current_path: str, old_data: str, @@ -615,6 +761,15 @@ def refresh_release_bundle( f"version {release_manifest_data_version!r}, expected {new_data!r}." ) new_release_manifest_revision = release_manifest_fetch.repo_commit + _validate_populace_critical_calibration_targets( + country=country, + release_manifest_json=release_manifest_json, + release_manifest_path=new_release_manifest_path, + data_package_name=data_package_json.get("name", ""), + data_repo_id=repo_id, + data_repo_type=data_package_json.get("repo_type", "model"), + default_revision=new_release_manifest_revision, + ) certified_dataset = ( current.certified_data_artifact.dataset diff --git a/tests/test_bundle_refresh.py b/tests/test_bundle_refresh.py index 5e491684..38da76e8 100644 --- a/tests/test_bundle_refresh.py +++ b/tests/test_bundle_refresh.py @@ -132,6 +132,50 @@ def _data_release_manifest_response( ) +_INCOME_TAX_TARGET = ( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024" +) +_INCOME_TAX_RETURNS_TARGET = ( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_returns@2024" +) +_SOCIAL_SECURITY_TARGET = "ssa_supplement.cy2024.oasdi_ssi_payments.social_security_benefits.payment_amount@2024" + + +def _json_bytes(payload: dict) -> bytes: + return json.dumps(payload).encode() + + +def _json_sha256(payload: dict) -> str: + return hashlib.sha256(_json_bytes(payload)).hexdigest() + + +def _json_response(payload: dict, headers: dict | None = None) -> _FakeHFResponse: + return _FakeHFResponse(_json_bytes(payload), headers=headers) + + +def _populace_calibration_diagnostics( + *, + income_tax_relative_error: float = -0.02, + income_tax_returns_relative_error: float = -0.07, + social_security_relative_error: float = 0.04, +) -> dict: + def target(name: str, relative_error: float) -> dict: + return { + "name": name, + "target": 100.0, + "final_estimate": 100.0 * (1.0 + relative_error), + "relative_error": relative_error, + } + + return { + "targets": [ + target(_INCOME_TAX_TARGET, income_tax_relative_error), + target(_INCOME_TAX_RETURNS_TARGET, income_tax_returns_relative_error), + target(_SOCIAL_SECURITY_TARGET, social_security_relative_error), + ] + } + + @pytest.fixture def sandbox(tmp_path: Path) -> dict: """A writable scratch copy of the US release manifest + a stub @@ -805,6 +849,99 @@ def fake_urlopen(request, *args, **kwargs): ) +def _write_populace_refresh_manifest(sandbox, *, old_release: str) -> None: + manifest_path = sandbox["manifest_dir"] / "us.json" + manifest = json.loads(manifest_path.read_text()) + manifest["data_package"] = { + "name": "populace-data", + "version": "0.1.0", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "release_manifest_path": f"releases/{old_release}/release_manifest.json", + "release_manifest_revision": old_release, + } + manifest["certified_data_artifact"] = { + "data_package": {"name": "populace-data", "version": "0.1.0"}, + "build_id": old_release, + "dataset": "populace_us_2024", + "uri": f"hf://policyengine/populace-us/populace_us_2024.h5@{old_release}", + "sha256": "1" * 64, + } + manifest["certification"] = { + "compatibility_basis": "exact_build_model_version", + "data_build_id": old_release, + "built_with_model_version": "1.600.0", + "certified_for_model_version": "1.600.0", + "certified_by": "test fixture", + } + manifest["default_dataset"] = "populace_us_2024" + manifest["datasets"] = { + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": old_release, + "sha256": "1" * 64, + }, + "populace_us_2024_calibration": { + "path": "populace_us_2024_calibration.npz", + "repo_id": "policyengine/populace-us", + "revision": old_release, + "sha256": "2" * 64, + }, + "calibration_diagnostics": { + "path": f"releases/{old_release}/calibration_diagnostics.json", + "repo_id": "policyengine/populace-us", + "revision": old_release, + "sha256": "3" * 64, + }, + } + manifest_path.write_text(json.dumps(manifest, indent=2)) + + +def _populace_release_payload( + *, + new_release: str, + diagnostics: dict, + include_diagnostics_artifact: bool = True, +) -> dict: + artifacts = { + "populace_us_2024": { + "kind": "microdata", + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": new_release, + "sha256": "5" * 64, + }, + "populace_us_2024_calibration": { + "kind": "calibration", + "path": "populace_us_2024_calibration.npz", + "repo_id": "policyengine/populace-us", + "revision": new_release, + "sha256": "6" * 64, + }, + } + if include_diagnostics_artifact: + artifacts["calibration_diagnostics"] = { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": "policyengine/populace-us", + "revision": new_release, + "sha256": _json_sha256(diagnostics), + } + return { + "schema_version": 1, + "data_package": {"name": "populace-data", "version": "0.1.0"}, + "build": { + "build_id": new_release, + "built_with_model_package": { + "name": "policyengine-us", + "version": "1.600.0", + }, + }, + "artifacts": artifacts, + } + + def test__same_data_version_release_override_refreshes_revisioned_populace_artifacts( sandbox, ) -> None: @@ -863,6 +1000,7 @@ def test__same_data_version_release_override_refreshes_revisioned_populace_artif } manifest_path.write_text(json.dumps(manifest, indent=2)) + diagnostics = _populace_calibration_diagnostics() payload = { "schema_version": 1, "data_package": {"name": "populace-data", "version": "0.1.0"}, @@ -893,7 +1031,7 @@ def test__same_data_version_release_override_refreshes_revisioned_populace_artif "path": "calibration_diagnostics.json", "repo_id": "policyengine/populace-us", "revision": new_release, - "sha256": "7" * 64, + "sha256": _json_sha256(diagnostics), }, "reform_validation": { "kind": "diagnostics", @@ -911,10 +1049,15 @@ def fake_urlopen(request, *args, **kwargs): f"/resolve/{new_release}/releases/{new_release}/release_manifest.json" in url ): - return _FakeHFResponse( - json.dumps(payload).encode(), + return _json_response( + payload, headers={"x-repo-commit": "new-release-manifest-commit"}, ) + if ( + f"/resolve/{new_release}/releases/{new_release}/calibration_diagnostics.json" + in url + ): + return _json_response(diagnostics) raise AssertionError(f"Unexpected URL fetched: {url}") with patch("policyengine.provenance.bundle.urlopen", side_effect=fake_urlopen): @@ -943,7 +1086,7 @@ def fake_urlopen(request, *args, **kwargs): "path": f"releases/{new_release}/calibration_diagnostics.json", "repo_id": "policyengine/populace-us", "revision": new_release, - "sha256": "7" * 64, + "sha256": _json_sha256(diagnostics), } assert written["datasets"]["reform_validation"] == { "path": f"releases/{new_release}/reform_validation.json", @@ -959,6 +1102,92 @@ def fake_urlopen(request, *args, **kwargs): } +def test__populace_release_requires_calibration_diagnostics( + sandbox, +) -> None: + old_release = "populace-us-2024-old" + new_release = "populace-us-2024-new" + _write_populace_refresh_manifest(sandbox, old_release=old_release) + diagnostics = _populace_calibration_diagnostics() + payload = _populace_release_payload( + new_release=new_release, + diagnostics=diagnostics, + include_diagnostics_artifact=False, + ) + + def fake_urlopen(request, *args, **kwargs): + url = request.full_url + if ( + f"/resolve/{new_release}/releases/{new_release}/release_manifest.json" + in url + ): + return _json_response( + payload, + headers={"x-repo-commit": "new-release-manifest-commit"}, + ) + raise AssertionError(f"Unexpected URL fetched: {url}") + + with patch("policyengine.provenance.bundle.urlopen", side_effect=fake_urlopen): + with pytest.raises( + ValueError, + match="missing calibration_diagnostics", + ): + refresh_release_bundle( + country="us", + data_version="0.1.0", + release_manifest_path=f"releases/{new_release}/release_manifest.json", + release_manifest_revision=new_release, + manifest_dir=sandbox["manifest_dir"], + pyproject_path=sandbox["pyproject_path"], + ) + + +def test__populace_release_rejects_critical_calibration_target_miss( + sandbox, +) -> None: + old_release = "populace-us-2024-old" + new_release = "populace-us-2024-new" + _write_populace_refresh_manifest(sandbox, old_release=old_release) + diagnostics = _populace_calibration_diagnostics( + income_tax_relative_error=-0.6508063496056629 + ) + payload = _populace_release_payload( + new_release=new_release, + diagnostics=diagnostics, + ) + + def fake_urlopen(request, *args, **kwargs): + url = request.full_url + if ( + f"/resolve/{new_release}/releases/{new_release}/release_manifest.json" + in url + ): + return _json_response( + payload, + headers={"x-repo-commit": "new-release-manifest-commit"}, + ) + if ( + f"/resolve/{new_release}/releases/{new_release}/calibration_diagnostics.json" + in url + ): + return _json_response(diagnostics) + raise AssertionError(f"Unexpected URL fetched: {url}") + + with patch("policyengine.provenance.bundle.urlopen", side_effect=fake_urlopen): + with pytest.raises( + ValueError, + match="critical calibration target gate failed", + ): + refresh_release_bundle( + country="us", + data_version="0.1.0", + release_manifest_path=f"releases/{new_release}/release_manifest.json", + release_manifest_revision=new_release, + manifest_dir=sandbox["manifest_dir"], + pyproject_path=sandbox["pyproject_path"], + ) + + def test__custom_release_manifest_requires_existing_long_term_dataset_sha( sandbox, ) -> None: diff --git a/tests/test_models.py b/tests/test_models.py index 1cba6907..a302ec08 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -119,7 +119,7 @@ def test_has_release_manifest_metadata(self): assert ( us_latest.default_dataset_uri == "hf://policyengine/populace-us/populace_us_2024.h5" - "@822609aa834528162d530c1c2bfca2cb347a01c2" + "@populace-us-2024-a912aea-76666318a202-20260616T175345Z" ) def test_has_hundreds_of_parameters(self): diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index ceeb3e9c..0656e635 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -44,8 +44,8 @@ US_MODEL_VERSION = "1.729.0" US_BUILT_WITH_MODEL_VERSION = "1.729.0" US_DATA_RELEASE_VERSION = "0.1.0" -US_DATA_RELEASE_ID = "populace-us-2024-f0d2ef6-09292aa0d5db-20260617T032307Z" -US_DATA_RELEASE_REVISION = "822609aa834528162d530c1c2bfca2cb347a01c2" +US_DATA_RELEASE_ID = "populace-us-2024-a912aea-76666318a202-20260616T175345Z" +US_DATA_RELEASE_REVISION = "c4e2fd454ddce0e1889ab77abff178a7bdd72b18" US_DATA_RELEASE_PATH = f"releases/{US_DATA_RELEASE_ID}/release_manifest.json" US_DATA_ARTIFACT_REVISION = US_DATA_RELEASE_ID US_CERTIFICATION_SOURCE = "populace-data release manifest" @@ -53,7 +53,7 @@ f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_ARTIFACT_REVISION}" ) US_CERTIFIED_DATASET_URI = ( - f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_RELEASE_REVISION}" + f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_ARTIFACT_REVISION}" ) US_RELEASE_MANIFEST_DATASET_URI = ( f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_RELEASE_REVISION}" diff --git a/uv.lock b/uv.lock index d99d9ac7..ddb94c9d 100644 --- a/uv.lock +++ b/uv.lock @@ -2820,7 +2820,7 @@ wheels = [ [[package]] name = "policyengine" -version = "4.17.5" +version = "4.17.6" source = { editable = "." } dependencies = [ { name = "diskcache" },