From 9f7f246966d16eb26b484aca9c4a8132aa302b8c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 17 Jun 2026 07:16:21 -0400 Subject: [PATCH] Gate critical US Populace target fit --- .../tests/test_us_fiscal_refresh_builder.py | 101 +++++++- .../src/populace/data/contract.py | 134 +++++++++++ packages/populace-data/tests/test_contract.py | 218 +++++++++++++++--- packages/populace-data/tests/test_release.py | 122 ++++++++-- tools/build_us_fiscal_refresh_release.py | 105 +++++++++ 5 files changed, 624 insertions(+), 56 deletions(-) diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index 169026f..9d5e0d3 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -23,6 +23,41 @@ def _load_builder_module(): return module +def _passing_critical_diagnostics(builder) -> tuple[SimpleNamespace, ...]: + return ( + SimpleNamespace( + name=( + "irs_soi.ty2022.historic_table_2.us.all." + f"income_tax_liability_amount@{builder.PERIOD}" + ), + target=2_105_345_646_000.0, + initial_estimate=2_000_000_000_000.0, + final_estimate=2_067_762_165_736.424, + relative_error=-0.0178514536722185, + ), + SimpleNamespace( + name=( + "irs_soi.ty2022.historic_table_2.us.all." + f"income_tax_liability_returns@{builder.PERIOD}" + ), + target=113_562_590.0, + initial_estimate=105_421_734.40619682, + final_estimate=105_437_267.69738781, + relative_error=-0.07154928663226319, + ), + SimpleNamespace( + name=( + "ssa_supplement.cy2024.oasdi_ssi_payments." + f"social_security_benefits.payment_amount@{builder.PERIOD}" + ), + target=1_471_195_000_000.0, + initial_estimate=1_541_646_703_291.2527, + final_estimate=1_541_540_768_722.367, + relative_error=0.047815394099604024, + ), + ) + + def test_soi_component_amounts_use_source_specific_signs() -> None: builder = _load_builder_module() @@ -192,7 +227,7 @@ def test_release_gate_failures_are_not_unconditional() -> None: builder = _load_builder_module() result = SimpleNamespace( skipped=(), - diagnostics=(object(),), + diagnostics=_passing_critical_diagnostics(builder), initial_loss=10.0, final_loss=5.0, ) @@ -207,7 +242,7 @@ def test_release_gate_failures_are_not_unconditional() -> None: skipped = SimpleNamespace(target=SimpleNamespace(name="skipped"), reason="bad") with_skipped = SimpleNamespace( skipped=(skipped,), - diagnostics=(object(),), + diagnostics=_passing_critical_diagnostics(builder), initial_loss=10.0, final_loss=5.0, ) @@ -218,7 +253,7 @@ def test_release_gate_failures_are_not_unconditional() -> None: worse = SimpleNamespace( skipped=(), - diagnostics=(object(),), + diagnostics=_passing_critical_diagnostics(builder), initial_loss=5.0, final_loss=10.0, ) @@ -231,7 +266,7 @@ def test_release_gate_failures_include_target_profile_coverage() -> None: builder = _load_builder_module() result = SimpleNamespace( skipped=(), - diagnostics=(object(),), + diagnostics=_passing_critical_diagnostics(builder), initial_loss=10.0, final_loss=5.0, ) @@ -254,7 +289,7 @@ def test_release_gate_failures_include_health_input_signal() -> None: builder = _load_builder_module() result = SimpleNamespace( skipped=(), - diagnostics=(object(),), + diagnostics=_passing_critical_diagnostics(builder), initial_loss=10.0, final_loss=5.0, ) @@ -290,6 +325,7 @@ def test_release_gate_failures_reject_positive_zero_support_targets() -> None: initial_estimate=10.0, final_estimate=20.0, ), + *_passing_critical_diagnostics(builder), ), initial_loss=10.0, final_loss=5.0, @@ -301,6 +337,61 @@ def test_release_gate_failures_reject_positive_zero_support_targets() -> None: ] +def test_release_gate_failures_reject_bad_critical_target_fit() -> None: + builder = _load_builder_module() + result = SimpleNamespace( + skipped=(), + diagnostics=( + SimpleNamespace( + name=( + "irs_soi.ty2022.historic_table_2.us.all." + f"income_tax_liability_amount@{builder.PERIOD}" + ), + target=2_105_345_646_000.0, + initial_estimate=2_000_000_000_000.0, + final_estimate=735_173_331_468.564, + relative_error=0.0, + ), + *_passing_critical_diagnostics(builder)[1:], + ), + initial_loss=10.0, + final_loss=5.0, + ) + + failures = builder._release_gate_failures( + result, + {"dropped_target_names": []}, + ) + + assert len(failures) == 2 + assert "stale relative_error" in failures[0] + assert "federal income tax liability amount" in failures[1] + assert "relative_error=-0.650806" in failures[1] + + +def test_release_gate_failures_reject_missing_critical_targets() -> None: + builder = _load_builder_module() + result = SimpleNamespace( + skipped=(), + diagnostics=_passing_critical_diagnostics(builder)[1:], + initial_loss=10.0, + final_loss=5.0, + ) + + failures = builder._release_gate_failures( + result, + {"dropped_target_names": []}, + ) + + assert failures == [ + "Critical fiscal target " + "'irs_soi.ty2022.historic_table_2.us.all." + f"income_tax_liability_amount@{builder.PERIOD}' " + "(federal income tax liability amount) is missing from calibration " + "diagnostics." + ] + + def test_health_input_signal_gate_rejects_degenerate_aca_inputs() -> None: builder = _load_builder_module() diff --git a/packages/populace-data/src/populace/data/contract.py b/packages/populace-data/src/populace/data/contract.py index cd4c81c..bf68a0e 100644 --- a/packages/populace-data/src/populace/data/contract.py +++ b/packages/populace-data/src/populace/data/contract.py @@ -20,6 +20,7 @@ import hashlib import json +import math import re from collections.abc import Mapping from pathlib import Path @@ -55,6 +56,39 @@ SOURCE_COVERAGE_DIAGNOSTICS_SCHEMA_VERSION = 1 _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") _GIT_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_US_CRITICAL_TARGET_FIT_REQUIREMENTS = ( + { + "requirement_id": "federal_income_tax_amount", + "label": "federal income tax liability amount", + "max_abs_relative_error": 0.05, + "names": ( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024", + ), + "families": ("irs_soi",), + "target_roles": ("federal_income_tax_total",), + }, + { + "requirement_id": "income_tax_liability_returns", + "label": "income tax liability returns", + "max_abs_relative_error": 0.10, + "names": ( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_returns@2024", + ), + "families": ("irs_soi",), + "target_roles": (), + }, + { + "requirement_id": "social_security_benefits", + "label": "Social Security benefits", + "max_abs_relative_error": 0.05, + "names": ( + "ssa_supplement.cy2024.oasdi_ssi_payments." + "social_security_benefits.payment_amount@2024", + ), + "families": ("ssa",), + "target_roles": ("social_security_total",), + }, +) def required_release_files(release_id: str) -> tuple[str, ...]: @@ -622,6 +656,104 @@ def _check_calibration_diagnostics(diagnostics: Mapping, failures: list[str]) -> ) +def _check_us_critical_target_fit(diagnostics: Mapping, failures: list[str]) -> None: + targets = diagnostics.get("targets") + if not isinstance(targets, list): + return + for requirement in _US_CRITICAL_TARGET_FIT_REQUIREMENTS: + names = set(requirement["names"]) + target_roles = set(requirement["target_roles"]) + families = set(requirement["families"]) + matches = [ + target + for target in targets + if isinstance(target, Mapping) + and ( + target.get("name") in names + or ( + isinstance(target.get("metadata"), Mapping) + and target["metadata"].get("target_role") in target_roles + and _target_registry_family(target) in families + ) + ) + ] + if not matches: + failures.append( + "calibration_diagnostics.json is missing required US critical " + f"target {requirement['requirement_id']!r} " + f"({requirement['label']})." + ) + continue + for target in matches: + relative_error = target.get("relative_error") + computed_relative_error = _target_relative_error(target, failures) + if computed_relative_error is None: + continue + if not isinstance(relative_error, int | float): + failures.append( + "calibration_diagnostics.json critical target " + f"{target.get('name')!r} has non-numeric relative_error " + f"{relative_error!r}." + ) + elif not math.isclose( + float(relative_error), + computed_relative_error, + rel_tol=1e-9, + abs_tol=1e-9, + ): + failures.append( + "calibration_diagnostics.json critical target " + f"{target.get('name')!r} has stale relative_error " + f"{relative_error!r}; computed " + f"{computed_relative_error:.6g} from target and " + "final_estimate." + ) + max_abs = float(requirement["max_abs_relative_error"]) + if abs(computed_relative_error) > max_abs: + failures.append( + "calibration_diagnostics.json critical target " + f"{target.get('name')!r} ({requirement['label']}) has " + f"relative_error={computed_relative_error:.6g}, exceeding " + f"{max_abs:.6g}; target={target.get('target')!r}, " + f"final_estimate={target.get('final_estimate')!r}." + ) + + +def _target_relative_error(target: Mapping, failures: list[str]) -> float | None: + target_value = target.get("target") + final_estimate = target.get("final_estimate") + if not isinstance(target_value, int | float) or not isinstance( + final_estimate, int | float + ): + failures.append( + "calibration_diagnostics.json critical target " + f"{target.get('name')!r} has non-numeric target/final_estimate: " + f"target={target_value!r}, final_estimate={final_estimate!r}." + ) + return None + target_value = float(target_value) + final_estimate = float(final_estimate) + if not math.isfinite(target_value) or not math.isfinite(final_estimate): + failures.append( + "calibration_diagnostics.json critical target " + f"{target.get('name')!r} has non-finite target/final_estimate: " + f"target={target_value!r}, final_estimate={final_estimate!r}." + ) + return None + if target_value == 0.0: + return final_estimate - target_value + return (final_estimate - target_value) / target_value + + +def _target_registry_family(target: Mapping) -> str: + registry = target.get("registry") + if isinstance(registry, Mapping): + family = registry.get("family") + if family is not None: + return str(family) + return "" + + def _check_source_coverage_diagnostics( diagnostics: Mapping, failures: list[str] ) -> None: @@ -806,6 +938,8 @@ def validate_release_dir(release_dir: Path | str) -> None: if diagnostics is not None: calibration_diagnostics = diagnostics _check_calibration_diagnostics(diagnostics, failures) + if release_id.startswith("populace-us-"): + _check_us_critical_target_fit(diagnostics, failures) _check_cross_manifest_consistency( build_manifest, diff --git a/packages/populace-data/tests/test_contract.py b/packages/populace-data/tests/test_contract.py index 8f45762..58844ec 100644 --- a/packages/populace-data/tests/test_contract.py +++ b/packages/populace-data/tests/test_contract.py @@ -30,6 +30,7 @@ SOURCE_COVERAGE_SHA = "9" * 64 TARGET_SURFACE_SHA = "e" * 64 REGISTRY_VERSION = "registryabc123" +TARGET_COUNT = 4 def _model_package(release_id: str) -> tuple[str, str]: @@ -58,8 +59,8 @@ def _build_manifest(release_id: str = RELEASE_ID) -> dict: "calibration": { "filename": "populace_us_2024_calibration.npz", "sha256": CALIBRATION_SHA, - "target_surface": {"sha256": TARGET_SURFACE_SHA, "n_targets": 1}, - "target_registry": {"version": REGISTRY_VERSION, "n_specs": 1}, + "target_surface": {"sha256": TARGET_SURFACE_SHA, "n_targets": TARGET_COUNT}, + "target_registry": {"version": REGISTRY_VERSION, "n_specs": TARGET_COUNT}, }, "gates": {"exported_nonzero": {"passed": True}}, } @@ -136,7 +137,7 @@ def _calibration_diagnostics() -> dict: "target_surface": { "schema_version": 1, "weight_entity": "household", - "n_targets": 1, + "n_targets": TARGET_COUNT, "n_records": 2, "constraint_matrix": {"rows": 1, "columns": 2, "nnz": 2}, "sha256": TARGET_SURFACE_SHA, @@ -146,33 +147,96 @@ def _calibration_diagnostics() -> dict: "target_registry": { "country": "us", "version": REGISTRY_VERSION, - "n_specs": 1, + "n_specs": TARGET_COUNT, }, "loss_trajectory": [1.0, 0.5], "skipped": [], "targets": [ - { - "name": "population@2024", - "target_name": "population", - "period": 2024, - "entity": "household", - "aggregation": "count", - "measure": None, - "filter": None, - "source": "Census PEP 2024", - "metadata": {}, - "target": 1.0, - "compiled_target": 1.0, - "initial_estimate": 0.8, - "final_estimate": 1.0, - "relative_error": 0.0, - "within_tolerance": True, - "registry": {"family": "cbo"}, - } + _target_row( + "population@2024", + target_name="population", + target=1.0, + initial_estimate=0.8, + final_estimate=1.0, + relative_error=0.0, + family="cbo", + ), + _target_row( + "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_amount@2024", + target_name=( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount" + ), + target=2_105_345_646_000.0, + initial_estimate=2_000_000_000_000.0, + final_estimate=2_067_762_165_736.424, + relative_error=-0.0178514536722185, + family="irs_soi", + target_role="federal_income_tax_total", + ), + _target_row( + "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_returns@2024", + target_name=( + "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_returns" + ), + target=113_562_590.0, + initial_estimate=105_421_734.40619682, + final_estimate=105_437_267.69738781, + relative_error=-0.07154928663226319, + family="irs_soi", + ), + _target_row( + "ssa_supplement.cy2024.oasdi_ssi_payments." + "social_security_benefits.payment_amount@2024", + target_name=( + "ssa_supplement.cy2024.oasdi_ssi_payments." + "social_security_benefits.payment_amount" + ), + target=1_471_195_000_000.0, + initial_estimate=1_541_646_703_291.2527, + final_estimate=1_541_540_768_722.367, + relative_error=0.047815394099604024, + family="ssa", + target_role="social_security_total", + ), ], } +def _target_row( + name: str, + *, + target_name: str, + target: float, + initial_estimate: float, + final_estimate: float, + relative_error: float, + family: str, + target_role: str | None = None, +) -> dict: + metadata = {"target_role": target_role} if target_role else {} + return { + "name": name, + "target_name": target_name, + "period": 2024, + "entity": "household", + "aggregation": "sum", + "measure": None, + "filter": None, + "source": "Fixture admin target", + "metadata": metadata, + "target": target, + "compiled_target": target, + "initial_estimate": initial_estimate, + "final_estimate": final_estimate, + "relative_error": relative_error, + "within_tolerance": None, + "registry": {"family": family}, + } + + def _source_coverage_diagnostics() -> dict: return { "schema_version": 1, @@ -211,7 +275,19 @@ def _source_coverage_diagnostics() -> dict: "target_count": 1, "sources": ["Census PEP 2024"], "reference_urls": ["https://example.test/source"], - } + }, + "irs_soi": { + "label": "IRS Statistics of Income", + "target_count": 2, + "sources": ["IRS SOI Historic Table 2"], + "reference_urls": ["https://example.test/soi"], + }, + "ssa": { + "label": "Social Security Administration", + "target_count": 1, + "sources": ["SSA Annual Statistical Supplement"], + "reference_urls": ["https://example.test/ssa"], + }, }, } @@ -263,6 +339,92 @@ def test_a_complete_release_passes(release_dir: Path) -> None: validate_release_dir(release_dir) +def test_us_release_rejects_bad_critical_target_fit(release_dir: Path) -> None: + diagnostics = _calibration_diagnostics() + target = next( + row + for row in diagnostics["targets"] + if row["name"] == "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_amount@2024" + ) + target["final_estimate"] = 735_173_331_468.564 + target["relative_error"] = -0.6508063496056629 + _write_json_and_refresh_manifest_hash( + release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + + failures = "\n".join(excinfo.value.failures) + assert "federal income tax liability amount" in failures + assert "relative_error=-0.650806" in failures + + +def test_us_release_recomputes_critical_target_fit(release_dir: Path) -> None: + diagnostics = _calibration_diagnostics() + target = next( + row + for row in diagnostics["targets"] + if row["name"] == "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_amount@2024" + ) + target["final_estimate"] = 735_173_331_468.564 + target["relative_error"] = 0.0 + _write_json_and_refresh_manifest_hash( + release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + + failures = "\n".join(excinfo.value.failures) + assert "stale relative_error" in failures + assert "relative_error=-0.650806" in failures + + +def test_us_release_requires_critical_targets(release_dir: Path) -> None: + diagnostics = _calibration_diagnostics() + diagnostics["targets"] = [ + row + for row in diagnostics["targets"] + if row["name"] != "ssa_supplement.cy2024.oasdi_ssi_payments." + "social_security_benefits.payment_amount@2024" + ] + diagnostics["target_surface"]["n_targets"] = len(diagnostics["targets"]) + _write_json_and_refresh_manifest_hash( + release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + source_coverage = _source_coverage_diagnostics() + source_coverage["fiscal_target_sources"].pop("ssa") + _write_json_and_refresh_manifest_hash( + release_dir, + filename=US_SOURCE_COVERAGE_DIAGNOSTICS_FILE, + artifact_key="us_source_coverage", + payload=source_coverage, + ) + build_manifest = _build_manifest() + build_manifest["calibration"]["target_surface"]["n_targets"] = len( + diagnostics["targets"] + ) + (release_dir / "build_manifest.json").write_text(json.dumps(build_manifest)) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + + failures = "\n".join(excinfo.value.failures) + assert "social_security_benefits" in failures + + @pytest.mark.parametrize("filename", required_release_files(RELEASE_ID)) def test_each_required_file_is_named_when_missing( release_dir: Path, filename: str @@ -536,11 +698,11 @@ def test_us_source_coverage_must_not_claim_uncalibrated_families( release_dir: Path, ) -> None: payload = _source_coverage_diagnostics() - payload["fiscal_target_sources"]["irs_soi"] = { - "label": "Internal Revenue Service Statistics of Income", + payload["fiscal_target_sources"]["jct"] = { + "label": "Joint Committee on Taxation", "target_count": 1, - "sources": ["SOI publication"], - "reference_urls": ["https://example.test/soi"], + "sources": ["JCT tax expenditures"], + "reference_urls": ["https://example.test/jct"], } _write_json_and_refresh_manifest_hash( release_dir, @@ -551,7 +713,7 @@ def test_us_source_coverage_must_not_claim_uncalibrated_families( with pytest.raises(ReleaseContractError) as excinfo: validate_release_dir(release_dir) failures = "\n".join(excinfo.value.failures) - assert "unexpected ['irs_soi']" in failures + assert "unexpected ['jct']" in failures def test_us_source_coverage_target_counts_match_calibration( diff --git a/packages/populace-data/tests/test_release.py b/packages/populace-data/tests/test_release.py index 8fc6f3d..e62f989 100644 --- a/packages/populace-data/tests/test_release.py +++ b/packages/populace-data/tests/test_release.py @@ -33,6 +33,7 @@ CALIBRATION_SHA = "ac31f2be76a0f8dc4da89b6935aa4b8b1b2e1bd4eb3d03b809333084f25b376e" TARGET_SURFACE_SHA = "e" * 64 REGISTRY_VERSION = "registryabc123" +TARGET_COUNT = 4 def _calibration_diagnostics() -> dict: @@ -43,7 +44,7 @@ def _calibration_diagnostics() -> dict: "target_surface": { "schema_version": 1, "weight_entity": "household", - "n_targets": 1, + "n_targets": TARGET_COUNT, "n_records": 2, "constraint_matrix": {"rows": 1, "columns": 2, "nnz": 2}, "sha256": TARGET_SURFACE_SHA, @@ -53,33 +54,96 @@ def _calibration_diagnostics() -> dict: "target_registry": { "country": "us", "version": REGISTRY_VERSION, - "n_specs": 1, + "n_specs": TARGET_COUNT, }, "loss_trajectory": [1.0, 0.5], "skipped": [], "targets": [ - { - "name": "population@2024", - "target_name": "population", - "period": 2024, - "entity": "household", - "aggregation": "count", - "measure": None, - "filter": None, - "source": "Census PEP 2024", - "metadata": {}, - "target": 1.0, - "compiled_target": 1.0, - "initial_estimate": 0.8, - "final_estimate": 1.0, - "relative_error": 0.0, - "within_tolerance": True, - "registry": {"family": "cbo"}, - } + _target_row( + "population@2024", + target_name="population", + target=1.0, + initial_estimate=0.8, + final_estimate=1.0, + relative_error=0.0, + family="cbo", + ), + _target_row( + "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_amount@2024", + target_name=( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount" + ), + target=2_105_345_646_000.0, + initial_estimate=2_000_000_000_000.0, + final_estimate=2_067_762_165_736.424, + relative_error=-0.0178514536722185, + family="irs_soi", + target_role="federal_income_tax_total", + ), + _target_row( + "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_returns@2024", + target_name=( + "irs_soi.ty2022.historic_table_2.us.all." + "income_tax_liability_returns" + ), + target=113_562_590.0, + initial_estimate=105_421_734.40619682, + final_estimate=105_437_267.69738781, + relative_error=-0.07154928663226319, + family="irs_soi", + ), + _target_row( + "ssa_supplement.cy2024.oasdi_ssi_payments." + "social_security_benefits.payment_amount@2024", + target_name=( + "ssa_supplement.cy2024.oasdi_ssi_payments." + "social_security_benefits.payment_amount" + ), + target=1_471_195_000_000.0, + initial_estimate=1_541_646_703_291.2527, + final_estimate=1_541_540_768_722.367, + relative_error=0.047815394099604024, + family="ssa", + target_role="social_security_total", + ), ], } +def _target_row( + name: str, + *, + target_name: str, + target: float, + initial_estimate: float, + final_estimate: float, + relative_error: float, + family: str, + target_role: str | None = None, +) -> dict: + metadata = {"target_role": target_role} if target_role else {} + return { + "name": name, + "target_name": target_name, + "period": 2024, + "entity": "household", + "aggregation": "sum", + "measure": None, + "filter": None, + "source": "Fixture admin target", + "metadata": metadata, + "target": target, + "compiled_target": target, + "initial_estimate": initial_estimate, + "final_estimate": final_estimate, + "relative_error": relative_error, + "within_tolerance": None, + "registry": {"family": family}, + } + + def _source_coverage_diagnostics() -> dict: return { "schema_version": 1, @@ -118,7 +182,19 @@ def _source_coverage_diagnostics() -> dict: "target_count": 1, "sources": ["Census PEP 2024"], "reference_urls": ["https://example.test/source"], - } + }, + "irs_soi": { + "label": "IRS Statistics of Income", + "target_count": 2, + "sources": ["IRS SOI Historic Table 2"], + "reference_urls": ["https://example.test/soi"], + }, + "ssa": { + "label": "Social Security Administration", + "target_count": 1, + "sources": ["SSA Annual Statistical Supplement"], + "reference_urls": ["https://example.test/ssa"], + }, }, } @@ -191,11 +267,11 @@ def release_dir(tmp_path: Path) -> Path: "sha256": CALIBRATION_SHA, "target_surface": { "sha256": TARGET_SURFACE_SHA, - "n_targets": 1, + "n_targets": TARGET_COUNT, }, "target_registry": { "version": REGISTRY_VERSION, - "n_specs": 1, + "n_specs": TARGET_COUNT, }, }, "gates": {"parity_gaps": 0}, diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index f470010..68e24ad 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -68,6 +68,32 @@ CALIBRATION_FILENAME = "populace_us_2024_calibration.npz" POST_EXPORT_ABSOLUTE_TOLERANCE = 1_000_000.0 POST_EXPORT_RELATIVE_TOLERANCE = 5e-4 +US_CRITICAL_TARGET_FIT_REQUIREMENTS = ( + { + "name": ( + "irs_soi.ty2022.historic_table_2.us.all." + f"income_tax_liability_amount@{PERIOD}" + ), + "label": "federal income tax liability amount", + "max_abs_relative_error": 0.05, + }, + { + "name": ( + "irs_soi.ty2022.historic_table_2.us.all." + f"income_tax_liability_returns@{PERIOD}" + ), + "label": "income tax liability returns", + "max_abs_relative_error": 0.10, + }, + { + "name": ( + "ssa_supplement.cy2024.oasdi_ssi_payments." + f"social_security_benefits.payment_amount@{PERIOD}" + ), + "label": "Social Security benefits", + "max_abs_relative_error": 0.05, + }, +) DIRECT_ACTIVE_ALIASES = ( "cms-aca-oep-state-level", @@ -1289,6 +1315,7 @@ def _release_gate_failures( f"{len(zero_support)} positive fiscal targets have zero " f"materialized support (examples: {examples}{suffix})." ) + failures.extend(_critical_target_fit_failures(result)) if not math.isfinite(result.initial_loss) or not math.isfinite(result.final_loss): failures.append("Calibration loss is non-finite.") elif result.final_loss > result.initial_loss: @@ -1299,6 +1326,84 @@ def _release_gate_failures( return failures +def _critical_target_fit_failures(result) -> list[str]: + diagnostics_by_name = { + getattr(diagnostic, "name", None): diagnostic + for diagnostic in getattr(result, "diagnostics", ()) + } + failures: list[str] = [] + for requirement in US_CRITICAL_TARGET_FIT_REQUIREMENTS: + diagnostic = diagnostics_by_name.get(requirement["name"]) + if diagnostic is None: + failures.append( + "Critical fiscal target " + f"{requirement['name']!r} ({requirement['label']}) is missing " + "from calibration diagnostics." + ) + continue + relative_error = getattr(diagnostic, "relative_error", None) + computed_relative_error = _diagnostic_relative_error(diagnostic, failures) + if computed_relative_error is None: + continue + if not isinstance(relative_error, int | float): + failures.append( + "Critical fiscal target " + f"{requirement['name']!r} ({requirement['label']}) has " + f"non-numeric relative_error {relative_error!r}." + ) + elif not math.isclose( + float(relative_error), + computed_relative_error, + rel_tol=1e-9, + abs_tol=1e-9, + ): + failures.append( + "Critical fiscal target " + f"{requirement['name']!r} ({requirement['label']}) has " + f"stale relative_error {relative_error!r}; computed " + f"{computed_relative_error:.6g} from target and final_estimate." + ) + max_abs = float(requirement["max_abs_relative_error"]) + if abs(computed_relative_error) > max_abs: + failures.append( + "Critical fiscal target " + f"{requirement['name']!r} ({requirement['label']}) has " + f"relative_error={computed_relative_error:.6g}, exceeding " + f"{max_abs:.6g}; target={getattr(diagnostic, 'target', None)!r}, " + "final_estimate=" + f"{getattr(diagnostic, 'final_estimate', None)!r}." + ) + return failures + + +def _diagnostic_relative_error(diagnostic, failures: list[str]) -> float | None: + target_value = getattr(diagnostic, "target", None) + final_estimate = getattr(diagnostic, "final_estimate", None) + if not isinstance(target_value, int | float) or not isinstance( + final_estimate, int | float + ): + failures.append( + "Critical fiscal target " + f"{getattr(diagnostic, 'name', None)!r} has non-numeric " + f"target/final_estimate: target={target_value!r}, " + f"final_estimate={final_estimate!r}." + ) + return None + target_value = float(target_value) + final_estimate = float(final_estimate) + if not math.isfinite(target_value) or not math.isfinite(final_estimate): + failures.append( + "Critical fiscal target " + f"{getattr(diagnostic, 'name', None)!r} has non-finite " + f"target/final_estimate: target={target_value!r}, " + f"final_estimate={final_estimate!r}." + ) + return None + if target_value == 0.0: + return final_estimate - target_value + return (final_estimate - target_value) / target_value + + def _assert_release_gates( result, compilation: Mapping[str, object],