Skip to content

Gate critical US Populace target fit #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MaxGhenis merged 1 commit into from
Jun 17, 2026
+624 −56
Merged

Gate critical US Populace target fit #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 96 additions & 5 deletions packages/populace-build/tests/test_us_fiscal_refresh_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,41 @@ def _load_builder_module():
return module


def _passing_critical_diagnostics(builder) -> tuple[SimpleNamespace, ...]:
return (
SimpleNamespace(
name=(
"irs_soi.ty2022.historic_table_2.us.all."
f"income_tax_liability_amount@{builder.PERIOD}"
),
target=2_105_345_646_000.0,
initial_estimate=2_000_000_000_000.0,
final_estimate=2_067_762_165_736.424,
relative_error=-0.0178514536722185,
),
SimpleNamespace(
name=(
"irs_soi.ty2022.historic_table_2.us.all."
f"income_tax_liability_returns@{builder.PERIOD}"
),
target=113_562_590.0,
initial_estimate=105_421_734.40619682,
final_estimate=105_437_267.69738781,
relative_error=-0.07154928663226319,
),
SimpleNamespace(
name=(
"ssa_supplement.cy2024.oasdi_ssi_payments."
f"social_security_benefits.payment_amount@{builder.PERIOD}"
),
target=1_471_195_000_000.0,
initial_estimate=1_541_646_703_291.2527,
final_estimate=1_541_540_768_722.367,
relative_error=0.047815394099604024,
),
)


def test_soi_component_amounts_use_source_specific_signs() -> None:
builder = _load_builder_module()

Expand Down Expand Up @@ -192,7 +227,7 @@ def test_release_gate_failures_are_not_unconditional() -> None:
builder = _load_builder_module()
result = SimpleNamespace(
skipped=(),
diagnostics=(object(),),
diagnostics=_passing_critical_diagnostics(builder),
initial_loss=10.0,
final_loss=5.0,
)
Expand All @@ -207,7 +242,7 @@ def test_release_gate_failures_are_not_unconditional() -> None:
skipped = SimpleNamespace(target=SimpleNamespace(name="skipped"), reason="bad")
with_skipped = SimpleNamespace(
skipped=(skipped,),
diagnostics=(object(),),
diagnostics=_passing_critical_diagnostics(builder),
initial_loss=10.0,
final_loss=5.0,
)
Expand All @@ -218,7 +253,7 @@ def test_release_gate_failures_are_not_unconditional() -> None:

worse = SimpleNamespace(
skipped=(),
diagnostics=(object(),),
diagnostics=_passing_critical_diagnostics(builder),
initial_loss=5.0,
final_loss=10.0,
)
Expand All @@ -231,7 +266,7 @@ def test_release_gate_failures_include_target_profile_coverage() -> None:
builder = _load_builder_module()
result = SimpleNamespace(
skipped=(),
diagnostics=(object(),),
diagnostics=_passing_critical_diagnostics(builder),
initial_loss=10.0,
final_loss=5.0,
)
Expand All @@ -254,7 +289,7 @@ def test_release_gate_failures_include_health_input_signal() -> None:
builder = _load_builder_module()
result = SimpleNamespace(
skipped=(),
diagnostics=(object(),),
diagnostics=_passing_critical_diagnostics(builder),
initial_loss=10.0,
final_loss=5.0,
)
Expand Down Expand Up @@ -290,6 +325,7 @@ def test_release_gate_failures_reject_positive_zero_support_targets() -> None:
initial_estimate=10.0,
final_estimate=20.0,
),
*_passing_critical_diagnostics(builder),
),
initial_loss=10.0,
final_loss=5.0,
Expand All @@ -301,6 +337,61 @@ def test_release_gate_failures_reject_positive_zero_support_targets() -> None:
]


def test_release_gate_failures_reject_bad_critical_target_fit() -> None:
builder = _load_builder_module()
result = SimpleNamespace(
skipped=(),
diagnostics=(
SimpleNamespace(
name=(
"irs_soi.ty2022.historic_table_2.us.all."
f"income_tax_liability_amount@{builder.PERIOD}"
),
target=2_105_345_646_000.0,
initial_estimate=2_000_000_000_000.0,
final_estimate=735_173_331_468.564,
relative_error=0.0,
),
*_passing_critical_diagnostics(builder)[1:],
),
initial_loss=10.0,
final_loss=5.0,
)

failures = builder._release_gate_failures(
result,
{"dropped_target_names": []},
)

assert len(failures) == 2
assert "stale relative_error" in failures[0]
assert "federal income tax liability amount" in failures[1]
assert "relative_error=-0.650806" in failures[1]


def test_release_gate_failures_reject_missing_critical_targets() -> None:
builder = _load_builder_module()
result = SimpleNamespace(
skipped=(),
diagnostics=_passing_critical_diagnostics(builder)[1:],
initial_loss=10.0,
final_loss=5.0,
)

failures = builder._release_gate_failures(
result,
{"dropped_target_names": []},
)

assert failures == [
"Critical fiscal target "
"'irs_soi.ty2022.historic_table_2.us.all."
f"income_tax_liability_amount@{builder.PERIOD}' "
"(federal income tax liability amount) is missing from calibration "
"diagnostics."
]


def test_health_input_signal_gate_rejects_degenerate_aca_inputs() -> None:
builder = _load_builder_module()

Expand Down
134 changes: 134 additions & 0 deletions packages/populace-data/src/populace/data/contract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import hashlib
import json
import math
import re
from collections.abc import Mapping
from pathlib import Path
Expand Down Expand Up @@ -55,6 +56,39 @@
SOURCE_COVERAGE_DIAGNOSTICS_SCHEMA_VERSION = 1
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
_GIT_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
_US_CRITICAL_TARGET_FIT_REQUIREMENTS = (
{
"requirement_id": "federal_income_tax_amount",
"label": "federal income tax liability amount",
"max_abs_relative_error": 0.05,
"names": (
"irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024",
),
"families": ("irs_soi",),
"target_roles": ("federal_income_tax_total",),
},
{
"requirement_id": "income_tax_liability_returns",
"label": "income tax liability returns",
"max_abs_relative_error": 0.10,
"names": (
"irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_returns@2024",
),
"families": ("irs_soi",),
"target_roles": (),
},
{
"requirement_id": "social_security_benefits",
"label": "Social Security benefits",
"max_abs_relative_error": 0.05,
"names": (
"ssa_supplement.cy2024.oasdi_ssi_payments."
"social_security_benefits.payment_amount@2024",
),
"families": ("ssa",),
"target_roles": ("social_security_total",),
},
)


def required_release_files(release_id: str) -> tuple[str, ...]:
Expand Down Expand Up @@ -622,6 +656,104 @@ def _check_calibration_diagnostics(diagnostics: Mapping, failures: list[str]) ->
)


def _check_us_critical_target_fit(diagnostics: Mapping, failures: list[str]) -> None:
targets = diagnostics.get("targets")
if not isinstance(targets, list):
return
for requirement in _US_CRITICAL_TARGET_FIT_REQUIREMENTS:
names = set(requirement["names"])
target_roles = set(requirement["target_roles"])
families = set(requirement["families"])
matches = [
target
for target in targets
if isinstance(target, Mapping)
and (
target.get("name") in names
or (
isinstance(target.get("metadata"), Mapping)
and target["metadata"].get("target_role") in target_roles
and _target_registry_family(target) in families
)
)
]
if not matches:
failures.append(
"calibration_diagnostics.json is missing required US critical "
f"target {requirement['requirement_id']!r} "
f"({requirement['label']})."
)
continue
for target in matches:
relative_error = target.get("relative_error")
computed_relative_error = _target_relative_error(target, failures)
if computed_relative_error is None:
continue
if not isinstance(relative_error, int | float):
failures.append(
"calibration_diagnostics.json critical target "
f"{target.get('name')!r} has non-numeric relative_error "
f"{relative_error!r}."
)
elif not math.isclose(
float(relative_error),
computed_relative_error,
rel_tol=1e-9,
abs_tol=1e-9,
):
failures.append(
"calibration_diagnostics.json critical target "
f"{target.get('name')!r} has stale relative_error "
f"{relative_error!r}; computed "
f"{computed_relative_error:.6g} from target and "
"final_estimate."
)
max_abs = float(requirement["max_abs_relative_error"])
if abs(computed_relative_error) > max_abs:
failures.append(
"calibration_diagnostics.json critical target "
f"{target.get('name')!r} ({requirement['label']}) has "
f"relative_error={computed_relative_error:.6g}, exceeding "
f"{max_abs:.6g}; target={target.get('target')!r}, "
f"final_estimate={target.get('final_estimate')!r}."
)


def _target_relative_error(target: Mapping, failures: list[str]) -> float | None:
target_value = target.get("target")
final_estimate = target.get("final_estimate")
if not isinstance(target_value, int | float) or not isinstance(
final_estimate, int | float
):
failures.append(
"calibration_diagnostics.json critical target "
f"{target.get('name')!r} has non-numeric target/final_estimate: "
f"target={target_value!r}, final_estimate={final_estimate!r}."
)
return None
target_value = float(target_value)
final_estimate = float(final_estimate)
if not math.isfinite(target_value) or not math.isfinite(final_estimate):
failures.append(
"calibration_diagnostics.json critical target "
f"{target.get('name')!r} has non-finite target/final_estimate: "
f"target={target_value!r}, final_estimate={final_estimate!r}."
)
return None
if target_value == 0.0:
return final_estimate - target_value
return (final_estimate - target_value) / target_value


def _target_registry_family(target: Mapping) -> str:
registry = target.get("registry")
if isinstance(registry, Mapping):
family = registry.get("family")
if family is not None:
return str(family)
return ""


def _check_source_coverage_diagnostics(
diagnostics: Mapping, failures: list[str]
) -> None:
Expand Down Expand Up @@ -806,6 +938,8 @@ def validate_release_dir(release_dir: Path | str) -> None:
if diagnostics is not None:
calibration_diagnostics = diagnostics
_check_calibration_diagnostics(diagnostics, failures)
if release_id.startswith("populace-us-"):
_check_us_critical_target_fit(diagnostics, failures)

_check_cross_manifest_consistency(
build_manifest,
Expand Down
Loading
Loading