diff --git a/gm_bench/environment.py b/gm_bench/environment.py index b2b0fde..dd6db59 100644 --- a/gm_bench/environment.py +++ b/gm_bench/environment.py @@ -14,7 +14,14 @@ def load_environment_files(directory: str | Path | None = None) -> list[Path]: The local file wins over the shared file, while values explicitly exported by the caller win over both. Values are never printed or returned. + + Setting ``GM_BENCH_DISABLE_ENV_FILES`` (to any non-empty value) makes this + a no-op. The test suite sets it so no entry point -- in-process, subprocess, + or a future script with its own call site -- can silently inherit a live + credential from ``.env.local``. """ + if os.environ.get("GM_BENCH_DISABLE_ENV_FILES"): + return [] root = Path(directory) if directory is not None else Path.cwd() loaded: list[Path] = [] for name in (".env.local", ".env"): diff --git a/gm_bench/publication.py b/gm_bench/publication.py index 13a063c..c644526 100644 --- a/gm_bench/publication.py +++ b/gm_bench/publication.py @@ -17,6 +17,8 @@ PUBLICATION_FORMAT = "gm-bench-result-summary-v1" +_REPO_ROOT = Path(__file__).resolve().parents[1] + def canonical_sha256(payload: dict[str, Any]) -> str: encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode() @@ -146,8 +148,13 @@ def v3_route_acceptance_issues(registry: dict[str, Any]) -> list[str]: if not isinstance(evidence_artifact, str) or not evidence_artifact.strip(): issues.append("sota-v3 exact-route evidence artifact is missing") else: + evidence_path = Path(evidence_artifact) + if not evidence_path.is_absolute(): + # Registries record repo-relative artifact paths; resolving against + # the CWD would make acceptance depend on where the caller ran. + evidence_path = _REPO_ROOT / evidence_path try: - loaded = json.loads(Path(evidence_artifact).read_text()) + loaded = json.loads(evidence_path.read_text()) if not isinstance(loaded, dict): raise ValueError("evidence artifact must contain a JSON object") evidence = loaded diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index d6ced4b..d4da673 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -499,15 +499,17 @@ def _enforce_operator_ceiling(max_spend_usd: float, contract: str | None) -> Non only thing standing between a typo and an unbounded run was the operator retyping the right number. A committed ceiling that nothing enforces is a comment. A null ceiling stays permissive: contracts that have not - committed to a number are not silently given one. + committed to a number are not silently given one. An unreadable protocol + file is not the same as a null ceiling: it may hide a committed number, so + it fails closed like every other malformed input to this gate. """ _, _, _, protocol_path, _ = CONTRACT_CONFIGS.get(contract or "", (None,) * 5) if protocol_path is None: protocol_path = PROTOCOL_CONFIG try: budget_policy = (_read_json(protocol_path) or {}).get("budget_policy") or {} - except (OSError, ValueError, json.JSONDecodeError): - return + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read {protocol_path.name} to enforce the operator ceiling: {exc}") from exc ceiling = budget_policy.get("operator_ceiling_usd") if ceiling is None: return diff --git a/tests/conftest.py b/tests/conftest.py index 89ec216..2abda49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,6 @@ import pytest -import scripts.run_publication_matrix as publication_runner - @pytest.fixture(autouse=True) def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -20,19 +18,24 @@ def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N def block_real_provider_credentials(monkeypatch: pytest.MonkeyPatch) -> None: """Stop the test suite from ever authenticating against a paid provider. - The publication runner calls ``load_environment_files(ROOT)`` at startup, - which reads the gitignored ``.env.local`` out of the working tree. Any test - that drives ``main()`` through a paid phase without stubbing out the child - process therefore runs the real benchmark against real routes and bills a - real account -- with no failure to signal it, because the run succeeds. - That is exactly what happened on 2026-08-04: a test written to assert that - a spend ceiling *blocks* a run instead spent $0.44 across 38 live calls, - because the fixture lane it resolved to had every gate already unlocked. - - Neutralising the loader is enough. Tests that need a credential present - still set one explicitly with ``monkeypatch.setenv``, which continues to - work; what they cannot do any more is silently inherit a live key. + Every entry point (the publication runner, the CLI, the route-evidence + collector) calls ``load_environment_files`` at startup, which reads the + gitignored ``.env.local`` out of the working tree. Any test that reaches a + paid phase without stubbing out the provider call therefore runs the real + benchmark against real routes and bills a real account -- with no failure + to signal it, because the run succeeds. That is exactly what happened on + 2026-08-04: a test written to assert that a spend ceiling *blocks* a run + instead spent $0.44 across 38 live calls, because the fixture lane it + resolved to had every gate already unlocked. + + ``GM_BENCH_DISABLE_ENV_FILES`` neutralises the loader at its source module + rather than patching one importer's reference, so it also covers tests + that drive the CLI as a subprocess (the child inherits the variable) and + any future script that adds its own ``load_environment_files`` call. + Tests that need a credential present still set one explicitly with + ``monkeypatch.setenv``, which continues to work; what they cannot do any + more is silently inherit a live key. """ - monkeypatch.setattr(publication_runner, "load_environment_files", lambda _root: []) + monkeypatch.setenv("GM_BENCH_DISABLE_ENV_FILES", "1") for name in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"): monkeypatch.delenv(name, raising=False) diff --git a/tests/test_environment.py b/tests/test_environment.py index aea0e68..e2de0d6 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,11 +1,28 @@ from __future__ import annotations import os +import subprocess +import sys +from pathlib import Path + +import pytest from gm_bench.environment import load_environment_files -def test_local_env_loads_before_shared_env_without_overriding_process(tmp_path, monkeypatch) -> None: +@pytest.fixture +def enable_env_file_loading(monkeypatch) -> None: + """Opt back in to real env-file loading for the loader's own tests. + + The autouse ``block_real_provider_credentials`` fixture disables the + loader suite-wide; the tests in this module exist to exercise it. + """ + monkeypatch.delenv("GM_BENCH_DISABLE_ENV_FILES", raising=False) + + +def test_local_env_loads_before_shared_env_without_overriding_process( + tmp_path, monkeypatch, enable_env_file_loading +) -> None: (tmp_path / ".env").write_text("SHARED=shared\nLOCAL_WINS=shared\nPROCESS_WINS=shared\n") (tmp_path / ".env.local").write_text( "# local secrets\nexport LOCAL_WINS=local\nQUOTED='secret value'\nPROCESS_WINS=local\n" @@ -24,7 +41,9 @@ def test_local_env_loads_before_shared_env_without_overriding_process(tmp_path, assert os.environ["PROCESS_WINS"] == "process" -def test_env_loader_ignores_comments_invalid_names_and_missing_files(tmp_path, monkeypatch) -> None: +def test_env_loader_ignores_comments_invalid_names_and_missing_files( + tmp_path, monkeypatch, enable_env_file_loading +) -> None: (tmp_path / ".env.local").write_text("# comment\nnot an assignment\nBAD-NAME=x\nVALID=ok\n") monkeypatch.delenv("VALID", raising=False) @@ -35,7 +54,7 @@ def test_env_loader_ignores_comments_invalid_names_and_missing_files(tmp_path, m assert "BAD-NAME" not in os.environ -def test_cli_provider_readiness_uses_local_env_file(tmp_path, monkeypatch, capsys) -> None: +def test_cli_provider_readiness_uses_local_env_file(tmp_path, monkeypatch, capsys, enable_env_file_loading) -> None: from gm_bench import cli (tmp_path / ".env.local").write_text("OPENROUTER_API_KEY=test-secret\n") @@ -48,3 +67,48 @@ def test_cli_provider_readiness_uses_local_env_file(tmp_path, monkeypatch, capsy assert '"provider": "openrouter"' in output assert '"credential_present": true' in output assert "test-secret" not in output + + +def test_disable_switch_makes_the_loader_a_no_op(tmp_path, monkeypatch) -> None: + """The suite-wide credential guard must hold at the loader's source. + + Patching one importer's reference (the pre-2026-08-05 guard) left every + other ``from gm_bench.environment import load_environment_files`` call + site live -- including the route-evidence collector and any subprocess. + """ + (tmp_path / ".env.local").write_text("OPENROUTER_API_KEY=sk-live-should-never-load\n") + monkeypatch.setenv("GM_BENCH_DISABLE_ENV_FILES", "1") + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + + assert load_environment_files(tmp_path) == [] + assert "OPENROUTER_API_KEY" not in os.environ + + +def test_disable_switch_is_inherited_by_subprocess_entry_points(tmp_path) -> None: + """A child process re-imports everything, so in-process patches vanish. + + Only something carried in the environment survives the fork; this pins + that the guard actually crosses the process boundary. + """ + (tmp_path / ".env.local").write_text("OPENROUTER_API_KEY=sk-live-should-never-load\n") + env = {key: value for key, value in os.environ.items() if key != "OPENROUTER_API_KEY"} + env["GM_BENCH_DISABLE_ENV_FILES"] = "1" + repo_root = str(Path(__file__).resolve().parents[1]) + env["PYTHONPATH"] = os.pathsep.join(filter(None, (repo_root, env.get("PYTHONPATH")))) + + result = subprocess.run( + [ + sys.executable, + "-c", + "import os\n" + "from gm_bench.environment import load_environment_files\n" + "loaded = load_environment_files(os.getcwd())\n" + "print(len(loaded), 'OPENROUTER_API_KEY' in os.environ)", + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.split() == ["0", "False"] diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 908b07d..0e07c03 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1874,6 +1874,34 @@ def test_operator_ceiling_stays_permissive_when_no_cap_is_committed( publication_runner._enforce_operator_ceiling(1.00, "sota-test") +def test_operator_ceiling_fails_closed_when_the_protocol_cannot_be_read( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An unreadable protocol is not a null ceiling: it may hide a committed cap. + + Every other malformed input to this gate raises; a corrupt or missing + protocol file silently disabling the ceiling was the one fail-open path. + """ + corrupt = tmp_path / "corrupt-protocol.json" + corrupt.write_text("{this is not json") + monkeypatch.setitem( + publication_runner.CONTRACT_CONFIGS, + "sota-corrupt-protocol", + (corrupt,) * 5, + ) + with pytest.raises(ValueError, match="operator ceiling"): + publication_runner._enforce_operator_ceiling(1.00, "sota-corrupt-protocol") + + monkeypatch.setitem( + publication_runner.CONTRACT_CONFIGS, + "sota-missing-protocol", + (tmp_path / "does-not-exist.json",) * 5, + ) + with pytest.raises(ValueError, match="operator ceiling"): + publication_runner._enforce_operator_ceiling(1.00, "sota-missing-protocol") + + def test_paid_run_above_the_ceiling_is_refused_before_any_cell_runs( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_sota_v3_route_catalog.py b/tests/test_sota_v3_route_catalog.py index b47c6e4..9761048 100644 --- a/tests/test_sota_v3_route_catalog.py +++ b/tests/test_sota_v3_route_catalog.py @@ -166,6 +166,18 @@ def test_v3_route_acceptance_is_bound_to_public_zero_completion_evidence() -> No assert any("privacy evidence digest does not match" in issue for issue in v3_route_acceptance_issues(registry)) +def test_v3_route_acceptance_resolves_relative_evidence_from_repo_root(tmp_path: Path, monkeypatch) -> None: + """Acceptance must not depend on which directory the caller ran from. + + The registry records a repo-relative evidence path; resolving it against + the CWD only worked because pytest and the runner both happen to start at + the repo root. + """ + registry = _read("sota_v3_models.json") + monkeypatch.chdir(tmp_path) + assert v3_route_acceptance_issues(registry) == [] + + def test_smoke_authorization_still_cannot_unlock_panel_or_publication() -> None: lane = _read("sota_v3_lane.json") registry = _read("sota_v3_models.json")