Close the dormant holes from the post-merge review

gm_bench/environment.py

@@ -14,7 +14,14 @@ def load_environment_files(directory: str | Path | None = None) -> list[Path]:
 
     The local file wins over the shared file, while values explicitly exported
     by the caller win over both. Values are never printed or returned.
+
+    Setting ``GM_BENCH_DISABLE_ENV_FILES`` (to any non-empty value) makes this
+    a no-op. The test suite sets it so no entry point -- in-process, subprocess,
+    or a future script with its own call site -- can silently inherit a live
+    credential from ``.env.local``.
     """
+    if os.environ.get("GM_BENCH_DISABLE_ENV_FILES"):
+        return []
     root = Path(directory) if directory is not None else Path.cwd()
     loaded: list[Path] = []
     for name in (".env.local", ".env"):

gm_bench/publication.py

@@ -17,6 +17,8 @@
 
 PUBLICATION_FORMAT = "gm-bench-result-summary-v1"
 
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+
 
 def canonical_sha256(payload: dict[str, Any]) -> str:
     encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode()
@@ -146,8 +148,13 @@ def v3_route_acceptance_issues(registry: dict[str, Any]) -> list[str]:
     if not isinstance(evidence_artifact, str) or not evidence_artifact.strip():
         issues.append("sota-v3 exact-route evidence artifact is missing")
     else:
+        evidence_path = Path(evidence_artifact)
+        if not evidence_path.is_absolute():
+            # Registries record repo-relative artifact paths; resolving against
+            # the CWD would make acceptance depend on where the caller ran.
+            evidence_path = _REPO_ROOT / evidence_path
         try:
-            loaded = json.loads(Path(evidence_artifact).read_text())
+            loaded = json.loads(evidence_path.read_text())
             if not isinstance(loaded, dict):
                 raise ValueError("evidence artifact must contain a JSON object")
             evidence = loaded

scripts/run_publication_matrix.py

@@ -499,15 +499,17 @@ def _enforce_operator_ceiling(max_spend_usd: float, contract: str | None) -> Non
     only thing standing between a typo and an unbounded run was the operator
     retyping the right number. A committed ceiling that nothing enforces is a
     comment.  A null ceiling stays permissive: contracts that have not
-    committed to a number are not silently given one.
+    committed to a number are not silently given one.  An unreadable protocol
+    file is not the same as a null ceiling: it may hide a committed number, so
+    it fails closed like every other malformed input to this gate.
     """
     _, _, _, protocol_path, _ = CONTRACT_CONFIGS.get(contract or "", (None,) * 5)
     if protocol_path is None:
         protocol_path = PROTOCOL_CONFIG
     try:
         budget_policy = (_read_json(protocol_path) or {}).get("budget_policy") or {}
-    except (OSError, ValueError, json.JSONDecodeError):
-        return
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        raise ValueError(f"cannot read {protocol_path.name} to enforce the operator ceiling: {exc}") from exc
     ceiling = budget_policy.get("operator_ceiling_usd")
     if ceiling is None:
         return

tests/conftest.py

@@ -4,8 +4,6 @@
 
 import pytest
 
-import scripts.run_publication_matrix as publication_runner
-
 
 @pytest.fixture(autouse=True)
 def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -20,19 +18,24 @@ def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N
 def block_real_provider_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
     """Stop the test suite from ever authenticating against a paid provider.
 
-    The publication runner calls ``load_environment_files(ROOT)`` at startup,
-    which reads the gitignored ``.env.local`` out of the working tree. Any test
-    that drives ``main()`` through a paid phase without stubbing out the child
-    process therefore runs the real benchmark against real routes and bills a
-    real account -- with no failure to signal it, because the run succeeds.
-    That is exactly what happened on 2026-08-04: a test written to assert that
-    a spend ceiling *blocks* a run instead spent $0.44 across 38 live calls,
-    because the fixture lane it resolved to had every gate already unlocked.
-
-    Neutralising the loader is enough. Tests that need a credential present
-    still set one explicitly with ``monkeypatch.setenv``, which continues to
-    work; what they cannot do any more is silently inherit a live key.
+    Every entry point (the publication runner, the CLI, the route-evidence
+    collector) calls ``load_environment_files`` at startup, which reads the
+    gitignored ``.env.local`` out of the working tree. Any test that reaches a
+    paid phase without stubbing out the provider call therefore runs the real
+    benchmark against real routes and bills a real account -- with no failure
+    to signal it, because the run succeeds. That is exactly what happened on
+    2026-08-04: a test written to assert that a spend ceiling *blocks* a run
+    instead spent $0.44 across 38 live calls, because the fixture lane it
+    resolved to had every gate already unlocked.
+
+    ``GM_BENCH_DISABLE_ENV_FILES`` neutralises the loader at its source module
+    rather than patching one importer's reference, so it also covers tests
+    that drive the CLI as a subprocess (the child inherits the variable) and
+    any future script that adds its own ``load_environment_files`` call.
+    Tests that need a credential present still set one explicitly with
+    ``monkeypatch.setenv``, which continues to work; what they cannot do any
+    more is silently inherit a live key.
     """
-    monkeypatch.setattr(publication_runner, "load_environment_files", lambda _root: [])
+    monkeypatch.setenv("GM_BENCH_DISABLE_ENV_FILES", "1")
     for name in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"):
         monkeypatch.delenv(name, raising=False)

tests/test_environment.py

@@ -1,11 +1,28 @@
 from __future__ import annotations
 
 import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
 
 from gm_bench.environment import load_environment_files
 
 
-def test_local_env_loads_before_shared_env_without_overriding_process(tmp_path, monkeypatch) -> None:
+@pytest.fixture
+def enable_env_file_loading(monkeypatch) -> None:
+    """Opt back in to real env-file loading for the loader's own tests.
+
+    The autouse ``block_real_provider_credentials`` fixture disables the
+    loader suite-wide; the tests in this module exist to exercise it.
+    """
+    monkeypatch.delenv("GM_BENCH_DISABLE_ENV_FILES", raising=False)
+
+
+def test_local_env_loads_before_shared_env_without_overriding_process(
+    tmp_path, monkeypatch, enable_env_file_loading
+) -> None:
     (tmp_path / ".env").write_text("SHARED=shared\nLOCAL_WINS=shared\nPROCESS_WINS=shared\n")
     (tmp_path / ".env.local").write_text(
         "# local secrets\nexport LOCAL_WINS=local\nQUOTED='secret value'\nPROCESS_WINS=local\n"
@@ -24,7 +41,9 @@ def test_local_env_loads_before_shared_env_without_overriding_process(tmp_path,
     assert os.environ["PROCESS_WINS"] == "process"
 
 
-def test_env_loader_ignores_comments_invalid_names_and_missing_files(tmp_path, monkeypatch) -> None:
+def test_env_loader_ignores_comments_invalid_names_and_missing_files(
+    tmp_path, monkeypatch, enable_env_file_loading
+) -> None:
     (tmp_path / ".env.local").write_text("# comment\nnot an assignment\nBAD-NAME=x\nVALID=ok\n")
     monkeypatch.delenv("VALID", raising=False)
 
@@ -35,7 +54,7 @@ def test_env_loader_ignores_comments_invalid_names_and_missing_files(tmp_path, m
     assert "BAD-NAME" not in os.environ
 
 
-def test_cli_provider_readiness_uses_local_env_file(tmp_path, monkeypatch, capsys) -> None:
+def test_cli_provider_readiness_uses_local_env_file(tmp_path, monkeypatch, capsys, enable_env_file_loading) -> None:
     from gm_bench import cli
 
     (tmp_path / ".env.local").write_text("OPENROUTER_API_KEY=test-secret\n")
@@ -48,3 +67,48 @@ def test_cli_provider_readiness_uses_local_env_file(tmp_path, monkeypatch, capsy
     assert '"provider": "openrouter"' in output
     assert '"credential_present": true' in output
     assert "test-secret" not in output
+
+
+def test_disable_switch_makes_the_loader_a_no_op(tmp_path, monkeypatch) -> None:
+    """The suite-wide credential guard must hold at the loader's source.
+
+    Patching one importer's reference (the pre-2026-08-05 guard) left every
+    other ``from gm_bench.environment import load_environment_files`` call
+    site live -- including the route-evidence collector and any subprocess.
+    """
+    (tmp_path / ".env.local").write_text("OPENROUTER_API_KEY=sk-live-should-never-load\n")
+    monkeypatch.setenv("GM_BENCH_DISABLE_ENV_FILES", "1")
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+
+    assert load_environment_files(tmp_path) == []
+    assert "OPENROUTER_API_KEY" not in os.environ
+
+
+def test_disable_switch_is_inherited_by_subprocess_entry_points(tmp_path) -> None:
+    """A child process re-imports everything, so in-process patches vanish.
+
+    Only something carried in the environment survives the fork; this pins
+    that the guard actually crosses the process boundary.
+    """
+    (tmp_path / ".env.local").write_text("OPENROUTER_API_KEY=sk-live-should-never-load\n")
+    env = {key: value for key, value in os.environ.items() if key != "OPENROUTER_API_KEY"}
+    env["GM_BENCH_DISABLE_ENV_FILES"] = "1"
+    repo_root = str(Path(__file__).resolve().parents[1])
+    env["PYTHONPATH"] = os.pathsep.join(filter(None, (repo_root, env.get("PYTHONPATH"))))
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import os\n"
+            "from gm_bench.environment import load_environment_files\n"
+            "loaded = load_environment_files(os.getcwd())\n"
+            "print(len(loaded), 'OPENROUTER_API_KEY' in os.environ)",
+        ],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert result.stdout.split() == ["0", "False"]

tests/test_publication_runner.py

@@ -1874,6 +1874,34 @@ def test_operator_ceiling_stays_permissive_when_no_cap_is_committed(
         publication_runner._enforce_operator_ceiling(1.00, "sota-test")
 
 
+def test_operator_ceiling_fails_closed_when_the_protocol_cannot_be_read(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """An unreadable protocol is not a null ceiling: it may hide a committed cap.
+
+    Every other malformed input to this gate raises; a corrupt or missing
+    protocol file silently disabling the ceiling was the one fail-open path.
+    """
+    corrupt = tmp_path / "corrupt-protocol.json"
+    corrupt.write_text("{this is not json")
+    monkeypatch.setitem(
+        publication_runner.CONTRACT_CONFIGS,
+        "sota-corrupt-protocol",
+        (corrupt,) * 5,
+    )
+    with pytest.raises(ValueError, match="operator ceiling"):
+        publication_runner._enforce_operator_ceiling(1.00, "sota-corrupt-protocol")
+
+    monkeypatch.setitem(
+        publication_runner.CONTRACT_CONFIGS,
+        "sota-missing-protocol",
+        (tmp_path / "does-not-exist.json",) * 5,
+    )
+    with pytest.raises(ValueError, match="operator ceiling"):
+        publication_runner._enforce_operator_ceiling(1.00, "sota-missing-protocol")
+
+
 def test_paid_run_above_the_ceiling_is_refused_before_any_cell_runs(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,

tests/test_sota_v3_route_catalog.py

@@ -166,6 +166,18 @@ def test_v3_route_acceptance_is_bound_to_public_zero_completion_evidence() -> No
     assert any("privacy evidence digest does not match" in issue for issue in v3_route_acceptance_issues(registry))
 
 
+def test_v3_route_acceptance_resolves_relative_evidence_from_repo_root(tmp_path: Path, monkeypatch) -> None:
+    """Acceptance must not depend on which directory the caller ran from.
+
+    The registry records a repo-relative evidence path; resolving it against
+    the CWD only worked because pytest and the runner both happen to start at
+    the repo root.
+    """
+    registry = _read("sota_v3_models.json")
+    monkeypatch.chdir(tmp_path)
+    assert v3_route_acceptance_issues(registry) == []
+
+
 def test_smoke_authorization_still_cannot_unlock_panel_or_publication() -> None:
     lane = _read("sota_v3_lane.json")
     registry = _read("sota_v3_models.json")