From 2ce880d392c4d039e52f1dddd089d7d6badae56f Mon Sep 17 00:00:00 2001 From: 2st Date: Fri, 4 Sep 2026 20:48:21 -0600 Subject: [PATCH 1/3] Let the project shape the generated inputs through recast_inputs.py A subprogram whose domain is a relation between its arguments -- a packed triangular workspace whose extent is n(n+1)/2, a pivot vector that must be a permutation, a mode that must be 1 or 2 -- cannot be drawn independently per argument, and the redraw rules only report how often that failed. The project now says what it accepts: a `recast_inputs.py` at the source root defines `prepare(unit, subprogram, inputs, rng)`, which returns one trial's draw shaped into the domain, or None to leave it to the generated rules. It lives in the source tree, so the report's identity covers it with nothing new to digest. A shaped draw is the project asserting the reference takes it, and is judged that way: the reference runs first, and if it raises or produces a NaN the verification stops with InputProfileError naming the file, subprogram and trial -- that is the profile's fault, never the candidate's, and it is never drawn again. A candidate that raises on a shaped draw the reference took has failed the translation, no redraw. A profile that does not import, defines no prepare, edits the draw in place and returns None, or renames the arguments stops the gate the same way. Replays never run the profile: recorded inputs are already in the domain, and letting anything edit them would rewrite the exam. The candidate-side `_PREPARE_INPUTS` hook is gone; the emitted module is what is being judged and does not get to choose its own inputs. Metrics carry `input_profile` and the `shaped` subprograms, and each subprogram reports how many trials were shaped. On the MINPACK corpus a 70-line profile takes dogleg and r1updt from 10 of 10 trials reshaped to 165 and 351 bit-exact points at their real packed extents, with every subprogram at zero redraws. (cherry picked from commit 28bfc8e13106b12e014929670a62fd13dcc02c5c) Signed-off-by: 2st --- docs/roadmap.md | 8 +- src/recast/errors.py | 11 ++ src/recast/verify/bitexact.py | 258 +++++++++++++++++++++++++++------- tests/test_bitexact_draws.py | 187 ++++++++++++++++++++++++ tests/test_dump_replay.py | 28 ++-- tests/test_f2py_oracle.py | 32 +++-- 6 files changed, 450 insertions(+), 74 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 2adda5e..b24f6e4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -551,10 +551,10 @@ byte-identical. Three consequences are worth naming because none is obvious: * **`trials` does not apply.** A recording holds the points it holds; asking for ten against a three-sample recording would be seven invented ones or seven copies. -* **`_PREPARE_INPUTS` is skipped.** The hook exists to drag *generated* inputs - into the physical domain, and recorded ones are already there. It also ships - inside the artifact under test, so running it on a replay would let the - candidate edit the production run's own numbers before being judged on them. +* **`recast_inputs.py` is skipped.** The project's input profile exists to + drag *generated* inputs into the physical domain, and recorded ones are + already there. Running it on a replay would let the production run's own + numbers be edited before the artifact is judged on them. * **Reference-side `setup` is skipped.** A replayed reference has no state to set: whatever the run's module state was is folded into what it recorded. An operator whose `setup` does not match the run's own initialization gets a diff --git a/src/recast/errors.py b/src/recast/errors.py index 8318e4a..482d1c0 100644 --- a/src/recast/errors.py +++ b/src/recast/errors.py @@ -36,6 +36,17 @@ class OracleUnavailable(RecastError): """The reference could not be materialized. Verdicts must be FAILED, not skipped.""" +class InputProfileError(RecastError): + """The project's ``recast_inputs.py`` shaped inputs the reference does not take. + + A shaped draw is the project asserting that these are inputs the source + accepts. When the reference cannot run them -- it raises, or computes a + NaN -- the assertion is what is wrong, not the translation, so the + verification stops here instead of charging the candidate with it or + drawing again. Fix the profile; nothing about the engine is being judged. + """ + + class ScannerUnavailable(RecastError): """A Scanner or Adjudicator could not run at all. The stage is INCOMPLETE. diff --git a/src/recast/verify/bitexact.py b/src/recast/verify/bitexact.py index 41e4187..d6e1c24 100644 --- a/src/recast/verify/bitexact.py +++ b/src/recast/verify/bitexact.py @@ -22,9 +22,13 @@ table, values from per-name ``ranges``. The physical ranges that make a model kernels behave -- temperatures in kelvin, pressures in pascals -- are domain knowledge and arrive in config; the engine's defaults are only wide, not -wise. Subprograms with deferred blocks are skipped and said so: their -translation raises ``NotImplementedError`` by construction, and the gate's -job is to judge translations, not queues. +wise. Structure that no per-name range can express -- a packed workspace +whose extent is ``n(n+1)/2``, a mode the source stops on, a column that must +be monotone -- comes from the project itself: a ``recast_inputs.py`` at the +root, whose ``prepare(unit, subprogram, inputs, rng)`` shapes each generated +draw before both sides receive it. Subprograms with deferred blocks are +skipped and said so: their translation raises ``NotImplementedError`` by +construction, and the gate's job is to judge translations, not queues. """ from __future__ import annotations @@ -38,6 +42,7 @@ from pathlib import Path from typing import Any +from recast.errors import InputProfileError from recast.model import Candidate, Confidence, OracleRef, Unit, Verdict from recast.plugins.executor import Executor from recast.plugins.verifier import Verifier @@ -50,6 +55,16 @@ DEFAULT_DIMENSION = 8 SUPPORTED_DTYPES = frozenset({"float32", "float64", "int32", "int64", "bool"}) PROCEDURE_DTYPE = "PROCEDURE" +INPUT_PROFILE = "recast_inputs.py" +"""The project's input profile, looked for at the root the run was given. + +Its ``prepare(unit, subprogram, inputs, rng)`` receives the inputs this +harness drew for one trial, by argument name, and returns them shaped into +the source's domain -- or ``None`` to leave that subprogram's draw as it is. +A shaped draw is an assertion that the reference takes it, and is judged as +one: it is never redrawn, a candidate that refuses it has failed, and a +reference that refuses it means the profile is wrong. +""" """What a dummy *procedure* argument is declared as. Not a value the harness can sample: it is something to call, and what both @@ -254,6 +269,13 @@ class BitexactVerifier(Verifier): evidence at the extents it landed on, not the configured ones: a subprogram that passes mostly that way fails by name, with the number of such trials recorded as ``reshaped``. + + None of this applies to a draw the project's ``recast_inputs.py`` shaped. + That draw is the project saying the source takes it, so there is nothing + to draw again: the reference is called first, and if it refuses, the + profile is wrong and the verification stops there + (``InputProfileError``); if it answers and the candidate refuses, the + candidate has failed on inputs the source takes. """ dominant_at: float | None = None @@ -352,6 +374,8 @@ def _compare_all( candidate, Confidence.FAILED, {}, f"candidate does not import: {error}" ) + profile = None if recorded else self._load_input_profile(config) + table = getattr(translated, "_SIGNATURES", None) if not isinstance(table, dict) or not table: return self._verdict( @@ -466,7 +490,8 @@ def generable(name: str) -> bool: trials, dims, ranges, - prepare=getattr(translated, "_PREPARE_INPUTS", None), + profile=profile, + unit_uid=unit.uid, dominant_at=config.get("dominant_at", self.dominant_at), dominant_axis=config.get("dominant_axis", -1), rel_scale=str(config.get("rel_scale", "element")), @@ -529,6 +554,8 @@ def generable(name: str) -> bool: "trials": trials, "skipped": skipped, "uncovered": uncovered, + "input_profile": INPUT_PROFILE if profile is not None else None, + "shaped": sorted(name for name, out in per_subprogram.items() if out.get("shaped")), "max_rel": worst_rel, **({"ungated": declared} if declared else {}), **totals, @@ -662,7 +689,8 @@ def _compare_subprogram( trials: int, dims: dict[str, int], ranges: dict[str, tuple[float, float]], - prepare: Any = None, + profile: Any = None, + unit_uid: str = "", dominant_at: float | None = None, dominant_axis: Any = -1, rel_scale: str = "element", @@ -741,6 +769,8 @@ def _compare_subprogram( # Trials that were compared only after a shape refusal moved the free # extents off the configured ones. reshaped = 0 + # Trials the project's input profile shaped. Those are never redrawn. + shaped_trials = 0 # Extents no operator pinned. A redraw varies these along with the # values, because some of what a subprogram will not accept is a # *shape*: a packed triangular workspace wants an extent that is a @@ -838,24 +868,19 @@ def _compare_subprogram( f"{', '.join(missing_outputs)}; partial output evidence is not a pass" } - if prepare is not None and samples is None: - # The candidate may carry a ``_PREPARE_INPUTS(name, inputs, - # rng)`` hook, the way it carries ``_SIGNATURES``: per-name - # ranges cannot express structure -- a pressure column must - # be monotone, an interface field must bracket its levels -- - # and unphysical inputs drive both sides into error paths - # the production model aborts out of. The hook shapes inputs - # into the defined domain; it cannot bias the verdict, - # because both sides receive the same shaped inputs. - # - # It is skipped for a replay, and that is the important half: - # the hook exists to drag *generated* inputs into the physical - # domain, and recorded inputs are already there by - # construction. Running it would let the candidate edit the - # production run's own numbers before being judged on them, - # which is the one thing a hook supplied by the artifact under - # test must never be able to do. - prepare(name, inputs, rng) + shaped = False + if profile is not None and samples is None: + # The project's profile shapes *generated* inputs only. A + # recording's inputs are already in the domain by + # construction, and letting anything edit the production + # run's own numbers before the artifact is judged on them + # would let the exam be rewritten. + inputs, shaped = self._shape_inputs( + np, profile, unit_uid, name, round_index, inputs, rng + ) + if shaped: + shaped_trials += 1 + site = _profile_site(unit_uid, name, round_index) # Keyword calls on both sides: f2py reorders inferred-dimension # scalars into trailing keywords, so positional order is not a @@ -881,39 +906,67 @@ def _compare_subprogram( if a["intent"] != "OUT" } except Exception as error: + if shaped: + raise InputProfileError( + f"{site} returned inputs the reference does not take: " + f"{type(error).__name__}: {error}" + ) from error return { "error": f"oracle input preparation failed: {type(error).__name__}: {error}" } truth_args = [ truth_kwargs[spell(a["name"])] for a in required if a["intent"] != "OUT" ] - try: - translated_out = translated_fn(**translated_kwargs) - except (SystemExit, IndexError) as error: - # Not a comparison that failed -- a draw the subprogram does - # not take. ``SystemExit`` is a translated ERROR STOP: the - # source itself saying these arguments are not its own, and - # the reference would say the same by ending the process, - # taking every other unit's verdict with it. ``IndexError`` is - # a subscript past a dummy array's declared extent, where the - # reference, compiled without bounds checking, reads memory - # the call does not own. Either way the reference must not be - # called on this draw; draw again. - declined = f"candidate raised: {type(error).__name__}: {error}" - reshape = reshape or isinstance(error, IndexError) - redrawn += 1 - continue - except Exception as error: - return {"error": f"candidate raised: {type(error).__name__}: {error}"} - if samples is not None: - # Nothing to call: the reference already ran, in production, - # and what it produced is the recording. - truth_out = recorded_outputs - else: + if shaped: + # The profile asserts the reference takes this draw, so the + # reference goes first and decides. Refused there, the + # assertion is what is wrong and nothing about the + # candidate is being judged; taken there and refused by the + # candidate, the candidate has failed on inputs the source + # accepts -- not a draw to make again. try: truth_out = truth_fn(**truth_kwargs) except Exception as error: - return {"error": f"oracle raised: {type(error).__name__}: {error}"} + raise InputProfileError( + f"{site} returned inputs the reference does not take: " + f"{type(error).__name__}: {error}" + ) from error + try: + translated_out = translated_fn(**translated_kwargs) + except (Exception, SystemExit) as error: + return { + "error": "candidate raised on shaped inputs the reference took: " + f"{type(error).__name__}: {error}" + } + else: + try: + translated_out = translated_fn(**translated_kwargs) + except (SystemExit, IndexError) as error: + # Not a comparison that failed -- a draw the subprogram + # does not take. ``SystemExit`` is a translated ERROR + # STOP: the source itself saying these arguments are not + # its own, and the reference would say the same by + # ending the process, taking every other unit's verdict + # with it. ``IndexError`` is a subscript past a dummy + # array's declared extent, where the reference, compiled + # without bounds checking, reads memory the call does + # not own. Either way the reference must not be called + # on this draw; draw again. + declined = f"candidate raised: {type(error).__name__}: {error}" + reshape = reshape or isinstance(error, IndexError) + redrawn += 1 + continue + except Exception as error: + return {"error": f"candidate raised: {type(error).__name__}: {error}"} + if samples is not None: + # Nothing to call: the reference already ran, in + # production, and what it produced is the recording. + truth_out = recorded_outputs + else: + try: + truth_out = truth_fn(**truth_kwargs) + except Exception as error: + return {"error": f"oracle raised: {type(error).__name__}: {error}"} pairs = self._paired_outputs( sub, @@ -987,6 +1040,13 @@ def _compare_subprogram( if samples is None: nan_ours = np.isnan(a) nan_theirs = np.isnan(b) + if shaped and nan_theirs.any(): + raise InputProfileError( + f"{site} returned inputs on which the reference produced " + f"NaN in {label}; a NaN-tainted trial compares the compiler's " + "scheduling rather than the translation, and a shaped draw " + "is not drawn again" + ) if nan_ours.any() and bool((nan_ours == nan_theirs).all()): # A draw that put the subprogram outside its numeric # domain on *both* sides: a square root of a negative, @@ -1074,6 +1134,7 @@ def _compare_subprogram( "integer_mismatch": integer_mismatch, "redrawn": redrawn, "reshaped": reshaped, + "shaped": shaped_trials, } if dominant_at is not None: outcome["max_ulp_dominant"] = max_ulp_dominant @@ -1091,7 +1152,7 @@ def _devices(translated: Any, handle: dict[str, Any]) -> dict[str, str]: A verdict that does not record it cannot be re-argued later. Asked for rather than detected, and by the same convention as - ``_SIGNATURES`` and ``_PREPARE_INPUTS``: the emitted module declares + ``_SIGNATURES``: the emitted module declares ``_DEVICE`` if it knows, and an Oracle puts one on its handle. Reaching for ``jax.devices()`` here instead would put an accelerator import in the core, which is the one thing the core does not do. @@ -1447,6 +1508,89 @@ def _value( # -- loading -------------------------------------------------------------- + @staticmethod + def _load_input_profile(config: dict[str, Any]) -> Callable[..., Any] | None: + """The project's ``recast_inputs.py``, when the tree carries one. + + Looked for at the root the run was given -- the same root the + sources were read from, so the profile is part of the source + artifact and travels, and is digested, with it. A tree without one + is the generated path unchanged. A tree with one that does not + import, or defines no ``prepare``, is a project whose assertion + about its own inputs cannot be read, and that stops the gate rather + than being taken as "no profile". + """ + root = config.get("root") + if root is None: + return None + path = Path(root) / INPUT_PROFILE + if not path.is_file(): + return None + spec = importlib.util.spec_from_file_location("recast_inputs", path) + if spec is None or spec.loader is None: + raise InputProfileError(f"{INPUT_PROFILE} could not be loaded as a module") + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except (Exception, SystemExit) as error: + raise InputProfileError( + f"{INPUT_PROFILE} does not import: {type(error).__name__}: {error}" + ) from error + prepare: Callable[..., Any] | None = getattr(module, "prepare", None) + if not callable(prepare): + raise InputProfileError( + f"{INPUT_PROFILE} defines no callable prepare(unit, subprogram, inputs, rng)" + ) + return prepare + + @staticmethod + def _shape_inputs( + np: Any, + prepare: Callable[..., Any], + unit_uid: str, + name: str, + trial: int, + drawn: dict[str, Any], + rng: Any, + ) -> tuple[dict[str, Any], bool]: + """Hand one trial's draw to the profile; say whether it shaped it. + + The profile sees a copy, so the only way its work reaches the + comparison is by returning it: a hook that edits the draw in place + and returns ``None`` would otherwise be a shaped trial judged under + the unshaped rules, silently. Returned inputs must name exactly the + arguments drawn -- the profile shapes values, it does not rewrite + the interface. + """ + site = _profile_site(unit_uid, name, trial) + offered = {key: _copy_input(value) for key, value in drawn.items()} + try: + shaped = prepare(unit_uid, name, offered, rng) + except Exception as error: + raise InputProfileError(f"{site} raised {type(error).__name__}: {error}") from error + if shaped is None: + edited = sorted(key for key in drawn if not _same_input(np, offered[key], drawn[key])) + if edited: + raise InputProfileError( + f"{site} edited {', '.join(edited)} in place and returned None; " + "return the shaped inputs, or None to leave the draw as it is" + ) + return drawn, False + if not isinstance(shaped, dict): + raise InputProfileError( + f"{site} returned {type(shaped).__name__}; return the inputs by argument " + "name, or None" + ) + if set(shaped) != set(drawn): + missing = sorted(set(drawn) - set(shaped)) + extra = sorted(set(shaped) - set(drawn)) + raise InputProfileError( + f"{site} returned inputs that do not name the arguments drawn" + + (f"; missing {', '.join(missing)}" if missing else "") + + (f"; unknown {', '.join(extra)}" if extra else "") + ) + return shaped, True + @staticmethod def _load_candidate(candidate: Candidate, workspace: Path, suffix: str = "_numpy.py") -> Any: """Write the candidate's files and import its generated module. @@ -1502,5 +1646,23 @@ def _verdict( ) +def _profile_site(unit_uid: str, name: str, trial: int) -> str: + return f"{INPUT_PROFILE}: prepare({unit_uid!r}, {name!r}) at trial {trial}" + + +def _copy_input(value: Any) -> Any: + copy = getattr(value, "copy", None) + return copy() if callable(copy) else value + + +def _same_input(np: Any, offered: Any, drawn: Any) -> bool: + if offered is drawn: + return True + try: + return bool(np.array_equal(np.asarray(offered), np.asarray(drawn), equal_nan=True)) + except (TypeError, ValueError): + return False + + def factory(**_config: Any) -> BitexactVerifier: return BitexactVerifier() diff --git a/tests/test_bitexact_draws.py b/tests/test_bitexact_draws.py index 5109ebc..471e25f 100644 --- a/tests/test_bitexact_draws.py +++ b/tests/test_bitexact_draws.py @@ -224,3 +224,190 @@ def test_a_draw_that_needs_no_redrawing_is_the_one_the_seed_names(tmp_path: Path verdict = judge(tmp_path, plain, SimpleNamespace(w_probe=lambda x: x * 2.0)) assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail assert verdict.metrics["subprograms"]["probe"]["redrawn"] == 0 + + +# -- the project's input profile ---------------------------------------------- + + +def profile(tmp_path: Path, body: str) -> Path: + """Write ``recast_inputs.py`` at a project root and return that root.""" + root = tmp_path / "project" + root.mkdir(exist_ok=True) + (root / "recast_inputs.py").write_text(body) + return root + + +PACKED_PROFILE = """\ +import numpy as np + + +def prepare(unit, subprogram, inputs, rng): + assert unit == "draw:m" and subprogram == "probe" + n = int(inputs["n"]) + lr = n * (n + 1) // 2 + inputs["lr"] = np.int32(lr) + inputs["r"] = np.asfortranarray(rng.uniform(-1.0, 1.0, size=lr)) + return inputs +""" + + +def test_a_shaped_draw_is_compared_as_shaped_and_never_redrawn(tmp_path: Path) -> None: + """The packed workspace that fails by name under the generated rules is + compared as drawn once the project says how ``lr`` follows ``n``: no + redraw, nothing moved, and the trials are recorded as shaped.""" + root = profile(tmp_path, PACKED_PROFILE) + verdict = judge( + tmp_path, + PACKED, + SimpleNamespace(w_probe=lambda n, lr, r: float(r[(int(n) * (int(n) + 1)) // 2 - 1])), + root=str(root), + ) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + probe = verdict.metrics["subprograms"]["probe"] + assert (probe["redrawn"], probe["reshaped"], probe["shaped"]) == (0, 0, 10) + assert verdict.metrics["input_profile"] == "recast_inputs.py" + assert verdict.metrics["shaped"] == ["probe"] + + +def test_a_candidate_that_refuses_a_shaped_draw_has_failed(tmp_path: Path) -> None: + """Under the generated rules a translated ERROR STOP is a draw to make + again. Under a profile that fixed ``mode`` to a value the source takes, + the reference answers and the candidate stops: that is the translation + refusing inputs the source accepts, and it fails by name.""" + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['mode'] = np.int32(1)\n" + " return inputs\n", + ) + wrong = MODE.replace("if int(mode) not in (1, 2):", "if int(mode) != 2:") + verdict = judge( + tmp_path, wrong, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root) + ) + assert verdict.confidence is Confidence.FAILED + detail = verdict.detail or "" + assert "probe: candidate raised on shaped inputs the reference took: SystemExit" in detail + assert "redrawn" not in detail + + +def test_shaped_inputs_the_reference_refuses_are_the_profiles_fault(tmp_path: Path) -> None: + """The profile asserts the reference takes the draw, so the reference is + called first. When it refuses, nothing about the candidate is being + judged: the gate stops with the profile, subprogram and trial named.""" + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['mode'] = np.int32(7)\n" + " return inputs\n", + ) + + def w_probe(mode: Any, x: Any) -> Any: + # Standing in for an ERROR STOP the reference would end the process on. + if int(mode) not in (1, 2): + raise ValueError("invalid mode in probe") + return x * 2.0 + + with pytest.raises(InputProfileError) as caught: + judge(tmp_path, MODE, SimpleNamespace(w_probe=w_probe), root=str(root)) + message = str(caught.value) + assert message.startswith("recast_inputs.py: prepare('draw:m', 'probe') at trial 0") + assert "the reference does not take: ValueError: invalid mode in probe" in message + + +def test_a_reference_nan_on_shaped_inputs_is_the_profiles_fault(tmp_path: Path) -> None: + """Both sides going to NaN is a redraw under the generated rules. A + shaped draw is not drawn again, so a reference NaN on it is the profile + having put the source outside its numeric domain.""" + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['x'] = np.float64(-4.0)\n" + " return inputs\n", + ) + + def w_probe(x: Any) -> Any: + with np.errstate(invalid="ignore"): + return np.sqrt(x) + + with pytest.raises(InputProfileError, match="the reference produced NaN in y"): + judge(tmp_path, NAN, SimpleNamespace(w_probe=w_probe), root=str(root)) + + +def test_a_profile_that_returns_none_leaves_the_draw_to_the_generated_rules( + tmp_path: Path, +) -> None: + root = profile( + tmp_path, + "def prepare(unit, subprogram, inputs, rng):\n return None\n", + ) + + def w_probe(mode: Any, x: Any) -> Any: + assert int(mode) in (1, 2), "the reference was called on a refused draw" + return x * 2.0 + + verdict = judge(tmp_path, MODE, SimpleNamespace(w_probe=w_probe), root=str(root)) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + probe = verdict.metrics["subprograms"]["probe"] + assert probe["redrawn"] > 0 and probe["shaped"] == 0 + assert verdict.metrics["input_profile"] == "recast_inputs.py" + assert verdict.metrics["shaped"] == [] + + +def test_a_profile_that_edits_in_place_and_returns_none_is_refused(tmp_path: Path) -> None: + """The profile sees a copy; the only way its work reaches the comparison + is by returning it. Otherwise an edited draw would be judged under the + unshaped rules with nobody told.""" + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['mode'] = np.int32(1)\n", + ) + with pytest.raises(InputProfileError, match="edited mode in place and returned None"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + + +def test_a_profile_that_renames_the_arguments_is_refused(tmp_path: Path) -> None: + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "def prepare(unit, subprogram, inputs, rng):\n" + " return {'mode': inputs['mode'], 'xx': inputs['x']}\n", + ) + with pytest.raises(InputProfileError, match="missing x; unknown xx"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + + +def test_a_profile_that_does_not_import_stops_the_gate(tmp_path: Path) -> None: + """A tree with a profile that cannot be read is not a tree without one.""" + from recast.errors import InputProfileError + + root = profile(tmp_path, "def prepare(unit, subprogram, inputs, rng)\n return None\n") + with pytest.raises(InputProfileError, match=r"recast_inputs\.py does not import: SyntaxError"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + root = profile(tmp_path, "PREPARE = None\n") + with pytest.raises(InputProfileError, match="defines no callable prepare"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + + +def test_a_root_without_a_profile_is_the_generated_path(tmp_path: Path) -> None: + root = tmp_path / "bare" + root.mkdir() + verdict = judge( + tmp_path, + NAN.replace("return np.sqrt(x)", "return x * 2.0"), + SimpleNamespace(w_probe=lambda x: x * 2.0), + root=str(root), + ) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + assert verdict.metrics["input_profile"] is None diff --git a/tests/test_dump_replay.py b/tests/test_dump_replay.py index 75df905..41dcc26 100644 --- a/tests/test_dump_replay.py +++ b/tests/test_dump_replay.py @@ -279,22 +279,27 @@ def test_an_oracle_that_supplies_no_samples_fails_closed(tmp_path: Path) -> None assert "handed over no samples" in verdict.detail -def test_a_candidate_hook_cannot_edit_the_recorded_inputs(tmp_path: Path) -> None: - """``_PREPARE_INPUTS`` shapes *generated* inputs and must not touch these. +def test_the_project_profile_cannot_edit_the_recorded_inputs(tmp_path: Path) -> None: + """``recast_inputs.py`` shapes *generated* inputs and must not touch these. - The hook ships inside the artifact under test. Letting it rewrite the - production run's own numbers before the artifact is judged on them would - let the candidate choose its own exam. + A recording's inputs are the production run's own numbers. Letting + anything rewrite them before the artifact is judged on them would let + the exam be rewritten, so a replay never loads the profile -- not even + one that would refuse to load. """ path = tmp_path / "toy_numpy.py" path.write_text( "import numpy as np\n" f"_SIGNATURES = {SIGNATURES!r}\n" - "def _PREPARE_INPUTS(name, inputs, rng):\n" - " inputs['x'] = inputs['x'] * 0.0\n" "def scale_by_two(n, x):\n" " return x * 2.0\n" ) + root = tmp_path / "project" + root.mkdir() + (root / "recast_inputs.py").write_text( + "def prepare(unit, subprogram, inputs, rng):\n" + " raise AssertionError('the profile ran on a replay')\n" + ) candidate = Candidate( unit="toy", transform="translate.numpy", @@ -308,10 +313,11 @@ def test_a_candidate_hook_cannot_edit_the_recorded_inputs(tmp_path: Path) -> Non ) from recast.verify.bitexact import BitexactVerifier - verdict = BitexactVerifier().verify(_unit(), candidate, ref, tmp_path, _executor(), {}) - # If the hook had run, x would be zeros and the recording's 2/4/6 would - # not be reproduced. - assert verdict.confidence is Confidence.BIT_EXACT + verdict = BitexactVerifier().verify( + _unit(), candidate, ref, tmp_path, _executor(), {"root": str(root)} + ) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + assert verdict.metrics["input_profile"] is None # -- the example, end to end -------------------------------------------------- diff --git a/tests/test_f2py_oracle.py b/tests/test_f2py_oracle.py index 0a348f1..d164233 100644 --- a/tests/test_f2py_oracle.py +++ b/tests/test_f2py_oracle.py @@ -1321,12 +1321,13 @@ def test_wrappers_serve_a_file_of_bare_subprograms() -> None: assert "real(8), intent(inout) :: t(pcols, pver)" in text -def test_the_gate_lets_a_candidate_shape_its_own_inputs(tmp_path: Path) -> None: +def test_the_project_profile_shapes_the_generated_inputs(tmp_path: Path) -> None: """Per-name ranges cannot express structure -- a monotone pressure - column, a consistent thickness field. A candidate may carry - ``_PREPARE_INPUTS`` the way it carries ``_SIGNATURES``; both sides then - receive the same shaped arrays, so it chooses the sampled region without - touching the verdict.""" + column, a consistent thickness field. The project carries a + ``recast_inputs.py`` at its root, and its ``prepare`` shapes every + generated draw before both sides receive it, so it chooses the sampled + region without touching the verdict -- and the candidate, which is the + thing under judgement, has no say in it.""" import numpy as np module = tmp_path / "candidate" @@ -1350,15 +1351,21 @@ def test_the_gate_lets_a_candidate_shape_its_own_inputs(tmp_path: Path) -> None: SEEN = [] -def _PREPARE_INPUTS(name, inputs, rng): - inputs["x"][:] = 2.0 # every trial sees the same shaped input - - def step(x): SEEN.append(float(x[0])) return np.asarray(x) * 3.0 """ ) + root = tmp_path / "project" + root.mkdir() + (root / "recast_inputs.py").write_text( + "import numpy as np\n" + "\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " assert unit == 'fortran:shaped' and subprogram == 'step'\n" + " inputs['x'] = np.full_like(inputs['x'], 2.0) # every trial sees the same input\n" + " return inputs\n" + ) class Truth: @staticmethod @@ -1382,15 +1389,18 @@ def w_step(x): ref, tmp_path / "ws", LocalExecutor(), - {"trials": 3, "dims": {"n": 4}, "ranges": {"x": (100.0, 200.0)}}, + {"root": str(root), "trials": 3, "dims": {"n": 4}, "ranges": {"x": (100.0, 200.0)}}, ) assert verdict.confidence is Confidence.BIT_EXACT + assert verdict.metrics["input_profile"] == "recast_inputs.py" + assert verdict.metrics["shaped"] == ["step"] + assert verdict.metrics["subprograms"]["step"]["shaped"] == 3 staged = tmp_path / "ws" / "candidate" sys.path.insert(0, str(staged)) try: import shaped_numpy - # The hook ran: every trial saw 2.0, not a value from the range. + # The profile ran: every trial saw 2.0, not a value from the range. assert shaped_numpy.SEEN and all(v == 2.0 for v in shaped_numpy.SEEN) finally: sys.path.remove(str(staged)) From f64f09d9e6b99627861f1043b1b90455a581ad59 Mon Sep 17 00:00:00 2001 From: 2st Date: Fri, 4 Sep 2026 20:48:21 -0600 Subject: [PATCH 2/3] Read recast_inputs.py without writing bytecode into the project root The profile was loaded through a SourceFileLoader, which cached its bytecode as __pycache__/recast_inputs.cpython-311.pyc beside it. The root is a checkout whose cleanliness is checked -- the training lab refuses to onboard a dirty tree, and its prepare-inputs command refused to commit the profile because the engine's own self-check had littered the worktree. Compile and run the source by hand into a fresh module namespace instead. (cherry picked from commit 21367454aa6aa13832782c16e7fc3fc2e6e534c2) Signed-off-by: 2st --- src/recast/verify/bitexact.py | 18 +++++++++++++----- tests/test_bitexact_draws.py | 3 +++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/recast/verify/bitexact.py b/src/recast/verify/bitexact.py index d6e1c24..e8fd2de 100644 --- a/src/recast/verify/bitexact.py +++ b/src/recast/verify/bitexact.py @@ -38,6 +38,7 @@ import operator import re import sys +import types from collections.abc import Callable from pathlib import Path from typing import Any @@ -1526,12 +1527,19 @@ def _load_input_profile(config: dict[str, Any]) -> Callable[..., Any] | None: path = Path(root) / INPUT_PROFILE if not path.is_file(): return None - spec = importlib.util.spec_from_file_location("recast_inputs", path) - if spec is None or spec.loader is None: - raise InputProfileError(f"{INPUT_PROFILE} could not be loaded as a module") - module = importlib.util.module_from_spec(spec) + # Compiled and run by hand rather than through the import machinery: + # a SourceFileLoader would drop ``__pycache__/`` into the project + # root, and the root is a checkout whose cleanliness is checked. + module = types.ModuleType("recast_inputs") + module.__file__ = str(path) try: - spec.loader.exec_module(module) + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as error: + raise InputProfileError( + f"{INPUT_PROFILE} could not be read: {type(error).__name__}: {error}" + ) from error + try: + exec(compile(source, str(path), "exec"), module.__dict__) # noqa: S102 except (Exception, SystemExit) as error: raise InputProfileError( f"{INPUT_PROFILE} does not import: {type(error).__name__}: {error}" diff --git a/tests/test_bitexact_draws.py b/tests/test_bitexact_draws.py index 471e25f..8c52c22 100644 --- a/tests/test_bitexact_draws.py +++ b/tests/test_bitexact_draws.py @@ -267,6 +267,9 @@ def test_a_shaped_draw_is_compared_as_shaped_and_never_redrawn(tmp_path: Path) - assert (probe["redrawn"], probe["reshaped"], probe["shaped"]) == (0, 0, 10) assert verdict.metrics["input_profile"] == "recast_inputs.py" assert verdict.metrics["shaped"] == ["probe"] + # The root is a checkout whose cleanliness is checked: reading the + # profile must not drop bytecode into it. + assert sorted(entry.name for entry in root.iterdir()) == ["recast_inputs.py"] def test_a_candidate_that_refuses_a_shaped_draw_has_failed(tmp_path: Path) -> None: From d54c7c8599935a549df63122b3100896bb58090a Mon Sep 17 00:00:00 2001 From: 2st Date: Fri, 4 Sep 2026 20:48:21 -0600 Subject: [PATCH 3/3] Let a refused input profile end the run rather than fail the unit The verifier catch that turns a plugin crash into a unit failure also swallowed InputProfileError -- the project's recast_inputs.py shaping a draw the reference refuses. That is the profile's fault, not the unit's: recorded as a failed verdict it reads as a translation defect and sends a repair agent after the engine for a mistake in the corpus checkout. The error now escapes the walk as every other operator error does, through the unit and run "aborted" events. Signed-off-by: 2st --- src/recast/run.py | 8 +++++++- tests/test_run.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/recast/run.py b/src/recast/run.py index 2e97fa8..a5858bc 100644 --- a/src/recast/run.py +++ b/src/recast/run.py @@ -38,7 +38,7 @@ from typing import Any from recast import OUTPUT_DIRNAME, WORKSPACE_DIRNAME, __version__ -from recast.errors import ConfigError, RecastError, ScannerUnavailable +from recast.errors import ConfigError, InputProfileError, RecastError, ScannerUnavailable from recast.model import ( Candidate, Disclosure, @@ -1094,6 +1094,12 @@ def _walk_stage( verdict = verifier.verify( unit, unit_run.candidate, unit_run.oracle, workspace, executor, config ) + except InputProfileError: + # The project's recast_inputs.py shaped a draw the reference + # refused. That is the profile's fault, not this unit's: handed + # down as a unit failure it would read as a translation defect + # and send a repair agent after the engine. The walk ends on it. + raise except Exception as error: # Fail closed: a verifier that crashed has not compared anything, # and the unit fails on that rather than the walk ending here. diff --git a/tests/test_run.py b/tests/test_run.py index 9e2a5dd..27c5404 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -16,7 +16,7 @@ import pytest from recast import WORKSPACE_DIRNAME -from recast.errors import ConfigError, ScannerUnavailable +from recast.errors import ConfigError, InputProfileError, ScannerUnavailable from recast.model import ( Access, Candidate, @@ -140,6 +140,13 @@ def verify(self, unit, candidate, oracle, workspace, executor, config) -> Verdic raise RuntimeError("unexpected verifier bug") +class ProfileRefusedVerifier(PassVerifier): + name = "fake.verify-profile-refused" + + def verify(self, unit, candidate, oracle, workspace, executor, config) -> Verdict: + raise InputProfileError("recast_inputs.py: reference refused alpha[x]") + + class NeverOracle(FakeOracle): name = "fake-oracle.never" @@ -202,6 +209,7 @@ def _registry() -> Registry: registry.register("verifier", "fake.pass", PassVerifier) registry.register("verifier", "fake.fail", FailVerifier) registry.register("verifier", "fake.verify-explode", ExplodingVerifier) + registry.register("verifier", "fake.verify-profile-refused", ProfileRefusedVerifier) registry.register("store", "fake-store", MemoryStore) # Entry-point discovery would add the real plugins; a fake registry must # not, so mark every kind as already discovered. @@ -524,6 +532,31 @@ def test_a_verifier_bug_fails_closed_and_the_walk_goes_on(tmp_path: Path) -> Non assert (RunEventEntity.VERDICT, "failed", "verification_exception") in finished +def test_a_refused_input_profile_ends_the_run_instead_of_failing_the_unit( + tmp_path: Path, +) -> None: + """recast_inputs.py shaping a draw the reference refuses is the profile's + fault, not the unit's: recorded as a unit failure it would read as a + translation defect and send a repair agent after the engine.""" + observer = RecordingObserver() + with pytest.raises(InputProfileError, match="reference refused"): + run_recipe( + FakeRecipe(_stages(Stage("verifier", "fake.verify-profile-refused", gate=True))), + tmp_path, + {"units": ["fake:alpha"]}, + registry=_registry(), + observer=observer, + ) + finished = [ + (event.entity, event.status, event.reason_code) + for event in observer.events + if event.action is RunEventAction.FINISHED + ] + assert (RunEventEntity.VERDICT, "failed", "verification_exception") not in finished + assert (RunEventEntity.UNIT, "aborted", "unit_exception") in finished + assert finished[-1] == (RunEventEntity.RUN, "aborted", "run_exception") + + def test_an_oracle_that_does_not_apply_leaves_the_unit_incomplete(tmp_path: Path) -> None: """No reference for this unit's language is neither a failed reference nor a pass: the unit is incomplete, the gate never runs on nothing."""