diff --git a/docs/roadmap.md b/docs/roadmap.md index 2adda5e..b24f6e4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -551,10 +551,10 @@ byte-identical. Three consequences are worth naming because none is obvious: * **`trials` does not apply.** A recording holds the points it holds; asking for ten against a three-sample recording would be seven invented ones or seven copies. -* **`_PREPARE_INPUTS` is skipped.** The hook exists to drag *generated* inputs - into the physical domain, and recorded ones are already there. It also ships - inside the artifact under test, so running it on a replay would let the - candidate edit the production run's own numbers before being judged on them. +* **`recast_inputs.py` is skipped.** The project's input profile exists to + drag *generated* inputs into the physical domain, and recorded ones are + already there. Running it on a replay would let the production run's own + numbers be edited before the artifact is judged on them. * **Reference-side `setup` is skipped.** A replayed reference has no state to set: whatever the run's module state was is folded into what it recorded. An operator whose `setup` does not match the run's own initialization gets a diff --git a/src/recast/errors.py b/src/recast/errors.py index 8318e4a..482d1c0 100644 --- a/src/recast/errors.py +++ b/src/recast/errors.py @@ -36,6 +36,17 @@ class OracleUnavailable(RecastError): """The reference could not be materialized. Verdicts must be FAILED, not skipped.""" +class InputProfileError(RecastError): + """The project's ``recast_inputs.py`` shaped inputs the reference does not take. + + A shaped draw is the project asserting that these are inputs the source + accepts. When the reference cannot run them -- it raises, or computes a + NaN -- the assertion is what is wrong, not the translation, so the + verification stops here instead of charging the candidate with it or + drawing again. Fix the profile; nothing about the engine is being judged. + """ + + class ScannerUnavailable(RecastError): """A Scanner or Adjudicator could not run at all. The stage is INCOMPLETE. diff --git a/src/recast/run.py b/src/recast/run.py index 2e97fa8..a5858bc 100644 --- a/src/recast/run.py +++ b/src/recast/run.py @@ -38,7 +38,7 @@ from typing import Any from recast import OUTPUT_DIRNAME, WORKSPACE_DIRNAME, __version__ -from recast.errors import ConfigError, RecastError, ScannerUnavailable +from recast.errors import ConfigError, InputProfileError, RecastError, ScannerUnavailable from recast.model import ( Candidate, Disclosure, @@ -1094,6 +1094,12 @@ def _walk_stage( verdict = verifier.verify( unit, unit_run.candidate, unit_run.oracle, workspace, executor, config ) + except InputProfileError: + # The project's recast_inputs.py shaped a draw the reference + # refused. That is the profile's fault, not this unit's: handed + # down as a unit failure it would read as a translation defect + # and send a repair agent after the engine. The walk ends on it. + raise except Exception as error: # Fail closed: a verifier that crashed has not compared anything, # and the unit fails on that rather than the walk ending here. diff --git a/src/recast/verify/bitexact.py b/src/recast/verify/bitexact.py index 41e4187..e8fd2de 100644 --- a/src/recast/verify/bitexact.py +++ b/src/recast/verify/bitexact.py @@ -22,9 +22,13 @@ table, values from per-name ``ranges``. The physical ranges that make a model kernels behave -- temperatures in kelvin, pressures in pascals -- are domain knowledge and arrive in config; the engine's defaults are only wide, not -wise. Subprograms with deferred blocks are skipped and said so: their -translation raises ``NotImplementedError`` by construction, and the gate's -job is to judge translations, not queues. +wise. Structure that no per-name range can express -- a packed workspace +whose extent is ``n(n+1)/2``, a mode the source stops on, a column that must +be monotone -- comes from the project itself: a ``recast_inputs.py`` at the +root, whose ``prepare(unit, subprogram, inputs, rng)`` shapes each generated +draw before both sides receive it. Subprograms with deferred blocks are +skipped and said so: their translation raises ``NotImplementedError`` by +construction, and the gate's job is to judge translations, not queues. """ from __future__ import annotations @@ -34,10 +38,12 @@ import operator import re import sys +import types from collections.abc import Callable from pathlib import Path from typing import Any +from recast.errors import InputProfileError from recast.model import Candidate, Confidence, OracleRef, Unit, Verdict from recast.plugins.executor import Executor from recast.plugins.verifier import Verifier @@ -50,6 +56,16 @@ DEFAULT_DIMENSION = 8 SUPPORTED_DTYPES = frozenset({"float32", "float64", "int32", "int64", "bool"}) PROCEDURE_DTYPE = "PROCEDURE" +INPUT_PROFILE = "recast_inputs.py" +"""The project's input profile, looked for at the root the run was given. + +Its ``prepare(unit, subprogram, inputs, rng)`` receives the inputs this +harness drew for one trial, by argument name, and returns them shaped into +the source's domain -- or ``None`` to leave that subprogram's draw as it is. +A shaped draw is an assertion that the reference takes it, and is judged as +one: it is never redrawn, a candidate that refuses it has failed, and a +reference that refuses it means the profile is wrong. +""" """What a dummy *procedure* argument is declared as. Not a value the harness can sample: it is something to call, and what both @@ -254,6 +270,13 @@ class BitexactVerifier(Verifier): evidence at the extents it landed on, not the configured ones: a subprogram that passes mostly that way fails by name, with the number of such trials recorded as ``reshaped``. + + None of this applies to a draw the project's ``recast_inputs.py`` shaped. + That draw is the project saying the source takes it, so there is nothing + to draw again: the reference is called first, and if it refuses, the + profile is wrong and the verification stops there + (``InputProfileError``); if it answers and the candidate refuses, the + candidate has failed on inputs the source takes. """ dominant_at: float | None = None @@ -352,6 +375,8 @@ def _compare_all( candidate, Confidence.FAILED, {}, f"candidate does not import: {error}" ) + profile = None if recorded else self._load_input_profile(config) + table = getattr(translated, "_SIGNATURES", None) if not isinstance(table, dict) or not table: return self._verdict( @@ -466,7 +491,8 @@ def generable(name: str) -> bool: trials, dims, ranges, - prepare=getattr(translated, "_PREPARE_INPUTS", None), + profile=profile, + unit_uid=unit.uid, dominant_at=config.get("dominant_at", self.dominant_at), dominant_axis=config.get("dominant_axis", -1), rel_scale=str(config.get("rel_scale", "element")), @@ -529,6 +555,8 @@ def generable(name: str) -> bool: "trials": trials, "skipped": skipped, "uncovered": uncovered, + "input_profile": INPUT_PROFILE if profile is not None else None, + "shaped": sorted(name for name, out in per_subprogram.items() if out.get("shaped")), "max_rel": worst_rel, **({"ungated": declared} if declared else {}), **totals, @@ -662,7 +690,8 @@ def _compare_subprogram( trials: int, dims: dict[str, int], ranges: dict[str, tuple[float, float]], - prepare: Any = None, + profile: Any = None, + unit_uid: str = "", dominant_at: float | None = None, dominant_axis: Any = -1, rel_scale: str = "element", @@ -741,6 +770,8 @@ def _compare_subprogram( # Trials that were compared only after a shape refusal moved the free # extents off the configured ones. reshaped = 0 + # Trials the project's input profile shaped. Those are never redrawn. + shaped_trials = 0 # Extents no operator pinned. A redraw varies these along with the # values, because some of what a subprogram will not accept is a # *shape*: a packed triangular workspace wants an extent that is a @@ -838,24 +869,19 @@ def _compare_subprogram( f"{', '.join(missing_outputs)}; partial output evidence is not a pass" } - if prepare is not None and samples is None: - # The candidate may carry a ``_PREPARE_INPUTS(name, inputs, - # rng)`` hook, the way it carries ``_SIGNATURES``: per-name - # ranges cannot express structure -- a pressure column must - # be monotone, an interface field must bracket its levels -- - # and unphysical inputs drive both sides into error paths - # the production model aborts out of. The hook shapes inputs - # into the defined domain; it cannot bias the verdict, - # because both sides receive the same shaped inputs. - # - # It is skipped for a replay, and that is the important half: - # the hook exists to drag *generated* inputs into the physical - # domain, and recorded inputs are already there by - # construction. Running it would let the candidate edit the - # production run's own numbers before being judged on them, - # which is the one thing a hook supplied by the artifact under - # test must never be able to do. - prepare(name, inputs, rng) + shaped = False + if profile is not None and samples is None: + # The project's profile shapes *generated* inputs only. A + # recording's inputs are already in the domain by + # construction, and letting anything edit the production + # run's own numbers before the artifact is judged on them + # would let the exam be rewritten. + inputs, shaped = self._shape_inputs( + np, profile, unit_uid, name, round_index, inputs, rng + ) + if shaped: + shaped_trials += 1 + site = _profile_site(unit_uid, name, round_index) # Keyword calls on both sides: f2py reorders inferred-dimension # scalars into trailing keywords, so positional order is not a @@ -881,39 +907,67 @@ def _compare_subprogram( if a["intent"] != "OUT" } except Exception as error: + if shaped: + raise InputProfileError( + f"{site} returned inputs the reference does not take: " + f"{type(error).__name__}: {error}" + ) from error return { "error": f"oracle input preparation failed: {type(error).__name__}: {error}" } truth_args = [ truth_kwargs[spell(a["name"])] for a in required if a["intent"] != "OUT" ] - try: - translated_out = translated_fn(**translated_kwargs) - except (SystemExit, IndexError) as error: - # Not a comparison that failed -- a draw the subprogram does - # not take. ``SystemExit`` is a translated ERROR STOP: the - # source itself saying these arguments are not its own, and - # the reference would say the same by ending the process, - # taking every other unit's verdict with it. ``IndexError`` is - # a subscript past a dummy array's declared extent, where the - # reference, compiled without bounds checking, reads memory - # the call does not own. Either way the reference must not be - # called on this draw; draw again. - declined = f"candidate raised: {type(error).__name__}: {error}" - reshape = reshape or isinstance(error, IndexError) - redrawn += 1 - continue - except Exception as error: - return {"error": f"candidate raised: {type(error).__name__}: {error}"} - if samples is not None: - # Nothing to call: the reference already ran, in production, - # and what it produced is the recording. - truth_out = recorded_outputs - else: + if shaped: + # The profile asserts the reference takes this draw, so the + # reference goes first and decides. Refused there, the + # assertion is what is wrong and nothing about the + # candidate is being judged; taken there and refused by the + # candidate, the candidate has failed on inputs the source + # accepts -- not a draw to make again. try: truth_out = truth_fn(**truth_kwargs) except Exception as error: - return {"error": f"oracle raised: {type(error).__name__}: {error}"} + raise InputProfileError( + f"{site} returned inputs the reference does not take: " + f"{type(error).__name__}: {error}" + ) from error + try: + translated_out = translated_fn(**translated_kwargs) + except (Exception, SystemExit) as error: + return { + "error": "candidate raised on shaped inputs the reference took: " + f"{type(error).__name__}: {error}" + } + else: + try: + translated_out = translated_fn(**translated_kwargs) + except (SystemExit, IndexError) as error: + # Not a comparison that failed -- a draw the subprogram + # does not take. ``SystemExit`` is a translated ERROR + # STOP: the source itself saying these arguments are not + # its own, and the reference would say the same by + # ending the process, taking every other unit's verdict + # with it. ``IndexError`` is a subscript past a dummy + # array's declared extent, where the reference, compiled + # without bounds checking, reads memory the call does + # not own. Either way the reference must not be called + # on this draw; draw again. + declined = f"candidate raised: {type(error).__name__}: {error}" + reshape = reshape or isinstance(error, IndexError) + redrawn += 1 + continue + except Exception as error: + return {"error": f"candidate raised: {type(error).__name__}: {error}"} + if samples is not None: + # Nothing to call: the reference already ran, in + # production, and what it produced is the recording. + truth_out = recorded_outputs + else: + try: + truth_out = truth_fn(**truth_kwargs) + except Exception as error: + return {"error": f"oracle raised: {type(error).__name__}: {error}"} pairs = self._paired_outputs( sub, @@ -987,6 +1041,13 @@ def _compare_subprogram( if samples is None: nan_ours = np.isnan(a) nan_theirs = np.isnan(b) + if shaped and nan_theirs.any(): + raise InputProfileError( + f"{site} returned inputs on which the reference produced " + f"NaN in {label}; a NaN-tainted trial compares the compiler's " + "scheduling rather than the translation, and a shaped draw " + "is not drawn again" + ) if nan_ours.any() and bool((nan_ours == nan_theirs).all()): # A draw that put the subprogram outside its numeric # domain on *both* sides: a square root of a negative, @@ -1074,6 +1135,7 @@ def _compare_subprogram( "integer_mismatch": integer_mismatch, "redrawn": redrawn, "reshaped": reshaped, + "shaped": shaped_trials, } if dominant_at is not None: outcome["max_ulp_dominant"] = max_ulp_dominant @@ -1091,7 +1153,7 @@ def _devices(translated: Any, handle: dict[str, Any]) -> dict[str, str]: A verdict that does not record it cannot be re-argued later. Asked for rather than detected, and by the same convention as - ``_SIGNATURES`` and ``_PREPARE_INPUTS``: the emitted module declares + ``_SIGNATURES``: the emitted module declares ``_DEVICE`` if it knows, and an Oracle puts one on its handle. Reaching for ``jax.devices()`` here instead would put an accelerator import in the core, which is the one thing the core does not do. @@ -1447,6 +1509,96 @@ def _value( # -- loading -------------------------------------------------------------- + @staticmethod + def _load_input_profile(config: dict[str, Any]) -> Callable[..., Any] | None: + """The project's ``recast_inputs.py``, when the tree carries one. + + Looked for at the root the run was given -- the same root the + sources were read from, so the profile is part of the source + artifact and travels, and is digested, with it. A tree without one + is the generated path unchanged. A tree with one that does not + import, or defines no ``prepare``, is a project whose assertion + about its own inputs cannot be read, and that stops the gate rather + than being taken as "no profile". + """ + root = config.get("root") + if root is None: + return None + path = Path(root) / INPUT_PROFILE + if not path.is_file(): + return None + # Compiled and run by hand rather than through the import machinery: + # a SourceFileLoader would drop ``__pycache__/`` into the project + # root, and the root is a checkout whose cleanliness is checked. + module = types.ModuleType("recast_inputs") + module.__file__ = str(path) + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as error: + raise InputProfileError( + f"{INPUT_PROFILE} could not be read: {type(error).__name__}: {error}" + ) from error + try: + exec(compile(source, str(path), "exec"), module.__dict__) # noqa: S102 + except (Exception, SystemExit) as error: + raise InputProfileError( + f"{INPUT_PROFILE} does not import: {type(error).__name__}: {error}" + ) from error + prepare: Callable[..., Any] | None = getattr(module, "prepare", None) + if not callable(prepare): + raise InputProfileError( + f"{INPUT_PROFILE} defines no callable prepare(unit, subprogram, inputs, rng)" + ) + return prepare + + @staticmethod + def _shape_inputs( + np: Any, + prepare: Callable[..., Any], + unit_uid: str, + name: str, + trial: int, + drawn: dict[str, Any], + rng: Any, + ) -> tuple[dict[str, Any], bool]: + """Hand one trial's draw to the profile; say whether it shaped it. + + The profile sees a copy, so the only way its work reaches the + comparison is by returning it: a hook that edits the draw in place + and returns ``None`` would otherwise be a shaped trial judged under + the unshaped rules, silently. Returned inputs must name exactly the + arguments drawn -- the profile shapes values, it does not rewrite + the interface. + """ + site = _profile_site(unit_uid, name, trial) + offered = {key: _copy_input(value) for key, value in drawn.items()} + try: + shaped = prepare(unit_uid, name, offered, rng) + except Exception as error: + raise InputProfileError(f"{site} raised {type(error).__name__}: {error}") from error + if shaped is None: + edited = sorted(key for key in drawn if not _same_input(np, offered[key], drawn[key])) + if edited: + raise InputProfileError( + f"{site} edited {', '.join(edited)} in place and returned None; " + "return the shaped inputs, or None to leave the draw as it is" + ) + return drawn, False + if not isinstance(shaped, dict): + raise InputProfileError( + f"{site} returned {type(shaped).__name__}; return the inputs by argument " + "name, or None" + ) + if set(shaped) != set(drawn): + missing = sorted(set(drawn) - set(shaped)) + extra = sorted(set(shaped) - set(drawn)) + raise InputProfileError( + f"{site} returned inputs that do not name the arguments drawn" + + (f"; missing {', '.join(missing)}" if missing else "") + + (f"; unknown {', '.join(extra)}" if extra else "") + ) + return shaped, True + @staticmethod def _load_candidate(candidate: Candidate, workspace: Path, suffix: str = "_numpy.py") -> Any: """Write the candidate's files and import its generated module. @@ -1502,5 +1654,23 @@ def _verdict( ) +def _profile_site(unit_uid: str, name: str, trial: int) -> str: + return f"{INPUT_PROFILE}: prepare({unit_uid!r}, {name!r}) at trial {trial}" + + +def _copy_input(value: Any) -> Any: + copy = getattr(value, "copy", None) + return copy() if callable(copy) else value + + +def _same_input(np: Any, offered: Any, drawn: Any) -> bool: + if offered is drawn: + return True + try: + return bool(np.array_equal(np.asarray(offered), np.asarray(drawn), equal_nan=True)) + except (TypeError, ValueError): + return False + + def factory(**_config: Any) -> BitexactVerifier: return BitexactVerifier() diff --git a/tests/test_bitexact_draws.py b/tests/test_bitexact_draws.py index 5109ebc..8c52c22 100644 --- a/tests/test_bitexact_draws.py +++ b/tests/test_bitexact_draws.py @@ -224,3 +224,193 @@ def test_a_draw_that_needs_no_redrawing_is_the_one_the_seed_names(tmp_path: Path verdict = judge(tmp_path, plain, SimpleNamespace(w_probe=lambda x: x * 2.0)) assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail assert verdict.metrics["subprograms"]["probe"]["redrawn"] == 0 + + +# -- the project's input profile ---------------------------------------------- + + +def profile(tmp_path: Path, body: str) -> Path: + """Write ``recast_inputs.py`` at a project root and return that root.""" + root = tmp_path / "project" + root.mkdir(exist_ok=True) + (root / "recast_inputs.py").write_text(body) + return root + + +PACKED_PROFILE = """\ +import numpy as np + + +def prepare(unit, subprogram, inputs, rng): + assert unit == "draw:m" and subprogram == "probe" + n = int(inputs["n"]) + lr = n * (n + 1) // 2 + inputs["lr"] = np.int32(lr) + inputs["r"] = np.asfortranarray(rng.uniform(-1.0, 1.0, size=lr)) + return inputs +""" + + +def test_a_shaped_draw_is_compared_as_shaped_and_never_redrawn(tmp_path: Path) -> None: + """The packed workspace that fails by name under the generated rules is + compared as drawn once the project says how ``lr`` follows ``n``: no + redraw, nothing moved, and the trials are recorded as shaped.""" + root = profile(tmp_path, PACKED_PROFILE) + verdict = judge( + tmp_path, + PACKED, + SimpleNamespace(w_probe=lambda n, lr, r: float(r[(int(n) * (int(n) + 1)) // 2 - 1])), + root=str(root), + ) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + probe = verdict.metrics["subprograms"]["probe"] + assert (probe["redrawn"], probe["reshaped"], probe["shaped"]) == (0, 0, 10) + assert verdict.metrics["input_profile"] == "recast_inputs.py" + assert verdict.metrics["shaped"] == ["probe"] + # The root is a checkout whose cleanliness is checked: reading the + # profile must not drop bytecode into it. + assert sorted(entry.name for entry in root.iterdir()) == ["recast_inputs.py"] + + +def test_a_candidate_that_refuses_a_shaped_draw_has_failed(tmp_path: Path) -> None: + """Under the generated rules a translated ERROR STOP is a draw to make + again. Under a profile that fixed ``mode`` to a value the source takes, + the reference answers and the candidate stops: that is the translation + refusing inputs the source accepts, and it fails by name.""" + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['mode'] = np.int32(1)\n" + " return inputs\n", + ) + wrong = MODE.replace("if int(mode) not in (1, 2):", "if int(mode) != 2:") + verdict = judge( + tmp_path, wrong, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root) + ) + assert verdict.confidence is Confidence.FAILED + detail = verdict.detail or "" + assert "probe: candidate raised on shaped inputs the reference took: SystemExit" in detail + assert "redrawn" not in detail + + +def test_shaped_inputs_the_reference_refuses_are_the_profiles_fault(tmp_path: Path) -> None: + """The profile asserts the reference takes the draw, so the reference is + called first. When it refuses, nothing about the candidate is being + judged: the gate stops with the profile, subprogram and trial named.""" + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['mode'] = np.int32(7)\n" + " return inputs\n", + ) + + def w_probe(mode: Any, x: Any) -> Any: + # Standing in for an ERROR STOP the reference would end the process on. + if int(mode) not in (1, 2): + raise ValueError("invalid mode in probe") + return x * 2.0 + + with pytest.raises(InputProfileError) as caught: + judge(tmp_path, MODE, SimpleNamespace(w_probe=w_probe), root=str(root)) + message = str(caught.value) + assert message.startswith("recast_inputs.py: prepare('draw:m', 'probe') at trial 0") + assert "the reference does not take: ValueError: invalid mode in probe" in message + + +def test_a_reference_nan_on_shaped_inputs_is_the_profiles_fault(tmp_path: Path) -> None: + """Both sides going to NaN is a redraw under the generated rules. A + shaped draw is not drawn again, so a reference NaN on it is the profile + having put the source outside its numeric domain.""" + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['x'] = np.float64(-4.0)\n" + " return inputs\n", + ) + + def w_probe(x: Any) -> Any: + with np.errstate(invalid="ignore"): + return np.sqrt(x) + + with pytest.raises(InputProfileError, match="the reference produced NaN in y"): + judge(tmp_path, NAN, SimpleNamespace(w_probe=w_probe), root=str(root)) + + +def test_a_profile_that_returns_none_leaves_the_draw_to_the_generated_rules( + tmp_path: Path, +) -> None: + root = profile( + tmp_path, + "def prepare(unit, subprogram, inputs, rng):\n return None\n", + ) + + def w_probe(mode: Any, x: Any) -> Any: + assert int(mode) in (1, 2), "the reference was called on a refused draw" + return x * 2.0 + + verdict = judge(tmp_path, MODE, SimpleNamespace(w_probe=w_probe), root=str(root)) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + probe = verdict.metrics["subprograms"]["probe"] + assert probe["redrawn"] > 0 and probe["shaped"] == 0 + assert verdict.metrics["input_profile"] == "recast_inputs.py" + assert verdict.metrics["shaped"] == [] + + +def test_a_profile_that_edits_in_place_and_returns_none_is_refused(tmp_path: Path) -> None: + """The profile sees a copy; the only way its work reaches the comparison + is by returning it. Otherwise an edited draw would be judged under the + unshaped rules with nobody told.""" + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "import numpy as np\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " inputs['mode'] = np.int32(1)\n", + ) + with pytest.raises(InputProfileError, match="edited mode in place and returned None"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + + +def test_a_profile_that_renames_the_arguments_is_refused(tmp_path: Path) -> None: + from recast.errors import InputProfileError + + root = profile( + tmp_path, + "def prepare(unit, subprogram, inputs, rng):\n" + " return {'mode': inputs['mode'], 'xx': inputs['x']}\n", + ) + with pytest.raises(InputProfileError, match="missing x; unknown xx"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + + +def test_a_profile_that_does_not_import_stops_the_gate(tmp_path: Path) -> None: + """A tree with a profile that cannot be read is not a tree without one.""" + from recast.errors import InputProfileError + + root = profile(tmp_path, "def prepare(unit, subprogram, inputs, rng)\n return None\n") + with pytest.raises(InputProfileError, match=r"recast_inputs\.py does not import: SyntaxError"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + root = profile(tmp_path, "PREPARE = None\n") + with pytest.raises(InputProfileError, match="defines no callable prepare"): + judge(tmp_path, MODE, SimpleNamespace(w_probe=lambda mode, x: x * 2.0), root=str(root)) + + +def test_a_root_without_a_profile_is_the_generated_path(tmp_path: Path) -> None: + root = tmp_path / "bare" + root.mkdir() + verdict = judge( + tmp_path, + NAN.replace("return np.sqrt(x)", "return x * 2.0"), + SimpleNamespace(w_probe=lambda x: x * 2.0), + root=str(root), + ) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + assert verdict.metrics["input_profile"] is None diff --git a/tests/test_dump_replay.py b/tests/test_dump_replay.py index 75df905..41dcc26 100644 --- a/tests/test_dump_replay.py +++ b/tests/test_dump_replay.py @@ -279,22 +279,27 @@ def test_an_oracle_that_supplies_no_samples_fails_closed(tmp_path: Path) -> None assert "handed over no samples" in verdict.detail -def test_a_candidate_hook_cannot_edit_the_recorded_inputs(tmp_path: Path) -> None: - """``_PREPARE_INPUTS`` shapes *generated* inputs and must not touch these. +def test_the_project_profile_cannot_edit_the_recorded_inputs(tmp_path: Path) -> None: + """``recast_inputs.py`` shapes *generated* inputs and must not touch these. - The hook ships inside the artifact under test. Letting it rewrite the - production run's own numbers before the artifact is judged on them would - let the candidate choose its own exam. + A recording's inputs are the production run's own numbers. Letting + anything rewrite them before the artifact is judged on them would let + the exam be rewritten, so a replay never loads the profile -- not even + one that would refuse to load. """ path = tmp_path / "toy_numpy.py" path.write_text( "import numpy as np\n" f"_SIGNATURES = {SIGNATURES!r}\n" - "def _PREPARE_INPUTS(name, inputs, rng):\n" - " inputs['x'] = inputs['x'] * 0.0\n" "def scale_by_two(n, x):\n" " return x * 2.0\n" ) + root = tmp_path / "project" + root.mkdir() + (root / "recast_inputs.py").write_text( + "def prepare(unit, subprogram, inputs, rng):\n" + " raise AssertionError('the profile ran on a replay')\n" + ) candidate = Candidate( unit="toy", transform="translate.numpy", @@ -308,10 +313,11 @@ def test_a_candidate_hook_cannot_edit_the_recorded_inputs(tmp_path: Path) -> Non ) from recast.verify.bitexact import BitexactVerifier - verdict = BitexactVerifier().verify(_unit(), candidate, ref, tmp_path, _executor(), {}) - # If the hook had run, x would be zeros and the recording's 2/4/6 would - # not be reproduced. - assert verdict.confidence is Confidence.BIT_EXACT + verdict = BitexactVerifier().verify( + _unit(), candidate, ref, tmp_path, _executor(), {"root": str(root)} + ) + assert verdict.confidence is Confidence.BIT_EXACT, verdict.detail + assert verdict.metrics["input_profile"] is None # -- the example, end to end -------------------------------------------------- diff --git a/tests/test_f2py_oracle.py b/tests/test_f2py_oracle.py index 0a348f1..d164233 100644 --- a/tests/test_f2py_oracle.py +++ b/tests/test_f2py_oracle.py @@ -1321,12 +1321,13 @@ def test_wrappers_serve_a_file_of_bare_subprograms() -> None: assert "real(8), intent(inout) :: t(pcols, pver)" in text -def test_the_gate_lets_a_candidate_shape_its_own_inputs(tmp_path: Path) -> None: +def test_the_project_profile_shapes_the_generated_inputs(tmp_path: Path) -> None: """Per-name ranges cannot express structure -- a monotone pressure - column, a consistent thickness field. A candidate may carry - ``_PREPARE_INPUTS`` the way it carries ``_SIGNATURES``; both sides then - receive the same shaped arrays, so it chooses the sampled region without - touching the verdict.""" + column, a consistent thickness field. The project carries a + ``recast_inputs.py`` at its root, and its ``prepare`` shapes every + generated draw before both sides receive it, so it chooses the sampled + region without touching the verdict -- and the candidate, which is the + thing under judgement, has no say in it.""" import numpy as np module = tmp_path / "candidate" @@ -1350,15 +1351,21 @@ def test_the_gate_lets_a_candidate_shape_its_own_inputs(tmp_path: Path) -> None: SEEN = [] -def _PREPARE_INPUTS(name, inputs, rng): - inputs["x"][:] = 2.0 # every trial sees the same shaped input - - def step(x): SEEN.append(float(x[0])) return np.asarray(x) * 3.0 """ ) + root = tmp_path / "project" + root.mkdir() + (root / "recast_inputs.py").write_text( + "import numpy as np\n" + "\n" + "def prepare(unit, subprogram, inputs, rng):\n" + " assert unit == 'fortran:shaped' and subprogram == 'step'\n" + " inputs['x'] = np.full_like(inputs['x'], 2.0) # every trial sees the same input\n" + " return inputs\n" + ) class Truth: @staticmethod @@ -1382,15 +1389,18 @@ def w_step(x): ref, tmp_path / "ws", LocalExecutor(), - {"trials": 3, "dims": {"n": 4}, "ranges": {"x": (100.0, 200.0)}}, + {"root": str(root), "trials": 3, "dims": {"n": 4}, "ranges": {"x": (100.0, 200.0)}}, ) assert verdict.confidence is Confidence.BIT_EXACT + assert verdict.metrics["input_profile"] == "recast_inputs.py" + assert verdict.metrics["shaped"] == ["step"] + assert verdict.metrics["subprograms"]["step"]["shaped"] == 3 staged = tmp_path / "ws" / "candidate" sys.path.insert(0, str(staged)) try: import shaped_numpy - # The hook ran: every trial saw 2.0, not a value from the range. + # The profile ran: every trial saw 2.0, not a value from the range. assert shaped_numpy.SEEN and all(v == 2.0 for v in shaped_numpy.SEEN) finally: sys.path.remove(str(staged)) diff --git a/tests/test_run.py b/tests/test_run.py index 9e2a5dd..27c5404 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -16,7 +16,7 @@ import pytest from recast import WORKSPACE_DIRNAME -from recast.errors import ConfigError, ScannerUnavailable +from recast.errors import ConfigError, InputProfileError, ScannerUnavailable from recast.model import ( Access, Candidate, @@ -140,6 +140,13 @@ def verify(self, unit, candidate, oracle, workspace, executor, config) -> Verdic raise RuntimeError("unexpected verifier bug") +class ProfileRefusedVerifier(PassVerifier): + name = "fake.verify-profile-refused" + + def verify(self, unit, candidate, oracle, workspace, executor, config) -> Verdict: + raise InputProfileError("recast_inputs.py: reference refused alpha[x]") + + class NeverOracle(FakeOracle): name = "fake-oracle.never" @@ -202,6 +209,7 @@ def _registry() -> Registry: registry.register("verifier", "fake.pass", PassVerifier) registry.register("verifier", "fake.fail", FailVerifier) registry.register("verifier", "fake.verify-explode", ExplodingVerifier) + registry.register("verifier", "fake.verify-profile-refused", ProfileRefusedVerifier) registry.register("store", "fake-store", MemoryStore) # Entry-point discovery would add the real plugins; a fake registry must # not, so mark every kind as already discovered. @@ -524,6 +532,31 @@ def test_a_verifier_bug_fails_closed_and_the_walk_goes_on(tmp_path: Path) -> Non assert (RunEventEntity.VERDICT, "failed", "verification_exception") in finished +def test_a_refused_input_profile_ends_the_run_instead_of_failing_the_unit( + tmp_path: Path, +) -> None: + """recast_inputs.py shaping a draw the reference refuses is the profile's + fault, not the unit's: recorded as a unit failure it would read as a + translation defect and send a repair agent after the engine.""" + observer = RecordingObserver() + with pytest.raises(InputProfileError, match="reference refused"): + run_recipe( + FakeRecipe(_stages(Stage("verifier", "fake.verify-profile-refused", gate=True))), + tmp_path, + {"units": ["fake:alpha"]}, + registry=_registry(), + observer=observer, + ) + finished = [ + (event.entity, event.status, event.reason_code) + for event in observer.events + if event.action is RunEventAction.FINISHED + ] + assert (RunEventEntity.VERDICT, "failed", "verification_exception") not in finished + assert (RunEventEntity.UNIT, "aborted", "unit_exception") in finished + assert finished[-1] == (RunEventEntity.RUN, "aborted", "run_exception") + + def test_an_oracle_that_does_not_apply_leaves_the_unit_incomplete(tmp_path: Path) -> None: """No reference for this unit's language is neither a failed reference nor a pass: the unit is incomplete, the gate never runs on nothing."""