diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5d11c24..69c5a63e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,6 +213,25 @@ jobs: # subprocess from other jobs. The literal below is pinned in claims.yaml # against the same number in scripts/repro/ORACLE_WIRING.md, so the gate # and the doc cannot drift apart. + # RQ-56-CITE / #911 — the SAME shape as the oracle-wiring gate below, on + # the artifacts surface, which never had one. `cargo test -- ` + # exits 0 when the filter matches NOTHING, so a rivet artifact could claim + # verification by a test that does not exist. v0.55.0 shipped two such + # citations; a post-release sweep found them, no gate did. + # + # Scoped to CLAIMING statuses (implemented/verified/accepted): under + # draft/proposed a forward-looking citation is a plan, and flagging it + # would make this gate noisy. A noisy gate gets ignored — that is how + # codecov/patch stopped being read (#923). + - name: Artifact citation gate — no artifact claims a test that does not exist (911) + run: | + set -euo pipefail + python3 scripts/artifact_citation_check.py | tee /tmp/artifact-cites.log + # Non-vacuity: assert it examined a NON-EMPTY population rather than + # merely exiting 0 over nothing. + grep -qE '^artifact citations: [1-9][0-9]* cited filters over [1-9][0-9]* test names' \ + /tmp/artifact-cites.log + - name: Oracle wiring gate — every repro script declares a CI status (890) run: | set -euo pipefail diff --git a/artifacts/e2e-verification.yaml b/artifacts/e2e-verification.yaml index 00f17ce9..c25b8209 100644 --- a/artifacts/e2e-verification.yaml +++ b/artifacts/e2e-verification.yaml @@ -350,7 +350,7 @@ artifacts: signature_hash, flags), (3) string table construction with correct offsets, (4) FNV-1a hash computation for import type signatures, (5) empty import table for modules with no imports (section omitted). - status: implemented + status: proposed note: > Covered by crates/synth-cli/tests/wast_compile.rs (ELF structure validation tests) tags: [e2e, unit-test, import-table, synth-backend] @@ -360,7 +360,18 @@ artifacts: fields: method: automated-test steps: - run: "cargo test -p synth-backend -- test_meld_import_table" + # RQ-56-CITE / #911: this cited `test_meld_import_table`, which exists + # in ZERO files. `cargo test` exits 0 on a filter matching nothing, so + # the artifact claimed verification by evidence that never ran — and + # shipped that way in v0.55.0. + # + # NOT re-pointed at a weaker test that happens to touch the section. + # What exists is `linker_script.rs` asserting the generated script + # CONTAINS the string ".meld_import_table" — that is not the ElfBuilder + # emission this artifact describes (import_count, ImportEntry packing, + # string-table offsets, FNV hash). Citing it would restate the same lie + # more carefully. The status is what was wrong, so the status moves. + run: "TODO(#911): tests described below are NOT written" coverage: > Import table emission, string table construction, hash computation, empty table omission. Byte-level verification diff --git a/scripts/artifact_citation_check.py b/scripts/artifact_citation_check.py new file mode 100644 index 00000000..44ac9306 --- /dev/null +++ b/scripts/artifact_citation_check.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""RQ-56-CITE (#911) — a verification artifact may not cite a test that does not exist. + +# The class this makes unrepresentable + +`cargo test -- ` where nothing matches prints + + test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 56 filtered out + +and **exits 0**. A rivet artifact citing such a filter therefore claims +verification by evidence that never runs, and no gate noticed: v0.55.0 SHIPPED +with two such citations (`test_meld_import_table`, `test_startup_memory_base`), +found by a post-release hygiene sweep rather than by CI. + +`scripts/oracle_wiring_check.py` already enforces exactly this shape for +`scripts/repro/*` — every oracle must declare a status, and a `wired` one must +really be referenced. The *artifacts* surface simply never got the equivalent, +so the same defect class stayed expressible one layer over. This is that +equivalent. + +# What "the test exists" means here, precisely + +`cargo test -- FILTER` matches FILTER as a **substring of the full test path** +(`module::path::test_name`). So a citation is satisfied iff at least one +`#[test]` function in the workspace has a path containing FILTER. + +Test names are collected by SOURCE SCAN, not by running cargo: this check runs +in `Claim Check`, which does not build the workspace, and a gate that needs a +20-minute build to answer a 200ms question would not survive. The scan reads +`#[test]` / `#[tokio::test]` attributes and the `fn ` that follows, plus +the enclosing `mod` path. + +**The approximation is deliberately in the SAFE direction.** A test the scan +misses would produce a FALSE FAILURE (loud, fixable, visible), never a false +pass. It cannot silently bless a citation that names nothing — which is the +whole failure being closed. Test names generated by a macro are the known blind +spot; if one appears, widen the scan rather than exempting the citation. + +Exit 0 iff every citation resolves to >= 1 test. +""" + +from __future__ import annotations + +import glob +import re +import sys +from pathlib import Path + +import yaml + +# `cargo test [flags] -- FILTER` — the filter is the first non-flag token after `--`. +CARGO_TEST_FILTER = re.compile(r"cargo\s+test\b[^\n]*?--\s+(?!-)(\S+)") +# `#[test]`, `#[tokio::test]`, `#[test_case(...)]` etc., then the fn it decorates. +TEST_ATTR = re.compile(r"#\[(?:\w+::)?test(?:_case)?\b") +FN_NAME = re.compile(r"\bfn\s+([A-Za-z_][A-Za-z0-9_]*)") +MOD_DECL = re.compile(r"^\s*(?:pub\s+)?mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{") + + +def collect_test_paths(root: Path) -> set[str]: + """Every `module::path::test_fn` the workspace defines. + + Both the bare name and the module-qualified path are recorded, because a + citation may reasonably use either. + """ + names: set[str] = set() + for rs in glob.glob(str(root / "crates" / "**" / "*.rs"), recursive=True): + try: + text = Path(rs).read_text(encoding="utf-8", errors="replace") + except OSError: + continue + # Track a simple mod stack so `mod tests { #[test] fn foo }` yields + # `tests::foo` as well as `foo`. + mod_stack: list[str] = [] + depth_of: list[int] = [] + depth = 0 + pending_test = False + for line in text.splitlines(): + m = MOD_DECL.match(line) + if m: + mod_stack.append(m.group(1)) + depth_of.append(depth) + depth += line.count("{") - line.count("}") + while depth_of and depth <= depth_of[-1]: + mod_stack.pop() + depth_of.pop() + if TEST_ATTR.search(line): + pending_test = True + continue + if pending_test: + f = FN_NAME.search(line) + if f: + names.add(f.group(1)) + if mod_stack: + names.add("::".join(mod_stack + [f.group(1)])) + pending_test = False + elif line.strip() and not line.strip().startswith("#"): + # An attribute run ended without an fn — reset rather than + # attach the next unrelated fn. + pending_test = False + return names + + +# Statuses that CLAIM the work is done. A dangling citation under one of these +# is a false claim of verification — the #911 defect. +# +# Under `draft` / `proposed` / `approved` the artifact is describing tests that +# are INTENDED, so a forward-looking citation is a plan, not a lie. Flagging +# those too would make this gate noisy, and a noisy gate gets ignored — which is +# how `codecov/patch` stopped being read (#923). Scoping it to claims keeps +# every failure real. +CLAIMING_STATUSES = {"implemented", "verified", "accepted"} + + +def citations(root: Path) -> list[tuple[str, str, str, str]]: + """-> [(artifact_file, artifact_id, status, filter)] for every cited filter.""" + out: list[tuple[str, str, str, str]] = [] + for f in sorted(glob.glob(str(root / "artifacts" / "*.yaml"))): + try: + doc = yaml.safe_load(Path(f).read_text()) + except Exception: + continue + rel = str(Path(f).relative_to(root)) + + def walk(node, current_id="", status=""): + if isinstance(node, dict): + current_id = node.get("id", current_id) + status = node.get("status", status) + for k, v in node.items(): + if k == "run" and isinstance(v, str): + for m in CARGO_TEST_FILTER.finditer(v): + out.append((rel, current_id, status, m.group(1))) + else: + walk(v, current_id, status) + elif isinstance(node, list): + for item in node: + walk(item, current_id, status) + + walk(doc) + return out + + +def main() -> int: + root = Path(__file__).resolve().parent.parent + tests = collect_test_paths(root) + cites = citations(root) + + if not tests: + print("FAIL: collected ZERO test names — the scan is broken, not the tree") + return 1 + # Non-vacuity: a check over an empty citation set would pass forever. + if not cites: + print("FAIL: found ZERO cited test filters — the parser is broken, or the") + print(" artifacts stopped citing tests. Either way this gate is inert.") + return 1 + + dangling = [c for c in cites if not any(c[3] in t for t in tests)] + bad = [c for c in dangling if c[2] in CLAIMING_STATUSES] + planned = [c for c in dangling if c[2] not in CLAIMING_STATUSES] + + for f, i, st, flt in bad: + print( + f"FAIL {f} [{i}] status={st}: cites `cargo test -- {flt}`, which " + f"matches NO test in the workspace. `cargo test` exits 0 on a filter " + f"that matches nothing, so this artifact CLAIMS verification by " + f"evidence that never runs." + ) + for f, i, st, flt in planned: + print( + f"note {f} [{i}] status={st}: cites `{flt}`, not yet written — " + f"a plan, not a claim. Becomes a FAILURE if the status advances to " + f"{'/'.join(sorted(CLAIMING_STATUSES))} before the test exists." + ) + print( + f"artifact citations: {len(cites)} cited filters over {len(tests)} test " + f"names — {len(cites) - len(dangling)} resolve, {len(bad)} false claim(s), " + f"{len(planned)} planned-but-unwritten" + ) + if bad: + print(f"{len(bad)} artifact(s) claim verification by a test that does not exist.") + return 1 + print("every cited test filter resolves to at least one test.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())