RQ-56-CITE (#911): an artifact cannot claim a test that does not exist

.github/workflows/ci.yml

@@ -213,6 +213,25 @@ jobs:
       # subprocess from other jobs. The literal below is pinned in claims.yaml
       # against the same number in scripts/repro/ORACLE_WIRING.md, so the gate
       # and the doc cannot drift apart.
+      # RQ-56-CITE / #911 — the SAME shape as the oracle-wiring gate below, on
+      # the artifacts surface, which never had one. `cargo test -- <filter>`
+      # exits 0 when the filter matches NOTHING, so a rivet artifact could claim
+      # verification by a test that does not exist. v0.55.0 shipped two such
+      # citations; a post-release sweep found them, no gate did.
+      #
+      # Scoped to CLAIMING statuses (implemented/verified/accepted): under
+      # draft/proposed a forward-looking citation is a plan, and flagging it
+      # would make this gate noisy. A noisy gate gets ignored — that is how
+      # codecov/patch stopped being read (#923).
+      - name: Artifact citation gate — no artifact claims a test that does not exist (911)
+        run: |
+          set -euo pipefail
+          python3 scripts/artifact_citation_check.py | tee /tmp/artifact-cites.log
+          # Non-vacuity: assert it examined a NON-EMPTY population rather than
+          # merely exiting 0 over nothing.
+          grep -qE '^artifact citations: [1-9][0-9]* cited filters over [1-9][0-9]* test names' \
+            /tmp/artifact-cites.log
+
       - name: Oracle wiring gate — every repro script declares a CI status (890)
         run: |
           set -euo pipefail

artifacts/e2e-verification.yaml

@@ -350,7 +350,7 @@ artifacts:
       signature_hash, flags), (3) string table construction with correct
       offsets, (4) FNV-1a hash computation for import type signatures,
       (5) empty import table for modules with no imports (section omitted).
-    status: implemented
+    status: proposed
     note: >
       Covered by crates/synth-cli/tests/wast_compile.rs (ELF structure validation tests)
     tags: [e2e, unit-test, import-table, synth-backend]
@@ -360,7 +360,18 @@ artifacts:
     fields:
       method: automated-test
       steps:
-        run: "cargo test -p synth-backend -- test_meld_import_table"
+        # RQ-56-CITE / #911: this cited `test_meld_import_table`, which exists
+        # in ZERO files. `cargo test` exits 0 on a filter matching nothing, so
+        # the artifact claimed verification by evidence that never ran — and
+        # shipped that way in v0.55.0.
+        #
+        # NOT re-pointed at a weaker test that happens to touch the section.
+        # What exists is `linker_script.rs` asserting the generated script
+        # CONTAINS the string ".meld_import_table" — that is not the ElfBuilder
+        # emission this artifact describes (import_count, ImportEntry packing,
+        # string-table offsets, FNV hash). Citing it would restate the same lie
+        # more carefully. The status is what was wrong, so the status moves.
+        run: "TODO(#911): tests described below are NOT written"
         coverage: >
           Import table emission, string table construction, hash
           computation, empty table omission. Byte-level verification

scripts/artifact_citation_check.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+"""RQ-56-CITE (#911) — a verification artifact may not cite a test that does not exist.
+
+# The class this makes unrepresentable
+
+`cargo test -- <filter>` where nothing matches prints
+
+    test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 56 filtered out
+
+and **exits 0**. A rivet artifact citing such a filter therefore claims
+verification by evidence that never runs, and no gate noticed: v0.55.0 SHIPPED
+with two such citations (`test_meld_import_table`, `test_startup_memory_base`),
+found by a post-release hygiene sweep rather than by CI.
+
+`scripts/oracle_wiring_check.py` already enforces exactly this shape for
+`scripts/repro/*` — every oracle must declare a status, and a `wired` one must
+really be referenced. The *artifacts* surface simply never got the equivalent,
+so the same defect class stayed expressible one layer over. This is that
+equivalent.
+
+# What "the test exists" means here, precisely
+
+`cargo test -- FILTER` matches FILTER as a **substring of the full test path**
+(`module::path::test_name`). So a citation is satisfied iff at least one
+`#[test]` function in the workspace has a path containing FILTER.
+
+Test names are collected by SOURCE SCAN, not by running cargo: this check runs
+in `Claim Check`, which does not build the workspace, and a gate that needs a
+20-minute build to answer a 200ms question would not survive. The scan reads
+`#[test]` / `#[tokio::test]` attributes and the `fn <name>` that follows, plus
+the enclosing `mod` path.
+
+**The approximation is deliberately in the SAFE direction.** A test the scan
+misses would produce a FALSE FAILURE (loud, fixable, visible), never a false
+pass. It cannot silently bless a citation that names nothing — which is the
+whole failure being closed. Test names generated by a macro are the known blind
+spot; if one appears, widen the scan rather than exempting the citation.
+
+Exit 0 iff every citation resolves to >= 1 test.
+"""
+
+from __future__ import annotations
+
+import glob
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+# `cargo test [flags] -- FILTER` — the filter is the first non-flag token after `--`.
+CARGO_TEST_FILTER = re.compile(r"cargo\s+test\b[^\n]*?--\s+(?!-)(\S+)")
+# `#[test]`, `#[tokio::test]`, `#[test_case(...)]` etc., then the fn it decorates.
+TEST_ATTR = re.compile(r"#\[(?:\w+::)?test(?:_case)?\b")
+FN_NAME = re.compile(r"\bfn\s+([A-Za-z_][A-Za-z0-9_]*)")
+MOD_DECL = re.compile(r"^\s*(?:pub\s+)?mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{")
+
+
+def collect_test_paths(root: Path) -> set[str]:
+    """Every `module::path::test_fn` the workspace defines.
+
+    Both the bare name and the module-qualified path are recorded, because a
+    citation may reasonably use either.
+    """
+    names: set[str] = set()
+    for rs in glob.glob(str(root / "crates" / "**" / "*.rs"), recursive=True):
+        try:
+            text = Path(rs).read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        # Track a simple mod stack so `mod tests { #[test] fn foo }` yields
+        # `tests::foo` as well as `foo`.
+        mod_stack: list[str] = []
+        depth_of: list[int] = []
+        depth = 0
+        pending_test = False
+        for line in text.splitlines():
+            m = MOD_DECL.match(line)
+            if m:
+                mod_stack.append(m.group(1))
+                depth_of.append(depth)
+            depth += line.count("{") - line.count("}")
+            while depth_of and depth <= depth_of[-1]:
+                mod_stack.pop()
+                depth_of.pop()
+            if TEST_ATTR.search(line):
+                pending_test = True
+                continue
+            if pending_test:
+                f = FN_NAME.search(line)
+                if f:
+                    names.add(f.group(1))
+                    if mod_stack:
+                        names.add("::".join(mod_stack + [f.group(1)]))
+                    pending_test = False
+                elif line.strip() and not line.strip().startswith("#"):
+                    # An attribute run ended without an fn — reset rather than
+                    # attach the next unrelated fn.
+                    pending_test = False
+    return names
+
+
+# Statuses that CLAIM the work is done. A dangling citation under one of these
+# is a false claim of verification — the #911 defect.
+#
+# Under `draft` / `proposed` / `approved` the artifact is describing tests that
+# are INTENDED, so a forward-looking citation is a plan, not a lie. Flagging
+# those too would make this gate noisy, and a noisy gate gets ignored — which is
+# how `codecov/patch` stopped being read (#923). Scoping it to claims keeps
+# every failure real.
+CLAIMING_STATUSES = {"implemented", "verified", "accepted"}
+
+
+def citations(root: Path) -> list[tuple[str, str, str, str]]:
+    """-> [(artifact_file, artifact_id, status, filter)] for every cited filter."""
+    out: list[tuple[str, str, str, str]] = []
+    for f in sorted(glob.glob(str(root / "artifacts" / "*.yaml"))):
+        try:
+            doc = yaml.safe_load(Path(f).read_text())
+        except Exception:
+            continue
+        rel = str(Path(f).relative_to(root))
+
+        def walk(node, current_id="<unknown>", status="<none>"):
+            if isinstance(node, dict):
+                current_id = node.get("id", current_id)
+                status = node.get("status", status)
+                for k, v in node.items():
+                    if k == "run" and isinstance(v, str):
+                        for m in CARGO_TEST_FILTER.finditer(v):
+                            out.append((rel, current_id, status, m.group(1)))
+                    else:
+                        walk(v, current_id, status)
+            elif isinstance(node, list):
+                for item in node:
+                    walk(item, current_id, status)
+
+        walk(doc)
+    return out
+
+
+def main() -> int:
+    root = Path(__file__).resolve().parent.parent
+    tests = collect_test_paths(root)
+    cites = citations(root)
+
+    if not tests:
+        print("FAIL: collected ZERO test names — the scan is broken, not the tree")
+        return 1
+    # Non-vacuity: a check over an empty citation set would pass forever.
+    if not cites:
+        print("FAIL: found ZERO cited test filters — the parser is broken, or the")
+        print("      artifacts stopped citing tests. Either way this gate is inert.")
+        return 1
+
+    dangling = [c for c in cites if not any(c[3] in t for t in tests)]
+    bad = [c for c in dangling if c[2] in CLAIMING_STATUSES]
+    planned = [c for c in dangling if c[2] not in CLAIMING_STATUSES]
+
+    for f, i, st, flt in bad:
+        print(
+            f"FAIL {f} [{i}] status={st}: cites `cargo test -- {flt}`, which "
+            f"matches NO test in the workspace. `cargo test` exits 0 on a filter "
+            f"that matches nothing, so this artifact CLAIMS verification by "
+            f"evidence that never runs."
+        )
+    for f, i, st, flt in planned:
+        print(
+            f"note {f} [{i}] status={st}: cites `{flt}`, not yet written — "
+            f"a plan, not a claim. Becomes a FAILURE if the status advances to "
+            f"{'/'.join(sorted(CLAIMING_STATUSES))} before the test exists."
+        )
+    print(
+        f"artifact citations: {len(cites)} cited filters over {len(tests)} test "
+        f"names — {len(cites) - len(dangling)} resolve, {len(bad)} false claim(s), "
+        f"{len(planned)} planned-but-unwritten"
+    )
+    if bad:
+        print(f"{len(bad)} artifact(s) claim verification by a test that does not exist.")
+        return 1
+    print("every cited test filter resolves to at least one test.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())