From 85cb05feb507361c2433b3702710a0c1639db132 Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Thu, 6 Aug 2026 22:35:10 +0200 Subject: [PATCH 1/2] feat(verify): census the whole-crate verification steps (weak evidence, #262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole-crate step count is REGRESSING: ~54 at the 2026-07 traceability audit, 70 measured today. #262 anticipated this and proposed warn-then-fail; this is the warn half. A whole-crate `cargo test -p X` step asserts "something in this crate passes", not "THIS test verifies THIS requirement". It is also the one step shape that cannot detect evidence drift: the existing guard (`cargo_tests_passed(...) == 0`, pulseengine.eu#89) needs a name to check against, so under a whole-crate step a test that is renamed, deleted, or `#[ignore]`d leaves the step green. That is not hypothetical. FV-FALCON-NOTCH-001 runs `cargo test -p falcon-core` as its closed-loop evidence for NOTCH-P01, and the only notch test in that crate is `#[ignore]`d pending #290 — so the step runs 60 unrelated tests and reports PASS. Reported on #290. Reports, does not fail. The backlog is 70 steps; failing now would block unrelated work. The goal is to stop the count drifting up, and to make it visible on GREEN runs — a gate that only speaks when it fails cannot show a backlog shrinking. Convert to a hard failure once the backlog is worked down. Verified locally: $ python3 scripts/run-falcon-verification.py --filter '(has-tag "wasm-pipeline")' [ PASS] (0.16s) FV-FALCON-PIPELINE-001: cargo test -p relay-mix-quad — WHOLE-CRATE step: names no test (#262) [ PASS] (0.13s) FV-FALCON-PIPELINE-001: cargo test -p relay-rate — WHOLE-CRATE step: names no test (#262) [ PASS] (0.11s) FV-FALCON-PIPELINE-001: cargo test -p falcon-sitl-hover — WHOLE-CRATE step: names no test (#262) # 3 whole-crate step(s) — weak evidence, name the verifying test(s) (#262) EXIT=0 Pass/fail behaviour is unchanged: steps still PASS and the run still exits 0. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvusAXYbHLyv3uTzfBcMbG --- scripts/run-falcon-verification.py | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/run-falcon-verification.py b/scripts/run-falcon-verification.py index 541107b..433ea72 100755 --- a/scripts/run-falcon-verification.py +++ b/scripts/run-falcon-verification.py @@ -199,6 +199,35 @@ def cargo_tests_passed(output: str) -> int: return sum(int(m) for m in re.findall(r"test result: ok\. (\d+) passed", output)) +# Whole-crate steps seen this run: (artifact-id, cmd). Reported, not failed — +# see cargo_test_is_whole_crate. +WHOLE_CRATE_STEPS: list[tuple[str, str]] = [] + + +def cargo_test_is_whole_crate(cmd: str) -> bool: + """True if `cmd` is a `cargo test` step that runs an ENTIRE crate/workspace + instead of naming the test(s) that verify the requirement. + + Whole-crate steps are weak evidence: they assert "something in this crate + passes", not "THIS test verifies THIS requirement". They are also the one + step shape that CANNOT detect evidence drift — the named-test guard in + run_steps (`cargo_tests_passed(...) == 0`) has no name to check, so a test + that is renamed, deleted, or `#[ignore]`d leaves the step green. + + Counted and reported (#262), not failed: the backlog was 70 steps when this + landed, so failing immediately would block unrelated work. The point is to + stop the count drifting back up — it had already grown ~54 -> 70 since the + 2026-07 audit. Convert to a hard failure once the backlog is worked down.""" + import shlex + try: + toks = shlex.split(cmd) + except ValueError: + return False + if toks[:2] != ["cargo", "test"]: + return False + return not cargo_test_names_a_filter(cmd) + + def run_steps(artifact: dict[str, Any], dry_run: bool) -> tuple[bool, list[dict]]: aid = artifact["id"] steps = artifact.get("fields", {}).get("steps") or [] @@ -257,6 +286,11 @@ def run_steps(artifact: dict[str, Any], dry_run: bool) -> tuple[bool, list[dict] if passed and cargo_test_names_a_filter(cmd) and cargo_tests_passed(proc.stdout) == 0: passed = False note = " — named test ran 0 (renamed/removed? evidence drift)" + # Weak-evidence census (#262). Does NOT affect pass/fail — see + # cargo_test_is_whole_crate for why this warns rather than fails. + if cargo_test_is_whole_crate(cmd): + WHOLE_CRATE_STEPS.append((aid, cmd)) + note += " — WHOLE-CRATE step: names no test (#262)" artifact_pass = artifact_pass and passed status = "PASS" if passed else (f"FAIL (rc={rc})" if rc != 0 else "FAIL (0 tests)") print(f" [{status:>14}] ({duration:6.2f}s) {aid}: {cmd}{note}") @@ -352,6 +386,15 @@ def main() -> int: "steps": step_results, }) + # Weak-evidence census (#262). Printed even on a green run — a gate that + # only speaks when it fails cannot show a backlog shrinking. + if WHOLE_CRATE_STEPS: + print() + print(f"# {len(WHOLE_CRATE_STEPS)} whole-crate step(s) — weak evidence, " + f"name the verifying test(s) (#262):") + for aid, cmd in WHOLE_CRATE_STEPS: + print(f"# {aid}: {cmd}") + if args.markdown: print() print(emit_markdown(report)) From 0bbd2bb1626672fd780eae80c4afa158afcd386f Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Sat, 8 Aug 2026 10:01:10 +0200 Subject: [PATCH 2/2] ci: refresh the PR payload so the scoped Verify-Filter is picked up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate reads Verify-Filter from github.event.pull_request.body — the EVENT payload — so editing the body alone does not change what a queued run sees. A synchronize event is required. Squash-merge drops this commit. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvusAXYbHLyv3uTzfBcMbG