From 812879afcb562301759fd116a8de8e0bdf9d21d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 20:52:44 +0000 Subject: [PATCH] feat(guard): the REQUIRED fuzz context's target list, not only the advisory one (#406) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REQ-GUARD-GATE-EVIDENCE-002 (h) added check_fuzz_targets.py to stop the fuzz target list drifting from its fuzz/Cargo.toml [[bin]] declaration. It audited the wrong one of two lists. `fuzz-nightly.yml`'s matrix (what the tool compared) is ADVISORY — its job name is templated, so it can never be a required context. `ci.yml`'s fuzz-smoke job, which hand-writes each harness as a `cargo fuzz run ` step, IS the required context (`Fuzz smoke (60s/target)`) and was compared against nothing. Add a [[bin]] plus a nightly matrix entry but forget the fuzz-smoke step and the tool reported `All 4 declared fuzz targets are run` while the REQUIRED gate fuzzed 3 of 4, green, forever. Extending the nightly `- target:` regex to ci.yml does not work — the targets sit as the positional of a `cargo fuzz run` command inside `run:` steps. A second extractor (smoke_targets) parses them, scoped to the fuzz-smoke: job block and skipping full-line comments so a `fuzz run` string in prose cannot poison the scan, dropping the +toolchain token, value flags with their argument (so --target is not read as a harness), and libfuzzer args after `--`; a command not resolving to exactly one positional is a broken scan (exit 2). The declaration is now audited against BOTH lists, worst exit wins. Self-test grows to 18 rows, including the #406 acceptance criterion (a [[bin]] absent from fuzz-smoke exits 1), the comment-poisoning regression, and a real-tree row asserting the committed ci.yml is clean under both audits. REQ-GUARD-FUZZ-SMOKE-001 + TEST-GUARD-FUZZ-SMOKE (with runnable steps and satisfies/verifies links) record it. The human-scoped REQ-GUARD-GATE-EVIDENCE-002 is left untouched. Set equality proves DECLARATION parity, not execution — a continue-on-error/if:false leg still counts as run; that is a distinct obligation left to a successor. Closes #406. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 16 +- artifacts/requirements.yaml | 69 ++++++++ artifacts/verification.yaml | 70 ++++++++ tools/check_fuzz_targets.py | 330 ++++++++++++++++++++++++++++++++---- 4 files changed, 442 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd776ff..4082d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -548,12 +548,16 @@ jobs: # every filter is vacuous. - name: Verification-filter guardrail run: tools/check_verification_filters.py - # REQ-GUARD-GATE-EVIDENCE-002 (h). fuzz-nightly names its targets in a - # hand-written matrix `include` (each carries its own extra_args, #361), - # so the declared set and the run set are maintained separately and - # nothing compared them. A target dropped from either side leaves the - # nightly green having fuzzed a strict subset — no error, no warning, no - # smaller number in the log. Cheap: two file reads, no build. + # REQ-GUARD-GATE-EVIDENCE-002 (h) + REQ-GUARD-FUZZ-SMOKE-001 (#406). Two + # workflows run the fuzz harnesses from lists maintained apart from + # fuzz/Cargo.toml's `[[bin]]` declarations: fuzz-nightly's matrix `include` + # (each target carries its own extra_args, #361) and — the one that + # actually gates merges — THIS file's `fuzz-smoke` job, which hand-writes + # one `cargo fuzz run ` step per harness. The original tool audited + # only the ADVISORY nightly, so a target added to fuzz/Cargo.toml but not + # to fuzz-smoke left the REQUIRED context green having fuzzed a strict + # subset (#406). The tool now asserts the declaration against BOTH lists; + # `--ci-workflow` defaults to this file. Cheap: three file reads, no build. - name: Fuzz-target guardrail self-test run: tools/check_fuzz_targets.py --self-test - name: Fuzz-target guardrail diff --git a/artifacts/requirements.yaml b/artifacts/requirements.yaml index 77bf4bc..342c86e 100644 --- a/artifacts/requirements.yaml +++ b/artifacts/requirements.yaml @@ -5024,3 +5024,72 @@ artifacts: status: proposed release: v0.36.0 tags: [process, guardrail, ci, tooling, human-scoped] + + - id: REQ-GUARD-FUZZ-SMOKE-001 + type: requirement + title: The fuzz-target guardrail shall audit the REQUIRED context, not only the advisory one + description: > + REQ-GUARD-GATE-EVIDENCE-002 (h) added `tools/check_fuzz_targets.py` to stop + the fuzz target list drifting from its `[[bin]]` declaration in + `fuzz/Cargo.toml`. It audited the wrong one of two lists (#406). + + There are two run-lists, maintained apart from the declaration: + + * `fuzz-nightly.yml`'s `matrix.include` — one entry per target. This is + the list the original tool compared (`--workflow` default). It is + ADVISORY: the job name is templated (`Fuzz ${{ matrix.target }}`), + which by `check_required_contexts.py`'s own rules can NEVER be a + required context. + + * `ci.yml`'s `fuzz-smoke` job — the same three targets hand-written as + one `cargo +nightly fuzz run … ` step each. THIS is the list + that gates: `Fuzz smoke (60s/target)` is in + `.github/required-contexts.txt`. Nothing compared it to anything. + + So the covered list was advisory and the uncovered list blocked merges. + Add a `[[bin]]` plus a nightly matrix entry but forget the `fuzz-smoke` + step, and the tool reported `All 4 declared fuzz targets are run` while the + REQUIRED gate fuzzed 3 of 4, green, forever. This is -002's own defect + shape one level up: the shortfall in the blocking gate rendered as the + ideal reading of a non-blocking one. + + OBLIGATION. The declared `[[bin]]` set shall be asserted equal — both + directions — to the target set RUN by the required `fuzz-smoke` job, not + only to the advisory nightly matrix. Extending the nightly's `- target:` + regex to `ci.yml` does not work: the fuzz-smoke targets sit as the + positional of a `cargo … fuzz run` command inside `- run:` steps, so a + SECOND extractor is required. That extractor drops the `+toolchain` token, + value-taking flags with their argument (so the real `--target ` is + never read as a harness), and the libfuzzer args after `--`; a step that + does not resolve to exactly one target positional is a broken scan + (exit 2), never a silently dropped leg. It is scoped to the `fuzz-smoke:` + job block and skips full-line comments — because `fuzz run` also appears in + prose, and a whole-file scan matched this file's own guard-step comment and + read the required gate as a broken scan. + + ORACLE (executed, non-vacuous). `tools/check_fuzz_targets.py --self-test` + carries an 18-row decision table; the fuzz-smoke rows include the #406 + acceptance criterion itself — a `[[bin]]` added to `fuzz/Cargo.toml` but + not to the `fuzz-smoke` job must exit 1. That row's exit (1) differs from + the base row's (0) on the same tool with a distinct input, which is the + discrimination the advisory-only tool lacked. A REAL-TREE row runs the + check against the committed `ci.yml` and asserts exit 0, so a `fuzz run` + string landing in a comment (the regression an earlier whole-file scan + shipped) fails the self-test rather than silently reading the gate as + broken. Demonstrated live against the real `ci.yml`: appending a fourth + `[[bin]]` to the real `fuzz/Cargo.toml` makes the fuzz-smoke check exit 1 + (`fuzz_new_harness … NOT run by the fuzz-smoke job`), while the unmodified + tree exits 0. + + PROVEN vs ASSUMED, stated precisely. PROVEN: the required gate now fails + when its executed target set diverges from the declaration, in either + direction, and fails closed on an unparsable step. NOT CLAIMED: that a + declared, listed target is actually FUZZED. Set equality proves + DECLARATION parity, not execution — a `fuzz-smoke` step carrying + `continue-on-error: true` or `if: false` still counts as "run" here. That + is a distinct obligation (execution evidence, not list parity) and is left + to a successor; this requirement closes the list-parity hole #406 names and + no more. + status: implemented + release: v0.36.0 + tags: [process, guardrail, ci, tooling, fuzzing] diff --git a/artifacts/verification.yaml b/artifacts/verification.yaml index 97e4c00..2d9b214 100644 --- a/artifacts/verification.yaml +++ b/artifacts/verification.yaml @@ -4268,6 +4268,76 @@ artifacts: release: v0.36.0 tags: [process, guardrail, ci, tooling, fuzzing] + - id: TEST-GUARD-FUZZ-SMOKE + type: feature + title: The REQUIRED fuzz-smoke target list is proven equal to the fuzz targets declared + description: > + Verifies REQ-GUARD-FUZZ-SMOKE-001. Closes #406. + + TEST-GUARD-FUZZ-TARGETS above audits the ADVISORY `fuzz-nightly.yml` + matrix. The list that actually blocks merges is `ci.yml`'s `fuzz-smoke` + job (`Fuzz smoke (60s/target)`, in `.github/required-contexts.txt`), which + hand-writes each harness as a `cargo +nightly fuzz run … ` step — + and it was compared against nothing. `tools/check_fuzz_targets.py` now + asserts the `[[bin]]` declaration against BOTH run-lists; `--ci-workflow` + defaults to `ci.yml` and the worst exit wins, so a broken scan is never + masked by a clean sibling. + + The fuzz-smoke extractor cannot reuse the nightly's `- target:` regex — the + targets sit as the positional of a `cargo … fuzz run` command inside + `- run:` steps. It drops the `+toolchain` token, value-taking flags with + their argument (so the real `--target ` is not read as a harness), + and the libfuzzer args after `--`. A step not resolving to exactly one + target positional exits 2 (broken scan), not a silent skip. + + The extractor is SCOPED to the `fuzz-smoke:` job block and skips full-line + comments — because `fuzz run` also occurs in prose. An earlier whole-file + version matched the guard step's own explanatory comment + (`cargo … fuzz run … `) in this same ci.yml, resolved it to + several tokens, and read the REQUIRED `Rivet validate (artifacts)` context + as a broken scan (exit 2) — the fix's own comment turning the gate it adds + red. The self-test's synthetic fixtures could not see that (they carry no + comment), so the suite carries a REAL-TREE row that runs the check against + the committed ci.yml and asserts exit 0; a poisoning comment fails it + loudly. + + `tools/check_fuzz_targets.py --self-test` runs an 18-row decision table + before the gate judges anything. The fuzz-smoke rows include: + + * the #406 acceptance criterion — a `[[bin]]` added to `fuzz/Cargo.toml` + but not to the fuzz-smoke job must exit 1. Its exit (1) differs from + the base row's (0) under a distinct input, which is the discrimination + the advisory-only tool lacked and the non-vacuity proof for this gate; + * the reverse — fuzz-smoke running a target not declared exits 1; + * a step with no `fuzz run` at all, and a step whose command leaves an + ambiguous positional, each exit 2 (broken scan), so the extractor + cannot pass by reading nothing; + * the `--flag=value` equals form is a single hyphen token, so it is a + flag and does not leak a positional. + + The base row only passes because the `--target x86_64-unknown-linux-gnu` + triple is NOT counted as a third harness. Demonstrated live against the + real `ci.yml`: a fourth `[[bin]]` on the real `fuzz/Cargo.toml` makes the + fuzz-smoke check exit 1; the unmodified tree exits 0. Three file reads, no + build. + + NOT claimed: that a listed target is actually fuzzed. Set equality is + declaration parity, not execution — a `continue-on-error`/`if: false` leg + still counts as "run". Distinct obligation, left to a successor. + fields: + method: automated-test + steps: + - run: tools/check_fuzz_targets.py --self-test + - run: tools/check_fuzz_targets.py + status: implemented + release: v0.36.0 + tags: [process, guardrail, ci, tooling, fuzzing, v0360] + links: + - type: satisfies + target: REQ-GUARD-FUZZ-SMOKE-001 + - type: verifies + target: REQ-GUARD-FUZZ-SMOKE-001 + - id: TEST-GUARD-VERIFICATION-FILTERS type: feature title: A verification step whose test filter selects nothing is proven to fail diff --git a/tools/check_fuzz_targets.py b/tools/check_fuzz_targets.py index 2bfe1ba..2d068e8 100755 --- a/tools/check_fuzz_targets.py +++ b/tools/check_fuzz_targets.py @@ -1,23 +1,42 @@ #!/usr/bin/env python3 """Fail when the fuzz targets RUN are not exactly the fuzz targets DECLARED. -REQ-GUARD-GATE-EVIDENCE-002 (h). +REQ-GUARD-GATE-EVIDENCE-002 (h) — the advisory nightly. +REQ-GUARD-FUZZ-SMOKE-001 — the REQUIRED `Fuzz smoke` context (#406). WHY THIS EXISTS =============== -`fuzz/Cargo.toml` declares the harnesses as `[[bin]]` entries. `fuzz-nightly.yml` -runs them from a hand-written `matrix.include` list — one entry per target, -because each carries its own `extra_args` (only `fuzz_scheduler_solver` needs -`-max_len=128`, and a flat list would force that cap onto two healthy fuzzers; -see #361). That per-target detail is a good reason for the `include` form and is -staying. - -The cost of the `include` form is that the two lists are maintained separately -and nothing compares them. Add a harness to `fuzz/Cargo.toml` and forget the -workflow, or delete a matrix entry, and the job stays GREEN having fuzzed a -strict subset. There is no error, no warning, and no smaller number anywhere in -the log — the run simply says `success` for the legs it did run. +`fuzz/Cargo.toml` declares the harnesses as `[[bin]]` entries. TWO workflows run +them from separately-maintained lists, and this gate compares the declaration to +BOTH: + + * `fuzz-nightly.yml` runs them from a hand-written `matrix.include` list — one + entry per target, because each carries its own `extra_args` (only + `fuzz_scheduler_solver` needs `-max_len=128`, and a flat list would force + that cap onto two healthy fuzzers; see #361). That per-target detail is a + good reason for the `include` form and is staying. This workflow is + ADVISORY — its job name is templated (`Fuzz ${{ matrix.target }}`), which by + `check_required_contexts.py`'s rules can never be a required context. + + * `ci.yml`'s `fuzz-smoke` job hand-writes the SAME targets as one + `cargo +nightly fuzz run … ` step each. THIS job is a REQUIRED + context (`Fuzz smoke (60s/target)` in `.github/required-contexts.txt`), so + it is the one that actually blocks merges. + +#406: the original tool audited only the nightly matrix — the advisory list — +and left the required `fuzz-smoke` job compared against nothing. Add a `[[bin]]` +and a nightly matrix entry but forget the `fuzz-smoke` step, and this tool would +report `All 4 declared fuzz targets are run` while the required gate fuzzed 3 of +4, green, forever. Auditing the advisory list and not the blocking one is the +requirement's own defect shape: the covered list does not gate, and the gating +list was uncovered. + +The cost of both list forms is the same: the two lists are maintained separately +and nothing compared them. Add a harness to `fuzz/Cargo.toml` and forget a +workflow, or delete an entry, and the job stays GREEN having fuzzed a strict +subset. There is no error, no warning, and no smaller number anywhere in the +log — the run simply says `success` for the legs it did run. That is the family this requirement is about: the shortfall renders as the ideal reading. A fuzz job that fuzzed two of three targets looks exactly like a fuzz @@ -32,7 +51,8 @@ WHAT IT CHECKS ============== -Set equality, both directions, because the two failure modes are different bugs: +Declared-set equality against EACH run-set, both directions, because the two +failure modes are different bugs: * declared but not run -> a harness exists and is never exercised. * run but not declared -> the workflow names a target cargo-fuzz cannot @@ -43,10 +63,26 @@ says nothing on success cannot distinguish "compared three" from "compared none", the same reason `check_fmt_workspaces.py` always prints its count. -stdlib-only, and both parsers are deliberately narrow: `[[bin]] name = "..."` -from the manifest, `- target: ...` from the matrix. Neither file is arbitrary -YAML/TOML in practice, and a dependency this gate cannot install is a gate that -can fail to run — which reads as approval. +stdlib-only, and all three parsers are deliberately narrow: `[[bin]] name = "…"` +from the manifest, `- target: …` from the nightly matrix, and the target +positional of a `cargo … fuzz run … [-- …]` command from `ci.yml`. + +The fuzz-smoke extractor is SCOPED to the `fuzz-smoke:` job block (from its +2-space-indented key to the next top-level job) and SKIPS full-line comments — +because the string `fuzz run` also occurs in prose comments and could occur in +other jobs, and a whole-file scan that matched one of those would resolve to +several positional tokens and wrongly read the required gate as a broken scan. +That is not hypothetical: the comment above the guard step in this same `ci.yml` +contains `cargo … fuzz run … `, and an unscoped scan turned the required +`Rivet validate (artifacts)` context red on the very PR that added this tool. +Within the block it drops the toolchain token (`+nightly`), the value-taking +flags a command may carry (`--target `, so the triple is never mistaken +for a harness), and the libfuzzer args after `--`; a `fuzz run` command that does +not resolve to exactly one target positional is a broken scan (exit 2), not a +silent skip, so an unrecognised flag form fails loud and is fixed here +deliberately rather than dropping a target unnoticed. None of the three files is +arbitrary YAML/TOML in practice, and a dependency this gate cannot install is a +gate that can fail to run — which reads as approval. """ from __future__ import annotations @@ -61,8 +97,24 @@ # `[[bin]]` … `name = "fuzz_x"` — the first `name` after each [[bin]] header. _BIN_HEADER = re.compile(r"^\s*\[\[bin\]\]\s*$", re.M) _NAME = re.compile(r'^\s*name\s*=\s*"([^"]+)"\s*$', re.M) -# `- target: fuzz_x` inside the workflow matrix. +# `- target: fuzz_x` inside the nightly workflow matrix. _MATRIX_TARGET = re.compile(r"^\s*-\s+target:\s*(\S+)\s*$", re.M) +# The `fuzz-smoke:` job key (2-space indent) and any other top-level job key, so +# the smoke scan can be confined to that one job — the string `fuzz run` also +# appears in prose comments and could appear in other jobs. +_SMOKE_JOB = re.compile(r"^ fuzz-smoke:\s*$", re.M) +_TOP_JOB = re.compile(r"^ \S") +# `cargo … fuzz run …` inside a step of that job. Everything after `fuzz run` +# is captured and picked apart in code. +_FUZZ_RUN = re.compile(r"\bfuzz\s+run\b(.*)$") +# cargo-fuzz `run` options that take a following value token; their argument +# must not be mistaken for the target positional. `--target x86_64-…` is the one +# actually used, but the run subcommand accepts these too, and an equals form +# (`--target=…`) is handled separately as a single hyphen-prefixed token. +_VALUE_FLAGS = frozenset({ + "--target", "-s", "--sanitizer", "-j", "--jobs", "--features", + "--build-dir", "--target-dir", "-D", "--dev", "--release", +}) def declared_targets(manifest_text: str) -> set[str]: @@ -76,28 +128,109 @@ def declared_targets(manifest_text: str) -> set[str]: def matrix_targets(workflow_text: str) -> set[str]: - """Every `- target:` in the fuzz workflow matrix.""" + """Every `- target:` in the nightly fuzz workflow matrix.""" return {m.group(1).strip('"\'') for m in _MATRIX_TARGET.finditer(workflow_text)} -def check(manifest: Path, workflow: Path, out=sys.stdout) -> int: - declared = declared_targets(manifest.read_text(encoding="utf-8")) - run = matrix_targets(workflow.read_text(encoding="utf-8")) +def _one_smoke_target(after_run: str) -> str | None: + """The single target positional of one `cargo … fuzz run …` command. + + Returns the harness name, or None if the command does not resolve to + EXACTLY one positional — which is a broken scan, handled loudly by the + caller rather than dropped. Drops the libfuzzer args after `--`, the + `+toolchain` token, value-taking flags together with their argument, and any + remaining hyphen-prefixed flag (covering the `--flag=value` form). + """ + head = after_run.split(" -- ", 1)[0] + toks = head.split() + positionals: list[str] = [] + i = 0 + while i < len(toks): + t = toks[i] + if t in _VALUE_FLAGS: + i += 2 # skip the flag and its value + continue + if t.startswith("-") or t.startswith("+"): + i += 1 # a bare flag, or the `--flag=value` / `+toolchain` form + continue + positionals.append(t) + i += 1 + return positionals[0] if len(positionals) == 1 else None + + +def _smoke_job_block(workflow_text: str) -> str | None: + """The `fuzz-smoke:` job block, key line to the next top-level job key. + + None if there is no `fuzz-smoke:` job — which the caller treats as a broken + scan (the required job vanished), not an empty pass. + """ + lines = workflow_text.splitlines() + start = None + for i, ln in enumerate(lines): + if _SMOKE_JOB.match(ln): + start = i + break + if start is None: + return None + end = len(lines) + for j in range(start + 1, len(lines)): + if _TOP_JOB.match(lines[j]): + end = j + break + return "\n".join(lines[start:end]) + + +def smoke_targets(workflow_text: str) -> set[str] | None: + """Every target run by a `cargo … fuzz run` step in ci.yml's fuzz-smoke job. + + Scoped to the `fuzz-smoke:` job block and skipping full-line comments, so a + `fuzz run` string in a prose comment or an unrelated job cannot poison the + scan (an earlier whole-file version matched the guard step's own comment and + turned the required gate red, #406). + + None signals a broken scan: no `fuzz-smoke:` job at all, or a `fuzz run` + command that did not resolve to exactly one target positional (an + unrecognised flag form, say). Either must fail the gate rather than silently + drop a leg — the whole point of #406. + """ + block = _smoke_job_block(workflow_text) + if block is None: + return None + out: set[str] = set() + for raw in block.splitlines(): + if raw.lstrip().startswith("#"): + continue # a full-line YAML/shell comment is never a run command + m = _FUZZ_RUN.search(raw) + if not m: + continue + tgt = _one_smoke_target(m.group(1)) + if tgt is None: + return None + out.add(tgt) + return out + - print("== fuzz-target guardrail ==", file=out) +def _compare(declared: set[str], run: set[str] | None, manifest: Path, + run_src: str, run_desc: str, out=sys.stdout) -> int: + print(f"== fuzz-target guardrail: {run_desc} ==", file=out) print(f"declared in {manifest}: {len(declared)} {sorted(declared)}", file=out) - print(f"run by {workflow}: {len(run)} {sorted(run)}", file=out) + if run is None: + print(f"run by {run_src}: ", file=out) + print(f"::error::a `fuzz run` step in {run_src} did not resolve to " + "exactly one target — a broken scan, not a disabled job.", file=out) + return 2 + print(f"run by {run_src}: {len(run)} {sorted(run)}", file=out) # An empty side is a broken scan, not a finding. Without this, deleting the - # [[bin]] section or renaming the matrix key would make both sets empty and - # the equality check would pass — the gate reporting perfect agreement - # about nothing. + # [[bin]] section or renaming the matrix key / step form would make both + # sets empty and the equality check would pass — the gate reporting perfect + # agreement about nothing. if not declared: print("::error::no `[[bin]]` targets found — the manifest parse read " "nothing, which is a broken scan, not an empty fuzz suite.", file=out) return 2 if not run: - print("::error::no `- target:` entries found — the workflow parse read " + print(f"::error::no fuzz targets found in {run_src} — the parse read " "nothing, which is a broken scan, not a disabled job.", file=out) return 2 @@ -106,18 +239,35 @@ def check(manifest: Path, workflow: Path, out=sys.stdout) -> int: if missing: for t in missing: print(f"::error::{t} is declared in fuzz/Cargo.toml but NOT run by " - f"the workflow — it is never fuzzed and nothing says so.", file=out) + f"{run_desc} — it is never fuzzed and nothing says so.", file=out) if extra: for t in extra: - print(f"::error::{t} is run by the workflow but NOT declared in " + print(f"::error::{t} is run by {run_desc} but NOT declared in " f"fuzz/Cargo.toml — that leg cannot build.", file=out) if missing or extra: return 1 - print(f"\nAll {len(declared)} declared fuzz targets are run.", file=out) + print(f"\nAll {len(declared)} declared fuzz targets are run by {run_desc}.", + file=out) return 0 +def check(manifest: Path, workflow: Path, out=sys.stdout) -> int: + """Declared set vs the nightly matrix `include` list.""" + declared = declared_targets(manifest.read_text(encoding="utf-8")) + run = matrix_targets(workflow.read_text(encoding="utf-8")) + return _compare(declared, run, manifest, str(workflow), + "the nightly matrix", out=out) + + +def check_smoke(manifest: Path, ci_workflow: Path, out=sys.stdout) -> int: + """Declared set vs the REQUIRED ci.yml fuzz-smoke steps (#406).""" + declared = declared_targets(manifest.read_text(encoding="utf-8")) + run = smoke_targets(ci_workflow.read_text(encoding="utf-8")) + return _compare(declared, run, manifest, str(ci_workflow), + "the fuzz-smoke job", out=out) + + _MANIFEST = """[package] name = "spar-fuzz" @@ -141,11 +291,24 @@ def check(manifest: Path, workflow: Path, out=sys.stdout) -> int: extra_args: "-max_len=128" """ +# ci.yml's fuzz-smoke job form: one `cargo … fuzz run … ` step each, +# carrying the real `--target ` flag whose value must not be read as a +# harness, and libfuzzer args after `--`. Matches the two _MANIFEST bins. +_CI = """ fuzz-smoke: + name: Fuzz smoke (60s/target) + steps: + - name: fuzz_aadl_parse (60s) + run: cargo +nightly fuzz run --target x86_64-unknown-linux-gnu fuzz_aadl_parse -- -max_total_time=60 -timeout=10 + - name: fuzz_scheduler_solver (60s) + run: cargo +nightly fuzz run --target x86_64-unknown-linux-gnu fuzz_scheduler_solver -- -max_total_time=60 -timeout=5 +""" + def self_test() -> int: passed = failed = 0 - def case(desc: str, want: int, manifest: str, workflow: str) -> None: + def case(desc: str, want: int, manifest: str, workflow: str, + checker=check) -> None: nonlocal passed, failed with tempfile.TemporaryDirectory() as td: m = Path(td) / "Cargo.toml" @@ -153,17 +316,18 @@ def case(desc: str, want: int, manifest: str, workflow: str) -> None: m.write_text(manifest, encoding="utf-8") w.write_text(workflow, encoding="utf-8") buf = io.StringIO() - got = check(m, w, out=buf) + got = checker(m, w, out=buf) if got == want: passed += 1 print(f" ok {desc}") else: failed += 1 print(f" FAIL {desc}: got {got}, want {want}") - for line in buf.getvalue().splitlines()[:5]: + for line in buf.getvalue().splitlines()[:6]: print(f" {line}") print("check_fuzz_targets self-test") + print(" nightly matrix (REQ-GUARD-GATE-EVIDENCE-002 h)") case("declared set == run set", 0, _MANIFEST, _WORKFLOW) # The bug this exists for: a harness that is never fuzzed. case("REGRESSION: declared but NOT run must FAIL", 1, @@ -186,6 +350,91 @@ def case(desc: str, want: int, manifest: str, workflow: str) -> None: """) case("quoted matrix target is unquoted before comparing", 0, _MANIFEST, _WORKFLOW.replace("- target: fuzz_aadl_parse", '- target: "fuzz_aadl_parse"')) + + print(" fuzz-smoke steps (REQ-GUARD-FUZZ-SMOKE-001, #406)") + # Base case doubles as the non-vacuity proof for the extractor: it only + # passes if the `--target x86_64-unknown-linux-gnu` triple is NOT read as a + # third harness (that would be an `extra`, exit 1) and the libfuzzer args + # after `--` are dropped. + case("declared set == fuzz-smoke run set", 0, _MANIFEST, _CI, + checker=check_smoke) + # THE #406 ACCEPTANCE CRITERION: a [[bin]] added to fuzz/Cargo.toml but not + # to ci.yml's fuzz-smoke job must fail a gate. Distinct input (3 declared), + # distinct output (exit 1) from the base case (2 declared, exit 0) — the + # property the old advisory-only tool lacked. + case("REGRESSION #406: declared but NOT in fuzz-smoke must FAIL", 1, + _MANIFEST + '\n[[bin]]\nname = "fuzz_codegen_roundtrip"\n', _CI, + checker=check_smoke) + case("fuzz-smoke runs a target NOT declared must FAIL", 1, _MANIFEST, + _CI + " - name: fuzz_ghost\n run: cargo +nightly fuzz run " + "--target x86_64-unknown-linux-gnu fuzz_ghost -- -max_total_time=60\n", + checker=check_smoke) + # No `fuzz run` step at all is a broken scan, not a disabled job. + case("no fuzz-smoke steps is a broken scan", 2, _MANIFEST, + " fuzz-smoke:\n steps:\n - name: checkout\n run: echo hi\n", + checker=check_smoke) + # A `fuzz run` step that does not resolve to one target fails loud (exit 2) + # rather than dropping the leg — e.g. an unrecognised value-flag form that + # leaves two positionals. + case("fuzz run with an ambiguous command is a broken scan", 2, _MANIFEST, + " fuzz-smoke:\n steps:\n - run: cargo +nightly fuzz run " + "--unknown-value-flag stray_value fuzz_aadl_parse\n", + checker=check_smoke) + # The `--flag=value` equals form is a single hyphen token, correctly a flag. + case("equals-form flag does not leak a positional", 0, _MANIFEST, + _CI.replace("--target x86_64-unknown-linux-gnu", + "--target=x86_64-unknown-linux-gnu"), + checker=check_smoke) + # THE REGRESSION AN EARLIER VERSION SHIPPED: a `fuzz run` string in a prose + # comment of ANOTHER job (here `guard:`) must not poison the scan. A + # whole-file scan matched exactly this and read the required gate as a + # broken scan; the block scoping must exit 0 here, not 2. + case("a `fuzz run` comment in another job does not poison the scan", 0, + _MANIFEST, + _CI + " guard:\n steps:\n # one `cargo fuzz run ` " + "step per harness — poisoning comment\n - run: echo audit\n", + checker=check_smoke) + # A comment INSIDE the fuzz-smoke block is skipped too (defense in depth). + case("a `fuzz run` comment inside the block is skipped", 0, _MANIFEST, + _CI.replace(" steps:\n", + " steps:\n # cargo fuzz run — inside-block " + "comment must be skipped\n"), + checker=check_smoke) + # No fuzz-smoke job at all is a broken scan (the required job vanished), not + # an empty pass. + case("ci.yml with no fuzz-smoke job is a broken scan", 2, _MANIFEST, + "jobs:\n other:\n steps:\n - run: echo hi\n", + checker=check_smoke) + + # Real-tree guard. The synthetic fixtures above cannot catch a `fuzz run` + # string in a COMMENT of the ACTUAL committed ci.yml — the exact regression + # an earlier whole-file scan shipped (#406). If the repo files are reachable + # from cwd, assert the shipped tree is clean under BOTH audits; this is the + # row that fails loudly the moment a poisoning comment lands in ci.yml. + print(" real committed tree (if reachable)") + + def real_case(desc: str, checker, run_path: str) -> None: + nonlocal passed, failed + manifest = Path("fuzz/Cargo.toml") + wf = Path(run_path) + if not (manifest.exists() and wf.exists()): + print(f" skip {desc} (repo files not reachable from cwd)") + return + buf = io.StringIO() + got = checker(manifest, wf, out=buf) + if got == 0: + passed += 1 + print(f" ok {desc}") + else: + failed += 1 + print(f" FAIL {desc}: got {got}, want 0") + for line in buf.getvalue().splitlines()[:6]: + print(f" {line}") + + real_case("real ci.yml fuzz-smoke is clean (no comment poisons it)", + check_smoke, ".github/workflows/ci.yml") + real_case("real fuzz-nightly matrix is clean", + check, ".github/workflows/fuzz-nightly.yml") print(f"\n{passed} passed, {failed} failed") return 1 if failed else 0 @@ -193,12 +442,19 @@ def case(desc: str, want: int, manifest: str, workflow: str) -> None: def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--manifest", default="fuzz/Cargo.toml") - ap.add_argument("--workflow", default=".github/workflows/fuzz-nightly.yml") + ap.add_argument("--workflow", default=".github/workflows/fuzz-nightly.yml", + help="advisory nightly workflow (matrix include form)") + ap.add_argument("--ci-workflow", default=".github/workflows/ci.yml", + help="workflow carrying the REQUIRED fuzz-smoke job (#406)") ap.add_argument("--self-test", action="store_true") a = ap.parse_args() if a.self_test: return self_test() - return check(Path(a.manifest), Path(a.workflow)) + # Audit the declaration against BOTH run-lists; the worst exit wins so a + # broken scan (2) is never masked by a clean sibling (0). + rc_nightly = check(Path(a.manifest), Path(a.workflow)) + rc_smoke = check_smoke(Path(a.manifest), Path(a.ci_workflow)) + return max(rc_nightly, rc_smoke) if __name__ == "__main__":