diff --git a/CONTEXT.md b/CONTEXT.md index 883952f..38fa6d4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -21,7 +21,7 @@ A security-analysis tool whose engine is a maintained Rust crate compiled into B _Avoid_: Built-in tool, internal scanner **External scanner**: -A third-party analysis binary whose value is its curated **rule corpus** rather than its engine, run only under an inode-pinned `exec.allow` grant with argv Bee constructs from typed inputs. Bee never reimplements a corpus and never hands the model a command line. +A third-party analysis binary whose value is its curated **rule corpus** rather than its engine, run only under an inode-pinned `exec.allow` grant with argv Bee constructs from typed inputs. Bee never reimplements a corpus and never hands the model a command line. One scan may be several children — a version pin verified before use, then the analysis itself — but every one of them is a child, because asking a binary what it is, running it, and reading what it wrote are all things the harness does from outside the scope if it does them itself. _Avoid_: Plugin, integration, shell-out **Finding ledger**: diff --git a/README.md b/README.md index da57b25..cdc1a66 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ cargo build --features astgrep,astgrep-rust # just structural searc | `astgrep` + `astgrep-` | `ast_grep` | Structural search over the parse tree, so a match in a comment or a string literal is not a match. One feature per grammar — each is compiled C, so a build pays only for the languages it scans. | | `findings` | `record_finding`, `list_findings` | The durable finding ledger. | | `cvss` | `cvss` | Severity **computed** from a vector, never asserted by the model. | -| `scanners` | `scan` | External scanner adapters (Opengrep today). Implies `findings`. | +| `scanners` | `scan` | External scanner adapters — Opengrep for pattern rules, CodeQL for whole-program analysis. Implies `findings`. | | `gitlog` | `git_log` | Repository history via `gix` — `log` for the commits touching a path, `blame` for the commit that introduced one line. Read-only; runs in a scope-joined child like every other file tool, because `.git` is a directory of files. | | `sec` | — | Umbrella for all of the above. Grammars stay explicit. | @@ -144,6 +144,25 @@ allow = ["!/home/you/.local/bin/opengrep"] # `!` pins the inode rules = "/etc/bee/opengrep-rules" # `auto` is refused: a scanning scope has no egress ``` +**CodeQL is the same shape, plus a version pin.** The grant names the bundle's own CLI; the pin is +verified by running it before anything is analysed, so a bundle that is not the one the operator +provisioned is an explicit unavailability rather than a thin set of results. + +```toml +[exec] +allow = ["!/opt/codeql-bundle/codeql/codeql"] + +[security.scanners.codeql] +bundle_version = "codeql-bundle-v2.26.1" # or "2.26.1" — either spelling of the same pin +``` + +Databases are built one way, `--build-mode=none`, so `actions`, `csharp`, `java`, `javascript`, +`python`, and `ruby` are analysable and everything else is **declined by name**. Extracting a +compiled language means intercepting the build's process spawns, which would mean admitting every +compiler and linker that build happens to invoke — an unbounded widening of the very scope bee +exists to hold. Half-support would be worse than none: a thin database yields few findings, which +reads exactly like clean code. + **Findings outlive the session.** They land in `.bee/findings/ledger.jsonl` — append-only, one JSON object per line, meant to be committed and reviewed in a pull request. Re-running merges rather than duplicating, and identity excludes the line number so code movement does not mint a duplicate. diff --git a/specs/016-native-tools/contracts/scanner-adapter.md b/specs/016-native-tools/contracts/scanner-adapter.md index 06414cb..e9e4529 100644 --- a/specs/016-native-tools/contracts/scanner-adapter.md +++ b/specs/016-native-tools/contracts/scanner-adapter.md @@ -10,13 +10,35 @@ pub trait ScannerAdapter: Send + Sync { /// Stable name, used in policy grants and in `FindingSource::Scanner(name)`. fn name(&self) -> &'static str; - /// Can this scanner run right now? Checks existence, pin, and any provisioned artefact. + /// Can this scanner answer THIS request right now? Existence, pin, provisioned artefacts, + /// and whether the thing being asked for is something this adapter will do at all. /// Runs **before** argv construction, so unavailability is reported without spawning. - fn probe(&self, grant: &ScannerGrant) -> Result<(), UnavailableReason>; - - /// Build the child's argv from typed, validated inputs. The ONLY place argv is authored. + /// + /// It takes the request because availability is not purely a property of the installation: + /// a CodeQL bundle that is present, pinned, and correct still cannot analyse a language + /// whose extraction requires observing a build (FR-011). + fn probe(&self, grant: &ScannerGrant, req: &ScanRequest) -> Result<(), UnavailableReason>; + + /// A child to run before the scan, whose stdout `verify_preflight` reads. `None` — the + /// default — means there is nothing to ask the binary before using it. Some facts can only + /// be had by asking the tool, and asking it is running it; it is spawned in scope like any + /// other child rather than probed from the harness (Constitution III). + fn preflight(&self, grant: &ScannerGrant) -> Option> { None } + + /// Judge what `preflight` printed. An `Err` stops the scan before its first step, so a + /// failed check is an unavailability and never a scan that found nothing. + fn verify_preflight(&self, grant: &ScannerGrant, stdout: &str) + -> Result<(), UnavailableReason> { Ok(()) } + + /// Build the scan's children from typed, validated inputs. The ONLY place argv is authored. /// Returns an error — never a partially-built command — if any input fails validation. - fn argv(&self, grant: &ScannerGrant, req: &ScanRequest, out: &Path) -> Result, String>; + /// One entry per child, run in order; every step must succeed before the next one runs. + /// Opengrep needs one step, CodeQL needs create-then-analyse. + /// + /// `scratch` is a per-call directory inside the scope, created before this is called and + /// removed afterwards. `out` is where the last step must leave its report. + fn steps(&self, grant: &ScannerGrant, req: &ScanRequest, scratch: &Path, out: &Path) + -> Result>, String>; /// Where this scanner writes its report, relative to the scratch dir handed to it. fn report_kind(&self) -> ReportKind; // Sarif for both Opengrep and CodeQL @@ -45,7 +67,7 @@ This is the discipline `src/search.rs` already states for ripgrep, generalised: > None of ripgrep's command-executing options (`--pre`, `--search-zip`) are exposed — the model > drives this only through `SearchArgs`, and there is nothing here that runs another program. -Validation performed in `argv()`, all of which return `Err` rather than a degraded command: +Validation performed in `steps()`, all of which return `Err` rather than a degraded command: - `target` resolves inside the scope after symlink resolution - `grant.rules`, when set, resolves inside the scope and is not `auto` (research R6) - no argument begins with `-` unless the adapter itself emitted it @@ -149,29 +171,61 @@ the network (cached under `~/.opengrep/cli`), and a scanning scope has no egress opaquely. Measured: the same target scanned with a local ruleset produced a ~600-byte report versus 1.9 MB with `auto`. -## CodeQL adapter (US6, deferred) +## CodeQL adapter (US6) Consumes an operator-provisioned, version-pinned **bundle** — `codeql-bundle-.tar.zst` from `github/codeql-action` releases, which ships the CLI, matching queries, and precompiled query packs. `codeql-action` v4.37.3 pins `codeql-bundle-v2.26.1` / CLI `2.26.1` (`src/defaults.json`). ```text -probe : /codeql/codeql version --format=json → compare against grant.bundle_version - mismatch or absent ⇒ Unavailable { BundleMismatch { expected, found } } +probe : pin + bundle presence + a version pinned at all + is this language analysable + no process is spawned to answer any of it + +preflight : codeql version --format=json → compare against grant.bundle_version + mismatch, unreadable, or unpinned ⇒ Unavailable { BundleMismatch { expected, found } } -argv : database create --language= --build-mode=none --source-root= - database analyze --format=sarif-latest --output= +steps : database create --language= --build-mode=none --source-root= + database analyze --format=sarif-latest --output= ``` +The granted binary **is** the bundle's CLI, so the inode pin already covers what runs; the +`[security.scanners.codeql] bundle` path is optional, and when set it is checked to contain the +granted binary — otherwise the version verified belongs to a different CodeQL than the one that +scans. `bundle_version` accepts either spelling (`codeql-bundle-v2.26.1` or `2.26.1`), because the +operator has the bundle version to hand while the CLI only ever reports the CLI version. An +**unpinned** bundle yields no preflight and no command: nothing unverified is reached by any route. +`grant.rules`, when set, names a query suite and replaces the default `codeql/-queries` pack. + +The database lives in the per-call scratch directory and is removed with it — it is large, and it is +built out of the code under analysis. + **Only `build-mode: none` languages are supported.** `databaseInitCluster` (`codeql-action/src/codeql.ts:547`) pushes `--begin-tracing` and `--trace-process-name` for compiled languages, because extraction works by **intercepting the build's process spawns**. That requires admitting every compiler, linker, and build tool the target's build happens to invoke — an unbounded, un-attenuable widening that surrenders SC-006 and violates Constitution II. A request for a traced -language is declined explicitly (`Unavailable`), because half-support is worse than none. - -`rust` is a builtin CodeQL language (`codeql-action/src/languages/builtin.json`), so bee can -eventually analyse itself. +language is declined explicitly (`Unavailable { LanguageRequiresBuild }`), because half-support is +worse than none: a traced language extracted without tracing yields a thin database, and a thin +database yields few findings, which reads exactly like clean code. + +**How the supported set is decided, and why it is a static list.** CodeQL answers this by a +filesystem fact — `isTracedLanguage` (`codeql-action/src/codeql.ts:535`) stats +`/tools/tracing-config.lua` in the provisioned distribution. bee cannot read that from the +harness (Constitution III), and asking the CLI would cost two more children per scan, so +`BUILDLESS_LANGUAGES` is a list in `src/scanners/codeql.rs` instead. Safe to trade because it +**fails closed**: an unlisted language is declined, so staleness costs coverage and never +correctness. + +| Language | Analysable | Why | +|---|---|---| +| `actions`, `javascript`, `python`, `ruby` | yes | scanned languages — never traced under any build mode | +| `java`, `csharp` | yes | first-class `build-mode: none` extractors (`src/analyze.ts:129-147`) | +| `cpp`, `swift` | no | traced | +| `go` | no | documented as not yet supporting build-mode none (`src/config-utils.ts:849`) | +| `rust` | not yet | a builtin language and very likely buildless, but "likely" is not evidence — it goes in when a real bundle says so, and then bee can analyse itself | + +CodeQL's own alias table is applied first (`src/languages/builtin.json`), so `typescript` reaches the +`javascript` extractor and `kotlin` the `java` one rather than being declined over spelling. ## Untrusted output (FR-015/016) diff --git a/specs/016-native-tools/quickstart.md b/specs/016-native-tools/quickstart.md index 848c4ce..7dd4185 100644 --- a/specs/016-native-tools/quickstart.md +++ b/specs/016-native-tools/quickstart.md @@ -355,11 +355,38 @@ cargo build -p bee-core # Expected: bee's own inode, plus exactly the scanners explicitly granted. Nothing else. ``` +## US6 — the CodeQL bundle + +Built, and testable without provisioning a bundle, because everything US6 turns on is a refusal: + +```bash +cargo test --features scanners --test codeql_adapter +# Expected: 10 passed. Each refusal asserts both the outcome and that the stub CLI logged no +# invocation — for a version mismatch, exactly one (the version check itself, and nothing after). +``` + +Configuration is the Opengrep shape plus a pin. The grant names the bundle's own CLI: + +```toml +[exec] +allow = ["!/opt/codeql-bundle/codeql/codeql"] + +[security.scanners.codeql] +bundle_version = "codeql-bundle-v2.26.1" # or "2.26.1"; both spell the same pin +``` + +Analysable: `actions`, `csharp`, `java`, `javascript`, `python`, `ruby` — plus CodeQL's own aliases, +so `typescript` reaches the `javascript` extractor. Everything else is declined **by name** with what +*is* analysable, because a traced language extracted without tracing yields a thin database, and a +thin database reads exactly like clean code. + +> **Not yet walked against a real bundle.** A stub proves bee's half — the argv it authors, the order +> it runs, every path on which it refuses, and the report reaching the ledger. It cannot prove that +> CodeQL, handed these arguments, produces useful findings. That walkthrough is **T073**, and it is +> also where `rust` gets settled: if the bundle's Rust extractor ships no `tools/tracing-config.lua`, +> it joins the list and bee can analyse itself. + ## Deferred to follow-on -- **US6** (CodeQL bundle) is planned in - [`contracts/scanner-adapter.md`](./contracts/scanner-adapter.md) but is not built. Its traced - languages would require admitting every compiler and linker a build invokes (research R7), which - is the widening SC-006 exists to prevent — so it stays deferred deliberately, not incidentally. - **VM case `scanner-escape-denied`** — landed on the matrix (38/38) once 017 made a scanner grant installable under enforcement; see `specs/017-enforceable-pins/`. diff --git a/specs/016-native-tools/tasks.md b/specs/016-native-tools/tasks.md index f4104ca..2a41b0e 100644 --- a/specs/016-native-tools/tasks.md +++ b/specs/016-native-tools/tasks.md @@ -198,16 +198,31 @@ malformed vector is rejected; a caller-supplied score is rejected rather than re --- -## Phase 8: User Story 6 — Deep whole-program analysis (P6) — DEFERRED +## Phase 8: User Story 6 — Deep whole-program analysis (P6) **Goal**: Run a provisioned, version-pinned CodeQL bundle over build-mode-`none` languages. -> Not in the first slice. Depends on the whole external tier (US3) being proven. - -- [ ] T058 [P] [US6] Write failing tests in `tests/codeql_adapter.rs`: an absent or version-mismatched bundle returns `Unavailable { BundleMismatch { expected, found } }` (US6 scenario 2); a traced (compiled) language is declined explicitly rather than half-supported (US6 scenario 3). -- [ ] T059 [US6] Implement bundle probing in `src/scanners/codeql.rs` — run `/codeql/codeql version --format=json` and compare against `grant.bundle_version` (pin `codeql-bundle-v2.26.1` / CLI `2.26.1`, per `codeql-action` v4.37.3 `src/defaults.json`). *(depends: T043)* -- [ ] T060 [US6] Implement the argv builder in `src/scanners/codeql.rs`: `database create --language= --build-mode=none --source-root=` then `database analyze --format=sarif-latest --output= `. *(depends: T059)* -- [ ] T061 [US6] Refuse traced languages in `src/scanners/codeql.rs` with an explanatory `Unavailable` — extraction for compiled languages intercepts the build's process spawns (`--begin-tracing`, `--trace-process-name`), which would require admitting every compiler and linker the build invokes, an unbounded widening that surrenders SC-006 and violates Constitution II (research R7). *(depends: T060)* +> Landed after the external tier was proven, as planned. Two things this phase changed that were not +> foreseen when it was written. The `ScannerAdapter` trait grew from one argv to three phases — +> `probe` / `preflight` / `steps` — because a CodeQL analysis is two commands and its version pin can +> only be read by *running* the CLI, which has to happen in a scope-joined child like everything +> else. And `probe` now takes the `ScanRequest`: a bundle can be present, pinned, and correct and +> still be unable to answer the question asked, which is a fact about the request, not the +> installation. +> +> **Not verified against a real bundle.** Scenarios 2 and 3 are refusals, and refusals are provable +> against a stub — indeed only provable that way, since a real bundle cannot be asked to report the +> wrong version on demand, and half of what `tests/codeql_adapter.rs` asserts is that a child was +> never spawned. Scenario 1's happy path is proven as far as bee's half goes (argv, order, the report +> reaching the ledger); that CodeQL handed these arguments produces useful findings needs a +> provisioned bundle and a `quickstart.md` walkthrough. See T073. + +- [X] T058 [P] [US6] Write failing tests in `tests/codeql_adapter.rs`: an absent or version-mismatched bundle returns `Unavailable { BundleMismatch { expected, found } }` (US6 scenario 2); a traced (compiled) language is declined explicitly rather than half-supported (US6 scenario 3). *(10 cases, each driven through `ScanTool` against a stub CLI rather than against the adapter directly, because the wiring is where a refusal would leak: every refusal case asserts not only the outcome but that the stub logged no invocation — or, for a version mismatch, exactly one. Also covers the two failure modes the task did not name and the stub made cheap: a CLI whose `version` output is unreadable is a mismatch rather than a pass, and an unpinned bundle produces no preflight and no command at all.)* +- [X] T059 [US6] Implement bundle probing in `src/scanners/codeql.rs` — run `/codeql/codeql version --format=json` and compare against `grant.bundle_version` (pin `codeql-bundle-v2.26.1` / CLI `2.26.1`, per `codeql-action` v4.37.3 `src/defaults.json`). *(depends: T043)* *(Split across the new `preflight` / `verify_preflight` pair: the version check is a child, because asking a binary what it is means running it, and running it from the harness would run it around the sandbox. What `probe` keeps is everything answerable without a process. Either spelling of the pin is accepted — the operator has `codeql-bundle-v2.26.1` to hand, the CLI only ever reports `2.26.1`, and rejecting one of those would turn a correct pin into a mismatch.)* +- [X] T060 [US6] Implement the argv builder in `src/scanners/codeql.rs`: `database create --language= --build-mode=none --source-root=` then `database analyze --format=sarif-latest --output= `. *(depends: T059)* *(The "then" is what forced `argv` to become `steps`. An intermediate step is judged by its exit status — it has no report to be judged by — while the last is still judged by the report, because an exit status conflates "findings exist" with "run failed" (research R6). The database goes in a per-call scratch directory that is removed with the call: it is large, and it is built out of the code under analysis.)* +- [X] T061 [US6] Refuse traced languages in `src/scanners/codeql.rs` with an explanatory `Unavailable` — extraction for compiled languages intercepts the build's process spawns (`--begin-tracing`, `--trace-process-name`), which would require admitting every compiler and linker the build invokes, an unbounded widening that surrenders SC-006 and violates Constitution II (research R7). *(depends: T060)* *(New `UnavailableReason::LanguageRequiresBuild`, carrying what **is** analysable so the refusal is actionable; kept distinct from `LanguageUnsupported`, which means "this build lacks that grammar" and which a rebuild fixes — this one no rebuild fixes. CodeQL decides tracedness by stat'ing `/tools/tracing-config.lua` (`codeql-action/src/codeql.ts:535`), a fact inside the provisioned bundle that bee cannot read from the harness, so `BUILDLESS_LANGUAGES` is a static evidence-backed list that fails closed: unlisted ⇒ declined, so staleness costs coverage, never correctness. `rust` is deliberately absent — very likely buildless, but "likely" is not evidence.)* + +- [ ] T073 [US6] Walk a provisioned `codeql-bundle-v2.26.1` end to end and record it in `quickstart.md` — one buildless language analysed for real, findings in the ledger, and the version pin confirmed against the CLI's actual `version --format=json` output. This is the only part of US6 a stub cannot stand in for. Settle `rust` while there: if the bundle's Rust extractor ships no `tools/tracing-config.lua`, add it to `BUILDLESS_LANGUAGES` and bee can analyse itself. --- diff --git a/src/scanners/codeql.rs b/src/scanners/codeql.rs new file mode 100644 index 0000000..2ca470b --- /dev/null +++ b/src/scanners/codeql.rs @@ -0,0 +1,550 @@ +//! The CodeQL adapter (016-native-tools US6). +//! +//! CodeQL is the deepest corpus bee can borrow — whole-program dataflow rather than pattern matching +//! — and it is also the one that asks the most of the operator: a provisioned, version-pinned +//! bundle, and a language whose facts can be extracted without watching a build. +//! +//! ## Why build-mode `none`, and nothing else +//! +//! CodeQL extracts a compiled language by **intercepting the build's process spawns** +//! (`--begin-tracing` / `--trace-process-name`, `codeql-action/src/codeql.ts:557`). Admitting that +//! means admitting every compiler, linker, and build tool the target's build happens to invoke — +//! an unbounded widening bee cannot attenuate and would not be able to describe to the operator it +//! asked. So bee builds databases one way, `--build-mode=none`, and declines the rest explicitly +//! (research R7, FR-011). Half-support would be worse than none: a traced language analysed without +//! tracing yields a thin database, and a thin database yields few findings, which reads exactly like +//! clean code. +//! +//! ## The supported set is a list, and it is deliberately short +//! +//! CodeQL itself decides tracedness by a filesystem fact — `isTracedLanguage` +//! (`codeql-action/src/codeql.ts:535`) stats `/tools/tracing-config.lua` inside the +//! provisioned distribution. bee cannot read that from the harness (Constitution III), and asking +//! the CLI would cost two more children per scan, so [`BUILDLESS_LANGUAGES`] is a static list +//! instead. That trades freshness for simplicity, and it is safe to trade because it **fails +//! closed**: a language bee does not list is declined, so the list going stale costs coverage and +//! never correctness. Adding to it is a code change with a bundle to check against. +//! +//! Measured against **`codeql-bundle-v2.26.1`** / CLI **2.26.1**, the pin `codeql-action` v4.37.3 +//! carries in `src/defaults.json`. + +use std::path::{Path, PathBuf}; + +use super::{probe_binary, validate_target, ScanRequest, ScannerAdapter, ScannerGrant}; +use crate::tools::outcome::UnavailableReason; + +pub struct CodeQl; + +/// The languages bee will build a database for, because none of them needs a build observed. +/// +/// Evidence, all from `codeql-action` v4.37.3: +/// +/// * `actions`, `javascript`, `python`, `ruby` are *scanned* languages — never traced under any +/// build mode, so `--build-mode=none` is the only mode they have. +/// * `java` and `csharp` have first-class `build-mode: none` extractors; the action carries +/// dedicated handling for where each puts its resolved dependencies (`src/analyze.ts:129-147`). +/// +/// Absent, and each for a reason: `cpp` and `swift` are traced; `go` is documented as not yet +/// supporting build-mode none (`src/config-utils.ts:849`); `rust` is a builtin language and very +/// likely buildless, but "likely" is not evidence, and the cost of being wrong is a scan that +/// quietly under-reports. It goes in when a real bundle says so. +pub const BUILDLESS_LANGUAGES: &[&str] = + &["actions", "csharp", "java", "javascript", "python", "ruby"]; + +/// Every language CodeQL itself knows, so bee can tell "I decline that" from "there is no such +/// thing" (`codeql-action/src/languages/builtin.json`). +const BUILTIN_LANGUAGES: &[&str] = &[ + "actions", + "cpp", + "csharp", + "go", + "java", + "javascript", + "python", + "ruby", + "rust", + "swift", +]; + +/// CodeQL's own spellings for the same extractor, from the `aliases` map in `builtin.json`. Applied +/// before the supported-set check so a caller asking for `typescript` is not told TypeScript is +/// unanalysable when `javascript` is the extractor that handles it. +const ALIASES: &[(&str, &str)] = &[ + ("c", "cpp"), + ("c-c++", "cpp"), + ("c-cpp", "cpp"), + ("c#", "csharp"), + ("c++", "cpp"), + ("java-kotlin", "java"), + ("javascript-typescript", "javascript"), + ("kotlin", "java"), + ("typescript", "javascript"), +]; + +/// Resolve a caller's language name to the extractor CodeQL would use. +fn canonical(lang: &str) -> String { + let lower = lang.trim().to_ascii_lowercase(); + ALIASES + .iter() + .find(|(from, _)| *from == lower) + .map(|(_, to)| (*to).to_string()) + .unwrap_or(lower) +} + +fn supported() -> Vec { + BUILDLESS_LANGUAGES.iter().map(|s| s.to_string()).collect() +} + +/// Reduce a bundle version to the CLI version it contains, so an operator may pin either spelling. +/// +/// `codeql-action` carries both — `"bundleVersion": "codeql-bundle-v2.26.1"` alongside +/// `"cliVersion": "2.26.1"` — and an operator provisioning a bundle has the first to hand while the +/// CLI only ever reports the second. Accepting one and rejecting the other would turn a correct pin +/// into a mismatch. +fn cli_version_of(pin: &str) -> &str { + let pin = pin.trim(); + let pin = pin.strip_prefix("codeql-bundle-").unwrap_or(pin); + pin.strip_prefix('v').unwrap_or(pin) +} + +impl ScannerAdapter for CodeQl { + fn name(&self) -> &'static str { + "codeql" + } + + /// Four questions, none of which costs a process: is the pinned binary still the pinned binary, + /// is the bundle where the operator said, was a version pinned at all, and is this a language + /// bee will analyse. + fn probe(&self, grant: &ScannerGrant, req: &ScanRequest) -> Result<(), UnavailableReason> { + probe_binary(grant)?; + + // A bundle path is optional — the granted binary is the bundle's CLI, so the pin already + // covers what actually runs. When the operator *does* name one, it has to be the bundle the + // granted binary came out of, or the version verified below belongs to a different CodeQL + // than the one that will do the scanning. + if let Some(bundle) = &grant.bundle { + let expected = grant + .bundle_version + .clone() + .unwrap_or_else(|| "(unpinned)".to_string()); + if !bundle.exists() { + return Err(UnavailableReason::BundleMismatch { + expected, + found: None, + }); + } + if !grant.path.starts_with(bundle) { + return Err(UnavailableReason::BundleMismatch { + expected, + found: Some(format!( + "the granted binary {} is not inside the configured bundle {}", + grant.path.display(), + bundle.display() + )), + }); + } + } + + let Some(lang) = req.lang.as_deref() else { + return Err(UnavailableReason::LanguageUnsupported { + lang: "(none specified)".to_string(), + compiled_in: supported(), + }); + }; + let lang = canonical(lang); + if BUILDLESS_LANGUAGES.contains(&lang.as_str()) { + return Ok(()); + } + // A language CodeQL has an extractor for, which bee declines on purpose, is a different + // answer from one nothing has ever heard of — and the caller can act on the difference. + if BUILTIN_LANGUAGES.contains(&lang.as_str()) { + return Err(UnavailableReason::LanguageRequiresBuild { + lang, + supported: supported(), + }); + } + Err(UnavailableReason::LanguageUnsupported { + lang, + compiled_in: supported(), + }) + } + + /// Ask the CLI its version. This is a child like any other, in scope, because asking a binary + /// what it is means running it — and running it from the harness would run it around the + /// sandbox (Constitution III). + /// + /// Returns `None` when no version was pinned: there is then nothing to compare against, and + /// [`ScannerAdapter::steps`] refuses to build a command at all, so an unverified bundle is never + /// reached by a different route. + fn preflight(&self, grant: &ScannerGrant) -> Option> { + grant.bundle_version.as_ref()?; + Some(vec!["version".to_string(), "--format=json".to_string()]) + } + + fn verify_preflight( + &self, + grant: &ScannerGrant, + stdout: &str, + ) -> Result<(), UnavailableReason> { + let Some(pin) = grant.bundle_version.as_deref() else { + return Ok(()); // no preflight was run; `steps` refuses below + }; + let expected = pin.to_string(); + + // Unreadable output is a mismatch, not a pass. A CLI that cannot say what version it is has + // not said it is the right one (Constitution I). + let found = serde_json::from_str::(stdout) + .ok() + .and_then(|v| v["version"].as_str().map(str::to_string)); + let Some(found) = found else { + return Err(UnavailableReason::BundleMismatch { + expected, + found: None, + }); + }; + + if cli_version_of(&found) == cli_version_of(pin) { + Ok(()) + } else { + Err(UnavailableReason::BundleMismatch { + expected, + found: Some(found), + }) + } + } + + /// Two children, because a CodeQL analysis is two operations: + /// + /// ```text + /// database create --language= --build-mode=none --source-root= + /// database analyze --format=sarif-latest --output= + /// ``` + /// + /// The database goes in the per-call scratch directory, so two scans in one episode cannot land + /// on each other and nothing survives the call. Every argument is a constant, a bee-chosen path, + /// or a value already checked against [`BUILDLESS_LANGUAGES`] — the model's `lang` cannot reach + /// the command line as anything but one of six known words. + fn steps( + &self, + grant: &ScannerGrant, + req: &ScanRequest, + scratch: &Path, + out: &Path, + ) -> Result>, String> { + validate_target(&req.target)?; + + if grant.bundle_version.is_none() { + return Err( + "no bundle version pinned for codeql: set `[security.scanners.codeql] \ + bundle_version` to the provisioned bundle's version (for example \ + `codeql-bundle-v2.26.1`). Running an unverified analysis bundle is refused, \ + because its queries decide what counts as a finding." + .to_string(), + ); + } + + // `probe` has already resolved and accepted this, but `steps` does not take that on trust: + // it is the function that authors the command line, so it checks what it is about to write. + let lang = canonical(req.lang.as_deref().unwrap_or_default()); + if !BUILDLESS_LANGUAGES.contains(&lang.as_str()) { + return Err(format!( + "`{lang}` is not analysable without a build; refusing to build the command" + )); + } + + let db: PathBuf = scratch.join("db"); + + // The query suite. Defaulting to the language's own pack is what makes the bundle worth + // provisioning — its precompiled queries are the corpus. An operator who wants a narrower + // set names one, and it is checked like any other configured path. + let suite = match &grant.rules { + Some(rules) => { + let s = rules.to_string_lossy(); + if s.starts_with('-') { + return Err(format!("query suite `{s}` would be read as a flag")); + } + if !rules.exists() { + return Err(format!("query suite `{s}` does not exist")); + } + s.into_owned() + } + None => format!("codeql/{lang}-queries"), + }; + + Ok(vec![ + vec![ + "database".to_string(), + "create".to_string(), + db.to_string_lossy().into_owned(), + format!("--language={lang}"), + "--build-mode=none".to_string(), + format!("--source-root={}", req.target.display()), + ], + vec![ + "database".to_string(), + "analyze".to_string(), + db.to_string_lossy().into_owned(), + "--format=sarif-latest".to_string(), + format!("--output={}", out.display()), + suite, + ], + ]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Fixture { + _tmp: tempfile::TempDir, + grant: ScannerGrant, + target: PathBuf, + scratch: PathBuf, + out: PathBuf, + } + + /// A grant that would work: the binary exists and is pinned, and a version is set. + fn fixture() -> Fixture { + let tmp = tempfile::tempdir().unwrap(); + let bin = tmp.path().join("codeql"); + std::fs::write(&bin, "#!/bin/sh\n").unwrap(); + let target = tmp.path().join("src"); + std::fs::create_dir_all(&target).unwrap(); + let scratch = tmp.path().join("scratch"); + std::fs::create_dir_all(&scratch).unwrap(); + let out = tmp.path().join("report.sarif"); + let mut grant = ScannerGrant::for_test("codeql", bin, None); + grant.bundle_version = Some("codeql-bundle-v2.26.1".to_string()); + Fixture { + _tmp: tmp, + grant, + target, + scratch, + out, + } + } + + fn req(f: &Fixture, lang: &str) -> ScanRequest { + let mut r = ScanRequest::new(f.target.clone()); + r.lang = Some(lang.to_string()); + r + } + + #[test] + fn a_bundle_and_a_cli_version_are_the_same_pin() { + assert_eq!(cli_version_of("codeql-bundle-v2.26.1"), "2.26.1"); + assert_eq!(cli_version_of("v2.26.1"), "2.26.1"); + assert_eq!(cli_version_of("2.26.1"), "2.26.1"); + } + + #[test] + fn an_alias_resolves_to_the_extractor_that_handles_it() { + assert_eq!(canonical("TypeScript"), "javascript"); + assert_eq!(canonical("kotlin"), "java"); + assert_eq!(canonical("C#"), "csharp"); + assert_eq!(canonical("python"), "python"); + } + + #[test] + fn typescript_is_analysable_because_javascript_is() { + let f = fixture(); + assert!(CodeQl.probe(&f.grant, &req(&f, "typescript")).is_ok()); + } + + #[test] + fn a_traced_language_is_declined_and_says_what_is_analysable() { + let f = fixture(); + for lang in ["cpp", "c++", "go", "swift", "rust"] { + let err = CodeQl.probe(&f.grant, &req(&f, lang)).unwrap_err(); + let UnavailableReason::LanguageRequiresBuild { supported, .. } = &err else { + panic!("{lang} should be declined as needing a build, got {err:?}"); + }; + assert!(supported.contains(&"python".to_string())); + // The refusal explains itself rather than just saying no. + let shown = err.to_string(); + assert!(shown.contains("build"), "{shown}"); + } + } + + #[test] + fn a_language_codeql_has_never_heard_of_is_a_different_answer() { + let f = fixture(); + assert!(matches!( + CodeQl.probe(&f.grant, &req(&f, "cobol")), + Err(UnavailableReason::LanguageUnsupported { .. }) + )); + } + + #[test] + fn no_language_at_all_is_refused_rather_than_guessed() { + let f = fixture(); + let r = ScanRequest::new(f.target.clone()); + assert!(matches!( + CodeQl.probe(&f.grant, &r), + Err(UnavailableReason::LanguageUnsupported { .. }) + )); + } + + #[test] + fn a_bundle_that_is_not_there_is_a_mismatch_naming_the_expected_version() { + let mut f = fixture(); + f.grant.bundle = Some(PathBuf::from("/nonexistent/codeql-bundle")); + let err = CodeQl.probe(&f.grant, &req(&f, "python")).unwrap_err(); + let UnavailableReason::BundleMismatch { expected, found } = &err else { + panic!("expected a bundle mismatch, got {err:?}"); + }; + assert_eq!(expected, "codeql-bundle-v2.26.1"); + assert!(found.is_none()); + assert!(err.to_string().contains("2.26.1")); + } + + #[test] + fn a_binary_outside_the_configured_bundle_is_refused() { + let mut f = fixture(); + // A real directory, but not the one the granted binary lives in: the version this would + // verify belongs to a different CodeQL than the one that would run. + let other = f.grant.path.parent().unwrap().join("other-bundle"); + std::fs::create_dir_all(&other).unwrap(); + f.grant.bundle = Some(other); + assert!(matches!( + CodeQl.probe(&f.grant, &req(&f, "python")), + Err(UnavailableReason::BundleMismatch { .. }) + )); + } + + #[test] + fn the_version_child_is_asked_for_machine_readable_output() { + let f = fixture(); + let argv = CodeQl.preflight(&f.grant).unwrap(); + assert_eq!(argv, vec!["version", "--format=json"]); + } + + #[test] + fn a_matching_version_passes_in_either_spelling() { + let f = fixture(); + assert!(CodeQl + .verify_preflight(&f.grant, r#"{"version":"2.26.1"}"#) + .is_ok()); + } + + #[test] + fn a_different_version_is_a_mismatch_that_names_both() { + let f = fixture(); + let err = CodeQl + .verify_preflight(&f.grant, r#"{"version":"2.20.0"}"#) + .unwrap_err(); + let UnavailableReason::BundleMismatch { expected, found } = &err else { + panic!("expected a bundle mismatch, got {err:?}"); + }; + assert_eq!(expected, "codeql-bundle-v2.26.1"); + assert_eq!(found.as_deref(), Some("2.20.0")); + } + + #[test] + fn a_cli_that_cannot_say_its_version_is_not_given_the_benefit_of_the_doubt() { + let f = fixture(); + for stdout in ["", "not json", "{}", r#"{"version":null}"#] { + assert!( + matches!( + CodeQl.verify_preflight(&f.grant, stdout), + Err(UnavailableReason::BundleMismatch { found: None, .. }) + ), + "output {stdout:?} should not pass the pin" + ); + } + } + + #[test] + fn the_two_commands_are_exactly_what_the_contract_says() { + let f = fixture(); + let steps = CodeQl + .steps(&f.grant, &req(&f, "python"), &f.scratch, &f.out) + .unwrap(); + assert_eq!(steps.len(), 2); + + let create = &steps[0]; + assert_eq!(create[0..2], ["database".to_string(), "create".to_string()]); + assert!(create.contains(&"--language=python".to_string())); + assert!(create.contains(&"--build-mode=none".to_string())); + assert!(create + .iter() + .any(|a| a.starts_with("--source-root=") && a.ends_with("src"))); + + let analyze = &steps[1]; + assert_eq!( + analyze[0..2], + ["database".to_string(), "analyze".to_string()] + ); + assert!(analyze.contains(&"--format=sarif-latest".to_string())); + assert!(analyze + .iter() + .any(|a| a.starts_with("--output=") && a.contains("report.sarif"))); + assert_eq!(analyze.last().unwrap(), "codeql/python-queries"); + + // Both steps name the same database, and it is inside the scratch bee chose. + assert_eq!(create[2], analyze[2]); + assert!(create[2].starts_with(&f.scratch.to_string_lossy().into_owned())); + } + + #[test] + fn nothing_here_ever_asks_for_tracing() { + let f = fixture(); + let steps = CodeQl + .steps(&f.grant, &req(&f, "java"), &f.scratch, &f.out) + .unwrap(); + for argv in &steps { + for arg in argv { + assert!( + !arg.contains("trace") + && !arg.contains("autobuild") + && !arg.contains("command"), + "argv must never ask CodeQL to observe a build: {arg}" + ); + } + } + } + + #[test] + fn an_unpinned_bundle_yields_no_command_and_no_preflight() { + let mut f = fixture(); + f.grant.bundle_version = None; + // Nothing to verify against... + assert!(CodeQl.preflight(&f.grant).is_none()); + // ...and therefore nothing to run. + let err = CodeQl + .steps(&f.grant, &req(&f, "python"), &f.scratch, &f.out) + .unwrap_err(); + assert!(err.contains("bundle_version"), "{err}"); + } + + #[test] + fn a_configured_suite_replaces_the_default_pack_and_is_checked() { + let mut f = fixture(); + let suite = f.scratch.join("custom.qls"); + std::fs::write(&suite, "- queries: .\n").unwrap(); + f.grant.rules = Some(suite.clone()); + let steps = CodeQl + .steps(&f.grant, &req(&f, "python"), &f.scratch, &f.out) + .unwrap(); + assert_eq!(steps[1].last().unwrap(), &suite.to_string_lossy()); + + f.grant.rules = Some(PathBuf::from("/nonexistent/custom.qls")); + assert!(CodeQl + .steps(&f.grant, &req(&f, "python"), &f.scratch, &f.out) + .is_err()); + f.grant.rules = Some(PathBuf::from("--rerun")); + assert!(CodeQl + .steps(&f.grant, &req(&f, "python"), &f.scratch, &f.out) + .is_err()); + } + + #[test] + fn steps_does_not_take_probes_word_for_the_language() { + // The two are separately reachable, so the one that writes the command line checks too. + let f = fixture(); + assert!(CodeQl + .steps(&f.grant, &req(&f, "cpp"), &f.scratch, &f.out) + .is_err()); + } +} diff --git a/src/scanners/mod.rs b/src/scanners/mod.rs index 1780c33..e6558c3 100644 --- a/src/scanners/mod.rs +++ b/src/scanners/mod.rs @@ -25,6 +25,7 @@ //! the kernel holds the same pin; in a host build this check is the only one, which is exactly why //! it lives at the tool layer rather than being left to the LSM. +pub mod codeql; pub mod opengrep; use std::path::{Path, PathBuf}; @@ -122,22 +123,61 @@ impl ScanRequest { } /// One external scanner bee knows how to drive. +/// +/// A scan is up to three phases, and an adapter opts into as much of that as it needs. Opengrep +/// uses one; CodeQL uses all three, which is why the shape is not simply "one argv" (US6). +/// +/// ```text +/// probe() no process at all — pin, provisioning, and whether this request is even answerable +/// preflight() one child whose stdout verify_preflight() reads — a version pin, checked in scope +/// steps() the scan itself, in order; every step must exit 0 before the next one runs +/// ``` pub trait ScannerAdapter: Send + Sync { /// Stable name, used in policy grants and in `FindingSource::Scanner(name)`. fn name(&self) -> &'static str; - /// Can this scanner run right now? Existence, pin, and any provisioned artefact. Runs **before** + /// Can this scanner answer *this request* right now? Existence, pin, provisioned artefacts, and + /// whether the thing being asked for is something this adapter will do at all. Runs **before** /// argv construction, so unavailability is reported without spawning anything. - fn probe(&self, grant: &ScannerGrant) -> Result<(), UnavailableReason>; + /// + /// It takes the request because availability is not purely a property of the installation: a + /// CodeQL bundle that is present, pinned, and correct still cannot analyse a language whose + /// extraction requires observing a build (FR-011). + fn probe(&self, grant: &ScannerGrant, req: &ScanRequest) -> Result<(), UnavailableReason>; + + /// A child to run before the scan, whose stdout [`ScannerAdapter::verify_preflight`] reads. + /// `None` — the default — means there is nothing to ask the binary before using it. + /// + /// This exists because some facts can only be had by asking the tool, and asking it is running + /// it: the tool is spawned in scope like any other child rather than probed from the harness + /// (Constitution III). + fn preflight(&self, _grant: &ScannerGrant) -> Option> { + None + } + + /// Judge what [`ScannerAdapter::preflight`] printed. An `Err` here stops the scan before its + /// first step, so a failed check is an unavailability and never a scan that found nothing. + fn verify_preflight( + &self, + _grant: &ScannerGrant, + _stdout: &str, + ) -> Result<(), UnavailableReason> { + Ok(()) + } - /// Build the child's argv from typed, validated inputs. The **only** place argv is authored. + /// Build the scan's children from typed, validated inputs. The **only** place argv is authored. /// Returns an error — never a partially-built command — if any input fails validation. - fn argv( + /// + /// `scratch` is a per-call directory inside the scope, created before this is called and removed + /// afterwards; an adapter that needs somewhere to put intermediate state puts it there. `out` is + /// where the last step must leave its report. + fn steps( &self, grant: &ScannerGrant, req: &ScanRequest, + scratch: &Path, out: &Path, - ) -> Result, String>; + ) -> Result>, String>; /// Where this scanner writes its report. fn report_kind(&self) -> ReportKind { @@ -149,12 +189,13 @@ pub trait ScannerAdapter: Send + Sync { pub fn adapter_for(name: &str) -> Option<&'static dyn ScannerAdapter> { match name { "opengrep" => Some(&opengrep::Opengrep), + "codeql" => Some(&codeql::CodeQl), _ => None, } } /// Every adapter bee ships. Used to decide which `exec.allow` entries are scanner grants. -pub const KNOWN_SCANNERS: &[&str] = &["opengrep"]; +pub const KNOWN_SCANNERS: &[&str] = &["opengrep", "codeql"]; /// The shared existence-and-pin check every adapter's `probe` delegates to. pub fn probe_binary(grant: &ScannerGrant) -> Result<(), UnavailableReason> { diff --git a/src/scanners/opengrep.rs b/src/scanners/opengrep.rs index bd6164e..fafec85 100644 --- a/src/scanners/opengrep.rs +++ b/src/scanners/opengrep.rs @@ -20,22 +20,28 @@ impl ScannerAdapter for Opengrep { "opengrep" } - fn probe(&self, grant: &ScannerGrant) -> Result<(), UnavailableReason> { + fn probe(&self, grant: &ScannerGrant, _req: &ScanRequest) -> Result<(), UnavailableReason> { + // Nothing about the request can make Opengrep unavailable: it needs no provisioned bundle, + // and its own rule corpus decides which languages it reads. probe_binary(grant) } + /// One step, because Opengrep needs one: + /// /// ```text /// scan --sarif --sarif-output= --quiet --config --timeout /// ``` /// /// Every argument here is either a constant or a validated, typed input. There is no path by - /// which a model-supplied string becomes a flag (FR-007). - fn argv( + /// which a model-supplied string becomes a flag (FR-007). The scratch directory goes unused — + /// Opengrep reads the tree and writes the report, with nothing in between to keep. + fn steps( &self, grant: &ScannerGrant, req: &ScanRequest, + _scratch: &Path, out: &Path, - ) -> Result, String> { + ) -> Result>, String> { validate_target(&req.target)?; let rules = grant.rules.as_ref().ok_or_else(|| { @@ -68,7 +74,7 @@ impl ScannerAdapter for Opengrep { // `--quiet` because Opengrep also prints the report to stdout when `--sarif` is set, and the // copy bee reads is the file. `--timeout` is Opengrep's own per-rule budget; the wall-clock // budget the tool enforces around the child is the one that actually bounds the call. - Ok(vec![ + Ok(vec![vec![ "scan".to_string(), "--sarif".to_string(), format!("--sarif-output={}", out.display()), @@ -78,7 +84,7 @@ impl ScannerAdapter for Opengrep { "--timeout".to_string(), req.timeout.as_secs().to_string(), req.target.to_string_lossy().into_owned(), - ]) + ]]) } } @@ -91,6 +97,7 @@ mod tests { _tmp: tempfile::TempDir, grant: ScannerGrant, target: PathBuf, + scratch: PathBuf, out: PathBuf, } @@ -100,22 +107,30 @@ mod tests { std::fs::write(&rules, "rules: []\n").unwrap(); let target = tmp.path().join("src"); std::fs::create_dir_all(&target).unwrap(); + let scratch = tmp.path().join("scratch"); + std::fs::create_dir_all(&scratch).unwrap(); let grant = ScannerGrant::for_test("opengrep", tmp.path().join("opengrep"), Some(rules)); let out = tmp.path().join("report.sarif"); Fixture { _tmp: tmp, grant, target, + scratch, out, } } + /// Opengrep is a one-step adapter; every case below asserts over that single command. + fn only_step(f: &Fixture, req: &ScanRequest) -> Result, String> { + let mut steps = Opengrep.steps(&f.grant, req, &f.scratch, &f.out)?; + assert_eq!(steps.len(), 1, "opengrep runs exactly one child"); + Ok(steps.remove(0)) + } + #[test] fn the_command_is_exactly_what_the_contract_says() { let f = fixture(); - let argv = Opengrep - .argv(&f.grant, &ScanRequest::new(f.target.clone()), &f.out) - .unwrap(); + let argv = only_step(&f, &ScanRequest::new(f.target.clone())).unwrap(); assert_eq!(argv[0], "scan"); assert!(argv.contains(&"--sarif".to_string())); assert!(argv.contains(&"--quiet".to_string())); @@ -131,9 +146,7 @@ mod tests { fn auto_is_refused_at_construction_with_the_reason() { let mut f = fixture(); f.grant.rules = Some(PathBuf::from("auto")); - let err = Opengrep - .argv(&f.grant, &ScanRequest::new(f.target.clone()), &f.out) - .unwrap_err(); + let err = only_step(&f, &ScanRequest::new(f.target.clone())).unwrap_err(); assert!(err.contains("auto"), "{err}"); assert!(err.contains("network"), "{err}"); } @@ -142,18 +155,14 @@ mod tests { fn no_ruleset_means_no_command() { let mut f = fixture(); f.grant.rules = None; - assert!(Opengrep - .argv(&f.grant, &ScanRequest::new(f.target.clone()), &f.out) - .is_err()); + assert!(only_step(&f, &ScanRequest::new(f.target.clone())).is_err()); } #[test] fn a_missing_ruleset_is_refused_before_spawning() { let mut f = fixture(); f.grant.rules = Some(PathBuf::from("/nonexistent/rules.yml")); - assert!(Opengrep - .argv(&f.grant, &ScanRequest::new(f.target.clone()), &f.out) - .is_err()); + assert!(only_step(&f, &ScanRequest::new(f.target.clone())).is_err()); } #[test] @@ -161,7 +170,7 @@ mod tests { let f = fixture(); let mut req = ScanRequest::new(f.target.clone()); req.timeout = std::time::Duration::from_secs(42); - let argv = Opengrep.argv(&f.grant, &req, &f.out).unwrap(); + let argv = only_step(&f, &req).unwrap(); let i = argv.iter().position(|a| a == "--timeout").unwrap(); assert_eq!(argv[i + 1], "42"); // The report path never comes from the caller — it is the one bee handed in. diff --git a/src/tools/outcome.rs b/src/tools/outcome.rs index d1dd50a..36a118d 100644 --- a/src/tools/outcome.rs +++ b/src/tools/outcome.rs @@ -71,6 +71,14 @@ pub enum UnavailableReason { }, /// The path is not inside a repository (US5 scenario 3). NotARepository { path: PathBuf }, + /// The requested language can only be analysed by observing a build, which bee will not do + /// (FR-011, US6 scenario 3). Carries the languages that *are* analysable, so a caller can retry + /// usefully. Distinct from [`UnavailableReason::LanguageUnsupported`], which is about a grammar + /// this binary was not built with: this one is a deliberate refusal, and no rebuild changes it. + LanguageRequiresBuild { + lang: String, + supported: Vec, + }, } impl std::fmt::Display for UnavailableReason { @@ -122,6 +130,20 @@ impl std::fmt::Display for UnavailableReason { UnavailableReason::NotARepository { path } => { write!(f, "{} is not inside a repository", path.display()) } + UnavailableReason::LanguageRequiresBuild { lang, supported } => { + write!( + f, + "`{lang}` can only be analysed by observing its build, which would mean \ + admitting every compiler and build tool that build happens to invoke; bee \ + will not widen a scope that far, so this is declined rather than half-run. \ + Analysable without a build: {}", + if supported.is_empty() { + "(none)".to_string() + } else { + supported.join(", ") + } + ) + } } } } @@ -137,6 +159,7 @@ impl UnavailableReason { UnavailableReason::BundleMismatch { .. } => "bundle_mismatch", UnavailableReason::LanguageUnsupported { .. } => "language_unsupported", UnavailableReason::NotARepository { .. } => "not_a_repository", + UnavailableReason::LanguageRequiresBuild { .. } => "language_requires_build", } } diff --git a/src/tools/scanner.rs b/src/tools/scanner.rs index 15656ad..e33d4ba 100644 --- a/src/tools/scanner.rs +++ b/src/tools/scanner.rs @@ -1,18 +1,26 @@ //! The `scan` tool (016-native-tools US3): run a granted external scanner, in scope, fail-closed. //! -//! ## The two-child pipeline, and why it is not one child +//! ## The pipeline, and why the report is a file rather than a stream //! //! ```text -//! ScannerGrant ──▶ child 1: the scanner argv built by the adapter, never by the model -//! (inode-pinned) │ writes -//! ▼ +//! ScannerGrant ──▶ preflight (optional) what is this binary? verified before it is used +//! (inode-pinned) │ +//! ▼ +//! the scan: one child per step, argv built by the adapter, never by the model +//! │ Opengrep needs one step; CodeQL needs create-then-analyse +//! │ writes +//! ▼ //! /report.sarif inside the scope -//! │ -//! child 2: `bee sarif-worker` reads and normalises IN SCOPE, bounded -//! ▼ +//! │ +//! `bee sarif-worker` reads and normalises IN SCOPE, bounded +//! ▼ //! ToolOutcome> //! ``` //! +//! Every one of those is a child. The harness never runs a scanner, never reads a report, and never +//! asks a binary what version it is — all three would be the harness reaching around the sandbox it +//! is meant to be imposing (Constitution III). +//! //! Forced by measurement (research R4): an Opengrep SARIF over one file is 1,912,546 bytes against a //! 102,400-byte `DEFAULT_OUTPUT_CAP`. Capturing it on stdout truncates it into unparseable JSON, //! which surfaces as "the scanner found nothing" — the failure FR-012 exists to prevent. Writing a @@ -183,14 +191,7 @@ impl Tool for ScanTool { }); }; - // ── 2. Is the binary there, and is it still the one that was granted? ─────────────────── - // Before argv construction and before any spawn, so unavailability costs no process — and, - // more importantly, so a swapped binary is never executed (FR-009, SC-010). - if let Err(reason) = adapter.probe(grant) { - return unavailable(reason); - } - - // ── 3. Build the command from typed inputs ────────────────────────────────────────────── + // ── 2. What is being asked, and within what budget? ───────────────────────────────────── let budget = match args.timeout_secs { // The operator's budget is a ceiling. A caller may ask for less, never for more. Some(secs) => Duration::from_secs(secs).min(grant.timeout), @@ -202,69 +203,149 @@ impl Tool for ScanTool { timeout: budget, }; + // ── 3. Can this scanner answer this request at all? ───────────────────────────────────── + // Before argv construction and before any spawn, so unavailability costs no process — and, + // more importantly, so a swapped binary is never executed (FR-009, SC-010). + if let Err(reason) = adapter.probe(grant, &req) { + return unavailable(reason); + } + // Unique per *call*, not per run: two scans in one episode, or two episodes sharing a // project, would otherwise write the same path and each would read a file the other was // still writing — producing a mid-document parse failure that looks like a broken scanner. static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let seq = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let report_path = PathBuf::from(SCAN_DIR).join(format!( - "{}-{}-{}-{}.sarif", + let stem = format!( + "{}-{}-{}-{}", args.scanner, self.run_id.replace(['/', ' ', '.'], "_"), std::process::id(), seq - )); - if let Some(parent) = report_path.parent() { - if let Err(e) = std::fs::create_dir_all(parent) { - return failed(format!( - "cannot create the scan directory {}: {e}", - parent.display() - )); - } + ); + let report_path = PathBuf::from(SCAN_DIR).join(format!("{stem}.sarif")); + // Intermediate state an adapter needs mid-scan — a CodeQL database, for instance — lives + // here and nowhere else, so it is bounded to the call and cleaned up with it. + let scratch = PathBuf::from(SCAN_DIR).join(&stem); + if let Err(e) = std::fs::create_dir_all(&scratch) { + return failed(format!( + "cannot create the scan directory {}: {e}", + scratch.display() + )); } // A stale report from an earlier call must never be mistaken for this call's output. let _ = std::fs::remove_file(&report_path); + let cleanup = || { + let _ = std::fs::remove_dir_all(&scratch); + }; - let argv = match adapter.argv(grant, &req, &report_path) { - Ok(a) => a, - Err(e) => return failed(e), + let steps = match adapter.steps(grant, &req, &scratch, &report_path) { + Ok(s) => s, + Err(e) => { + cleanup(); + return failed(e); + } }; - // ── 4. Child 1: the scanner ───────────────────────────────────────────────────────────── let program = grant.path.to_string_lossy().into_owned(); - let run = match run_child_timed(sandbox, &program, &argv, budget).await { - Ok(r) => r, - Err(ChildError::TimedOut(d)) => { - // No partial answer: whatever it wrote is by definition incomplete, and presenting - // an incomplete scan as a scan is the failure this feature is built against. - let _ = std::fs::remove_file(&report_path); + + // ── 4. Preflight: ask the binary what it is, in scope, before trusting its answers ────── + if let Some(argv) = adapter.preflight(grant) { + let probe = match run_child_timed(sandbox, &program, &argv, budget).await { + Ok(r) => r, + Err(ChildError::TimedOut(d)) => { + cleanup(); + return failed(format!( + "{} did not answer a version check within {}s", + args.scanner, + d.as_secs() + )); + } + Err(ChildError::Spawn(e)) => { + cleanup(); + return failed(e); + } + }; + // A check that did not run has not passed. Handing its empty stdout to the verifier + // would let a crashing binary look like an unreadable version — the right refusal by + // luck rather than by construction, so it is stated here instead. + if !probe.success { + cleanup(); return failed(format!( - "{} timed out after {}s; no partial findings are reported", + "{} could not report its version{}", args.scanner, - d.as_secs() + stderr_tail(&probe.stderr) )); } - Err(ChildError::Spawn(e)) => return failed(e), - }; - - // A signal kill (`code == None`) is not an ordinary non-zero exit and is never a clean scan. - if run.code.is_none() { - return failed(format!( - "{} was killed by a signal{}", - args.scanner, - stderr_tail(&run.stderr) - )); + if let Err(reason) = adapter.verify_preflight(grant, &probe.stdout) { + cleanup(); + return unavailable(reason); + } } - if !report_path.exists() { - return failed(format!( - "{} exited {} without writing a report{}", - args.scanner, - run.code.unwrap_or(-1), - stderr_tail(&run.stderr) - )); + + // ── 5. The scan itself, one child per step, in order ──────────────────────────────────── + // Every step must succeed before the next runs: a database that failed to build cannot be + // analysed, and analysing it anyway would produce an empty report — a clean scan by + // accident, which is the one outcome this tool may never manufacture (FR-012). + for (i, argv) in steps.iter().enumerate() { + let run = match run_child_timed(sandbox, &program, argv, budget).await { + Ok(r) => r, + Err(ChildError::TimedOut(d)) => { + // No partial answer: whatever it wrote is by definition incomplete, and + // presenting an incomplete scan as a scan is the failure this feature is built + // against. + let _ = std::fs::remove_file(&report_path); + cleanup(); + return failed(format!( + "{} timed out after {}s; no partial findings are reported", + args.scanner, + d.as_secs() + )); + } + Err(ChildError::Spawn(e)) => { + cleanup(); + return failed(e); + } + }; + + // A signal kill (`code == None`) is not an ordinary non-zero exit and is never a clean + // scan. + if run.code.is_none() { + cleanup(); + return failed(format!( + "{} was killed by a signal{}", + args.scanner, + stderr_tail(&run.stderr) + )); + } + // Intermediate steps are judged by their exit status because they have no report to be + // judged by; the final step is judged by the report, below, because an exit status + // conflates "findings exist" with "run failed" (research R6). + if i + 1 < steps.len() && !run.success { + cleanup(); + return failed(format!( + "{} step {} of {} exited {}{}", + args.scanner, + i + 1, + steps.len(), + run.code.unwrap_or(-1), + stderr_tail(&run.stderr) + )); + } + if i + 1 == steps.len() && !report_path.exists() { + cleanup(); + return failed(format!( + "{} exited {} without writing a report{}", + args.scanner, + run.code.unwrap_or(-1), + stderr_tail(&run.stderr) + )); + } } + // The database, or whatever else the scan needed on the way, has served its purpose. The + // report has not yet — child 2 still has to read it. + cleanup(); - // ── 5. Child 2: normalise in scope ────────────────────────────────────────────────────── + // ── 6. Child 2: normalise in scope ────────────────────────────────────────────────────── let exe = match self.exe() { Ok(e) => e, Err(e) => return failed(e), @@ -306,7 +387,7 @@ impl Tool for ScanTool { Err(e) => return failed(e), }; - // ── 6. Did the scan actually run? ─────────────────────────────────────────────────────── + // ── 7. Did the scan actually run? ─────────────────────────────────────────────────────── if !outcome.execution_successful { return failed(format!( "{} reported that its run did not complete successfully; its results are not \ @@ -315,7 +396,7 @@ impl Tool for ScanTool { )); } - // ── 7. Merge into the ledger ──────────────────────────────────────────────────────────── + // ── 8. Merge into the ledger ──────────────────────────────────────────────────────────── let mut recorded = 0usize; let mut rejected = Vec::new(); for f in &findings { diff --git a/tests/codeql_adapter.rs b/tests/codeql_adapter.rs new file mode 100644 index 0000000..b427e6a --- /dev/null +++ b/tests/codeql_adapter.rs @@ -0,0 +1,409 @@ +//! US6 — deep whole-program analysis (016-native-tools). +//! +//! CodeQL is the one adapter whose *availability* is a real question. Opengrep either runs or is +//! absent; a CodeQL bundle can be present, correctly pinned, and still be the wrong answer, because +//! the language asked for can only be extracted by watching a build. So these cases are about the +//! two refusals US6 turns on — a bundle that is not what was pinned (scenario 2), and a language bee +//! declines on purpose (scenario 3) — and about the thing both refusals share: **nothing runs**. +//! +//! Run with `cargo test --test codeql_adapter --features scanners`. No CodeQL bundle is needed. A +//! stub stands in for the CLI, which is what makes these testable at all: a real bundle cannot be +//! asked to report the wrong version on demand, and half of what is asserted here is that a +//! particular child was *never spawned*. +//! +//! What a stub cannot prove is that CodeQL, handed these arguments, produces useful results. That is +//! a walkthrough against a provisioned bundle, not a unit test — what is proven here is bee's half: +//! the argv it authors, the order it runs, and every path on which it refuses. + +#![cfg(feature = "scanners")] + +use std::path::{Path, PathBuf}; + +use bee::scanners::{self, ScannerGrant}; +use bee::security::{ScannerConfig, SecurityConfig}; +use bee::tools::outcome::CLEAN_PREFIX; +use bee::tools::scanner::ScanTool; +use bee::tools::Tool; + +const FIXTURE: &str = include_str!("fixtures/sarif/opengrep-shell-true.json"); + +/// The pin `codeql-action` v4.37.3 carries in `src/defaults.json`. +const PINNED: &str = "codeql-bundle-v2.26.1"; +const PINNED_CLI: &str = "2.26.1"; + +fn bee_exe() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_bee")) +} + +fn sandbox() -> bee::Sandbox { + bee::Sandbox::host(Vec::new()) +} + +/// A stub CodeQL CLI: answers `version`, creates a database directory, and writes a SARIF report +/// from `database analyze`. Every invocation is appended to `argv.log`, so a test can assert both +/// what bee built and — more often — that bee built nothing at all. +fn stub_codeql(dir: &Path, reported_version: &str) -> PathBuf { + let path = dir.join("codeql"); + let log = dir.join("argv.log"); + let version_json = format!(r#"{{"version":"{reported_version}"}}"#); + std::fs::write( + &path, + format!( + r#"#!/bin/sh +printf '%s\n' "$@" >> {log} +printf -- '--- end of invocation\n' >> {log} + +if [ "$1" = "version" ]; then + printf '%s\n' '{version}' + exit 0 +fi + +if [ "$1" = "database" ] && [ "$2" = "create" ]; then + mkdir -p "$3" + exit 0 +fi + +if [ "$1" = "database" ] && [ "$2" = "analyze" ]; then + out="" + for a in "$@"; do + case "$a" in + --output=*) out="${{a#--output=}}" ;; + esac + done + cat > "$out" <<'SARIF_EOF' +{body} +SARIF_EOF + exit 0 +fi + +exit 1 +"#, + log = log.display(), + version = version_json, + body = FIXTURE, + ), + ) + .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + path +} + +fn policy_granting(path: &Path) -> bee_core::Policy { + bee_core::Policy { + name: "scan".to_string(), + description: None, + mode: bee_core::Mode::default(), + filesystem: std::collections::BTreeMap::new(), + exec: bee_core::ExecPolicy { + allow: vec![format!("!{}", path.display())], + }, + network: Default::default(), + exfiltration: Default::default(), + } +} + +struct Fixture { + _tmp: tempfile::TempDir, + dir: PathBuf, + grants: Vec, + ledger: bee::findings::Ledger, + target: PathBuf, +} + +/// `reported_version` is what the stub CLI claims to be; `pinned` is what the operator configured; +/// `bundle` is the optional bundle root. Every case below is a combination of those three. +fn fixture(reported_version: &str, pinned: Option<&str>, bundle: Option) -> Fixture { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_path_buf(); + let bin = stub_codeql(&dir, reported_version); + let target = dir.join("src"); + std::fs::create_dir_all(&target).unwrap(); + std::fs::write(target.join("vuln.py"), "import subprocess\n").unwrap(); + + let mut security = SecurityConfig::default(); + security.scanners.insert( + "codeql".to_string(), + ScannerConfig { + bundle, + bundle_version: pinned.map(str::to_string), + timeout_secs: Some(30), + ..Default::default() + }, + ); + + let grants = scanners::grants_from_policy(Some(&policy_granting(&bin)), &security); + assert_eq!( + grants.len(), + 1, + "the pinned codeql entry must become a grant" + ); + Fixture { + _tmp: tmp, + dir: dir.clone(), + grants, + ledger: bee::findings::Ledger::at(dir.join("findings")), + target, + } +} + +/// The ordinary case: a correctly pinned bundle. +fn pinned_fixture() -> Fixture { + fixture(PINNED_CLI, Some(PINNED), None) +} + +async fn scan_lang(f: &Fixture, lang: &str) -> bee::ToolResult { + ScanTool::new(f.grants.clone(), f.ledger.clone(), "run-1") + .with_bee_exe(bee_exe()) + .call( + serde_json::json!({ + "scanner": "codeql", + "target": f.target.to_str().unwrap(), + "lang": lang, + }), + &sandbox(), + ) + .await +} + +/// Everything the stub was asked to do, one invocation per entry. +fn invocations(f: &Fixture) -> Vec> { + let log = f.dir.join("argv.log"); + if !log.exists() { + return Vec::new(); + } + let text = std::fs::read_to_string(log).unwrap(); + text.split("--- end of invocation\n") + .filter(|chunk| !chunk.trim().is_empty()) + .map(|chunk| chunk.lines().map(str::to_string).collect()) + .collect() +} + +// ── US6 scenario 2 · a bundle that is not the pinned bundle ────────────────────────────────────── + +#[tokio::test] +async fn a_version_mismatch_names_both_versions_and_analyses_nothing() { + let f = fixture("2.20.0", Some(PINNED), None); + let r = scan_lang(&f, "python").await; + + assert!(r.is_error, "a mismatched bundle must not succeed"); + assert!(r.content.contains(PINNED), "{}", r.content); + assert!(r.content.contains("2.20.0"), "{}", r.content); + assert!(!r.content.contains(CLEAN_PREFIX), "{}", r.content); + + // The load-bearing half: the version check ran, and then nothing else did. A mismatch that + // still analysed would be reporting results from a bundle nobody authorised. + let calls = invocations(&f); + assert_eq!( + calls.len(), + 1, + "only the version check may have run: {calls:?}" + ); + assert_eq!(calls[0][0], "version"); +} + +#[tokio::test] +async fn a_bundle_that_is_not_there_is_refused_before_the_cli_is_run_at_all() { + let f = fixture( + PINNED_CLI, + Some(PINNED), + Some(PathBuf::from("/nonexistent/codeql-bundle")), + ); + let r = scan_lang(&f, "python").await; + + assert!(r.is_error); + // Scenario 2 asks for the expected version by name, so an operator can compare it with what + // they provisioned without going to read bee's source. + assert!(r.content.contains(PINNED), "{}", r.content); + assert!(!r.content.contains(CLEAN_PREFIX), "{}", r.content); + assert!( + invocations(&f).is_empty(), + "an absent bundle is answerable without spawning anything" + ); +} + +#[tokio::test] +async fn a_cli_that_cannot_say_what_it_is_does_not_get_the_benefit_of_the_doubt() { + // Not a mismatch and not a crash — a version check that answers with something unreadable. The + // pin is unverified either way, so the scan does not proceed. + let f = fixture("not-json-at-all\", oops", Some(PINNED), None); + let r = scan_lang(&f, "python").await; + + assert!(r.is_error, "{}", r.content); + assert!(!r.content.contains(CLEAN_PREFIX), "{}", r.content); + let calls = invocations(&f); + assert_eq!( + calls.len(), + 1, + "nothing may run after an unverified pin: {calls:?}" + ); +} + +#[tokio::test] +async fn an_unpinned_bundle_yields_no_scan_and_says_which_setting_is_missing() { + let f = fixture(PINNED_CLI, None, None); + let r = scan_lang(&f, "python").await; + + assert!(r.is_error); + assert!(r.content.contains("bundle_version"), "{}", r.content); + assert!(!r.content.contains(CLEAN_PREFIX), "{}", r.content); + assert!( + invocations(&f).is_empty(), + "with nothing to verify against, there is nothing to run" + ); +} + +// ── US6 scenario 3 · a language that can only be analysed by watching a build ──────────────────── + +#[tokio::test] +async fn a_traced_language_is_declined_explicitly_rather_than_half_analysed() { + for lang in ["cpp", "c++", "go", "swift", "rust"] { + let f = pinned_fixture(); + let r = scan_lang(&f, lang).await; + + assert!(r.is_error, "{lang} must be declined"); + // "Declined", not "found nothing" — the distinction the whole feature exists to keep. + assert!(!r.content.contains(CLEAN_PREFIX), "{lang}: {}", r.content); + assert!(r.content.contains("build"), "{lang}: {}", r.content); + // And it says what *can* be analysed, so the refusal is actionable. + assert!(r.content.contains("python"), "{lang}: {}", r.content); + assert!( + invocations(&f).is_empty(), + "{lang}: a declined language must not start a database" + ); + } +} + +#[tokio::test] +async fn typescript_is_analysed_because_the_javascript_extractor_handles_it() { + // CodeQL's own alias table maps it (`src/languages/builtin.json`). Declining TypeScript because + // bee's list happens to spell it `javascript` would be a refusal with no reason behind it. + let f = pinned_fixture(); + let r = scan_lang(&f, "TypeScript").await; + assert!(!r.is_error, "{}", r.content); + + let calls = invocations(&f); + let create = calls + .iter() + .find(|c| c.get(1).map(String::as_str) == Some("create")); + let create = create.expect("a database should have been created"); + assert!( + create.contains(&"--language=javascript".to_string()), + "{create:?}" + ); +} + +// ── US6 scenario 1 · the analysis bee actually drives ──────────────────────────────────────────── + +#[tokio::test] +async fn a_buildless_language_is_created_then_analysed_and_lands_in_the_ledger() { + let f = pinned_fixture(); + let r = scan_lang(&f, "python").await; + assert!(!r.is_error, "{}", r.content); + + let calls = invocations(&f); + assert_eq!( + calls.len(), + 3, + "version, create, analyze — in that order: {calls:?}" + ); + assert_eq!(calls[0][0], "version"); + assert_eq!( + calls[1][0..2], + ["database".to_string(), "create".to_string()] + ); + assert_eq!( + calls[2][0..2], + ["database".to_string(), "analyze".to_string()] + ); + + assert!( + calls[1].contains(&"--build-mode=none".to_string()), + "{:?}", + calls[1] + ); + assert!( + calls[1].contains(&"--language=python".to_string()), + "{:?}", + calls[1] + ); + assert!( + calls[2].contains(&"--format=sarif-latest".to_string()), + "{:?}", + calls[2] + ); + assert_eq!(calls[2].last().unwrap(), "codeql/python-queries"); + + // Both database commands name the same database, or the second analysed nothing. + assert_eq!(calls[1][2], calls[2][2]); + + // The finding reached the ledger under CodeQL's name, which is what makes the tier worth having. + let view = bee::findings::fold(&f.ledger).unwrap(); + assert_eq!(view.findings.len(), 1, "{view:#?}"); + assert_eq!(view.findings[0].source.to_string(), "scanner:codeql"); +} + +#[tokio::test] +async fn bee_never_asks_codeql_to_observe_a_build() { + // SC-006 at the argv level. Tracing is the one thing that would force the scope open, and there + // is no request — no language, no flag combination — that makes bee write it. + let f = pinned_fixture(); + assert!(!scan_lang(&f, "python").await.is_error); + + for call in invocations(&f) { + for arg in &call { + assert!( + !arg.contains("trace") && !arg.contains("autobuild"), + "argv must never ask for tracing: {arg}" + ); + } + } +} + +#[tokio::test] +async fn the_database_does_not_outlive_the_call() { + // A CodeQL database is large and is built from the code under analysis. Leaving it behind would + // accumulate copies of the target in the project for no further benefit. + let f = pinned_fixture(); + assert!(!scan_lang(&f, "python").await.is_error); + + let calls = invocations(&f); + let db = calls + .iter() + .find(|c| c.get(1).map(String::as_str) == Some("create")) + .map(|c| PathBuf::from(&c[2])) + .expect("a database should have been created"); + assert!( + !db.exists(), + "the scratch database {} should have been removed", + db.display() + ); +} + +// ── The grant itself ───────────────────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn an_ungranted_codeql_refuses_however_much_of_it_is_installed() { + let tmp = tempfile::tempdir().unwrap(); + let bin = stub_codeql(tmp.path(), PINNED_CLI); + assert!(bin.exists()); + + let r = ScanTool::new(Vec::new(), bee::findings::Ledger::at(tmp.path()), "run-1") + .with_bee_exe(bee_exe()) + .call( + serde_json::json!({ + "scanner": "codeql", + "target": tmp.path().to_str().unwrap(), + "lang": "python", + }), + &sandbox(), + ) + .await; + + assert!(r.is_error); + assert!(r.content.contains("not granted"), "{}", r.content); + assert!(!r.content.contains(CLEAN_PREFIX), "{}", r.content); +} diff --git a/tests/scanner_adapter.rs b/tests/scanner_adapter.rs index db29071..caccbf6 100644 --- a/tests/scanner_adapter.rs +++ b/tests/scanner_adapter.rs @@ -281,9 +281,10 @@ fn config_auto_is_refused_when_the_command_is_built() { ScannerGrant::for_test("opengrep", tmp.path().join("opengrep"), Some("auto".into())); let adapter = scanners::adapter_for("opengrep").expect("opengrep adapter"); let err = adapter - .argv( + .steps( &grant, &ScanRequest::new(target), + tmp.path(), &tmp.path().join("out.sarif"), ) .expect_err("`auto` must be refused"); @@ -298,9 +299,10 @@ fn a_scanner_with_no_ruleset_configured_refuses_rather_than_defaulting_to_auto() let grant = ScannerGrant::for_test("opengrep", tmp.path().join("opengrep"), None); let adapter = scanners::adapter_for("opengrep").unwrap(); let err = adapter - .argv( + .steps( &grant, &ScanRequest::new(tmp.path().to_path_buf()), + tmp.path(), &tmp.path().join("out.sarif"), ) .expect_err("no rules ⇒ no scan"); @@ -317,9 +319,10 @@ fn the_model_cannot_smuggle_a_flag_through_the_target() { for hostile in ["--config=auto", "-e", "--pro"] { let err = adapter - .argv( + .steps( &grant, &ScanRequest::new(PathBuf::from(hostile)), + tmp.path(), &tmp.path().join("out.sarif"), ) .expect_err("a target that is really a flag must be refused");