From a395a8cd9f2b0ea61e41b51d25552a0df722bd8e Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Tue, 18 Aug 2026 02:18:22 +0200 Subject: [PATCH] docs: add verified researcher quickstart --- .github/workflows/release-please.yml | 39 +- CONTRIBUTING.md | 3 + README.md | 652 +++++++------------------- docs/index.md | 21 +- docs/maintainers/index.md | 91 ++++ docs/researcher-guide.md | 146 ++++++ mkdocs.yml | 2 + noxfile.py | 45 +- tools/check_project_services.py | 126 +++-- tools/check_readme_quickstart.py | 227 +++++++++ tools/tests/test_project_services.py | 120 ++++- tools/tests/test_readme_quickstart.py | 117 +++++ 12 files changed, 1018 insertions(+), 571 deletions(-) create mode 100644 docs/maintainers/index.md create mode 100644 docs/researcher-guide.md create mode 100644 tools/check_readme_quickstart.py create mode 100644 tools/tests/test_readme_quickstart.py diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 963f0c2..d8e7cb9 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -211,8 +211,9 @@ jobs: gh release upload "${TAG}" dist/* SHA256SUMS env-pack-assets/* ENV_PACK_SHA256SUMS - # Prove the published artifact installs and imports from the public index in a - # clean environment (no checkout, no local links). + # Prove the published artifact installs, imports, and executes the README + # quickstart from the public index in a clean environment (no checkout or + # local distribution links). smoke: needs: [release-please, publish] if: >- @@ -250,6 +251,40 @@ jobs: assert raes_adapters.__version__ == version("raes-adapters") PY test "$(/tmp/smoke/bin/python -c 'import raes_adapters; print(raes_adapters.__version__)')" = "${VERSION}" + - name: Run the published README quickstart + run: | + set -euo pipefail + quickstart_root="$(mktemp -d)" + cd "${quickstart_root}" + PYTHONPATH='' PYTHONSAFEPATH=1 /tmp/smoke/bin/raes-adapters run --mode conformance --suite pr --output cage2-quickstart > quickstart.json + /tmp/smoke/bin/python - <<'PY' + import json + from pathlib import Path + + expected = { + "disposition": "succeeded", + "evidence_basis": "hermetic-live", + "inventory": "inventory.json", + "mode": "conformance", + "run_count": 1, + } + assert json.loads(Path("quickstart.json").read_text()) == expected + artifacts = sorted( + path.as_posix() + for path in Path("cage2-quickstart").rglob("*") + if path.is_file() + ) + assert artifacts == [ + "cage2-quickstart/index.json", + "cage2-quickstart/inventory.json", + "cage2-quickstart/runs/cyborg-pr-seed-3/conformance/backend-conformance.json", + ] + inventory = json.loads(Path("cage2-quickstart/inventory.json").read_text()) + assert [item["path"] for item in inventory["artifacts"]] == [ + "index.json", + "runs/cyborg-pr-seed-3/conformance/backend-conformance.json", + ] + PY # After a release, the version bump + CHANGELOG land on `main`, so `dev` falls # one release commit behind. Open a back-merge PR so `dev` is resynced. This job diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ee2f74..69e4985 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,9 @@ ADR-069 §8). CAGE-2 replication is one backend served by the CybORG adapter rather than the scope of the repository. Contributions land through the Ground Control `/implement` workflow. +The [developer index](docs/maintainers/index.md) collects repository layout, +verification, packaging/release, governance, and adapter workflow references. + ## Branching - `main` — production. PRs only (from `dev`). No direct pushes, no force-push. diff --git a/README.md b/README.md index 001c00a..7dca7e4 100644 --- a/README.md +++ b/README.md @@ -1,509 +1,181 @@ -# raes-adapters +# RAES adapters [![Documentation](https://readthedocs.org/projects/raes-adapters/badge/?version=latest)](https://raes-adapters.readthedocs.io/en/latest/) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/OpenRAE/adapters/badge)](https://scorecard.dev/viewer/?uri=github.com/OpenRAE/adapters) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects?as=badge&url=https%3A%2F%2Fgithub.com%2FOpenRAE%2Fadapters)](https://www.bestpractices.dev/projects?as=entry&url=https%3A%2F%2Fgithub.com%2FOpenRAE%2Fadapters) -A single distribution, **`raes-adapters`**, that qualifies and realizes -[RAES](https://github.com/RAESystem/rae) scenarios against concrete simulator -backends. It ships shared adapter plumbing plus one importable module per -simulator. Implemented simulator dependencies are exposed as optional extras; -qualification evidence bounds the claims made for each selected backend. -Maintainer selection determines admission: a qualification record documents -source identity, attainable evidence, limitations, and claim strength, but it -does not veto implementation of a selected simulator. - -RAES — Reproducible Agentic Environments System — is the semantic authority. -Its scope is agentic environments generally: cyber, AI security, AI safety, -testing, research, and evaluation are examples of what an environment can model, -not the boundary of the model. Adapters here translate between a specific -simulator backend and the published RAES contracts; the shared surfaces speak in -participants, observations, actions, resources, controls, evaluation, provenance, -evidence, replay boundaries, and conformance. - -This repository hosts the adapter *implementations* and their build/CI mechanics. -It consumes published RAES contracts and never adds SDL, schemas, profiles, -vocabularies, or policy gates of its own (RAES ADR-069 §1). Backend-specific -concepts — including CybORG and CAGE-2 — stay inside the module that owns them -and never define the shared semantic boundary (ADR-002). - -## Install - -```bash -pip install raes-adapters # shared base plumbing + qualification evidence -``` - -The selected CybORG backend is admitted and usable through its documented -source installation. The `cyborg` extra key is dependency-light. Issue -[#12](https://github.com/OpenRAE/adapters/issues/12) qualified the official -CAGE Challenge 2 source and a packaging-only fix, but the upstream wheel omits -the version and Scenario2 runtime data. The fixed wheel passed a clean Python -3.12 smoke locally, but it is not a governed public artifact and the -distribution's declared Python/platform range is not yet qualified. Those -limitations bound installability and reproducibility claims; they do not veto -the maintainer-selected backend. Until that fix is published, install the -pinned CAGE-2 source at -commit `26ce1c1253fa9e2e73f25e6a7f2da32860c11257`, apply -`src/raes_adapters/cyborg/cage2-wheel-package-data.patch`, and install its -`CybORG/` package into the environment. The adapter validates the installed -version and selected source-file digests before constructing the backend. -The empty dependency list avoids advertising an editable checkout, -install-time clone, or unpublished wheel as an automatic installation route. - -The `cyberbattlesim` extra is likewise dependency-light. The -[qualification record](src/raes_adapters/cyberbattlesim/qualification.json) -binds Microsoft's legally usable, runnable source and admits the selected -profile. Because no official index/release artifact exists, users install the -pinned simulator source separately. Unbound random streams and open benchmark -findings remain explicit limits on deterministic-replay and outcome claims. - -The `primaite` extra is dependency-light for the same reason. The -[qualification record](src/raes_adapters/primaite/qualification.json) binds -DSTL's MIT-licensed PrimAITE source (tag `v4.0.0`) and admits the selected -`data_manipulation` profile through the source-native `PrimaiteGymEnv`. PrimAITE -publishes no index/release wheel, so users install the pinned source separately -(the qualified route pins `setuptools==75.6.0` to supply `pkg_resources`). A -broken public seed seam, undeclared runtime dependencies, and an unpinned -dependency graph remain explicit limits on deterministic-replay and -reproducibility claims. - -The `nasim` extra pins the published `nasim==0.12.0` distribution together with -its qualified runtime (`gymnasium==0.26.3`, `numpy==1.26.4`), so installing -`raes-adapters[nasim]` reproduces the admitted, runnable protocol. The -[qualification record](src/raes_adapters/nasim/qualification.json) binds Jonathon -Schwartz's MIT-licensed source and admits one `tiny`-benchmark bruteforce -protocol. The runtime pins gymnasium 0.26.3 because NASim's supplied agents -assert a Python-int action index that gymnasium >= 0.27 breaks; importing NASim -also requires the Tk system libraries. Those limitations bound reproducibility -claims; they do not veto the maintainer-selected backend. - -## One distribution, optional simulator extras - -RAES owns the *contracts* an adapter must honor; how this repository packages, -locks, and releases its code is a local decision (RAES ADR-069 §5 as amended — -see [RAESystem/rae#949](https://github.com/RAESystem/rae/issues/949) — and -[ADR-003](docs/decisions/adrs/adr-003-single-distribution-and-trusted-publishing.md)). -`raes-adapters` is one distribution: `raes_adapters.base` is always installed, -and each simulator is an optional module (`raes_adapters.cyborg`, ...) whose -heavy, mutually-incompatible dependencies live behind an extra. A single -`uv.lock` covers the tree; a future simulator with a conflicting stack is -isolated with uv's `conflicts` extras declaration, not a separate lockfile. - -## Shared adapter plumbing - -`raes_adapters.base` is available from the base installation and composes the -published RAES APIs directly: - -- `build_runtime_target` constructs `raes_runtime.RuntimeTarget` from a - published manifest and component set, leaving all shape checks to RAES. -- `apply_logical_clock_transition` dispatches a caller-selected transition - through `ReferenceTimeRuntime`; adapters must explicitly map native events and - supply exact coordinates. -- `apply_seed_controls` applies an ordered list of published stochastic-control - bindings through driver-local callables and returns RAES diagnostics for - applied, unbound, unsupported, and failed controls. Application alone is not - a replay claim. -- `execute_cleanup` admits a published cleanup plan, runs driver-local - operations in dependency order, and returns a validated - `TrialCleanupReceiptModel`. Operations are synchronous and remain responsible - for native timeout and verification mechanics. -- `project_action`, `project_observation`, and `project_evaluation` provide - typed direction-specific callable seams. Observation and evaluation outputs - must pass a caller-supplied RAES validator; native failures never become - portable fallback values. -- `redact_native_value` is default-deny and never renders arbitrary objects. - `bounded_context_label` admits only short, grammar-checked, intentionally safe - labels. -- `run_conformance_probe` returns the exact `BackendConformanceReport` from the - published RAES target runner without adding profiles, fixtures, cases, or - claims. - -These helpers do not define simulator concepts, portable DTOs, schemas, -backend protocols, diagnostic envelopes, stores, policy gates, or conformance -authority. - -## Layout +`raes-adapters` connects simulator backends to the published contracts of +[RAES](https://github.com/RAESystem/rae), the Reproducible Agentic +Environments System. A researcher can use an adapter to validate a packaged +scenario, inspect the selected simulator source and profile, exercise the +adapter boundary, and retain portable evidence from an admitted run. -```text -raes-adapters/ - pyproject.toml # the raes-adapters distribution (build + deps + extras) - noxfile.py # canonical verification graph - src/raes_adapters/ - base/ # shared adapter plumbing (ADR-069 §4) - cyberbattlesim/ # immutable qualification + selected public protocol - scenario/ # authored RAES SDL scenario (portable topology/objective truth) - experiment/ # published experiment contracts (reward/evaluator/stochastic intent) - mapping/ # pinned CyberBattleSim → RAES source ledger + loss disclosures - cyborg/ # CybORG qualification, patch evidence, and future backend - mapping/ # pinned CAGE-2 → RAES source ledger (REP-003) - profiles/ # conformance profile overrides - primaite/ # immutable qualification + selected public protocol - nasim/ # immutable NASim qualification + selected public protocol - tests/ # pytest suite for the distribution - release-please-config.json # Release Please: versioning + CHANGELOG from main - .github/workflows/ # CI + PR-title lint + Release Please publish - docs/decisions/adrs/ # repo-local ADRs (pinned) -``` +Those operations do not by themselves prove deterministic replay, scientific +validity, state or observation equivalence, outcome equivalence, or production +security. Each claim is limited by the selected environment pack, source pins, +participant artifacts, runtime controls, and retained evidence. -## Program status - -This repository was stood up under **REP-002** (RAES issue #636). Shared -plumbing and the CyberBattleSim backend are implemented; remaining simulator -backends land issue by issue: - -| Requirement | Scope | -|-------------|-------| -| REP-001 (#635) | Design: RAES ADR-069 + `cage-2-replication-design.md` | -| **REP-002 (#636)** | **This standup: distribution, CI, GC onboarding, strict Sonar** | -| REP-003 | CAGE-2 RAES SDL scenario + pinned mapping ledger | -| REP-004 | CybORG backend + `raes_adapters.base` implementation | -| REP-005 | Replicated runs + tiered equivalence evidence | - -## CyberBattleSim qualification - -Issue [#25](https://github.com/OpenRAE/adapters/issues/25) selects the -official Microsoft source at commit -`854d6966607fb68645651f55b0f97221bd293e0d` and one public -`CyberBattleChain-v0` protocol with the credential-cache baseline and basic -defender. The shipped -[protocol](src/raes_adapters/cyberbattlesim/public-protocol.md) fixes the exact -scenario, participant, evaluator, seed obligations, metrics, and termination -semantics. The separate -[architecture guardrails](docs/decisions/cyberbattlesim-qualification-guardrails.md) -explain why this evidence is not an adapter manifest or RAES conformance claim. - -Issue [#26](https://github.com/OpenRAE/adapters/issues/26) authors the -portable evidence set for that case: an authored RAES SDL scenario -(`scenario/cyberbattle-chain.sdl.yaml`) that validates and compiles against -`raes==2.0.0`, companion published experiment contracts -(`experiment/`) for the reward, evaluator, metric, episode/termination, and -descriptive stochastic intent that RAES excludes from SDL, and a pinned -[source → RAES mapping ledger](src/raes_adapters/cyberbattlesim/mapping/) whose -rows are each `mapped`, `excluded`, or `loss-disclosed`, with every disclosed -loss bound to the ADR-069 equivalence tier it weakens. Deterministic tests fail -CI on source drift, a missing category, a duplicate row, an unresolvable target, -a broken cross-artifact reference, an undisclosed loss, or leakage of a known -native identifier or object representation (raw native arrays, reward vectors, -and action ids are excluded structurally by the closed RAES models). The -evidence set does not turn source identity into a claim of dependency -installability, deterministic replay, or outcome equivalence; -the [scenario/ledger guardrails](docs/decisions/cyberbattlesim-scenario-ledger-guardrails.md) -fix its boundaries. - -## CyberBattleSim backend - -Issue [#27](https://github.com/OpenRAE/adapters/issues/27) implements a RAES -runtime target for the admitted size-10 `CyberBattleChain-v0` profile: - -```python -from raes_adapters.cyberbattlesim.backend import ( - cyberbattlesim_backend_conformance_payload, - cyberbattlesim_declared_weaknesses, - cyberbattlesim_source_protocol_diagnostics, - run_cyberbattlesim_conformance, -) - -report = run_cyberbattlesim_conformance(seed=20260729) -payload = cyberbattlesim_backend_conformance_payload(report) -diagnostics = cyberbattlesim_source_protocol_diagnostics() -weaknesses = cyberbattlesim_declared_weaknesses() -``` +## Quickstart: verify the CAGE-2 adapter boundary -Target creation is dependency-light and does not import the simulator. -Provisioning verifies the qualified simulator import-root identity and selected -dependency wheel identities, complete installed import-root trees (including -native libraries and unexpected files), -critical-file digests, dependency versions, and module origins for the directly -imported source, Gymnasium, and NumPy packages before importing and constructing -the separately installed source. The locally built simulator wheel is admitted -by its complete root because upstream publishes no reproducible wheel artifact; -direct Gymnasium/NumPy wheel installs must also match the recorded archive hash. -Editable/directory installations, symlinks, unqualified direct dependency -artifacts, absence, or an identity mismatch become a bounded -RAES diagnostic (and a source-backed conformance probe therefore fails rather -than pretending to run). The -reference processor compiles the checked-in scenario against the manifest, and -the shared target then realizes the applicable provisioning, orchestration, -participant, evaluation, observation, and cleanup surfaces. - -One admitted attacker action maps to at most one serialized native `env.step`. -The selected scan-and-reimage defender runs source-internally during that step; -it is not exposed as a second participant-admitted transition. -Native observations, masks, credentials, action coordinates, `info`, reward -vectors, and exceptions remain driver-private. Participant observations and -typed action results carry only RAES references admitted by their disclosure -boundary. Because the representative authored topology cannot identify the -selected native action coordinate, requested targets remain intent and are not -echoed as realized effect targets. Cumulative reward is evaluator-owned. Seed -bindings report the Gym environment and action-space streams as applied and the -Python/NumPy global streams as unbound. These controls improve run attestation -and bound repeatability without claiming byte-identical replay of a stochastic -experiment. - -The [backend architecture guardrails](docs/decisions/cyberbattlesim-backend-guardrails.md) -record the component ownership, failure hygiene, capability claims, and -acceptance-test mapping. - -Issue [#30](https://github.com/OpenRAE/adapters/issues/30) adds the frozen -[source-native/RAES baseline reproduction](docs/cyberbattlesim-baseline-reproduction.md). -Its content-addressed bundle preserves all 20 scheduled terminal attempts, -offline-recomputable aggregates, six separately cited tiers, and the known -stochastic, topology, evaluator, metric, and packaging limitations for the -OpenRAE/research#14 and #20 consumers. - -Issue [#28](https://github.com/OpenRAE/adapters/issues/28) composes that -runtime target with the published RAES conformance report and adapter-local -source-protocol probes. The backend conformance result remains the exact -`BackendConformanceReport` from RAES and is serialized only through the -published report projector. The CyberBattleSim probes add RAES diagnostics, -manifest-derived capability evidence links, source-ledger validation, and -declared weakness references; they do not create another profile, fixture -corpus, report schema, or research-validity claim. The -[conformance-composition guardrails](docs/decisions/cyberbattlesim-conformance-guardrails.md) -fix those boundaries. - -## PrimAITE qualification - -Issue [#39](https://github.com/OpenRAE/adapters/issues/39) selects the ARCD -PrimAITE source at tag `v4.0.0` (commit -`98617981d7f6ae2c3ffd9a8cc39944e05c9a09ea`) and one public `data_manipulation` -protocol driven through the source-native Gymnasium entrypoint -`primaite.session.environment.PrimaiteGymEnv`. The shipped -[protocol](src/raes_adapters/primaite/public-protocol.md) fixes the exact -scenario, participants (BLUE `proxy-agent`, scripted RED, probabilistic GREEN), -`Discrete(78)` action space, flattened `Box(1652,)` observation, seed -obligations, metrics, and the fixed-horizon truncation semantics (`terminated` -is always false; the episode truncates at `max_episode_length=128`). - -The [qualification record](src/raes_adapters/primaite/qualification.json) binds -the source identity, the canonical import-root digest (which matches the built -wheel exactly), the MIT/Crown-copyright legal disposition, the clean-install and -bounded do-nothing smoke, the resolved dependency graph and its permissive -license summary, and the maintainer admission with graded claim strength. It -also records the source's honest limitations: no index/release wheel, an -undeclared `pkg_resources`/setuptools runtime dependency (the qualified route -pins `setuptools==75.6.0`), a public seed seam that raises without the `rl`/torch -stack, a `requires-python` vs classifier inconsistency, and an unpinned upstream -dependency graph. The -[qualification guardrails](docs/decisions/primaite-qualification-guardrails.md) -explain why this evidence is not an adapter manifest or RAES conformance claim. - -## PrimAITE backend conformance - -Issue [#42](https://github.com/OpenRAE/adapters/issues/42) composes the PrimAITE -runtime target with the published RAES conformance report and adapter-local -source-protocol and leakage probes. Three claims stay distinct: backend -conformance is the exact `BackendConformanceReport` from RAES, serialized only -through the published report projector; source-protocol reproduction is -adapter-local executable evidence (source/ledger identity, reset and stochastic -dispositions, action representability, withheld observation and reward, terminal -semantics, and verified cleanup) emitted as RAES diagnostics; and the -research/readiness claim is bounded by the declared weakness and loss references. - -PrimAITE is deliberately **fail-closed**. The live `PrimaiteDriver` verifies -source identity and then refuses in-process construction (the qualified runtime is -CPython 3.11 and PrimAITE writes platform directories on import), so an injected -driver proves portable mechanics but never certifies a live-native capability. The -canonical report keeps its single published no-witness `realization-envelope-v1` -case, `native_conformance` stays false, capability evidence is empty, and every -affirmative manifest capability is disclosed as an open gap rather than certified. - -```python -from raes_adapters.primaite.backend import run_primaite_pr_conformance - -# `driver` is a deterministic PrimaiteDriverProtocol implementation for the PR -# lane. The live PrimaiteDriver is non-runnable in-process, so the bundle always -# keeps native_conformance=false and discloses capability gaps as non-claims. -bundle = run_primaite_pr_conformance(driver=deterministic_injected_driver) -# bundle: backend_conformance (published payload), source_diagnostics, -# capability_evidence ({}), capability_gaps (disclosed non-claims), -# declared_weaknesses. -``` +Start in a fresh Python 3.12 virtual environment on Linux or macOS. Install the +published distribution and its CAGE-2 pack-validation dependency: -The deterministic PR lane uses a fully constructed `RuntimeTarget` with an -explicit injected driver and never imports the simulator; the clean-install proof -composes the same bundle from the built `primaite`-extra wheel. The -hostile-failure and portable-output-leakage probes over the four surfaces -(provisioner, orchestrator, participant runtime, evaluator) plus cleanup live in -the deterministic PR test suite (`tests/test_primaite_conformance.py`). PrimAITE -exposes no RAES time surface, so clock control reports a validated unsupported -disposition rather than an affirmative time claim. Native readiness remains -blocked pending a reviewed worker/process isolation boundary and CPython 3.12 -qualification evidence. The -[conformance-composition guardrails](docs/decisions/primaite-conformance-guardrails.md) -fix those boundaries. - -## CybORG/CAGE-2 runtime qualification - -Issue [#12](https://github.com/OpenRAE/adapters/issues/12) selects the -official CAGE Challenge 2 repository at commit -`26ce1c1253fa9e2e73f25e6a7f2da32860c11257`, including its bundled CybORG 2.1, -Scenario2, evaluator, wrappers, and baseline agents as one source closure. The -[qualification record](src/raes_adapters/cyborg/qualification.json) binds the -source and file digests, dependency resolution, legal decisions, known defects, -and sanitized red/blue/green smoke result. The accompanying -[packaging patch](src/raes_adapters/cyborg/cage2-wheel-package-data.patch) is -qualification evidence only; it is not silently applied or published. - -Issue [#15](https://github.com/OpenRAE/adapters/issues/15) supplies the -provisioning path for that backend. `create_cyborg_target()` accepts -admitted RAES provisioning plans and deterministically generates the native -CybORG scenario: RAES switches become subnets, VM multiplicity becomes hosts, -infrastructure links become subnet membership, and supported OS families select -digest-verified CybORG images. The portable compiled plan entries and -configuration-bound realization-envelope identity remain in the RAES snapshot; -native CybORG objects stay private. Unsupported or lossy node facts fail before -construction. - -Issue [#16](https://github.com/OpenRAE/adapters/issues/16) adds aggregate -logical-turn execution. A validated blue action is translated by exact contract -address and drives one source-native turn; the resulting blue, green, and red -occurrences are recorded in declared source order with shared-state, joint-action, -and logical-time joins. B-line, Meander, and Sleep red selections and 30/50/100 -step limits are admitted through published RAES control contracts. Invalid input -has no native effect, unprojectable post-step output quarantines the session, and -the portable surfaces exclude native action identifiers, reward data, raw logs, -hidden state, and native object representations. Participant-relative -observations and evaluator-owned reward projections use published RAES -contracts and retain their source-ledger losses. - -The separate -[architecture guardrails](docs/decisions/cyborg-cage2-runtime-qualification-guardrails.md) -define the qualification boundary: source installation and known losses limit -strong replay/equivalence claims, but do not veto adapter construction or -permission to retain an honest partial reproducibility record. - -## CybORG conformance and disclosure - -Issue [#19](https://github.com/OpenRAE/adapters/issues/19) provides the -machine-readable CybORG conformance surface. It loads the live manifest through -the published registry, validates `backend-manifest-v2`, selects RAES's -`full-remote-control-plane` profile and canonical fixtures, and preserves the -exact `BackendConformanceReport` and projector. Adapter-local source-ledger, -pin, seed/clock, lifecycle, action/observation, reward/evaluation, cleanup, and -portable-output probes use RAES diagnostics and evidence references; they do -not append cases or invent a second aggregate pass result. - -```bash -# Offline PR/hermetic evidence, fixed seed 3 -uv run --frozen python -m raes_adapters.cyborg.conformance \ - --suite pr --output-dir artifacts/cyborg-conformance - -# Broader scheduled evidence, fixed ordered seeds 3 and 153 -uv run --frozen python -m raes_adapters.cyborg.conformance \ - --suite full --output-dir artifacts/cyborg-conformance + +```shell +python -m pip install 'raes-adapters[cyborg]' ``` -RAES 2.0.0's published realization-envelope witness algebra cannot construct -the required VM-to-network list binding. The canonical runner therefore emits -its own bounded `unsupported` no-witness case while all applicable fixtures and -adapter-local probes pass; the adapter does not rewrite that case. Both suites -remain `hermetic-live` with `native_conformance=false`. The existing qualified -source reproducer supplies the separate native readiness evidence. Seed 153 -does not erase `loss-evaluation-seed-unbound`, and neither suite claims -deterministic replay or scientific equivalence. See the -[conformance guardrails](docs/decisions/cyborg-conformance-guardrails.md). - -## NASim qualification - -Issue [#32](https://github.com/OpenRAE/adapters/issues/32) selects Jonathon -Schwartz's official -[NASim](https://github.com/Jjschwartz/NetworkAttackSimulator) source at tag -`v0.12.0` (commit `7c732bc4620d20a25b221a782adee29c2a89d800`, published as -`nasim==0.12.0`) and one public protocol: the `tiny` static benchmark run under -the supplied `bruteforce_agent` baseline. The shipped -[protocol](src/raes_adapters/nasim/public-protocol.md) fixes the exact scenario, -baseline, seed obligation, metrics, and Gymnasium `terminated`/`truncated` -semantics; the [qualification record](src/raes_adapters/nasim/qualification.json) -pins source, wheel, and import-root identities, the qualified dependency -resolution, and the attainable claim strength. The -[architecture guardrails](docs/decisions/nasim-qualification-guardrails.md) -explain why this evidence is not an adapter manifest or RAES conformance claim. - -The record discloses four upstream findings that bound the claims: the supplied -agents pass a NumPy integer action index that NASim's own `FlatActionSpace` -rejects on gymnasium >= 0.27 (so the qualified runtime pins gymnasium 0.26.3); -action success is drawn from the global NumPy RNG, so `make_benchmark`/`reset` -seeds do not bind a run; the static-benchmark seed argument is ignored; and -importing NASim requires the Tk system libraries. Source identity and protocol -configuration are attested; execution controls are partial and outcome -reproduction is stochastic-bounded, with no adapter, manifest, or -outcome-equivalence claim delivered by qualification. - -## NASim backend conformance - -Issue [#35](https://github.com/OpenRAE/adapters/issues/35) composes the NASim -runtime target with the published RAES conformance report and adapter-local -source-protocol probes. Three claims stay distinct: backend conformance is the -exact `BackendConformanceReport` from RAES, serialized only through the published -report projector; source-protocol reproduction is adapter-local executable -evidence (source/ledger identity, reset and stochastic dispositions, action and -observation projection, evaluator facts, independent terminal facts, and verified -cleanup) emitted as RAES diagnostics; and the research/readiness claim is bounded -by the declared weakness and loss references. The NASim probes add -manifest-derived capability-evidence links that fail closed when an affirmative -surface has no passing evidence; they do not create another profile, fixture -corpus, report schema, or research-validity claim. - -```python -from raes_adapters.nasim.backend import run_nasim_pr_conformance - -# `driver` is any deterministic NasimDriverProtocol implementation for the PR -# lane, or a real NasimDriver for the manual-live lane — the caller selects it. -bundle = run_nasim_pr_conformance(driver=deterministic_injected_driver) -# bundle: backend_conformance (published payload, which drives the four surfaces -# on the published fixtures), source_diagnostics, capability_evidence, -# declared_weaknesses. -``` +Run the short, hermetic CAGE-2 conformance suite from an empty working +directory: -The deterministic PR lane uses a fully constructed `RuntimeTarget` with an -explicit injected driver and never imports the simulator; the clean-install -proof composes the same evidence bundle from the built `nasim`-extra wheel, and -the manual-live lane runs the identical composition against a real `NasimDriver` -on the qualified runtime. The hostile-failure and portable-output-leakage probes -over the four surfaces are injected-driver constructs — a real `NasimDriver` -cannot be made to raise on a chosen surface — so they live in the deterministic -PR test suite (`tests/test_nasim_conformance.py`), which leak-tests each -surface's success and failure paths. NASim exposes no RAES time surface, so clock -control reports a validated -unsupported disposition rather than an affirmative time claim. The -[conformance-composition guardrails](docs/decisions/nasim-conformance-guardrails.md) -fix those boundaries. - -## Development - -Requires [`uv`](https://docs.astral.sh/uv/). Repo-wide gates run through nox: - -```bash -# full verification graph (hygiene, policy, lint, typecheck, tests, build) -uv tool run --from 'nox[uv]==2026.4.10' nox -s verify - -# just the tests (base plus all extras) -uv tool run --from 'nox[uv]==2026.4.10' nox -s tests + +```shell +raes-adapters run --mode conformance --suite pr --output cage2-quickstart ``` -Activate the git hooks on every fresh clone (hooks are not versioned): +The command prints one bounded JSON object: -```bash -uv run --project . pre-commit install --install-hooks + +```json +{"disposition":"succeeded","evidence_basis":"hermetic-live","inventory":"inventory.json","mode":"conformance","run_count":1} ``` -## Releases - -`raes-adapters` uses [Release Please](https://github.com/googleapis/release-please): -each push to `main` maintains a release PR that bumps the version and updates -`CHANGELOG.md` from Conventional Commit history. Merging it tags the release and -publishes the wheel and sdist to PyPI over OIDC Trusted Publishing (no stored -token), then opens a `main`→`dev` back-merge PR. Do not hand-edit `CHANGELOG.md`; -carry the release note in the Conventional Commit PR title. - -## Cross-repo workflow +It creates exactly these files: -Work here is issue-driven from RAES (ADR-069 §8). Adapter PRs reference the RAES -issue, `REP-001`, ADR-069, the design record, the source-ledger id, conformance -profile id, and seed suite. Cross-repo status is read from linked issues, PRs, -conformance reports, and evidence artifacts — not from comments or docs. - -## License + +```text +cage2-quickstart/index.json +cage2-quickstart/inventory.json +cage2-quickstart/runs/cyborg-pr-seed-3/conformance/backend-conformance.json +``` -MIT — see [LICENSE](LICENSE). +The portable Scenario 2 input remains inside the installed `cage2-research` +environment pack at `sdl/cage2-research.sdl.yaml`; the +[checked-in example pack](https://github.com/OpenRAE/adapters/tree/dev/src/raes_adapters/cyborg/examples/cage2-research) +is its reviewable source. `index.json` records the conformance run and its +declared gaps, `backend-conformance.json` is the portable RAES conformance +report, and `inventory.json` seals both files by relative path, size, and +SHA-256 digest. + +This is a successful adapter conformance run, not a native CybORG episode. Its +evidence basis is `hermetic-live`, every report states +`native_conformance=false`, and the fixed seed `3` belongs to the conformance +suite rather than a study design. The built-wheel gate executes this README +contract before merge; after publication, the release workflow repeats it +from the exact version on the public package index. + +## What a green result means + +| Question | Evidence needed | What the quickstart establishes | +| --- | --- | --- | +| Is the scenario valid? | The environment pack, SDL, task, experiment, participant joins, and their digests validate against published contracts. | The conformance probes validate the packaged CAGE-2 source ledger and adapter fixtures. They do not execute the packaged native study task. | +| Does the adapter conform? | A finite backend-conformance report records the exercised cases, execution basis, diagnostics, and capability gaps. | Yes, for the checked hermetic cases. It is not native-live conformance. | +| Did a simulator run complete? | The native runtime was admitted, the requested episodes completed, cleanup was verified, portable artifacts validated, and the inventory was sealed last. | No. The quickstart never imports or executes CybORG. | +| What research claim is supported? | A declared method joins source, scenario, participants, controls, observations, analysis, limitations, and retained evidence. | No study claim. `run_count: 1` is a conformance count, not an experimental result. | + +## Run or adapt a simulator study + +Native CAGE-2 `validate`, smoke, and study requests currently fail closed +before runtime planning. The packaged task requires semantic reward-component +evidence, while the pinned RAES contract cannot yet verify the required +artifact fields and negative data-quality states. The task is not weakened to +make a demo pass, and the `cyborg` extra does not install the unpublished +patched CybORG wheel. + +The [full installed-command and reproduction recipe](https://raes-adapters.readthedocs.io/en/latest/researcher-command/) +records the native source prerequisite, the complete command shapes, exit +codes, evidence layout, and the frozen 3 trial-length × 3 Red-policy × 1,000 +episode public CAGE-2 protocol. Use it to review or prepare a study; a native +run becomes admissible only when all declared evidence requirements validate. + +Study controls are authored artifacts, not free-form convenience flags: + +- **Agent:** select a declared Red variant. A different Blue implementation + needs its own manifest, selection, configuration, source provenance, and a + resealed pack. +- **Seed:** smoke and study seeds must be declared by the experiment. A seed + controls only the random streams the source and adapter can bind; it is not a + deterministic-replay guarantee. +- **Trial length:** it must match a declared experiment condition. Changing it + changes the study design and requires updated authored artifacts and digests. +- **Environment pack:** select a published pack identity and content digest. + Changing scenario or participant content creates a new pack version; it is + not an ambient path override. + +See the [researcher guide](https://raes-adapters.readthedocs.io/en/latest/researcher-guide/) +for a task-first explanation of those controls and the evidence needed before +interpreting a result. + +## Current adapters and evidence + +This table is a reader index, not a second support registry. The linked +qualification, pack, and conformance records remain authoritative. + +| Adapter/profile | Install and execution maturity | Current evidence status | +| --- | --- | --- | +| CybORG / CAGE-2 Scenario 2 | Adapter, example pack, and hermetic conformance are available. Native CybORG requires the separately installed pinned source plus an unpublished packaging-only fix. | Source qualified; pack status `built`; hermetic conformance with `native_conformance=false`; native researcher task evidence-gate blocked. | +| NASim / tiny | The `nasim` extra installs the pinned simulator stack and the source-backed conformance boundary is available. | Qualified source and conformance evidence; the current packaged researcher task is evidence-gate blocked before native study execution. | +| CyberBattleSim / chain | Adapter and reproduction tooling are implemented; native source is installed separately because upstream publishes no governed wheel. | Source/profile admitted, conformance evidence retained, and a bounded baseline reproduction record exists; the packaged researcher task is evidence-gate blocked. | +| PrimAITE / data manipulation | Qualified backend module; no researcher CLI selection is exposed. Native in-process execution remains fail-closed. | Conformance uses an injected non-native driver and makes no native-conformance claim. | + +## Limitations + +- **Simulator abstraction:** portable RAES records deliberately omit native + state, gym/PettingZoo tuples, action IDs, reward vectors, object + representations, raw logs, hidden truth, and full tracebacks. That protects + the portable boundary but cannot demonstrate native state equivalence. +- **Stochasticity:** declared seeds do not bind every simulator, policy, + Python, NumPy, or Gym random stream. Repeated outcomes can vary, and a seed + is not deterministic replay. +- **Source pins:** evidence applies to the qualified repository, commit, files, + patches, and package graph. A different source tree needs requalification. +- **Unsupported facts:** missing participant artifacts, unverifiable evidence + witnesses, capability gaps, and source inconsistencies remain explicit + losses. Successful validation never upgrades an unsupported claim. +- **Compute cost:** the full CAGE-2 reproduction schedules 9,000 episodes and + retains per-slot evidence. Estimate runtime and storage before starting it; + the quickstart is intentionally not that workload. +- **Non-production scope:** these adapters are research and evaluation + apparatus. Conformance is not a security certification, operational defense + guarantee, or authorization to deploy an agent in production. + +## Citation and provenance + +For a paper or evidence bundle, record the `raes-adapters` version, environment +pack name/version/content digest, adapter qualification profile and source +commit, participant artifact digests, experiment controls, and the retained +`inventory.json`. Cite the repository release and the upstream simulator; do +not cite a mutable branch as the executed identity. The CAGE-2 +[qualification record](https://github.com/OpenRAE/adapters/blob/dev/src/raes_adapters/cyborg/qualification.json) +and pack +[provenance ledger](https://github.com/OpenRAE/adapters/blob/dev/src/raes_adapters/cyborg/examples/cage2-research/docs/provenance-ledger.yaml) +show the current source and artifact bindings. + +## Troubleshooting + +- **Package version cannot be found:** use Python 3.12 and confirm that a + release exists on the configured public package index. Do not substitute an + editable checkout when claiming a published-distribution reproduction. +- **Output unavailable:** choose a new relative output directory. Existing, + absolute, traversing, or symlink-escaping paths are rejected. +- **Evidence unverifiable (exit 3):** this is the current fail-closed native + study boundary, not an installation failure. Review the researcher command + and task evidence requirements; do not remove them. +- **Native source unavailable:** the `cyborg` extra validates the pack but does + not download CybORG. Follow the pinned source qualification before attempting + native execution. +- **Unexpected internal failure (exit 70):** retain the bounded error code, + package version, command shape, and inventory if present. Do not publish + native logs or environment dumps in an issue. + +More diagnosis and all stable exit codes are in the +[researcher command reference](https://raes-adapters.readthedocs.io/en/latest/researcher-command/#exit-status). + +## Developing an adapter + +Build, CI, repository layout, packaging, release, governance, and contributor +mechanics are intentionally outside this researcher path. Start with the +[developer index](https://raes-adapters.readthedocs.io/en/latest/maintainers/) +and [CONTRIBUTING](https://github.com/OpenRAE/adapters/blob/dev/CONTRIBUTING.md). +RAES owns the semantic contracts; backend concepts remain inside their adapter +module, and `raes_adapters.base` remains plumbing rather than authority. diff --git a/docs/index.md b/docs/index.md index 76f9a03..d44791c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,28 +1,21 @@ # RAES adapters -This repository contains a single distribution, `raes-adapters`, that connects -concrete simulator backends to published RAES contracts. It ships shared base -plumbing plus backend modules whose dependencies can live behind separate -extras in one lock. Maintainer-selected backends are admitted; backend-local -qualification evidence records attainable claim strength and limitations. The -`cyborg` and `cyberbattlesim` extras are dependency-light because their selected -native sources have no governed publishable artifact, so users acquire those -simulators separately and the base install remains independent. The admitted -CybORG backend is supported through its documented source installation despite -the absence of an automatically installed native simulator; its extra installs -the published environment-pack validator used by the researcher command. - -The shared `raes_adapters.base` module provides plumbing only. RAES remains the -semantic and protocol authority. +`raes-adapters` connects concrete simulator backends to published RAES +contracts. Start with the installed conformance quickstart, then select the +researcher command or evidence recipe that matches the claim you need to make. +Scenario validity, adapter conformance, native run completion, and scientific +evidence are separate boundaries. ## Start here - [Repository overview](https://github.com/OpenRAE/adapters#readme) +- [Researcher guide](researcher-guide.md) - [Installed researcher command](researcher-command.md) - [NASim researcher command](nasim-researcher-command.md) - [CyberBattleSim researcher command](cyberbattlesim-researcher-command.md) - [CyberBattleSim baseline reproduction](cyberbattlesim-baseline-reproduction.md) - [Contribution guide](https://github.com/OpenRAE/adapters/blob/dev/CONTRIBUTING.md) +- [Developer index](maintainers/index.md) - [Architecture decisions](decisions/adrs/README.md) - [CybORG/CAGE-2 backend qualification guardrails](decisions/cyborg-cage2-runtime-qualification-guardrails.md) - [CybORG/CAGE-2 source-ledger guardrails](decisions/cyborg-cage2-source-ledger-guardrails.md) diff --git a/docs/maintainers/index.md b/docs/maintainers/index.md new file mode 100644 index 0000000..6e1e8f6 --- /dev/null +++ b/docs/maintainers/index.md @@ -0,0 +1,91 @@ +# Developer index + +This index is for adapter implementers, contributors, release maintainers, and +repository operators. Researchers should start with the +[README](https://github.com/OpenRAE/adapters#readme) and +[researcher guide](../researcher-guide.md). + +## Architecture boundary + +RAES owns the published semantic contracts. This repository adapts concrete +simulators to those contracts: + +- backend concepts stay under `src/raes_adapters//`; +- `raes_adapters.base` contains composition plumbing, not schemas, profiles, + backend protocols, diagnostic envelopes, stores, or policy authority; +- portable artifacts exclude native simulator objects, raw logs, hidden truth, + arguments/environment dumps, tokens, and full tracebacks; and +- accepted repository decisions are recorded under `docs/decisions/adrs/` and + content-pinned in `adr-index.yaml`. + +Start with [ADR-002](../decisions/adrs/adr-002-raes-authority-and-adapter-boundaries.md) +and [ADR-003](../decisions/adrs/adr-003-single-distribution-and-trusted-publishing.md). +Backend design notes under `docs/decisions/` record source-specific evidence +and guardrails; they are authority records, not researcher tutorials. + +## Repository layout + +| Path | Purpose | +| --- | --- | +| `src/raes_adapters/base/` | Shared adapter plumbing over published RAES APIs. | +| `src/raes_adapters//` | Backend implementation, qualification, mappings, profiles, and packaged examples. | +| `tests/` | Distribution and backend behavior tests. | +| `tools/` and `tools/tests/` | Repository policy and project-service checks. | +| `docs/decisions/` | Accepted ADRs plus backend-scoped design/evidence guardrails. | +| `noxfile.py` | Canonical local and CI verification graph. | +| `.github/workflows/` | Parallel PR gates and Trusted Publishing release automation. | + +## Local development + +Follow [CONTRIBUTING](https://github.com/OpenRAE/adapters/blob/dev/CONTRIBUTING.md) +for environment setup, hooks, change conventions, and backend additions. The +canonical completion command is: + +```shell +uv tool run --from 'nox[uv]==2026.4.10' nox -f noxfile.py -s verify -- --skip-requirement +``` + +Use `--skip-requirement` only for genuine requirement-free maintenance. Normal +issue work supplies the governing requirement UID through Ground Control. + +The graph runs hygiene, policy, lint, strict typing, tests with coverage, +clean-built distribution probes, and strict documentation. CI runs the same +sessions as independent jobs joined by the `PR Gate`; see +[continuous integration](ci.md) for targeted reproduction commands. + +## Packaging and releases + +`raes-adapters` is one distribution with optional per-simulator extras and one +`uv.lock`. A mutually incompatible simulator stack is isolated through uv +extras conflicts rather than a second distribution, workspace, or lockfile. + +Release Please owns the version and `CHANGELOG.md` from Conventional Commit +history. Trusted Publishing builds and verifies the tagged commit, publishes +through PyPI OIDC, attaches immutable distributions/checksums, and runs the +exact-version public-index smoke. Do not hand-edit the changelog, create a +changelog fragment, store a PyPI token, or silently replace an existing release +artifact. Service identities and workflow boundaries are listed under +[project services](project-services.md). + +## Add or change an adapter + +1. Keep native imports behind the backend module boundary so the base install + remains usable. +2. Reuse published RAES contracts and existing repository helpers; do not add a + local semantic model or shared backend registry. +3. Bind qualification to immutable source evidence and disclose unsupported + facts or losses rather than promoting them through static claims. +4. Add behavioral tests at the narrowest boundary and extend the clean-wheel + proof for installed behavior. +5. Update researcher documentation when a command, output, support statement, + limitation, or evidence interpretation changes. +6. Add or amend an ADR only for a durable repository decision; regenerate the + ADR pin in the same change. + +## Governance and services + +- [Contribution guide](https://github.com/OpenRAE/adapters/blob/dev/CONTRIBUTING.md) +- [Continuous integration](ci.md) +- [Project services](project-services.md) +- [ADR overview](../decisions/adrs/README.md) +- [Repository issue tracker](https://github.com/OpenRAE/adapters/issues) diff --git a/docs/researcher-guide.md b/docs/researcher-guide.md new file mode 100644 index 0000000..4526bc6 --- /dev/null +++ b/docs/researcher-guide.md @@ -0,0 +1,146 @@ +# Researcher guide + +Use this guide after completing the README conformance quickstart. The +quickstart proves that an installed adapter can emit bounded RAES conformance +evidence; it does not execute a simulator study. + +## Choose the evidence you need + +Before selecting a command, decide which statement the work must support: + +| Intended statement | Minimum evidence boundary | +| --- | --- | +| The authored scenario and experiment are well-formed. | Validate the environment pack, SDL, task, experiment, participants, controls, and all content digests. | +| The adapter exercises selected RAES backend contracts. | Retain a conformance report with its execution basis, cases, diagnostics, capability gaps, and declared non-claims. | +| A native run completed. | Admit the pinned simulator source, complete every scheduled run, verify cleanup, validate the portable records, and seal the inventory last. | +| A result reproduces or supports a research claim. | Predeclare the method and comparison rule, retain the complete evidence joins, account for losses and stochastic controls, and report only the strongest tier the evidence supports. | + +Do not infer a stronger row from a successful weaker row. In particular, +scenario validity is not conformance, conformance is not native completion, +and native completion is not scientific validity. + +## Work with the CAGE-2 pack + +The installed `cage2-research` pack carries the portable Scenario 2 SDL, task, +experiment, Blue participant manifest/selection/configuration, compatibility +record, content manifest, and provenance ledger. The +[checked-in pack](https://github.com/OpenRAE/adapters/tree/dev/src/raes_adapters/cyborg/examples/cage2-research) +is the review surface; installed package resources are the execution surface. + +The current pack has status `built`, not `golden`. Its native task remains +fail-closed because the pinned RAES contract cannot validate the task's +semantic reward-component witness. Pack validity and source qualification do +not override that gate. + +## Change a study deliberately + +### Participant implementation + +A Red policy can be selected only when the pack declares that variant. A new +Blue implementation needs a published participant manifest, selection, and +configuration whose identities and digests join the task and pack. Record the +implementation source or model bytes; do not represent an unavailable +submitted artifact through a similarly behaving substitute. + +### Seed allocation + +Seeds belong to the experiment design. Record which random streams each seed +actually controls and disclose the unbound streams. A shared numeric seed does +not imply deterministic replay across simulator, policy, Python, NumPy, or Gym +state. + +### Trial length + +Trial length is part of the declared condition, not an arbitrary cutoff. A new +length changes termination and comparison semantics, so update the experiment, +task joins, pack content manifest, and digests before execution. + +### Environment pack + +Changing the scenario, task, participant, or experiment creates a new pack +identity/version. Validate and reseal it through the published environment-pack +tools. The researcher CLI intentionally has no ambient profile root, arbitrary +driver import, hidden default, or runtime download seam. + +## CAGE-2 reproduction boundary + +The [installed researcher command](researcher-command.md) documents both the +two-seed authored example and the frozen public protocol reproduction. The +public protocol schedules 3 trial lengths × 3 Red variants × 1,000 episodes and +uses the declared study-scoped Python stream initialized with seed 153. It is a +substantial compute and storage workload, not a quickstart. + +The public submitted Blue artifact is unavailable. The retained comparison can +therefore support only the predeclared behavioral-baseline outcome tier; it +cannot establish submitted-agent identity, state or observation equivalence, +deterministic replay, or native conformance from score similarity. + +## Limitations to carry into a report + +- Portable artifacts exclude native state, observations, action identifiers, + reward vectors, object representations, raw logs, hidden truth, environment + dumps, and tracebacks. +- Stochastic controls are partial wherever the pinned source exposes no binding + seam. +- Qualification applies only to its repository, commit, selected files, + patches, interpreter/platform evidence, and dependency graph. +- Capability gaps and missing evidence witnesses remain negative facts; a + successful adjacent check does not satisfy them. +- Full studies can be expensive in episodes, wall time, memory, and evidence + storage. Estimate all four before execution. +- The apparatus is not a production control, security certification, or + operational defense guarantee. + +## Citation and provenance checklist + +Record these identities with the retained evidence: + +1. `raes-adapters` release version and artifact hash; +2. RAES and environment-pack dependency versions; +3. environment pack name, version, content digest, and compatibility record; +4. adapter qualification profile, upstream repository, source commit, and any + admitted patch digest; +5. scenario, task, experiment, and participant artifact digests; +6. run controls, seed allocation, runtime/software inventory, and declared + losses; and +7. the final `inventory.json` plus the archive or repository location that + preserves the referenced files. + +Cite an immutable release and the upstream simulator project. A branch name or +working-tree path is useful context but is not an executed artifact identity. + +## Troubleshooting + +### The package is not available + +Confirm Python 3.12, the configured public package index, and the requested +release. An editable checkout is suitable for development but must not be +reported as a published-distribution reproduction. + +### The output path is rejected + +Every output root is invocation-relative, new, and exclusively reserved. Use a +new directory name. Absolute paths, parent traversal, reuse, and resolved +symlink escape are rejected before evidence writes. + +### Validation exits with status 3 + +Read the stable error code. For the current CAGE-2 native task, +`researcher.validation.evidence-unverifiable` is expected: validation stopped +before runtime planning because the evidence witness cannot be verified. Do +not remove the requirement or replace it with a local allowlist. + +### Native source is unavailable + +The `cyborg` extra installs pack validation, not CybORG. Native execution needs +the exact separately installed qualified source and packaging fix recorded in +the qualification. The adapter does not clone, download, or select source from +an ambient environment variable. + +### A run fails + +Retain the bounded command error, package/source identities, and any sealed +portable inventory. Do not attach native logs, raw observations, environment +contents, tokens, or full tracebacks. The [exit-status table](researcher-command.md#exit-status) +separates usage, validation, output, native execution, artifact, and internal +failures. diff --git a/mkdocs.yml b/mkdocs.yml index e99ca36..0732710 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,11 +30,13 @@ markdown_extensions: nav: - Home: index.md + - Researcher guide: researcher-guide.md - Researcher command: researcher-command.md - NASim researcher command: nasim-researcher-command.md - CyberBattleSim researcher command: cyberbattlesim-researcher-command.md - CyberBattleSim baseline reproduction: cyberbattlesim-baseline-reproduction.md - Maintainers: + - Developer index: maintainers/index.md - Continuous integration: maintainers/ci.md - Project services: maintainers/project-services.md - Decisions: diff --git a/noxfile.py b/noxfile.py index fbde2ec..125067b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -363,9 +363,9 @@ def _run(session: nox.Session, *args: str, **kwargs: object) -> None: session.run(*args, external=True, **kwargs) -def _uv_run_root(session: nox.Session, *args: str) -> None: +def _uv_run_root(session: nox.Session, *args: str, **kwargs: object) -> None: """Run a tool from the project env (locked by the root uv.lock).""" - _run(session, "uv", "run", "--frozen", "--project", str(REPO_ROOT), *args) + _run(session, "uv", "run", "--frozen", "--project", str(REPO_ROOT), *args, **kwargs) def _expect_evidence_rejection(session: nox.Session, python: Path, *command: str) -> None: @@ -422,16 +422,28 @@ def _hygiene(session: nox.Session, paths: list[str]) -> None: session.log("hygiene: no files selected; skipping") return if text: - _uv_run_root(session, "trailing-whitespace-fixer", *text) - _uv_run_root(session, "end-of-file-fixer", *text) - _uv_run_root(session, "check-merge-conflict", *text) + session.log(f"hygiene: text files ({len(text)})") + _uv_run_root(session, "trailing-whitespace-fixer", *text, log=False) + _uv_run_root(session, "end-of-file-fixer", *text, log=False) + _uv_run_root(session, "check-merge-conflict", *text, log=False) if yaml: - _uv_run_root(session, "check-yaml", "--unsafe", *yaml) + session.log(f"hygiene: YAML files ({len(yaml)})") + _uv_run_root(session, "check-yaml", "--unsafe", *yaml, log=False) if json: - _uv_run_root(session, "check-json", *json) - _uv_run_root(session, "check-added-large-files", "--maxkb", MAX_LARGE_FILE_KB, *files) + session.log(f"hygiene: JSON files ({len(json)})") + _uv_run_root(session, "check-json", *json, log=False) + session.log(f"hygiene: tracked files ({len(files)})") + _uv_run_root( + session, + "check-added-large-files", + "--maxkb", + MAX_LARGE_FILE_KB, + *files, + log=False, + ) if priv: - _uv_run_root(session, "detect-private-key", *priv) + session.log(f"hygiene: non-test files ({len(priv)})") + _uv_run_root(session, "detect-private-key", *priv, log=False) def _lint(session: nox.Session) -> None: @@ -838,14 +850,15 @@ def _distributions(session: nox.Session) -> None: ) _run( session, + str(cyborg_venv / "bin" / "python"), + "-I", + str(REPO_ROOT / "tools" / "check_readme_quickstart.py"), + "--readme", + str(REPO_ROOT / "README.md"), + "--runner", str(cyborg_venv / "bin" / "raes-adapters"), - "run", - "--mode", - "conformance", - "--suite", - "pr", - "--output", - "researcher-conformance", + "--workdir", + str(probe_cwd / "readme-quickstart"), env={"PYTHONPATH": "", "PYTHONSAFEPATH": "1"}, ) diff --git a/tools/check_project_services.py b/tools/check_project_services.py index c2d2219..ecceddd 100644 --- a/tools/check_project_services.py +++ b/tools/check_project_services.py @@ -78,6 +78,77 @@ def _validate_docs_requirements(text: str, errors: list[str]) -> None: ) +def _validate_ground_control(text: str, errors: list[str]) -> None: + if "default_fallback:" in text: + errors.append(".ground-control.yaml: routing.default_fallback is retired") + + +def _validate_codeql(text: str, errors: list[str]) -> None: + if "continue-on-error:" in text: + errors.append(".github/workflows/codeql-analysis.yml: CodeQL must fail closed") + + +def _validate_ci(text: str, errors: list[str]) -> None: + # The verification graph runs as independent parallel jobs; `PR Gate` is the + # single aggregating required check that keeps every stage mandatory before a + # protected-branch merge. Pin its contract so no job can silently leave the + # gate: it must depend on every verification job plus Sonar, run on every PR, + # and fail closed unless each verification job succeeded (and Sonar passed on + # same-repository PRs). Adding a verification job means extending this list. + for expected in ( + "name: PR Gate", + "needs: [fast-checks, policy, tool-tests, typecheck, tests, distributions, docs, sonar]", + "if: ${{ always() && github.event_name == 'pull_request' }}", + '.value.result == "success"', + "A required verification job did not succeed", + "SonarCloud did not succeed for a same-repository PR", + ): + _require(text, expected, ".github/workflows/ci.yml", errors) + + +def _validate_release_workflow(text: str, errors: list[str]) -> None: + for expected in ( + "googleapis/release-please-action@", + "pypa/gh-action-pypi-publish@", + "environment: pypi", + "id-token: write", + "raes-pack-release build", + "env-pack-assets/cyberbattlesim-chain-1.0.0.tar.gz", + "env-pack-assets/cyberbattlesim-chain-1.0.0-views.tar.gz", + "(cd env-pack-assets && sha256sum", + "ENV_PACK_SHA256SUMS", + "Run the published README quickstart", + "/tmp/smoke/bin/raes-adapters run --mode conformance --suite pr --output cage2-quickstart", + '"evidence_basis": "hermetic-live"', + "cage2-quickstart/runs/cyborg-pr-seed-3/conformance/backend-conformance.json", + ): + _require(text, expected, ".github/workflows/release-please.yml", errors) + + # No stored PyPI credential, and no silent `skip-existing` recovery that could + # mask a partial or duplicate publication. + for forbidden in ( + "PYPI_API_TOKEN", + "TWINE_PASSWORD", + "skip-existing: true", + "sha256sum env-pack-assets/* | tee ENV_PACK_SHA256SUMS", + ): + if forbidden in text: + errors.append( + ".github/workflows/release-please.yml: forbidden release setting " + f"present: {forbidden}" + ) + + +def validate_workflow_policy(path: Path, repo_root: Path) -> list[str]: + """Return action-pin and retired GitHub Pages errors for one workflow.""" + + errors = validate_action_pins(path, repo_root) + text = path.read_text(encoding="utf-8") + if "actions/deploy-pages@" in text or "pages: write" in text: + errors.append(f"{path.relative_to(repo_root)}: GitHub Pages publishing is retired") + return errors + + def validate_repository(repo_root: Path) -> list[str]: """Validate the complete repository-owned project-services contract.""" errors: list[str] = [] @@ -89,8 +160,7 @@ def validate_repository(repo_root: Path) -> list[str]: _require(ground_control, "completion_command: make verify", ".ground-control.yaml", errors) _require(ground_control, "policy_command: make policy", ".ground-control.yaml", errors) _require(ground_control, "precommit_command: make precommit", ".ground-control.yaml", errors) - if "default_fallback:" in ground_control: - errors.append(".ground-control.yaml: routing.default_fallback is retired") + _validate_ground_control(ground_control, errors) root_pyproject = _read_required(repo_root, "pyproject.toml", errors) _require(root_pyproject, 'name = "raes-adapters"', "pyproject.toml", errors) @@ -137,28 +207,13 @@ def validate_repository(repo_root: Path) -> list[str]: "queries: security-extended", ): _require(codeql, expected, ".github/workflows/codeql-analysis.yml", errors) - if "continue-on-error:" in codeql: - errors.append(".github/workflows/codeql-analysis.yml: CodeQL must fail closed") + _validate_codeql(codeql, errors) title_lint = _read_required(repo_root, ".github/workflows/pr-title-lint.yml", errors) _require(title_lint, "name: Lint PR title", ".github/workflows/pr-title-lint.yml", errors) ci = _read_required(repo_root, ".github/workflows/ci.yml", errors) - # The verification graph runs as independent parallel jobs; `PR Gate` is the - # single aggregating required check that keeps every stage mandatory before a - # protected-branch merge. Pin its contract so no job can silently leave the - # gate: it must depend on every verification job plus Sonar, run on every PR, - # and fail closed unless each verification job succeeded (and Sonar passed on - # same-repository PRs). Adding a verification job means extending this list. - for expected in ( - "name: PR Gate", - "needs: [fast-checks, policy, tool-tests, typecheck, tests, distributions, docs, sonar]", - "if: ${{ always() && github.event_name == 'pull_request' }}", - '.value.result == "success"', - "A required verification job did not succeed", - "SonarCloud did not succeed for a same-repository PR", - ): - _require(ci, expected, ".github/workflows/ci.yml", errors) + _validate_ci(ci, errors) scorecard = _read_required(repo_root, ".github/workflows/scorecard.yml", errors) for expected in ( @@ -178,6 +233,8 @@ def validate_repository(repo_root: Path) -> list[str]: '"mkdocs",\n "build",\n "--strict",', "def _tool_tests(session: nox.Session)", '"tools/check_project_services.py"', + 'REPO_ROOT / "tools" / "check_readme_quickstart.py"', + '"--runner"', ): _require(noxfile, expected, "noxfile.py", errors) @@ -223,38 +280,11 @@ def validate_repository(repo_root: Path) -> list[str]: _read_required(repo_root, ".release-please-manifest.json", errors) release_wf = _read_required(repo_root, ".github/workflows/release-please.yml", errors) - for expected in ( - "googleapis/release-please-action@", - "pypa/gh-action-pypi-publish@", - "environment: pypi", - "id-token: write", - "raes-pack-release build", - "env-pack-assets/cyberbattlesim-chain-1.0.0.tar.gz", - "env-pack-assets/cyberbattlesim-chain-1.0.0-views.tar.gz", - "(cd env-pack-assets && sha256sum", - "ENV_PACK_SHA256SUMS", - ): - _require(release_wf, expected, ".github/workflows/release-please.yml", errors) - # No stored PyPI credential, and no silent `skip-existing` recovery that could - # mask a partial or duplicate publication. - for forbidden in ( - "PYPI_API_TOKEN", - "TWINE_PASSWORD", - "skip-existing: true", - "sha256sum env-pack-assets/* | tee ENV_PACK_SHA256SUMS", - ): - if forbidden in release_wf: - errors.append( - ".github/workflows/release-please.yml: forbidden release setting " - f"present: {forbidden}" - ) + _validate_release_workflow(release_wf, errors) workflows = sorted((repo_root / ".github/workflows").glob("*.y*ml")) for workflow in workflows: - errors.extend(validate_action_pins(workflow, repo_root)) - workflow_text = workflow.read_text(encoding="utf-8") - if "actions/deploy-pages@" in workflow_text or "pages: write" in workflow_text: - errors.append(f"{workflow.relative_to(repo_root)}: GitHub Pages publishing is retired") + errors.extend(validate_workflow_policy(workflow, repo_root)) return errors diff --git a/tools/check_readme_quickstart.py b/tools/check_readme_quickstart.py new file mode 100644 index 0000000..a496428 --- /dev/null +++ b/tools/check_readme_quickstart.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Validate and optionally execute the closed README quickstart contract.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shlex +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +_EXPECTED_INSTALL_ARGV = ( + "python", + "-m", + "pip", + "install", + "raes-adapters[cyborg]", +) +_EXPECTED_RUN_PREFIX = ( + "raes-adapters", + "run", + "--mode", + "conformance", + "--suite", + "pr", + "--output", +) +_EXPECTED_OUTPUT: dict[str, object] = { + "disposition": "succeeded", + "evidence_basis": "hermetic-live", + "inventory": "inventory.json", + "mode": "conformance", + "run_count": 1, +} +_REPORT_PATH = "runs/cyborg-pr-seed-3/conformance/backend-conformance.json" +_MARKERS = { + "install": "shell", + "run": "shell", + "output": "json", + "artifacts": "text", +} + + +class ContractError(ValueError): + """The README quickstart is absent, ambiguous, or unsafe to execute.""" + + +@dataclass(frozen=True) +class QuickstartContract: + """Closed command, result, and artifact contract extracted from the README.""" + + install_argv: tuple[str, ...] + run_argv: tuple[str, ...] + expected_output: dict[str, object] + expected_artifacts: tuple[str, ...] + output_root: PurePosixPath + + +def _marked_block(text: str, name: str, language: str) -> str: + marker = f"" + if text.count(marker) != 1: + raise ContractError(f"README must contain exactly one {name} quickstart marker") + pattern = re.compile( + rf"{re.escape(marker)}[ \t]*\n```{re.escape(language)}[ \t]*\n" + rf"(?P.*?)\n```", + re.DOTALL, + ) + match = pattern.search(text) + if match is None: + raise ContractError(f"README {name} marker must be followed by one {language} block") + return match.group("body").strip() + + +def _single_command(block: str, name: str) -> tuple[str, ...]: + lines = tuple(line.strip() for line in block.splitlines() if line.strip()) + if len(lines) != 1: + raise ContractError(f"README {name} block must contain exactly one command") + try: + return tuple(shlex.split(lines[0], posix=True)) + except ValueError as error: + raise ContractError(f"README {name} command is not valid closed argv") from error + + +def _relative_path(value: str, *, label: str) -> PurePosixPath: + path = PurePosixPath(value) + if path.is_absolute() or not value or any(part in {"", ".", ".."} for part in path.parts): + raise ContractError(f"README {label} must be a confined relative path") + return path + + +def parse_quickstart_contract(text: str) -> QuickstartContract: + """Parse and validate only the four explicitly marked README blocks.""" + + blocks = {name: _marked_block(text, name, language) for name, language in _MARKERS.items()} + install_argv = _single_command(blocks["install"], "install") + if install_argv != _EXPECTED_INSTALL_ARGV: + raise ContractError("README install command is outside the admitted published extra") + + run_argv = _single_command(blocks["run"], "run") + if len(run_argv) != len(_EXPECTED_RUN_PREFIX) + 1: + raise ContractError("README run command has an unexpected argument shape") + if run_argv[: len(_EXPECTED_RUN_PREFIX)] != _EXPECTED_RUN_PREFIX: + raise ContractError("README run command is outside the admitted conformance route") + output_root = _relative_path(run_argv[-1], label="output root") + if len(output_root.parts) != 1: + raise ContractError("README output root must be one invocation-relative directory") + + try: + expected_output: Any = json.loads(blocks["output"]) + except json.JSONDecodeError as error: + raise ContractError("README expected output must be one JSON object") from error + if expected_output != _EXPECTED_OUTPUT: + raise ContractError("README expected output does not match the bounded CLI contract") + + artifacts = tuple(line.strip() for line in blocks["artifacts"].splitlines() if line.strip()) + expected_artifacts = ( + f"{output_root.as_posix()}/index.json", + f"{output_root.as_posix()}/inventory.json", + f"{output_root.as_posix()}/{_REPORT_PATH}", + ) + if artifacts != expected_artifacts: + raise ContractError("README artifact list does not match the conformance bundle") + for artifact in artifacts: + _relative_path(artifact, label="artifact") + + return QuickstartContract( + install_argv=install_argv, + run_argv=run_argv, + expected_output=dict(expected_output), + expected_artifacts=artifacts, + output_root=output_root, + ) + + +def load_quickstart_contract(path: Path) -> QuickstartContract: + """Load the README contract from an explicit path.""" + + return parse_quickstart_contract(path.read_text(encoding="utf-8")) + + +def execute_quickstart( + contract: QuickstartContract, + *, + runner: Path, + workdir: Path, +) -> None: + """Execute the admitted command without a shell and verify its exact result.""" + + if not runner.is_file(): + raise ContractError("installed quickstart runner is missing") + workdir.mkdir(parents=True, exist_ok=True) + output = workdir / contract.output_root.as_posix() + if output.exists(): + raise ContractError("quickstart output root already exists") + + env = { + "PATH": f"{runner.parent}{os.pathsep}{os.defpath}", + "PYTHONPATH": "", + "PYTHONSAFEPATH": "1", + } + completed = subprocess.run( + [str(runner), *contract.run_argv[1:]], + cwd=workdir, + env=env, + capture_output=True, + check=False, + text=True, + ) + if completed.returncode != 0: + raise ContractError(f"installed quickstart exited {completed.returncode}") + if completed.stderr: + raise ContractError("installed quickstart wrote unexpected stderr") + try: + actual_output: Any = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise ContractError("installed quickstart did not emit JSON") from error + if actual_output != contract.expected_output: + raise ContractError("installed quickstart output differs from README") + + actual_artifacts = tuple( + sorted(path.relative_to(workdir).as_posix() for path in output.rglob("*") if path.is_file()) + ) + if actual_artifacts != contract.expected_artifacts: + raise ContractError("installed quickstart artifact tree differs from README") + + inventory_path = output / "inventory.json" + inventory: Any = json.loads(inventory_path.read_text(encoding="utf-8")) + inventory_paths = tuple(item["path"] for item in inventory["artifacts"]) + if inventory_paths != ("index.json", _REPORT_PATH): + raise ContractError("installed quickstart inventory does not seal the documented evidence") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--readme", type=Path, required=True) + parser.add_argument("--runner", type=Path) + parser.add_argument("--workdir", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Validate statically, and execute when both installed-run arguments are given.""" + + args = _parser().parse_args(argv) + if (args.runner is None) != (args.workdir is None): + print( + "README quickstart check: --runner and --workdir are required together", file=sys.stderr + ) + return 2 + try: + contract = load_quickstart_contract(args.readme) + if args.runner is not None and args.workdir is not None: + execute_quickstart(contract, runner=args.runner, workdir=args.workdir) + except (ContractError, OSError, KeyError, TypeError) as error: + print(f"README quickstart check: FAIL: {error}", file=sys.stderr) + return 1 + print("README quickstart check: OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_project_services.py b/tools/tests/test_project_services.py index b7ed66a..f4defa7 100644 --- a/tools/tests/test_project_services.py +++ b/tools/tests/test_project_services.py @@ -1,10 +1,23 @@ from __future__ import annotations +import importlib.util +import sys import tempfile import unittest from pathlib import Path +from types import ModuleType, SimpleNamespace -from tools.check_project_services import validate_action_pins, validate_repository +from tools.check_project_services import ( + _validate_ci, + _validate_codeql, + _validate_devmain, + _validate_docs_requirements, + _validate_ground_control, + _validate_release_workflow, + validate_action_pins, + validate_repository, + validate_workflow_policy, +) REPO_ROOT = Path(__file__).resolve().parents[2] @@ -30,6 +43,111 @@ def test_action_refs_must_be_full_commit_shas(self) -> None: validate_action_pins(workflow, Path(tmp)), ) + def test_hygiene_bulk_commands_suppress_the_repository_file_list(self) -> None: + calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] + + class FakeSession: + def run(self, *args: object, **kwargs: object) -> None: + calls.append((args, kwargs)) + + def log(self, _message: str) -> None: + pass + + fake_nox = ModuleType("nox") + fake_nox.Session = FakeSession # type: ignore[attr-defined] + fake_nox.options = SimpleNamespace() # type: ignore[attr-defined] + + def session_decorator(function: object | None = None, *, name: str | None = None) -> object: + del name + if function is not None: + return function + return lambda decorated: decorated + + fake_nox.session = session_decorator # type: ignore[attr-defined] + spec = importlib.util.spec_from_file_location( + "noxfile_under_test", REPO_ROOT / "noxfile.py" + ) + self.assertIsNotNone(spec) + self.assertIsNotNone(spec.loader if spec is not None else None) + + previous_nox = sys.modules.get("nox") + sys.modules["nox"] = fake_nox + try: + module = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + spec.loader.exec_module(module) # type: ignore[union-attr] + finally: + if previous_nox is None: + del sys.modules["nox"] + else: + sys.modules["nox"] = previous_nox + + module._hygiene( # type: ignore[attr-defined] + FakeSession(), + [ + "README.md", + "mkdocs.yml", + "release-please-config.json", + "tests/test_base_smoke.py", + ], + ) + + self.assertEqual(7, len(calls)) + self.assertTrue(all(kwargs.get("log") is False for _args, kwargs in calls)) + + def test_ground_control_retired_routing_fallback_is_detected(self) -> None: + errors: list[str] = [] + _validate_ground_control("routing:\n default_fallback: permissive\n", errors) + self.assertEqual([".ground-control.yaml: routing.default_fallback is retired"], errors) + + def test_devmain_restricted_git_command_is_detected(self) -> None: + errors: list[str] = [] + _validate_devmain("devmain:\n\tgit push origin dev\n", errors) + self.assertIn("Makefile: devmain must only open the promotion PR", errors) + + def test_unpinned_docs_requirement_is_detected(self) -> None: + errors: list[str] = [] + _validate_docs_requirements("mkdocs>=1.6\n", errors) + self.assertEqual( + ["docs/requirements.txt: dependencies must be exactly pinned: mkdocs>=1.6"], + errors, + ) + + def test_codeql_continue_on_error_is_detected(self) -> None: + errors: list[str] = [] + _validate_codeql("continue-on-error: true\n", errors) + self.assertIn(".github/workflows/codeql-analysis.yml: CodeQL must fail closed", errors) + + def test_ci_required_job_contract_violation_is_detected(self) -> None: + errors: list[str] = [] + _validate_ci("name: PR Gate\n", errors) + self.assertIn( + ".github/workflows/ci.yml: missing required configuration: " + "needs: [fast-checks, policy, tool-tests, typecheck, tests, " + "distributions, docs, sonar]", + errors, + ) + + def test_release_workflow_pypi_credential_is_detected(self) -> None: + errors: list[str] = [] + _validate_release_workflow("PYPI_API_TOKEN: forbidden\n", errors) + self.assertIn( + ".github/workflows/release-please.yml: forbidden release setting present: " + "PYPI_API_TOKEN", + errors, + ) + + def test_github_pages_permission_is_detected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + repo_root = Path(tmp) + workflow = repo_root / ".github" / "workflows" / "pages.yml" + workflow.parent.mkdir(parents=True) + workflow.write_text("permissions:\n pages: write\n", encoding="utf-8") + + self.assertEqual( + [".github/workflows/pages.yml: GitHub Pages publishing is retired"], + validate_workflow_policy(workflow, repo_root), + ) + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_readme_quickstart.py b/tools/tests/test_readme_quickstart.py new file mode 100644 index 0000000..bbf9f65 --- /dev/null +++ b/tools/tests/test_readme_quickstart.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from tools.check_readme_quickstart import ( + ContractError, + load_quickstart_contract, + parse_quickstart_contract, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _readme(*, run_command: str | None = None) -> str: + command = run_command or ( + "raes-adapters run --mode conformance --suite pr --output cage2-quickstart" + ) + summary = { + "disposition": "succeeded", + "evidence_basis": "hermetic-live", + "inventory": "inventory.json", + "mode": "conformance", + "run_count": 1, + } + return f"""\ + +```shell +python -m pip install 'raes-adapters[cyborg]' +``` + + +```shell +{command} +``` + + +```json +{json.dumps(summary, separators=(",", ":"))} +``` + + +```text +cage2-quickstart/index.json +cage2-quickstart/inventory.json +cage2-quickstart/runs/cyborg-pr-seed-3/conformance/backend-conformance.json +``` +""" + + +class ReadmeQuickstartTests(unittest.TestCase): + def test_contract_admits_only_the_documented_closed_commands(self) -> None: + contract = parse_quickstart_contract(_readme()) + + self.assertEqual( + ("python", "-m", "pip", "install", "raes-adapters[cyborg]"), + contract.install_argv, + ) + self.assertEqual( + ( + "raes-adapters", + "run", + "--mode", + "conformance", + "--suite", + "pr", + "--output", + "cage2-quickstart", + ), + contract.run_argv, + ) + self.assertEqual("hermetic-live", contract.expected_output["evidence_basis"]) + self.assertEqual( + ( + "cage2-quickstart/index.json", + "cage2-quickstart/inventory.json", + "cage2-quickstart/runs/cyborg-pr-seed-3/conformance/backend-conformance.json", + ), + contract.expected_artifacts, + ) + + def test_contract_rejects_shell_syntax_and_output_escape(self) -> None: + for command in ( + "raes-adapters run --mode conformance --suite pr --output evidence && whoami", + "raes-adapters run --mode conformance --suite pr --output ../evidence", + "raes-adapters run --mode conformance --suite pr --output /tmp/evidence", + ): + with self.subTest(command=command), self.assertRaises(ContractError): + parse_quickstart_contract(_readme(run_command=command)) + + def test_contract_requires_one_of_each_explicit_marker(self) -> None: + with self.assertRaises(ContractError): + parse_quickstart_contract(_readme().replace("", "")) + with self.assertRaises(ContractError): + parse_quickstart_contract(_readme() + _readme()) + + def test_repository_readme_exposes_the_checked_contract(self) -> None: + contract = load_quickstart_contract(REPO_ROOT / "README.md") + + self.assertEqual("cage2-quickstart", contract.output_root.as_posix()) + self.assertEqual(1, contract.expected_output["run_count"]) + + def test_contract_can_be_loaded_from_an_explicit_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + readme = Path(tmp) / "README.md" + readme.write_text(_readme(), encoding="utf-8") + + self.assertEqual( + "cage2-quickstart", + load_quickstart_contract(readme).output_root.as_posix(), + ) + + +if __name__ == "__main__": + unittest.main()