From 6ca77cba56d4d60ea90540b5f8ba614a4c74b52c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 11:17:17 +0000 Subject: [PATCH] =?UTF-8?q?feat(aux-audit):=20score=20agent-spec=20v1=20?= =?UTF-8?q?=E2=80=94=20authority,=20not=20labels=20(0.2.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the last half of #10. aux.H01 and aux.H03 stop asking "what did you call your autonomy level" and start asking what a v1 spec actually declares: per-action authority, the mechanism enforcing each row, and the five forms of human control. Architecture: both versions normalise into one evidence model (src/evidence.ts) that the rules read. One rule set instead of two, and the difference between what v0 and v1 can express becomes a visible, tested mapping rather than branching scattered through the rules. Where a version cannot express something, the field is absent and the rule says so — it never scores absence of a *field* as absence of a *mechanism*. What v1 makes computable that v0 could not: - aux.H05 reports the gap between what the credentials permit and what the mandate governs. The schema calls that the attack surface; it is now a number with the ungoverned capabilities named. - a row enforced by "the system prompt" is reported as unenforced, per trust-architecture: a prompt is a request, a tool boundary is a control. - aux.H04 reads consequence_scaled_approval and whether authority varies across actions at all — a single authority for every action is the slider the canon rejects, wearing a table. - aux.H07 requires an escalation to name both a recipient and what happens on timeout, because without on_timeout "ask a human" is a queue that fills up. Two findings that changed the design: 1. v1 has no memory field. Not "memory: false" — no field. aux.H08 is therefore reported as not scoreable rather than scored zero, and a quarter of the taxonomy (memory_amnesia, preference_ignored, context_leak) is unreachable from a v1 spec. Worth fixing in the schema; noted in #10. 2. A trust stage backed only by an unscoreable heuristic was being reported as EARNED. aux.T02 Contextual Trust depends solely on aux.H08, so every v1 spec was silently earning contextual trust on zero evidence — a claim dressed as a finding, which is the exact failure this taxonomy exists to name. Stages now carry `assessable`, an unassessable stage is never earned, and it stops the ladder the way a broken one does. Two regression tests pin it. v0.1.0 scoring is unchanged: same fixtures, same scores, same evidence strings. meta.spec_version records which format produced a report, since scores are not comparable across the two. 48 tests, up from 38. ajv added for real draft 2020-12 validation, with `format: date` implemented rather than ignored. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MhpqWEvwKp7NrmJfdXYq4M --- CHANGELOG.md | 15 + README.md | 6 +- docs/audit.md | 2 +- packages/aux-audit/README.md | 8 +- .../aux-audit/fixtures/v1-strong-spec.yaml | 212 ++++++++++ packages/aux-audit/fixtures/v1-weak-spec.yaml | 36 ++ packages/aux-audit/package-lock.json | 54 +++ packages/aux-audit/package.json | 3 +- packages/aux-audit/src/cli.ts | 15 +- packages/aux-audit/src/evidence.ts | 152 +++++++ packages/aux-audit/src/index.ts | 4 +- packages/aux-audit/src/report/markdown.ts | 24 +- packages/aux-audit/src/rules.ts | 397 +++++++++++++----- packages/aux-audit/src/score.ts | 38 +- packages/aux-audit/src/spec-v1.ts | 129 ++++++ packages/aux-audit/src/spec.ts | 59 ++- packages/aux-audit/src/types.ts | 8 + packages/aux-audit/test/audit.test.ts | 111 ++++- packages/aux-audit/test/report.test.ts | 35 +- packages/aux-audit/test/spec.test.ts | 75 +++- 20 files changed, 1174 insertions(+), 209 deletions(-) create mode 100644 packages/aux-audit/fixtures/v1-strong-spec.yaml create mode 100644 packages/aux-audit/fixtures/v1-weak-spec.yaml create mode 100644 packages/aux-audit/src/evidence.ts create mode 100644 packages/aux-audit/src/spec-v1.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1ee04..2670a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,21 @@ Publishing the GitHub release and tagging `v0.2` is a manual step after merge. - `CITATION.cff` for GitHub’s *Cite this repository* button. - Project-home links to [auxfirst.com](https://auxfirst.com). +## aux-audit 0.2.0 + +- Scores **agent-spec v1.0**. Both formats are read, detected from the document + rather than the filename, and `meta.spec_version` records which produced a + score — they are not comparable. +- Under v1 the rules ask a better question. `aux.H01` and `aux.H03` read + per-action authority and the five human controls instead of one label; + `aux.H05` computes the gap between what the credentials permit and what the + mandate governs; a row enforced by "the system prompt" counts as unenforced. +- `aux.H08` reports *not scoreable* under v1, which has no memory field, rather + than scoring zero. A trust stage whose entire backing is unscoreable is now + reported **not assessable** and is never counted as earned — previously an + empty shortfall read as success on zero evidence. +- v0.1.0 scoring is unchanged. + ### Notes - `agent-spec.schema.yaml` is now v1.0: real JSON Schema draft 2020-12, with a diff --git a/README.md b/README.md index 2a4031c..8ba4312 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ In CI, as a GitHub Action — SARIF into the Security tab, a scorecard on the PR fail-on: high ``` -It grades the **spec**, not the running product: it can prove a mechanism was never declared, never that a declared one works. A spec with no evaluation transcripts is capped at *present* on every heuristic, so the score cannot be gamed with better prose. See **[packages/aux-audit](packages/aux-audit/)**. +It grades the **spec**, not the running product: it can prove a mechanism was never declared, never that a declared one works. A spec with no evidence of testing is capped at *present* on every heuristic, so the score cannot be gamed with better prose. + +Under agent-spec v1 it computes the gap between what an agent's credentials permit and what its mandate governs — the attack surface — and treats a row enforced by "the system prompt" as unenforced. See **[packages/aux-audit](packages/aux-audit/)**. ## Start here @@ -80,7 +82,7 @@ Or read **[onboarding/README.md](onboarding/README.md)** — the router with all │ ├── aux-evolution-curve.yaml ← the 4 capability stages (teardowns only) │ ├── trust-contract.yaml ← example Trust Contract │ ├── agent-spec.schema.yaml ← v1.0, real JSON Schema — per-action mandate -│ ├── agent-spec.v0.yaml ← v0.1.0, frozen; what aux-audit 0.1.x reads +│ ├── agent-spec.v0.yaml ← v0.1.0, frozen; still scored, not comparable │ ├── MIGRATION.md ← v0 → v1, and why │ ├── migrate-v0-to-v1.py ← converter; leaves TODOs rather than guessing │ ├── brain-spec.schema.yaml ← Agency Brain input format (JSON Schema) diff --git a/docs/audit.md b/docs/audit.md index dc29a43..32eb7ae 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -16,7 +16,7 @@ npx aux-audit run ./agent-spec.yaml ## Contract -- **Input:** an agent spec matching [`schemas/agent-spec.v0.yaml`](../schemas/agent-spec.v0.yaml). The canonical [`agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) is now v1.0; aux-audit 0.1.x does not score it yet and says so plainly rather than reporting missing fields — see [MIGRATION.md](../schemas/MIGRATION.md) and [#10](https://github.com/auxfirst/trustkit/issues/10). +- **Input:** an agent spec in either format — [`agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) v1.0 or [`agent-spec.v0.yaml`](../schemas/agent-spec.v0.yaml). The version is detected from the document, not the filename, and `meta.spec_version` records which one was scored. Scores are not comparable across the two; see [MIGRATION.md](../schemas/MIGRATION.md). - **Output:** `score`, `grade`, `trust_stage`, `issues[]`, `recommendations[]`. - **Formats:** Markdown (default), JSON, SARIF 2.1.0. - **Exit codes:** `0` clean · `1` findings at or above `--fail-on` · `2` invalid spec (nothing graded). diff --git a/packages/aux-audit/README.md b/packages/aux-audit/README.md index bcab9e3..8f3cd41 100644 --- a/packages/aux-audit/README.md +++ b/packages/aux-audit/README.md @@ -32,9 +32,13 @@ Change the YAML in `schemas/`, and the audit changes with it. No heuristic text, **aux-audit grades the spec, not the running product.** It can prove a mechanism was never declared. It cannot prove a declared one works. -There is a sharper limit in the format this version reads, tracked in [trustkit#10](https://github.com/auxfirst/trustkit/issues/10): `agent-spec.v0.yaml` carries a single per-agent `autonomy` label and free-text `guarantees`, so a score reflects a **declared posture**, not an enforced boundary. +**Both spec versions are scored**, detected from the document rather than the filename. -The replacement, `agent-spec.schema.yaml` v1.0, is published and assigns authority action by action with a required `enforced_by` on every non-autonomous row. **aux-audit 0.1.x does not score it.** Hand it a v1 document and it says so in one line and exits 2, rather than reporting the fields v1 removed on purpose as missing. v1 scoring lands in 0.2.0. +Under **v1.0** the audit asks a better question. Instead of *"what did you call your autonomy level"*, `aux.H01` and `aux.H03` read per-action authority and the five human controls, and `aux.H05` computes the **gap between what the credentials permit and what the mandate governs** — the attack surface, in the schema's own words. A row enforced by "the system prompt" is reported as unenforced, because the canon is explicit that a prompt is a request and a tool boundary is a control. + +Under **v0.1.0** nothing changed: the same label and the same prose guarantees produce the same scores as before. Scores are not comparable across the two, so `meta.spec_version` records which one produced them. + +One thing v1 cannot express: **memory**. It has no field for persistence, scoping, or retention, so `aux.H08` is reported as not scoreable rather than scored zero — and `aux.T02` Contextual Trust, whose only backing heuristic that is, comes back *not assessable* rather than earned. See [trustkit#10](https://github.com/auxfirst/trustkit/issues/10). Two consequences, both deliberate: diff --git a/packages/aux-audit/fixtures/v1-strong-spec.yaml b/packages/aux-audit/fixtures/v1-strong-spec.yaml new file mode 100644 index 0000000..d3d23a3 --- /dev/null +++ b/packages/aux-audit/fixtures/v1-strong-spec.yaml @@ -0,0 +1,212 @@ +# Example agent spec — validates against agent-spec.schema.yaml +# +# A realistic, deliberately unglamorous agent: chasing overdue invoices. +# Note what the spec forces into the open — the send action is HIGH band on +# exposure and commitment alone, so it cannot be autonomous no matter how +# reliable the agent has proved to be on the read actions. +# +# Validate with: python3 schemas/validate.py schemas/example-agent-spec.yaml +# +# CC BY 4.0 · auxfirst agency 2026 + +spec_version: "1.0" +id: ar-followup +name: Accounts Receivable Follow-Up +purpose: >- + Chase overdue invoices on a schedule so finance staff spend their time only + on disputed and exceptional cases. + +owners: + business: + name: Anna Nowak + role: Head of Finance Operations + contact: anna.nowak@example.com + technical: + name: Piotr Lis + role: Platform Engineer + contact: piotr.lis@example.com + +trigger: + kind: schedule + detail: Weekdays 07:00 Europe/Warsaw + +users: + - role: Accounts receivable clerk + can_invoke: true + can_approve: true + - role: Account executive + can_invoke: false + can_approve: true + +systems: + data_sources: + - Billing database (invoices, payment status) + - CRM (contact of record, account owner) + connected: + - name: Billing service + auth: own_identity + scopes: [invoice:read, invoice:note:write] + - name: Mail gateway + auth: own_identity + scopes: [draft:write] # send scope deliberately withheld + - name: CRM + auth: delegated_identity + scopes: [contact:read] + +# What the credentials permit. Compare against mandate below — +# the difference is the attack surface. +capability: + can_read: + - invoices + - payment status + - contacts of record + can_change: + - invoice internal note + - email drafts + +mandate: + - action: Read invoice and payment status + authority: autonomous + enforced_by: Scoped read token, invoice:read only + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 0, band: LOW } + autonomy_level: L1 + + - action: Identify overdue invoices and rank by age + authority: autonomous + enforced_by: Read-only query path, no write scope + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 0, band: LOW } + autonomy_level: L1 + + - action: Write an internal note on the invoice + authority: autonomous + enforced_by: Field-scoped write API, note field only + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 1, band: LOW } + autonomy_level: L4 + + - action: Draft a reminder email + authority: autonomous + enforced_by: Mail gateway draft scope; send scope not granted + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 1, band: LOW } + autonomy_level: L3 + + # The hinge. Three dials are cool and the action is still HIGH, + # because it leaves the building and asks for money. + - action: Send a reminder email to the customer + authority: human_approval + enforced_by: Send scope withheld until an approval token is issued + approver: Account executive on the account + heat: { reversibility: 1, blast_radius: 0, exposure: 3, commitment: 3, authority: 1, band: HIGH } + autonomy_level: L3 + + - action: Apply a late-payment fee + authority: human_only + enforced_by: Agent has no write path to the fee service + approver: Head of Finance Operations + heat: { reversibility: 2, blast_radius: 0, exposure: 3, commitment: 4, authority: 2, band: CRITICAL } + autonomy_level: L2 + + - action: Escalate to a collections agency + authority: prohibited + enforced_by: Integration not provisioned for this agent identity + heat: { reversibility: 4, blast_radius: 1, exposure: 4, commitment: 4, authority: 3, band: CRITICAL } + +human_control: + observe: + available: true + mechanism: Per-run log with the invoices touched and drafts produced + audience: affected_user + interrupt: + available: true + mechanism: Pause the approval queue; in-flight drafts are held + audience: operator + approve: + available: true + mechanism: Approval queue, one entry per outbound email + audience: operator + override: + available: true + mechanism: Edit the draft before approving, or reject with a reason + audience: operator + disable: + available: true + mechanism: Feature flag plus token revocation + audience: admin_only + +exceptions: + - condition: identifier_missing + detail: No contact of record on the account + response: route + escalation: + recipient: Anna Nowak + deadline_minutes: 480 + on_timeout: stop + context_preserved: true + + - condition: records_conflict + detail: Billing and CRM disagree on payment status + response: ask_human + escalation: + recipient: Anna Nowak + deadline_minutes: 240 + on_timeout: stop + context_preserved: true + + - condition: result_disputed + detail: Customer replies contesting the invoice + response: route + escalation: + recipient: Account executive on the account + deadline_minutes: 120 + on_timeout: page_secondary + context_preserved: true + + - condition: tool_unavailable + detail: Billing service unreachable + response: stop + + - condition: confidence_low + response: ask_human + escalation: + recipient: Anna Nowak + deadline_minutes: 480 + on_timeout: stop + context_preserved: true + + - condition: sensitive_data_present + detail: Bank details appear in a customer reply + response: refuse + +escalation_default: + recipient: Anna Nowak + deadline_minutes: 480 + on_timeout: stop + context_preserved: true + +supervision: + agent_identity_disclosed: true + action_receipt: true + reversal: true + consequence_scaled_approval: true + escalation_handoff: true + provenance_at_decision: false # known gap — see note below + +shutdown: + procedure: >- + Disable the feature flag, revoke the billing and mail gateway tokens, drain + the approval queue, and notify the two owners. + tested: true + last_tested: "2026-08-20" + revokes_access: true + +model: + provider: anthropic + name: claude-sonnet-5 + version_pinned: true + +trust_stage: aux.T02 +status: live + +# Known gap, recorded rather than hidden: provenance_at_decision is false. +# The approver sees the draft but not the payment-history evidence behind the +# ranking. Tracked as tg.judgment.confident_nonsense risk; fix is the +# confidence-cues pattern on the approval queue. diff --git a/packages/aux-audit/fixtures/v1-weak-spec.yaml b/packages/aux-audit/fixtures/v1-weak-spec.yaml new file mode 100644 index 0000000..0440a25 --- /dev/null +++ b/packages/aux-audit/fixtures/v1-weak-spec.yaml @@ -0,0 +1,36 @@ +# A v1 spec that satisfies the schema and still fails the doctrine: everything +# autonomous, enforcement by prompt, no supervision primitives, untested kill +# switch. v1 makes these visible where v0 could not express them at all. +spec_version: "1.0" +id: inbox-autopilot +name: Inbox Autopilot +purpose: Clear the shared inbox without human involvement. +owners: + business: { name: "Unassigned", role: "TBD" } + technical: { name: "Platform Team", role: "Engineering" } +trigger: + kind: continuous +systems: + data_sources: ["mailbox"] + connected: + - name: gmail + auth: own_identity +capability: + can_read: ["mailbox", "contacts", "calendar"] + can_change: ["send_email", "delete_email", "calendar_event"] +mandate: + - action: send_email + authority: autonomous + enforced_by: "the system prompt tells it to be careful" +human_control: + observe: { available: false } + interrupt: { available: false } + approve: { available: false } + override: { available: false } + disable: { available: true, mechanism: "revoke the OAuth token", audience: admin_only } +exceptions: + - condition: tool_unavailable + response: retry +shutdown: + procedure: "Revoke the OAuth token in the admin console." + tested: false diff --git a/packages/aux-audit/package-lock.json b/packages/aux-audit/package-lock.json index 76e6472..9973473 100644 --- a/packages/aux-audit/package-lock.json +++ b/packages/aux-audit/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "MIT", "dependencies": { + "ajv": "^8.20.0", "yaml": "^2.9.0" }, "bin": { @@ -32,6 +33,59 @@ "undici-types": "~6.21.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/packages/aux-audit/package.json b/packages/aux-audit/package.json index 1f5e7fa..49e6382 100644 --- a/packages/aux-audit/package.json +++ b/packages/aux-audit/package.json @@ -1,6 +1,6 @@ { "name": "aux-audit", - "version": "0.1.0", + "version": "0.2.0", "description": "Score an agent product against the 10 AUX Heuristics and the Trust Architecture. Reproducible, CI-friendly, SARIF-capable.", "keywords": [ "aux", @@ -49,6 +49,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { + "ajv": "^8.20.0", "yaml": "^2.9.0" }, "devDependencies": { diff --git a/packages/aux-audit/src/cli.ts b/packages/aux-audit/src/cli.ts index 0d851b7..0ffa42b 100644 --- a/packages/aux-audit/src/cli.ts +++ b/packages/aux-audit/src/cli.ts @@ -13,14 +13,14 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, relative, resolve } from "node:path"; import { audit, shouldFail } from "./score.js"; -import { loadSpec, SpecError, SpecVersionError } from "./spec.js"; +import { loadEvidence, SpecError } from "./spec.js"; import { loadConfig } from "./config.js"; import { toJson } from "./report/json.js"; import { toMarkdown } from "./report/markdown.js"; import { toSarif } from "./report/sarif.js"; import { SEVERITY_ORDER, type Severity } from "./types.js"; -const VERSION = "0.1.0"; +const VERSION = "0.2.0"; const FORMATS = new Set(["md", "markdown", "json", "sarif"]); const SEVERITIES = new Set(["low", "medium", "high", "critical"]); @@ -36,6 +36,11 @@ OPTIONS --summary additionally write the Markdown scorecard here (use with --format sarif to get a PR comment body) --fail-on exit 1 on findings at or above: low | medium | high | critical + +VERSIONS + Both agent-spec formats are read, detected from the document rather than the + filename. v1.0 scores per-action authority and its enforcing mechanisms; v0.1.0 + scores the older single autonomy label. Scores are not comparable across them. -h, --help show this help -v, --version print the version @@ -165,14 +170,14 @@ function main(argv: string[]): number { let report; try { - const spec = loadSpec(args.spec); - report = audit(spec, { + const evidence = loadEvidence(args.spec); + report = audit(evidence, { ignore: config.ignore, severityOverrides: config.severity_overrides, toolVersion: VERSION, }); } catch (error) { - if (error instanceof SpecVersionError || error instanceof SpecError) { + if (error instanceof SpecError) { process.stderr.write(`aux-audit: ${error.message}\n`); return 2; } diff --git a/packages/aux-audit/src/evidence.ts b/packages/aux-audit/src/evidence.ts new file mode 100644 index 0000000..7c31841 --- /dev/null +++ b/packages/aux-audit/src/evidence.ts @@ -0,0 +1,152 @@ +/** + * The normalised model both spec versions map into. + * + * Rules read this, never a raw spec. Two reasons: one rule set instead of two, + * and the difference between what v0 and v1 can express becomes a visible, + * testable mapping rather than branching scattered through the rules. + * + * Where a version cannot express something at all, the field is `undefined` + * and the rule reports the heuristic as not applicable. It never guesses, and + * it never scores absence of a *field* as absence of a *mechanism*. + */ +import type { AgentSpec } from "./types.js"; +import type { AgentSpecV1, Authority, Control, ExceptionRow } from "./spec-v1.js"; + +export interface MemoryEvidence { + persistent: boolean; + scopeCount: number; + userVisible: boolean; + userEditable: boolean; +} + +export interface MandateEvidence { + action: string; + authority: Authority; + enforcedBy: string; + /** A prompt is a request, not a boundary — trust-architecture.yaml. */ + enforcementIsMechanism: boolean; +} + +export interface Evidence { + /** Which spec format this came from, not the agent's own release version. */ + specVersion: "v0.1.0" | "v1.0"; + name: string; + /** The agent's own version, as the spec declares it. */ + agentVersion: string; + + /** Per-action authority. v0 cannot express this; it has one label instead. */ + mandate?: MandateEvidence[]; + /** v0's single label, kept only so its rules can say what it was. */ + autonomyLabel?: string; + /** Capabilities the credentials grant but no mandate row governs. */ + ungovernedWrites?: string[]; + + /** Five forms of control, v1 only. */ + control?: Record<"observe" | "interrupt" | "approve" | "override" | "disable", Control>; + supervision?: Partial>; + exceptions?: ExceptionRow[]; + escalationDefault?: boolean; + shutdownTested?: boolean; + modelPinned?: boolean; + + /** v1 has no memory field at all. `undefined` means "cannot be expressed". */ + memory?: MemoryEvidence; + + /** Promises in prose. v0 only — v1 replaced them with enforced mandate rows. */ + guarantees: string[]; + tools: string[]; + flows: string[]; + goldenTranscripts: boolean; + failureTranscripts: boolean; + multiAgentSignal?: string; + + /** True once the spec carries evidence of testing, which gates level 3. */ + evidenced: boolean; +} + +/** + * "System prompt" is the canonical non-answer: trust-architecture.yaml states + * that a prompt is a request and a tool boundary is a control, and the v1 + * schema repeats it on the field itself. + */ +const NOT_A_MECHANISM = + /\b(prompt|instructions?|guidelines?|policy document|training|conventions?|documentation|we ask|asked to|told to|expected to)\b/i; + +function isMechanism(enforcedBy: string): boolean { + return enforcedBy.trim().length >= 3 && !NOT_A_MECHANISM.test(enforcedBy.trim()); +} + +const MULTI_AGENT = /agent|delegate|sub[-_]?agent|handoff|hand_off|swarm|crew/i; + +export function fromV0(spec: AgentSpec): Evidence { + const golden = (spec.evaluation?.golden_transcripts ?? []).length > 0; + const failure = (spec.evaluation?.failure_transcripts ?? []).length > 0; + return { + specVersion: "v0.1.0", + name: spec.name, + agentVersion: spec.version, + autonomyLabel: spec.autonomy, + memory: { + persistent: spec.memory.persistent, + scopeCount: spec.memory.scopes.length, + userVisible: spec.memory.user_visible, + userEditable: spec.memory.user_editable, + }, + guarantees: spec.guarantees ?? [], + tools: spec.tools ?? [], + flows: spec.flows ?? [], + goldenTranscripts: golden, + failureTranscripts: failure, + multiAgentSignal: + spec.surface === "multi-surface" + ? "multi-surface" + : (spec.tools ?? []).find((t) => MULTI_AGENT.test(t)) ?? + (spec.guarantees ?? []).find((g) => /\bagents?\b/i.test(g)), + evidenced: golden || failure, + }; +} + +export function fromV1(spec: AgentSpecV1): Evidence { + const mandate: MandateEvidence[] = spec.mandate.map((row) => ({ + action: row.action, + authority: row.authority, + enforcedBy: row.enforced_by, + enforcementIsMechanism: isMechanism(row.enforced_by), + })); + + // The gap between what the credentials permit and what the mandate governs + // is the attack surface — the v1 schema says so in its own header. + const governed = mandate.map((row) => row.action.toLowerCase()); + const ungovernedWrites = spec.capability.can_change.filter( + (write) => !governed.some((action) => action.includes(write.toLowerCase()) || write.toLowerCase().includes(action)), + ); + + const multiAgent = + spec.systems.connected.map((c) => c.name).find((n) => MULTI_AGENT.test(n)) ?? + mandate.map((m) => m.action).find((a) => MULTI_AGENT.test(a)); + + return { + specVersion: "v1.0", + name: spec.name, + agentVersion: spec.version ?? "unversioned", + mandate, + ungovernedWrites, + control: spec.human_control, + supervision: spec.supervision ?? {}, + exceptions: spec.exceptions, + escalationDefault: spec.escalation_default !== undefined, + shutdownTested: spec.shutdown.tested, + modelPinned: spec.model?.version_pinned ?? false, + // v1 declares no memory. Not false — absent. See trustkit#10. + memory: undefined, + guarantees: [], + tools: spec.capability.can_change, + flows: [], + goldenTranscripts: false, + failureTranscripts: false, + multiAgentSignal: multiAgent, + // A tested shutdown and a pinned model are v1's evidence of exercise, in + // the way transcripts are v0's. + evidenced: spec.shutdown.tested === true, + }; +} diff --git a/packages/aux-audit/src/index.ts b/packages/aux-audit/src/index.ts index d1d12f3..96725a2 100644 --- a/packages/aux-audit/src/index.ts +++ b/packages/aux-audit/src/index.ts @@ -1,5 +1,7 @@ export { audit, shouldFail, type AuditOptions } from "./score.js"; -export { loadSpec, validateSpec, SpecError } from "./spec.js"; +export { loadSpec, loadEvidence, evidenceFrom, validateSpec, SpecError } from "./spec.js"; +export { validateV1, type AgentSpecV1 } from "./spec-v1.js"; +export { fromV0, fromV1, type Evidence } from "./evidence.js"; export { loadConfig, type AuditConfig } from "./config.js"; export { loadCanon, gapsForHeuristic, heuristicsForStage } from "./canon.js"; export { RULES } from "./rules.js"; diff --git a/packages/aux-audit/src/report/markdown.ts b/packages/aux-audit/src/report/markdown.ts index 5c6a592..7c66340 100644 --- a/packages/aux-audit/src/report/markdown.ts +++ b/packages/aux-audit/src/report/markdown.ts @@ -30,7 +30,8 @@ export function toMarkdown(report: AuditReport): string { const lines: string[] = []; const applicable = report.heuristics.filter((h) => h.applicable); - lines.push(`## aux-audit — ${report.spec.name} \`${report.spec.version}\``); + const version = report.spec.version === "unversioned" ? "" : ` \`${report.spec.version}\``; + lines.push(`## aux-audit — ${report.spec.name}${version}`); lines.push(""); lines.push( `**Score ${report.score}/100 · Grade ${report.grade} · Trust stage: ${report.trust_stage ?? "none earned"}**`, @@ -42,13 +43,32 @@ export function toMarkdown(report: AuditReport): string { lines.push("| Stage | Earned | Blocked by |"); lines.push("|---|---|---|"); for (const stage of report.trust_stages) { - const blockers = stage.shortfall.length > 0 ? stage.shortfall.map((s) => `\`${s}\``).join(", ") : "—"; + if (!stage.assessable) { + lines.push( + `| ${stage.order}. ${stage.name} | — | not assessable: ${stage.depends_on + .map((s) => `\`${s}\``) + .join(", ")} could not be scored |`, + ); + continue; + } + const blockers = + stage.shortfall.length > 0 + ? stage.shortfall.map((s) => `\`${s}\``).join(", ") + : stage.earned + ? "—" + : "an earlier stage — trust is sequential"; lines.push(`| ${stage.order}. ${stage.name} | ${stage.earned ? "✅" : "❌"} | ${blockers} |`); } lines.push(""); lines.push( "> Trust is sequential. A stage is earned only when every heuristic its failure modes name scores *present* or better, and only if every earlier stage is earned.", ); + if (report.trust_stages.some((stage) => !stage.assessable)) { + lines.push(""); + lines.push( + "> A stage marked *not assessable* had no scoreable evidence at all. It is not earned and not failed — nothing was measured, and reporting it either way would be a claim rather than a finding.", + ); + } lines.push(""); lines.push(`### Heuristics (${applicable.length} scored)`); diff --git a/packages/aux-audit/src/rules.ts b/packages/aux-audit/src/rules.ts index ee01dbc..392fc9e 100644 --- a/packages/aux-audit/src/rules.ts +++ b/packages/aux-audit/src/rules.ts @@ -1,7 +1,7 @@ /** - * The rule set: one rule per AUX heuristic. + * The rule set: one rule per AUX heuristic, scoring the normalised evidence. * - * Every rule scores what the spec *declares*, on the 0-3 scale from + * Every rule scores what a spec *declares*, on the 0-3 scale from * schemas/aux-heuristics.yaml: * * 0 absent — not even acknowledged @@ -9,12 +9,13 @@ * 2 present — works in happy path, breaks under load * 3 robust — works under novel, adversarial, or edge cases * - * A spec is a claim, not a running product. aux-audit grades the claim: it can - * prove a mechanism was never declared, never that a declared one works. Every - * rule therefore returns the evidence it used, so a reader can disagree with a - * number by pointing at a line. + * Two spec versions reach these rules through one evidence model. v0 declares + * a single autonomy label and prose guarantees; v1 declares per-action + * authority with an enforcing mechanism. Where a rule can ask the better + * question, it does — and its evidence string names which version answered. */ -import type { AgentSpec, Level } from "./types.js"; +import type { Evidence } from "./evidence.js"; +import type { Level } from "./types.js"; export interface RuleOutcome { level: Level; @@ -24,93 +25,152 @@ export interface RuleOutcome { export interface Rule { id: string; - evaluate(spec: AgentSpec): RuleOutcome; + evaluate(e: Evidence): RuleOutcome; } const clamp = (n: number): Level => Math.max(0, Math.min(3, n)) as Level; +const join = (parts: (string | false | undefined)[]): string => + parts.filter(Boolean).join("; "); -/** First guarantee matching the pattern, for use as quotable evidence. */ -function guarantee(spec: AgentSpec, pattern: RegExp): string | undefined { - return (spec.guarantees ?? []).find((g) => pattern.test(g)); -} +const guarantee = (e: Evidence, pattern: RegExp): string | undefined => + e.guarantees.find((g) => pattern.test(g)); +const flowMatching = (e: Evidence, pattern: RegExp): string | undefined => + e.flows.find((f) => pattern.test(f)); -function flowMatching(spec: AgentSpec, pattern: RegExp): string | undefined { - return (spec.flows ?? []).find((f) => pattern.test(f)); -} +const isV1 = (e: Evidence): boolean => e.specVersion === "v1.0"; -const golden = (spec: AgentSpec) => - (spec.evaluation?.golden_transcripts ?? []).length > 0; -const failures = (spec: AgentSpec) => - (spec.evaluation?.failure_transcripts ?? []).length > 0; +/** Mandate rows that need an enforcing mechanism: everything but `autonomous`. */ +const gated = (e: Evidence) => (e.mandate ?? []).filter((r) => r.authority !== "autonomous"); +const unenforced = (e: Evidence) => gated(e).filter((r) => !r.enforcementIsMechanism); -function join(parts: string[]): string { - return parts.filter(Boolean).join("; "); -} +const control = (e: Evidence, which: "observe" | "interrupt" | "approve" | "override" | "disable") => + e.control?.[which]; +const has = (e: Evidence, which: Parameters[1]): boolean => + control(e, which)?.available === true; +const supervises = (e: Evidence, key: string): boolean => e.supervision?.[key] === true; export const RULES: Rule[] = [ { id: "aux.H01", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const checkpoints = (e.mandate ?? []).filter((r) => r.authority === "human_approval"); + const level = clamp( + (checkpoints.length > 0 ? 2 : 0) + + (supervises(e, "action_receipt") ? 1 : 0) + + (supervises(e, "agent_identity_disclosed") ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + checkpoints.length > 0 + ? `${checkpoints.length} of ${e.mandate?.length} action(s) require approval before execution` + : "no action requires approval — every mandate row is autonomous or barred", + supervises(e, "action_receipt") + ? "each action leaves a receipt" + : "no action receipt", + supervises(e, "agent_identity_disclosed") + ? "the agent discloses that it is an agent" + : "agent identity is not disclosed", + ]), + }; + } const base = - spec.autonomy === "human-in-the-loop" - ? 2 - : spec.autonomy === "human-on-the-loop" - ? 1 - : 0; + e.autonomyLabel === "human-in-the-loop" ? 2 : e.autonomyLabel === "human-on-the-loop" ? 1 : 0; const asks = guarantee( - spec, + e, /\b(ask|confirm|preview|propose|approval|before (send|act|writ|delet|purchas))/i, ); - const flows = (spec.flows ?? []).length > 0; - const level = clamp(base + (asks ? 1 : 0) + (flows ? 1 : 0)); + const flows = e.flows.length > 0; return { - level, + level: clamp(base + (asks ? 1 : 0) + (flows ? 1 : 0)), applicable: true, evidence: join([ - `autonomy is \`${spec.autonomy}\``, - asks ? `guarantee declares a checkpoint: "${asks}"` : "no guarantee declares a pre-action checkpoint", - flows ? `${spec.flows?.length} flow(s) documented` : "no flows documented", + `autonomy is \`${e.autonomyLabel}\``, + asks + ? `guarantee declares a checkpoint: "${asks}"` + : "no guarantee declares a pre-action checkpoint", + flows ? `${e.flows.length} flow(s) documented` : "no flows documented", ]), }; }, }, { id: "aux.H02", - evaluate(spec) { - const tools = (spec.tools ?? []).length > 0; - const flows = (spec.flows ?? []).length > 0; - const level = clamp( - (tools ? 1 : 0) + (flows ? 1 : 0) + (golden(spec) ? 1 : 0), - ); + evaluate(e) { + if (isV1(e)) { + const observe = control(e, "observe"); + const level = clamp( + (has(e, "observe") ? 1 : 0) + + (supervises(e, "provenance_at_decision") ? 1 : 0) + + (observe?.audience === "affected_user" ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + has(e, "observe") + ? `runs are observable${observe?.mechanism ? ` via ${observe.mechanism}` : ""}` + : "runs are not observable", + supervises(e, "provenance_at_decision") + ? "provenance is shown at the decision" + : "no provenance at the point of decision", + observe?.audience === "affected_user" + ? "visible to the affected user, not only operators" + : `observability reaches ${observe?.audience ?? "no one"}`, + ]), + }; + } + const tools = e.tools.length > 0; + const flows = e.flows.length > 0; return { - level, + level: clamp((tools ? 1 : 0) + (flows ? 1 : 0) + (e.goldenTranscripts ? 1 : 0)), applicable: true, evidence: join([ - tools ? `${spec.tools?.length} tool(s) named — calls can be attributed` : "no tools named, so tool calls cannot be shown", + tools + ? `${e.tools.length} tool(s) named — calls can be attributed` + : "no tools named, so tool calls cannot be shown", flows ? "flows documented" : "no flows documented", - golden(spec) ? "golden transcripts available as an evidence trail" : "no golden transcripts", + e.goldenTranscripts + ? "golden transcripts available as an evidence trail" + : "no golden transcripts", ]), }; }, }, { id: "aux.H03", - evaluate(spec) { - const base = spec.autonomy === "autonomous" ? 0 : 2; - const undo = guarantee( - spec, - /\b(undo|revert|cancel|stop|interrupt|rollback|pause)\b/i, - ); - const editable = spec.memory.user_editable; - const level = clamp(base + (undo ? 1 : 0) + (editable ? 1 : 0)); + evaluate(e) { + if (isV1(e)) { + const level = clamp( + (has(e, "interrupt") ? 1 : 0) + + (has(e, "override") ? 1 : 0) + + (supervises(e, "reversal") ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + has(e, "interrupt") ? "a run can be interrupted mid-flight" : "no interrupt control", + has(e, "override") ? "a decision can be overridden" : "no override control", + supervises(e, "reversal") ? "actions are reversible" : "actions are not reversible", + ]), + }; + } + const base = e.autonomyLabel === "autonomous" ? 0 : 2; + const undo = guarantee(e, /\b(undo|revert|cancel|stop|interrupt|rollback|pause)\b/i); + const editable = e.memory?.userEditable === true; return { - level, + level: clamp(base + (undo ? 1 : 0) + (editable ? 1 : 0)), applicable: true, evidence: join([ - spec.autonomy === "autonomous" + e.autonomyLabel === "autonomous" ? "autonomous agent — no inherent interruption point" - : `autonomy \`${spec.autonomy}\` provides a review point`, - undo ? `guarantee declares mid-flight control: "${undo}"` : "no undo/cancel/interrupt guarantee", + : `autonomy \`${e.autonomyLabel}\` provides a review point`, + undo + ? `guarantee declares mid-flight control: "${undo}"` + : "no undo/cancel/interrupt guarantee", editable ? "memory is user-editable" : "memory is not user-editable", ]), }; @@ -118,15 +178,30 @@ export const RULES: Rule[] = [ }, { id: "aux.H04", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const levels = new Set((e.mandate ?? []).map((r) => r.authority)); + const scaled = supervises(e, "consequence_scaled_approval"); + return { + level: clamp((scaled ? 2 : 0) + (levels.size > 1 ? 1 : 0)), + applicable: true, + evidence: join([ + scaled + ? "approval scales with consequence" + : "approval does not scale with consequence — the same gate for every stake", + levels.size > 1 + ? `authority varies across actions (${[...levels].join(", ")})` + : `every action carries the same authority (${[...levels][0] ?? "none"})`, + ]), + }; + } const ladder = guarantee( - spec, + e, /\b(tenure|earn|gradual|progressive|trust level|new user|first[- ]time|unlock|graduat)/i, ); - const onboarding = flowMatching(spec, /onboard|first[-_ ]?run|new[-_ ]?user/i); - const level = clamp((ladder ? 2 : 0) + (onboarding ? 1 : 0)); + const onboarding = flowMatching(e, /onboard|first[-_ ]?run|new[-_ ]?user/i); return { - level, + level: clamp((ladder ? 2 : 0) + (onboarding ? 1 : 0)), applicable: true, evidence: join([ ladder @@ -141,13 +216,41 @@ export const RULES: Rule[] = [ }, { id: "aux.H05", - evaluate(spec) { - const count = (spec.guarantees ?? []).length; + evaluate(e) { + if (isV1(e)) { + const rows = e.mandate ?? []; + const weak = unenforced(e); + const ungoverned = e.ungovernedWrites ?? []; + const level = clamp( + (rows.length > 0 ? 1 : 0) + + (gated(e).length > 0 && weak.length === 0 ? 1 : 0) + + (ungoverned.length === 0 ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + rows.length > 0 + ? `${rows.length} action(s) carry an explicit authority` + : "no mandate rows — authority is undefined", + weak.length === 0 + ? gated(e).length > 0 + ? "every gated action names an enforcing mechanism" + : "no gated actions to enforce" + : `${weak.length} gated action(s) enforced by something that is not a mechanism: ${weak + .map((r) => `"${r.enforcedBy}"`) + .join(", ")}`, + ungoverned.length === 0 + ? "no write capability sits outside the mandate" + : `${ungoverned.length} ${ungoverned.length === 1 ? "capability" : "capabilities"} the credentials grant but no mandate governs: ${ungoverned.join(", ")}`, + ]), + }; + } + const count = e.guarantees.length; const base = count === 0 ? 0 : count === 1 ? 1 : 2; - const explicit = guarantee(spec, /\bwill (never|not|always)\b/i); - const level = clamp(base + (explicit ? 1 : 0)); + const explicit = guarantee(e, /\bwill (never|not|always)\b/i); return { - level, + level: clamp(base + (explicit ? 1 : 0)), applicable: true, evidence: join([ count === 0 @@ -162,20 +265,36 @@ export const RULES: Rule[] = [ }, { id: "aux.H06", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const lowConfidence = (e.exceptions ?? []).find((x) => x.condition === "confidence_low"); + const acts = lowConfidence !== undefined && + ["ask_human", "route", "refuse", "stop"].includes(lowConfidence.response); + return { + level: clamp((lowConfidence ? 2 : 0) + (acts ? 1 : 0)), + applicable: true, + evidence: join([ + lowConfidence + ? `low confidence is a designed exception, answered with \`${lowConfidence.response}\`` + : "low confidence is not among the declared exceptions", + lowConfidence && !acts + ? `\`${lowConfidence.response}\` does not surface the uncertainty to anyone` + : acts && "the response hands the decision to a human rather than proceeding", + ]), + }; + } const confidence = guarantee( - spec, + e, /\b(confidence|uncertain|unsure|verify|cite|citation|source|evidence|caveat|flag when|don't know|do not know)/i, ); - const level = clamp((confidence ? 2 : 0) + (failures(spec) ? 1 : 0)); return { - level, + level: clamp((confidence ? 2 : 0) + (e.failureTranscripts ? 1 : 0)), applicable: true, evidence: join([ confidence ? `guarantee addresses confidence or sourcing: "${confidence}"` : "no guarantee addresses how confidence or uncertainty is communicated", - failures(spec) + e.failureTranscripts ? "failure transcripts exist — uncertainty behaviour is exercised" : "no failure transcripts, so uncertainty behaviour is untested", ]), @@ -184,20 +303,43 @@ export const RULES: Rule[] = [ }, { id: "aux.H07", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const escalations = (e.exceptions ?? []).filter((x) => x.escalation !== undefined); + const named = escalations.filter( + (x) => x.escalation!.recipient.trim().length > 0 && x.escalation!.on_timeout, + ); + const level = clamp( + (named.length > 0 ? 2 : 0) + + (e.escalationDefault || supervises(e, "escalation_handoff") ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + named.length > 0 + ? `${named.length} exception(s) escalate to a named recipient with a timeout` + : escalations.length > 0 + ? "escalations exist but none names both a recipient and what happens on timeout" + : "no exception escalates — the agent absorbs every failure itself", + e.escalationDefault + ? "a default escalation catches anything unlisted" + : "no default escalation for unlisted conditions", + ]), + }; + } const assertive = guarantee( - spec, + e, /\b(escalat|hand off|handoff|hand-off|human review|push back|pushback|refuse|decline|second opinion)/i, ); - const level = clamp((assertive ? 2 : 0) + (failures(spec) ? 1 : 0)); return { - level, + level: clamp((assertive ? 2 : 0) + (e.failureTranscripts ? 1 : 0)), applicable: true, evidence: join([ assertive ? `guarantee declares escalation or pushback: "${assertive}"` : "no guarantee declares when the agent escalates, refuses, or pushes back", - failures(spec) + e.failureTranscripts ? "failure transcripts exercise the escalation path" : "no failure transcripts covering escalation", ]), @@ -206,19 +348,27 @@ export const RULES: Rule[] = [ }, { id: "aux.H08", - evaluate(spec) { - if (!spec.memory.persistent) { + evaluate(e) { + // v1 declares no memory at all — not "no memory", but no field for it. + // Scoring absence of a field as absence of a mechanism would be a lie. + if (e.memory === undefined) { return { level: 0, - applicable: true, + applicable: false, evidence: - "memory.persistent is false — the agent starts every session as a stranger", + "not scoreable — agent-spec v1.0 has no memory field, so a spec cannot declare persistence, scoping, or retention (trustkit#10)", + }; + } + if (!e.memory.persistent) { + return { + level: 0, + applicable: true, + evidence: "memory.persistent is false — the agent starts every session as a stranger", }; } - const scopes = spec.memory.scopes.length; - const level = clamp(2 + (scopes >= 2 ? 1 : 0)); + const scopes = e.memory.scopeCount; return { - level, + level: clamp(2 + (scopes >= 2 ? 1 : 0)), applicable: true, evidence: join([ "memory is persistent", @@ -231,17 +381,8 @@ export const RULES: Rule[] = [ }, { id: "aux.H09", - evaluate(spec) { - const multiAgentTool = (spec.tools ?? []).find((t) => - /agent|delegate|sub[-_]?agent|handoff|hand_off|swarm|crew/i.test(t), - ); - const multiAgentGuarantee = guarantee(spec, /\bagents?\b/i); - const applicable = - spec.surface === "multi-surface" || - multiAgentTool !== undefined || - multiAgentGuarantee !== undefined; - - if (!applicable) { + evaluate(e) { + if (e.multiAgentSignal === undefined) { return { level: 3, applicable: false, @@ -249,21 +390,28 @@ export const RULES: Rule[] = [ "not applicable — the spec describes a single agent (no delegating tools, no multi-agent guarantees, single surface)", }; } - const attribution = guarantee( - spec, - /\b(attribut|which agent|responsib|named agent|on behalf)/i, - ); - const handoffFlow = flowMatching(spec, /handoff|hand[-_]off|delegat|escalat/i); - const level = clamp((attribution ? 2 : 0) + (handoffFlow ? 1 : 0)); + if (isV1(e)) { + const disclosed = supervises(e, "agent_identity_disclosed"); + const handoff = (e.exceptions ?? []).some((x) => x.response === "route"); + return { + level: clamp((disclosed ? 2 : 0) + (handoff ? 1 : 0)), + applicable: true, + evidence: join([ + `multi-agent signal present (${e.multiAgentSignal})`, + disclosed + ? "agent identity is disclosed, so actions can be attributed" + : "identity is not disclosed — the user cannot tell which agent acted", + handoff ? "a routing response documents the handoff" : "no routing exception documented", + ]), + }; + } + const attribution = guarantee(e, /\b(attribut|which agent|responsib|named agent|on behalf)/i); + const handoffFlow = flowMatching(e, /handoff|hand[-_]off|delegat|escalat/i); return { - level, + level: clamp((attribution ? 2 : 0) + (handoffFlow ? 1 : 0)), applicable: true, evidence: join([ - `multi-agent signals present (${ - spec.surface === "multi-surface" - ? "multi-surface" - : (multiAgentTool ?? "agent guarantee") - })`, + `multi-agent signals present (${e.multiAgentSignal})`, attribution ? `guarantee addresses attribution: "${attribution}"` : "no guarantee tells the user which agent did what", @@ -274,22 +422,43 @@ export const RULES: Rule[] = [ }, { id: "aux.H10", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const level = clamp( + (e.modelPinned ? 2 : 0) + + (e.shutdownTested ? 1 : 0) + + ((e.exceptions ?? []).length >= 3 ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + e.modelPinned + ? "the model version is pinned, so behaviour cannot change without a release" + : "the model version is not pinned — behaviour can change without a release", + e.shutdownTested + ? "the shutdown procedure has been tested" + : "the shutdown procedure is untested, so it is a claim rather than a control", + (e.exceptions ?? []).length >= 3 + ? `${e.exceptions?.length} failure conditions are designed for` + : "fewer than three failure conditions designed for", + ]), + }; + } const contract = guarantee( - spec, + e, /\b(deterministic|same (input|output)|consistent|reproducib|pinned|model version|regression)/i, ); - const level = clamp( - (golden(spec) ? 2 : 0) + (failures(spec) ? 1 : 0) + (contract ? 1 : 0), - ); return { - level, + level: clamp( + (e.goldenTranscripts ? 2 : 0) + (e.failureTranscripts ? 1 : 0) + (contract ? 1 : 0), + ), applicable: true, evidence: join([ - golden(spec) + e.goldenTranscripts ? "golden transcripts pin expected behaviour" : "no golden transcripts — nothing detects silent drift between model versions", - failures(spec) ? "failure transcripts present" : "no failure transcripts", + e.failureTranscripts ? "failure transcripts present" : "no failure transcripts", contract ? `guarantee declares a behavioural contract: "${contract}"` : "no guarantee about behavioural consistency over time", diff --git a/packages/aux-audit/src/score.ts b/packages/aux-audit/src/score.ts index 6fb44a1..6a5e3de 100644 --- a/packages/aux-audit/src/score.ts +++ b/packages/aux-audit/src/score.ts @@ -13,6 +13,7 @@ import { type Canon, } from "./canon.js"; import { RULES } from "./rules.js"; +import type { Evidence } from "./evidence.js"; import type { AgentSpec, AuditReport, @@ -45,11 +46,8 @@ const EARNED_AT = 2; */ const ROBUST_CAP = 2; -function hasEvidence(spec: AgentSpec): boolean { - return ( - (spec.evaluation?.golden_transcripts ?? []).length > 0 || - (spec.evaluation?.failure_transcripts ?? []).length > 0 - ); +function hasEvidence(e: Evidence): boolean { + return e.evidenced; } function grade(score: number): Grade { @@ -89,12 +87,12 @@ function recommend(heuristicName: string, fixPattern: string): string { return `${heuristicName}: adopt the \`${fixPattern}\` pattern.`; } -export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport { +export function audit(evidence: Evidence, options: AuditOptions = {}): AuditReport { const canon = loadCanon(); const ignore = new Set(options.ignore ?? []); const overrides = options.severityOverrides ?? {}; - const evidenced = hasEvidence(spec); + const evidenced = hasEvidence(evidence); const byId = new Map(canon.heuristics.map((h) => [h.id, h])); const heuristics: HeuristicResult[] = []; @@ -116,7 +114,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport }); continue; } - const outcome = rule.evaluate(spec); + const outcome = rule.evaluate(evidence); const capped = outcome.applicable && !evidenced && outcome.level > ROBUST_CAP; heuristics.push({ id: rule.id, @@ -143,11 +141,17 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport let ladderIntact = true; for (const stage of canon.stages) { const depends_on = heuristicsForStage(canon, stage.id); - const shortfall = depends_on.filter((id) => { - const result = levelOf.get(id); - return result !== undefined && result.applicable && result.level < EARNED_AT; - }); - const earnedHere = shortfall.length === 0; + const backing = depends_on + .map((id) => levelOf.get(id)) + .filter((result): result is HeuristicResult => result !== undefined); + // Nothing measured means nothing earned. Reporting a stage as earned + // because its only evidence could not be scored is the failure this whole + // taxonomy exists to name. + const assessable = backing.some((result) => result.applicable); + const shortfall = backing + .filter((result) => result.applicable && result.level < EARNED_AT) + .map((result) => result.id); + const earnedHere = assessable && shortfall.length === 0; const earned = ladderIntact && earnedHere; if (!earned) ladderIntact = false; trust_stages.push({ @@ -155,6 +159,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport name: stage.name, order: stage.order, earned, + assessable, depends_on, shortfall, }); @@ -181,7 +186,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport } for (const stage of trust_stages) { - if (stage.earned || stage.shortfall.length === 0) continue; + if (stage.earned || !stage.assessable || stage.shortfall.length === 0) continue; issues.push({ id: stage.id, type: `${stage.name.toLowerCase().replace(/ trust$/, "").replace(/\s+/g, "_")}_trust_gap`, @@ -197,7 +202,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport ); return { - spec: { name: spec.name, version: spec.version }, + spec: { name: evidence.name, version: evidence.agentVersion }, score, grade: grade(score), trust_stage, @@ -209,7 +214,8 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport recommendations: [...new Set(recommendations)], meta: { tool: "aux-audit", - tool_version: options.toolVersion ?? "0.1.0", + spec_version: evidence.specVersion, + tool_version: options.toolVersion ?? "0.2.0", heuristics_version: canon.heuristicsVersion, trust_architecture_version: canon.trustArchitectureVersion, taxonomy_version: canon.taxonomyVersion, diff --git a/packages/aux-audit/src/spec-v1.ts b/packages/aux-audit/src/spec-v1.ts new file mode 100644 index 0000000..53c7ef4 --- /dev/null +++ b/packages/aux-audit/src/spec-v1.ts @@ -0,0 +1,129 @@ +/** + * agent-spec v1.0 — real JSON Schema draft 2020-12, validated with ajv. + * + * v1 replaces v0's single `autonomy` label with a per-action mandate, each + * non-autonomous row naming the mechanism that enforces it. That is the whole + * reason it exists, and it is what makes the audit able to ask a better + * question than "what did you call your autonomy level". + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import ajvModule, { type ErrorObject, type ValidateFunction } from "ajv/dist/2020.js"; + +/** + * ajv ships CommonJS. Under NodeNext the constructor arrives on `.default`, + * but the type is the namespace, so both sides need coaxing. + */ +type AjvCtor = new (options?: Record) => { + compile(schema: object): ValidateFunction; + addFormat(name: string, format: RegExp): unknown; +}; +const Ajv2020 = ((ajvModule as unknown as { default?: unknown }).default ?? + ajvModule) as unknown as AjvCtor; +import { parse } from "yaml"; +import { schemaDir } from "./canon.js"; + +export type Authority = "autonomous" | "human_approval" | "human_only" | "prohibited"; + +export interface MandateRow { + action: string; + authority: Authority; + enforced_by: string; + approver?: string; +} + +export interface Control { + available: boolean; + mechanism?: string; + audience?: "affected_user" | "operator" | "admin_only" | "none"; +} + +export interface Escalation { + recipient: string; + deadline_minutes: number; + on_timeout: "stop" | "fallback" | "refuse" | "page_secondary"; + context_preserved?: boolean; +} + +export interface ExceptionRow { + condition: string; + detail?: string; + response: "retry" | "stop" | "ask_human" | "route" | "fallback" | "log" | "refuse"; + escalation?: Escalation; +} + +export interface AgentSpecV1 { + spec_version: string; + id: string; + name: string; + version?: string; + surface?: string; + purpose: string; + owners: { business: unknown; technical: unknown }; + trigger: { kind: string; detail?: string }; + users?: unknown[]; + systems: { data_sources: string[]; connected: { name: string; auth: unknown }[] }; + capability: { can_read: string[]; can_change: string[] }; + mandate: MandateRow[]; + human_control: Record<"observe" | "interrupt" | "approve" | "override" | "disable", Control>; + exceptions: ExceptionRow[]; + escalation_default?: Escalation; + supervision?: Partial< + Record< + | "agent_identity_disclosed" + | "action_receipt" + | "reversal" + | "consequence_scaled_approval" + | "escalation_handoff" + | "provenance_at_decision", + boolean + > + >; + shutdown: { procedure: string; tested: boolean; last_tested?: string; revokes_access?: boolean }; + model?: { provider?: string; name?: string; version_pinned?: boolean }; + trust_stage?: string; + status?: string; +} + +let compiled: ValidateFunction | undefined; + +function validator(): ValidateFunction { + if (compiled) return compiled; + const schema = parse( + readFileSync(join(schemaDir(), "agent-spec.schema.yaml"), "utf8"), + ) as object; + const ajv = new Ajv2020({ strict: false, allErrors: true }); + // `shutdown.last_tested` is `format: date`. Validate it rather than let ajv + // log that it is ignoring a format on every run. + ajv.addFormat("date", /^\d{4}-\d{2}-\d{2}$/); + const fn = ajv.compile(schema); + compiled = fn; + return fn; +} + +/** ajv's paths are terse; make them read like the field a person would fix. */ +function describe(error: ErrorObject): string { + const where = error.instancePath.replace(/^\//, "").replace(/\//g, ".") || "(root)"; + if (error.keyword === "required") { + const missing = (error.params as { missingProperty: string }).missingProperty; + return where === "(root)" + ? `\`${missing}\` is required` + : `\`${where}.${missing}\` is required`; + } + if (error.keyword === "enum") { + const allowed = (error.params as { allowedValues: unknown[] }).allowedValues; + return `\`${where}\` must be one of: ${allowed.join(", ")}`; + } + if (error.keyword === "additionalProperties") { + const extra = (error.params as { additionalProperty: string }).additionalProperty; + return `\`${where}\` has an unknown field: ${extra}`; + } + return `\`${where}\` ${error.message ?? "is invalid"}`; +} + +export function validateV1(input: unknown): { spec?: AgentSpecV1; problems: string[] } { + const validate = validator(); + if (validate(input)) return { spec: input as AgentSpecV1, problems: [] }; + const problems = (validate.errors ?? []).map(describe); + return { problems: [...new Set(problems)] }; +} diff --git a/packages/aux-audit/src/spec.ts b/packages/aux-audit/src/spec.ts index ef45baf..25e2849 100644 --- a/packages/aux-audit/src/spec.ts +++ b/packages/aux-audit/src/spec.ts @@ -9,6 +9,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { parse } from "yaml"; import { schemaDir } from "./canon.js"; +import { fromV0, fromV1, type Evidence } from "./evidence.js"; +import { validateV1 } from "./spec-v1.js"; import type { AgentSpec, Autonomy, MemorySpec, Surface } from "./types.js"; /** @@ -17,25 +19,6 @@ import type { AgentSpec, Autonomy, MemorySpec, Surface } from "./types.js"; */ const schemaPath = (): string => join(schemaDir(), "agent-spec.v0.yaml"); -/** - * Handed a v1 document, aux-audit 0.1.x used to reject it with four field - * errors — including "`autonomy` is required", demanding the very field v1 - * removes on purpose. That reads as a broken tool rather than a version - * mismatch, so it is detected before validation and reported as itself. - */ -export class SpecVersionError extends Error { - readonly detected = "v1.0"; - constructor() { - super( - "this is an agent-spec v1.0 document; aux-audit 0.1.x reads v0.1.0.\n" + - " Nothing was graded. v1 scoring lands in aux-audit 0.2.0.\n" + - " Migrating a v0 spec: python3 schemas/migrate-v0-to-v1.py your-spec.yaml\n" + - " Background: schemas/MIGRATION.md, trustkit#10", - ); - this.name = "SpecVersionError"; - } -} - /** v1 declares a per-action mandate and a spec_version; v0 declares neither. */ function looksLikeV1(raw: Record): boolean { return "mandate" in raw || "spec_version" in raw; @@ -128,8 +111,6 @@ export function validateSpec(input: unknown): AgentSpec { throw new SpecError(["the spec must be a YAML or JSON mapping"]); } const raw = input as Record; - if (looksLikeV1(raw)) throw new SpecVersionError(); - const { surface, autonomy } = schemaEnums(); for (const key of ["name", "version"] as const) { @@ -219,3 +200,39 @@ export function loadSpec(path: string): AgentSpec { } return validateSpec(parsed); } + + +/** + * Reads either spec version and returns the normalised evidence the rules + * score. Version is detected from the document, never from the filename, so a + * spec is graded as what it is rather than as what it was called. + */ +export function loadEvidence(path: string): Evidence { + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch { + throw new SpecError([`cannot read spec file: ${path}`]); + } + let parsed: unknown; + try { + parsed = parse(raw); + } catch (error) { + throw new SpecError([ + `spec is not valid YAML or JSON: ${(error as Error).message}`, + ]); + } + return evidenceFrom(parsed); +} + +export function evidenceFrom(parsed: unknown): Evidence { + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new SpecError(["the spec must be a YAML or JSON mapping"]); + } + if (looksLikeV1(parsed as Record)) { + const { spec, problems } = validateV1(parsed); + if (!spec) throw new SpecError(problems); + return fromV1(spec); + } + return fromV0(validateSpec(parsed)); +} diff --git a/packages/aux-audit/src/types.ts b/packages/aux-audit/src/types.ts index bd03cee..0effe44 100644 --- a/packages/aux-audit/src/types.ts +++ b/packages/aux-audit/src/types.ts @@ -57,6 +57,12 @@ export interface TrustStageResult { name: string; order: number; earned: boolean; + /** + * False when every heuristic backing this stage is inapplicable, so nothing + * was measured. A stage that cannot be assessed is never reported as earned: + * that would be a claim dressed as a finding. + */ + assessable: boolean; /** Heuristic ids that must score >= 2 for this stage, per the gap taxonomy. */ depends_on: string[]; shortfall: string[]; @@ -87,6 +93,8 @@ export interface AuditReport { recommendations: string[]; meta: { tool: string; + /** Which agent-spec format was scored. Scores are not comparable across it. */ + spec_version: "v0.1.0" | "v1.0"; tool_version: string; heuristics_version: string; trust_architecture_version: string; diff --git a/packages/aux-audit/test/audit.test.ts b/packages/aux-audit/test/audit.test.ts index acac779..487058b 100644 --- a/packages/aux-audit/test/audit.test.ts +++ b/packages/aux-audit/test/audit.test.ts @@ -1,13 +1,14 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { audit, shouldFail } from "../src/score.js"; -import { loadSpec } from "../src/spec.js"; +import { loadSpec, loadEvidence, evidenceFrom } from "../src/spec.js"; +import { fromV0 } from "../src/evidence.js"; import { loadCanon, heuristicsForStage } from "../src/canon.js"; import { RULES } from "../src/rules.js"; import { fixture } from "./helpers.js"; -const strong = () => audit(loadSpec(fixture("strong-spec.yaml"))); -const weak = () => audit(loadSpec(fixture("weak-spec.yaml"))); +const strong = () => audit(loadEvidence(fixture("strong-spec.yaml"))); +const weak = () => audit(loadEvidence(fixture("weak-spec.yaml"))); test("every rule maps to a heuristic in the canonical schema", () => { const canon = loadCanon(); @@ -48,13 +49,13 @@ test("aux.H09 is not applicable to a single-agent spec but is to a crew", () => const single = strong().heuristics.find((h) => h.id === "aux.H09")!; assert.equal(single.applicable, false); - const crew = audit(loadSpec(fixture("multi-agent-spec.yaml"))); + const crew = audit(loadEvidence(fixture("multi-agent-spec.yaml"))); const multi = crew.heuristics.find((h) => h.id === "aux.H09")!; assert.equal(multi.applicable, true, "agent.delegate should trigger H09"); }); test("non-applicable heuristics never produce issues", () => { - for (const report of [strong(), weak(), audit(loadSpec(fixture("multi-agent-spec.yaml")))]) { + for (const report of [strong(), weak(), audit(loadEvidence(fixture("multi-agent-spec.yaml")))]) { const skipped = report.heuristics.filter((h) => !h.applicable).map((h) => h.id); for (const id of skipped) { assert.ok(!report.issues.some((issue) => issue.id === id), `${id} should not be reported`); @@ -96,8 +97,7 @@ test("every issue carries evidence a reader can argue with", () => { }); test("config can ignore a heuristic and override a severity", () => { - const spec = loadSpec(fixture("weak-spec.yaml")); - const report = audit(spec, { + const report = audit(loadEvidence(fixture("weak-spec.yaml")), { ignore: ["aux.H04"], severityOverrides: { "aux.H01": "low" }, }); @@ -117,8 +117,8 @@ test("shouldFail respects the threshold", () => { test("the report is deterministic for the same spec", () => { const now = new Date("2026-01-01T00:00:00.000Z"); - const spec = loadSpec(fixture("strong-spec.yaml")); - assert.deepEqual(audit(spec, { now }), audit(spec, { now })); + const ev = loadEvidence(fixture("strong-spec.yaml")); + assert.deepEqual(audit(ev, { now }), audit(ev, { now })); }); test("evolution_stage is permanently out of scope, not pending", () => { @@ -131,9 +131,14 @@ test("evolution_stage is permanently out of scope, not pending", () => { }); test("level 3 requires evidence — a spec with no transcripts is capped at present", () => { - const spec = loadSpec(fixture("strong-spec.yaml")); - const withEvidence = audit(spec); - const withoutEvidence = audit({ ...spec, evaluation: { golden_transcripts: [], failure_transcripts: [] } }); + const ev = loadEvidence(fixture("strong-spec.yaml")); + const withEvidence = audit(ev); + const withoutEvidence = audit({ + ...ev, + goldenTranscripts: false, + failureTranscripts: false, + evidenced: false, + }); assert.ok(withEvidence.heuristics.some((h) => h.applicable && h.level === 3)); assert.ok( @@ -149,7 +154,7 @@ test("level 3 requires evidence — a spec with no transcripts is capped at pres test("the cap cannot be dodged by declaring more prose", () => { const spec = loadSpec(fixture("weak-spec.yaml")); - const wordy = audit({ + const wordy = audit(fromV0({ ...spec, guarantees: [ "will always ask before acting", @@ -161,7 +166,85 @@ test("the cap cannot be dodged by declaring more prose", () => { "responses are deterministic and pinned to a model version", ], flows: ["./flows/onboarding.md", "./flows/handoff.md"], - }); + })); assert.ok(wordy.heuristics.every((h) => !h.applicable || h.level <= 2)); assert.ok(wordy.score <= 67, `prose alone should not reach robust, got ${wordy.score}`); }); + +// --- agent-spec v1 ------------------------------------------------------ + +const v1strong = () => audit(loadEvidence(fixture("v1-strong-spec.yaml"))); +const v1weak = () => audit(loadEvidence(fixture("v1-weak-spec.yaml"))); + +test("v1: the report records which spec format produced the score", () => { + assert.equal(v1strong().meta.spec_version, "v1.0"); + assert.equal(strong().meta.spec_version, "v0.1.0"); +}); + +test("v1: a mandate with enforcement outscores one without", () => { + assert.ok(v1strong().score > v1weak().score); +}); + +test("v1: capability the mandate does not govern is reported as the attack surface", () => { + const h05 = v1weak().heuristics.find((h) => h.id === "aux.H05")!; + assert.match(h05.evidence, /capabilities the credentials grant but no mandate governs/); + assert.match(h05.evidence, /delete_email/); + assert.ok(h05.level < 2, "ungoverned write capability must not score as present"); +}); + +test("v1: a prompt is not an enforcing mechanism", () => { + const spec = { + spec_version: "1.0", + id: "prompt-enforcement-probe", + name: "Prompt Enforcement Probe", + purpose: "Exercise the rule that an enforcing mechanism is not a prompt.", + owners: { business: { name: "A", role: "r" }, technical: { name: "B", role: "r" } }, + trigger: { kind: "event" }, + systems: { data_sources: ["s"], connected: [{ name: "n", auth: "service_account" }] }, + capability: { can_read: [], can_change: ["refund"] }, + mandate: [{ action: "refund", authority: "human_approval", enforced_by: "the system prompt" }], + human_control: { + observe: { available: true }, + interrupt: { available: true }, + approve: { available: true }, + override: { available: true }, + disable: { available: true }, + }, + exceptions: [{ condition: "other", response: "stop" }], + shutdown: { procedure: "Disable the integration.", tested: true }, + }; + const report = audit(evidenceFrom(spec)); + const h05 = report.heuristics.find((h) => h.id === "aux.H05")!; + assert.match(h05.evidence, /enforced by something that is not a mechanism/); + assert.match(h05.evidence, /the system prompt/); +}); + +test("v1: aux.H08 is not scoreable, and says why rather than scoring zero", () => { + const h08 = v1strong().heuristics.find((h) => h.id === "aux.H08")!; + assert.equal(h08.applicable, false); + assert.match(h08.evidence, /no memory field/); + assert.ok(!v1strong().issues.some((i) => i.id === "aux.H08"), "must not be reported as a failing"); +}); + +test("a stage with no scoreable evidence is never reported as earned", () => { + // The bug this guards: aux.H08 is the only heuristic backing aux.T02. Under + // v1 it cannot be scored, and an empty shortfall was reading as "earned" — + // a claim dressed as a finding. + const contextual = v1strong().trust_stages.find((s) => s.id === "aux.T02")!; + assert.equal(contextual.assessable, false); + assert.equal(contextual.earned, false); + assert.ok(!v1strong().issues.some((i) => i.id === "aux.T02"), "not the product's failure to fix"); +}); + +test("an unassessable stage stops the ladder, as a broken one would", () => { + const stages = v1strong().trust_stages; + assert.equal(stages.find((s) => s.id === "aux.T01")!.earned, true); + for (const id of ["aux.T02", "aux.T03", "aux.T04"]) { + assert.equal(stages.find((s) => s.id === id)!.earned, false, `${id} must not be earned`); + } + assert.equal(v1strong().trust_stage, "functional"); +}); + +test("v0 stages stay assessable — the change is version-specific", () => { + assert.ok(strong().trust_stages.every((s) => s.assessable)); +}); diff --git a/packages/aux-audit/test/report.test.ts b/packages/aux-audit/test/report.test.ts index 90b9db5..374ee4f 100644 --- a/packages/aux-audit/test/report.test.ts +++ b/packages/aux-audit/test/report.test.ts @@ -5,13 +5,13 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { audit } from "../src/score.js"; -import { loadSpec } from "../src/spec.js"; +import { loadEvidence } from "../src/spec.js"; import { toJson } from "../src/report/json.js"; import { toMarkdown } from "../src/report/markdown.js"; import { toSarif } from "../src/report/sarif.js"; import { cliPath, fixture } from "./helpers.js"; -const weak = () => audit(loadSpec(fixture("weak-spec.yaml"))); +const weak = () => audit(loadEvidence(fixture("weak-spec.yaml"))); test("json output round-trips and keeps the v0.1 contract fields", () => { const report = weak(); @@ -128,7 +128,7 @@ test("CLI: --config is honoured", () => { test("CLI: --version and --help exit 0", () => { assert.match( execFileSync(process.execPath, [cliPath(), "--version"], { encoding: "utf8" }), - /^0\.1\.0/, + /^0\.2\.0/, ); assert.match( execFileSync(process.execPath, [cliPath(), "--help"], { encoding: "utf8" }), @@ -137,23 +137,32 @@ test("CLI: --version and --help exit 0", () => { }); -test("CLI: a v1 spec exits 2 with a version message, not field errors", () => { - const dir = mkdtempSync(join(tmpdir(), "aux-audit-v1-")); - const specPath = join(dir, "v1.yaml"); - writeFileSync( - specPath, - "spec_version: '1.0'\nid: collections\nname: Collections Agent\nmandate: []\n", +test("CLI: a v1 spec is scored, and the report says which format it was", () => { + const out = execFileSync( + process.execPath, + [cliPath(), "run", fixture("v1-strong-spec.yaml"), "--format", "json"], + { encoding: "utf8" }, ); + const report = JSON.parse(out); + assert.equal(report.meta.spec_version, "v1.0"); + assert.ok(report.score > 0); + assert.equal(report.spec.name, "Accounts Receivable Follow-Up"); +}); + +test("CLI: an invalid v1 spec reports v1 fields, never v0 ones", () => { + const dir = mkdtempSync(join(tmpdir(), "aux-audit-v1-")); + const specPath = join(dir, "broken.yaml"); + writeFileSync(specPath, "spec_version: '1.0'\nid: x\nname: X\nmandate: []\n"); try { execFileSync(process.execPath, [cliPath(), "run", specPath], { stdio: "pipe" }); assert.fail("expected a non-zero exit"); } catch (error) { - const err = error as { status: number; stderr: Buffer; stdout: Buffer }; + const err = error as { status: number; stderr: Buffer }; assert.equal(err.status, 2); const stderr = err.stderr.toString(); - assert.match(stderr, /agent-spec v1\.0 document/); - assert.match(stderr, /MIGRATION\.md/); + assert.match(stderr, /`shutdown` is required/); + // v0's vocabulary must never surface for a v1 document. assert.doesNotMatch(stderr, /`autonomy` is required/); - assert.doesNotMatch(stderr, /Score/); + assert.doesNotMatch(stderr, /`surface` is required/); } }); diff --git a/packages/aux-audit/test/spec.test.ts b/packages/aux-audit/test/spec.test.ts index a9907c1..f46b870 100644 --- a/packages/aux-audit/test/spec.test.ts +++ b/packages/aux-audit/test/spec.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { parse } from "yaml"; -import { loadSpec, validateSpec, SpecError, SpecVersionError } from "../src/spec.js"; +import { loadSpec, loadEvidence, evidenceFrom, validateSpec, SpecError } from "../src/spec.js"; import { fixture, repoSchema } from "./helpers.js"; test("loads a well-formed spec", () => { @@ -105,34 +105,75 @@ test("ISO-8601 durations are validated", () => { }); -test("a v1 document is named as a version mismatch, not a pile of field errors", () => { +test("a v1 document is detected and scored, not rejected", () => { const v1 = { spec_version: "1.0", id: "collections-agent", name: "Collections Agent", - mandate: [{ action: "send reminder", authority: "human_approval", enforced_by: "queue" }], + purpose: "Chase overdue invoices before they age past 90 days.", + owners: { + business: { name: "A. Novak", role: "Head of Credit" }, + technical: { name: "B. Reyes", role: "Staff Engineer" }, + }, + trigger: { kind: "schedule" }, + systems: { data_sources: ["ledger"], connected: [{ name: "erp", auth: "service_account" }] }, + capability: { can_read: ["invoices"], can_change: ["invoice_note"] }, + mandate: [ + { action: "invoice_note", authority: "autonomous", enforced_by: "scoped API token" }, + ], + human_control: { + observe: { available: true }, + interrupt: { available: true }, + approve: { available: false }, + override: { available: true }, + disable: { available: true }, + }, + exceptions: [{ condition: "records_conflict", response: "ask_human" }], + shutdown: { procedure: "Revoke the service account and drain the queue.", tested: true }, }; + const evidence = evidenceFrom(v1); + assert.equal(evidence.specVersion, "v1.0"); + assert.equal(evidence.name, "Collections Agent"); + assert.equal(evidence.mandate?.length, 1); + assert.equal(evidence.mandate?.[0]?.enforcementIsMechanism, true); + // v1 declares no memory, and the absence must not read as "no memory". + assert.equal(evidence.memory, undefined); +}); + +test("a v1 document that breaks the schema reports the real field", () => { assert.throws( - () => validateSpec(v1), + () => evidenceFrom({ spec_version: "1.0", mandate: [] }), (error: unknown) => { - assert.ok(error instanceof SpecVersionError, "should be a version error"); - assert.match(error.message, /agent-spec v1\.0 document/); - assert.match(error.message, /MIGRATION\.md/); - // The old behaviour demanded the very field v1 removes on purpose. - assert.doesNotMatch(error.message, /`autonomy` is required/); - assert.doesNotMatch(error.message, /is not valid/); + assert.ok(error instanceof SpecError); + const joined = error.problems.join("\n"); + assert.match(joined, /`shutdown` is required/); + assert.match(joined, /`mandate` must NOT have fewer than 1 items/); return true; }, ); }); -test("either v1 marker alone is enough to detect it", () => { - for (const marker of [{ mandate: [] }, { spec_version: "1.0" }]) { - assert.throws(() => validateSpec({ name: "x", ...marker }), SpecVersionError); - } +test("v1 rejects the autonomy label the canon forbids", () => { + assert.throws( + () => + evidenceFrom({ + spec_version: "1.0", + mandate: [{ action: "send", authority: "semi_autonomous", enforced_by: "queue" }], + }), + (error: unknown) => { + assert.ok(error instanceof SpecError); + assert.match( + error.problems.join("\n"), + /`mandate\.0\.authority` must be one of: autonomous, human_approval, human_only, prohibited/, + ); + return true; + }, + ); }); -test("a v0 spec is still graded, not mistaken for v1", () => { - const spec = loadSpec(fixture("strong-spec.yaml")); - assert.equal(spec.name, "Support Copilot v2"); +test("a v0 spec still loads as v0", () => { + const evidence = loadEvidence(fixture("strong-spec.yaml")); + assert.equal(evidence.specVersion, "v0.1.0"); + assert.equal(evidence.name, "Support Copilot v2"); + assert.equal(evidence.memory?.persistent, true); });