diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1ee04..2670a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,21 @@ Publishing the GitHub release and tagging `v0.2` is a manual step after merge. - `CITATION.cff` for GitHub’s *Cite this repository* button. - Project-home links to [auxfirst.com](https://auxfirst.com). +## aux-audit 0.2.0 + +- Scores **agent-spec v1.0**. Both formats are read, detected from the document + rather than the filename, and `meta.spec_version` records which produced a + score — they are not comparable. +- Under v1 the rules ask a better question. `aux.H01` and `aux.H03` read + per-action authority and the five human controls instead of one label; + `aux.H05` computes the gap between what the credentials permit and what the + mandate governs; a row enforced by "the system prompt" counts as unenforced. +- `aux.H08` reports *not scoreable* under v1, which has no memory field, rather + than scoring zero. A trust stage whose entire backing is unscoreable is now + reported **not assessable** and is never counted as earned — previously an + empty shortfall read as success on zero evidence. +- v0.1.0 scoring is unchanged. + ### Notes - `agent-spec.schema.yaml` is now v1.0: real JSON Schema draft 2020-12, with a diff --git a/README.md b/README.md index 2a4031c..8ba4312 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ In CI, as a GitHub Action — SARIF into the Security tab, a scorecard on the PR fail-on: high ``` -It grades the **spec**, not the running product: it can prove a mechanism was never declared, never that a declared one works. A spec with no evaluation transcripts is capped at *present* on every heuristic, so the score cannot be gamed with better prose. See **[packages/aux-audit](packages/aux-audit/)**. +It grades the **spec**, not the running product: it can prove a mechanism was never declared, never that a declared one works. A spec with no evidence of testing is capped at *present* on every heuristic, so the score cannot be gamed with better prose. + +Under agent-spec v1 it computes the gap between what an agent's credentials permit and what its mandate governs — the attack surface — and treats a row enforced by "the system prompt" as unenforced. See **[packages/aux-audit](packages/aux-audit/)**. ## Start here @@ -80,7 +82,7 @@ Or read **[onboarding/README.md](onboarding/README.md)** — the router with all │ ├── aux-evolution-curve.yaml ← the 4 capability stages (teardowns only) │ ├── trust-contract.yaml ← example Trust Contract │ ├── agent-spec.schema.yaml ← v1.0, real JSON Schema — per-action mandate -│ ├── agent-spec.v0.yaml ← v0.1.0, frozen; what aux-audit 0.1.x reads +│ ├── agent-spec.v0.yaml ← v0.1.0, frozen; still scored, not comparable │ ├── MIGRATION.md ← v0 → v1, and why │ ├── migrate-v0-to-v1.py ← converter; leaves TODOs rather than guessing │ ├── brain-spec.schema.yaml ← Agency Brain input format (JSON Schema) diff --git a/docs/audit.md b/docs/audit.md index dc29a43..32eb7ae 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -16,7 +16,7 @@ npx aux-audit run ./agent-spec.yaml ## Contract -- **Input:** an agent spec matching [`schemas/agent-spec.v0.yaml`](../schemas/agent-spec.v0.yaml). The canonical [`agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) is now v1.0; aux-audit 0.1.x does not score it yet and says so plainly rather than reporting missing fields — see [MIGRATION.md](../schemas/MIGRATION.md) and [#10](https://github.com/auxfirst/trustkit/issues/10). +- **Input:** an agent spec in either format — [`agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) v1.0 or [`agent-spec.v0.yaml`](../schemas/agent-spec.v0.yaml). The version is detected from the document, not the filename, and `meta.spec_version` records which one was scored. Scores are not comparable across the two; see [MIGRATION.md](../schemas/MIGRATION.md). - **Output:** `score`, `grade`, `trust_stage`, `issues[]`, `recommendations[]`. - **Formats:** Markdown (default), JSON, SARIF 2.1.0. - **Exit codes:** `0` clean · `1` findings at or above `--fail-on` · `2` invalid spec (nothing graded). diff --git a/packages/aux-audit/README.md b/packages/aux-audit/README.md index bcab9e3..8f3cd41 100644 --- a/packages/aux-audit/README.md +++ b/packages/aux-audit/README.md @@ -32,9 +32,13 @@ Change the YAML in `schemas/`, and the audit changes with it. No heuristic text, **aux-audit grades the spec, not the running product.** It can prove a mechanism was never declared. It cannot prove a declared one works. -There is a sharper limit in the format this version reads, tracked in [trustkit#10](https://github.com/auxfirst/trustkit/issues/10): `agent-spec.v0.yaml` carries a single per-agent `autonomy` label and free-text `guarantees`, so a score reflects a **declared posture**, not an enforced boundary. +**Both spec versions are scored**, detected from the document rather than the filename. -The replacement, `agent-spec.schema.yaml` v1.0, is published and assigns authority action by action with a required `enforced_by` on every non-autonomous row. **aux-audit 0.1.x does not score it.** Hand it a v1 document and it says so in one line and exits 2, rather than reporting the fields v1 removed on purpose as missing. v1 scoring lands in 0.2.0. +Under **v1.0** the audit asks a better question. Instead of *"what did you call your autonomy level"*, `aux.H01` and `aux.H03` read per-action authority and the five human controls, and `aux.H05` computes the **gap between what the credentials permit and what the mandate governs** — the attack surface, in the schema's own words. A row enforced by "the system prompt" is reported as unenforced, because the canon is explicit that a prompt is a request and a tool boundary is a control. + +Under **v0.1.0** nothing changed: the same label and the same prose guarantees produce the same scores as before. Scores are not comparable across the two, so `meta.spec_version` records which one produced them. + +One thing v1 cannot express: **memory**. It has no field for persistence, scoping, or retention, so `aux.H08` is reported as not scoreable rather than scored zero — and `aux.T02` Contextual Trust, whose only backing heuristic that is, comes back *not assessable* rather than earned. See [trustkit#10](https://github.com/auxfirst/trustkit/issues/10). Two consequences, both deliberate: diff --git a/packages/aux-audit/fixtures/v1-strong-spec.yaml b/packages/aux-audit/fixtures/v1-strong-spec.yaml new file mode 100644 index 0000000..d3d23a3 --- /dev/null +++ b/packages/aux-audit/fixtures/v1-strong-spec.yaml @@ -0,0 +1,212 @@ +# Example agent spec — validates against agent-spec.schema.yaml +# +# A realistic, deliberately unglamorous agent: chasing overdue invoices. +# Note what the spec forces into the open — the send action is HIGH band on +# exposure and commitment alone, so it cannot be autonomous no matter how +# reliable the agent has proved to be on the read actions. +# +# Validate with: python3 schemas/validate.py schemas/example-agent-spec.yaml +# +# CC BY 4.0 · auxfirst agency 2026 + +spec_version: "1.0" +id: ar-followup +name: Accounts Receivable Follow-Up +purpose: >- + Chase overdue invoices on a schedule so finance staff spend their time only + on disputed and exceptional cases. + +owners: + business: + name: Anna Nowak + role: Head of Finance Operations + contact: anna.nowak@example.com + technical: + name: Piotr Lis + role: Platform Engineer + contact: piotr.lis@example.com + +trigger: + kind: schedule + detail: Weekdays 07:00 Europe/Warsaw + +users: + - role: Accounts receivable clerk + can_invoke: true + can_approve: true + - role: Account executive + can_invoke: false + can_approve: true + +systems: + data_sources: + - Billing database (invoices, payment status) + - CRM (contact of record, account owner) + connected: + - name: Billing service + auth: own_identity + scopes: [invoice:read, invoice:note:write] + - name: Mail gateway + auth: own_identity + scopes: [draft:write] # send scope deliberately withheld + - name: CRM + auth: delegated_identity + scopes: [contact:read] + +# What the credentials permit. Compare against mandate below — +# the difference is the attack surface. +capability: + can_read: + - invoices + - payment status + - contacts of record + can_change: + - invoice internal note + - email drafts + +mandate: + - action: Read invoice and payment status + authority: autonomous + enforced_by: Scoped read token, invoice:read only + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 0, band: LOW } + autonomy_level: L1 + + - action: Identify overdue invoices and rank by age + authority: autonomous + enforced_by: Read-only query path, no write scope + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 0, band: LOW } + autonomy_level: L1 + + - action: Write an internal note on the invoice + authority: autonomous + enforced_by: Field-scoped write API, note field only + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 1, band: LOW } + autonomy_level: L4 + + - action: Draft a reminder email + authority: autonomous + enforced_by: Mail gateway draft scope; send scope not granted + heat: { reversibility: 0, blast_radius: 0, exposure: 0, commitment: 0, authority: 1, band: LOW } + autonomy_level: L3 + + # The hinge. Three dials are cool and the action is still HIGH, + # because it leaves the building and asks for money. + - action: Send a reminder email to the customer + authority: human_approval + enforced_by: Send scope withheld until an approval token is issued + approver: Account executive on the account + heat: { reversibility: 1, blast_radius: 0, exposure: 3, commitment: 3, authority: 1, band: HIGH } + autonomy_level: L3 + + - action: Apply a late-payment fee + authority: human_only + enforced_by: Agent has no write path to the fee service + approver: Head of Finance Operations + heat: { reversibility: 2, blast_radius: 0, exposure: 3, commitment: 4, authority: 2, band: CRITICAL } + autonomy_level: L2 + + - action: Escalate to a collections agency + authority: prohibited + enforced_by: Integration not provisioned for this agent identity + heat: { reversibility: 4, blast_radius: 1, exposure: 4, commitment: 4, authority: 3, band: CRITICAL } + +human_control: + observe: + available: true + mechanism: Per-run log with the invoices touched and drafts produced + audience: affected_user + interrupt: + available: true + mechanism: Pause the approval queue; in-flight drafts are held + audience: operator + approve: + available: true + mechanism: Approval queue, one entry per outbound email + audience: operator + override: + available: true + mechanism: Edit the draft before approving, or reject with a reason + audience: operator + disable: + available: true + mechanism: Feature flag plus token revocation + audience: admin_only + +exceptions: + - condition: identifier_missing + detail: No contact of record on the account + response: route + escalation: + recipient: Anna Nowak + deadline_minutes: 480 + on_timeout: stop + context_preserved: true + + - condition: records_conflict + detail: Billing and CRM disagree on payment status + response: ask_human + escalation: + recipient: Anna Nowak + deadline_minutes: 240 + on_timeout: stop + context_preserved: true + + - condition: result_disputed + detail: Customer replies contesting the invoice + response: route + escalation: + recipient: Account executive on the account + deadline_minutes: 120 + on_timeout: page_secondary + context_preserved: true + + - condition: tool_unavailable + detail: Billing service unreachable + response: stop + + - condition: confidence_low + response: ask_human + escalation: + recipient: Anna Nowak + deadline_minutes: 480 + on_timeout: stop + context_preserved: true + + - condition: sensitive_data_present + detail: Bank details appear in a customer reply + response: refuse + +escalation_default: + recipient: Anna Nowak + deadline_minutes: 480 + on_timeout: stop + context_preserved: true + +supervision: + agent_identity_disclosed: true + action_receipt: true + reversal: true + consequence_scaled_approval: true + escalation_handoff: true + provenance_at_decision: false # known gap — see note below + +shutdown: + procedure: >- + Disable the feature flag, revoke the billing and mail gateway tokens, drain + the approval queue, and notify the two owners. + tested: true + last_tested: "2026-08-20" + revokes_access: true + +model: + provider: anthropic + name: claude-sonnet-5 + version_pinned: true + +trust_stage: aux.T02 +status: live + +# Known gap, recorded rather than hidden: provenance_at_decision is false. +# The approver sees the draft but not the payment-history evidence behind the +# ranking. Tracked as tg.judgment.confident_nonsense risk; fix is the +# confidence-cues pattern on the approval queue. diff --git a/packages/aux-audit/fixtures/v1-weak-spec.yaml b/packages/aux-audit/fixtures/v1-weak-spec.yaml new file mode 100644 index 0000000..0440a25 --- /dev/null +++ b/packages/aux-audit/fixtures/v1-weak-spec.yaml @@ -0,0 +1,36 @@ +# A v1 spec that satisfies the schema and still fails the doctrine: everything +# autonomous, enforcement by prompt, no supervision primitives, untested kill +# switch. v1 makes these visible where v0 could not express them at all. +spec_version: "1.0" +id: inbox-autopilot +name: Inbox Autopilot +purpose: Clear the shared inbox without human involvement. +owners: + business: { name: "Unassigned", role: "TBD" } + technical: { name: "Platform Team", role: "Engineering" } +trigger: + kind: continuous +systems: + data_sources: ["mailbox"] + connected: + - name: gmail + auth: own_identity +capability: + can_read: ["mailbox", "contacts", "calendar"] + can_change: ["send_email", "delete_email", "calendar_event"] +mandate: + - action: send_email + authority: autonomous + enforced_by: "the system prompt tells it to be careful" +human_control: + observe: { available: false } + interrupt: { available: false } + approve: { available: false } + override: { available: false } + disable: { available: true, mechanism: "revoke the OAuth token", audience: admin_only } +exceptions: + - condition: tool_unavailable + response: retry +shutdown: + procedure: "Revoke the OAuth token in the admin console." + tested: false diff --git a/packages/aux-audit/package-lock.json b/packages/aux-audit/package-lock.json index 76e6472..9973473 100644 --- a/packages/aux-audit/package-lock.json +++ b/packages/aux-audit/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "MIT", "dependencies": { + "ajv": "^8.20.0", "yaml": "^2.9.0" }, "bin": { @@ -32,6 +33,59 @@ "undici-types": "~6.21.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/packages/aux-audit/package.json b/packages/aux-audit/package.json index 1f5e7fa..49e6382 100644 --- a/packages/aux-audit/package.json +++ b/packages/aux-audit/package.json @@ -1,6 +1,6 @@ { "name": "aux-audit", - "version": "0.1.0", + "version": "0.2.0", "description": "Score an agent product against the 10 AUX Heuristics and the Trust Architecture. Reproducible, CI-friendly, SARIF-capable.", "keywords": [ "aux", @@ -49,6 +49,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { + "ajv": "^8.20.0", "yaml": "^2.9.0" }, "devDependencies": { diff --git a/packages/aux-audit/src/cli.ts b/packages/aux-audit/src/cli.ts index 0d851b7..0ffa42b 100644 --- a/packages/aux-audit/src/cli.ts +++ b/packages/aux-audit/src/cli.ts @@ -13,14 +13,14 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, relative, resolve } from "node:path"; import { audit, shouldFail } from "./score.js"; -import { loadSpec, SpecError, SpecVersionError } from "./spec.js"; +import { loadEvidence, SpecError } from "./spec.js"; import { loadConfig } from "./config.js"; import { toJson } from "./report/json.js"; import { toMarkdown } from "./report/markdown.js"; import { toSarif } from "./report/sarif.js"; import { SEVERITY_ORDER, type Severity } from "./types.js"; -const VERSION = "0.1.0"; +const VERSION = "0.2.0"; const FORMATS = new Set(["md", "markdown", "json", "sarif"]); const SEVERITIES = new Set(["low", "medium", "high", "critical"]); @@ -36,6 +36,11 @@ OPTIONS --summary additionally write the Markdown scorecard here (use with --format sarif to get a PR comment body) --fail-on exit 1 on findings at or above: low | medium | high | critical + +VERSIONS + Both agent-spec formats are read, detected from the document rather than the + filename. v1.0 scores per-action authority and its enforcing mechanisms; v0.1.0 + scores the older single autonomy label. Scores are not comparable across them. -h, --help show this help -v, --version print the version @@ -165,14 +170,14 @@ function main(argv: string[]): number { let report; try { - const spec = loadSpec(args.spec); - report = audit(spec, { + const evidence = loadEvidence(args.spec); + report = audit(evidence, { ignore: config.ignore, severityOverrides: config.severity_overrides, toolVersion: VERSION, }); } catch (error) { - if (error instanceof SpecVersionError || error instanceof SpecError) { + if (error instanceof SpecError) { process.stderr.write(`aux-audit: ${error.message}\n`); return 2; } diff --git a/packages/aux-audit/src/evidence.ts b/packages/aux-audit/src/evidence.ts new file mode 100644 index 0000000..7c31841 --- /dev/null +++ b/packages/aux-audit/src/evidence.ts @@ -0,0 +1,152 @@ +/** + * The normalised model both spec versions map into. + * + * Rules read this, never a raw spec. Two reasons: one rule set instead of two, + * and the difference between what v0 and v1 can express becomes a visible, + * testable mapping rather than branching scattered through the rules. + * + * Where a version cannot express something at all, the field is `undefined` + * and the rule reports the heuristic as not applicable. It never guesses, and + * it never scores absence of a *field* as absence of a *mechanism*. + */ +import type { AgentSpec } from "./types.js"; +import type { AgentSpecV1, Authority, Control, ExceptionRow } from "./spec-v1.js"; + +export interface MemoryEvidence { + persistent: boolean; + scopeCount: number; + userVisible: boolean; + userEditable: boolean; +} + +export interface MandateEvidence { + action: string; + authority: Authority; + enforcedBy: string; + /** A prompt is a request, not a boundary — trust-architecture.yaml. */ + enforcementIsMechanism: boolean; +} + +export interface Evidence { + /** Which spec format this came from, not the agent's own release version. */ + specVersion: "v0.1.0" | "v1.0"; + name: string; + /** The agent's own version, as the spec declares it. */ + agentVersion: string; + + /** Per-action authority. v0 cannot express this; it has one label instead. */ + mandate?: MandateEvidence[]; + /** v0's single label, kept only so its rules can say what it was. */ + autonomyLabel?: string; + /** Capabilities the credentials grant but no mandate row governs. */ + ungovernedWrites?: string[]; + + /** Five forms of control, v1 only. */ + control?: Record<"observe" | "interrupt" | "approve" | "override" | "disable", Control>; + supervision?: Partial>; + exceptions?: ExceptionRow[]; + escalationDefault?: boolean; + shutdownTested?: boolean; + modelPinned?: boolean; + + /** v1 has no memory field at all. `undefined` means "cannot be expressed". */ + memory?: MemoryEvidence; + + /** Promises in prose. v0 only — v1 replaced them with enforced mandate rows. */ + guarantees: string[]; + tools: string[]; + flows: string[]; + goldenTranscripts: boolean; + failureTranscripts: boolean; + multiAgentSignal?: string; + + /** True once the spec carries evidence of testing, which gates level 3. */ + evidenced: boolean; +} + +/** + * "System prompt" is the canonical non-answer: trust-architecture.yaml states + * that a prompt is a request and a tool boundary is a control, and the v1 + * schema repeats it on the field itself. + */ +const NOT_A_MECHANISM = + /\b(prompt|instructions?|guidelines?|policy document|training|conventions?|documentation|we ask|asked to|told to|expected to)\b/i; + +function isMechanism(enforcedBy: string): boolean { + return enforcedBy.trim().length >= 3 && !NOT_A_MECHANISM.test(enforcedBy.trim()); +} + +const MULTI_AGENT = /agent|delegate|sub[-_]?agent|handoff|hand_off|swarm|crew/i; + +export function fromV0(spec: AgentSpec): Evidence { + const golden = (spec.evaluation?.golden_transcripts ?? []).length > 0; + const failure = (spec.evaluation?.failure_transcripts ?? []).length > 0; + return { + specVersion: "v0.1.0", + name: spec.name, + agentVersion: spec.version, + autonomyLabel: spec.autonomy, + memory: { + persistent: spec.memory.persistent, + scopeCount: spec.memory.scopes.length, + userVisible: spec.memory.user_visible, + userEditable: spec.memory.user_editable, + }, + guarantees: spec.guarantees ?? [], + tools: spec.tools ?? [], + flows: spec.flows ?? [], + goldenTranscripts: golden, + failureTranscripts: failure, + multiAgentSignal: + spec.surface === "multi-surface" + ? "multi-surface" + : (spec.tools ?? []).find((t) => MULTI_AGENT.test(t)) ?? + (spec.guarantees ?? []).find((g) => /\bagents?\b/i.test(g)), + evidenced: golden || failure, + }; +} + +export function fromV1(spec: AgentSpecV1): Evidence { + const mandate: MandateEvidence[] = spec.mandate.map((row) => ({ + action: row.action, + authority: row.authority, + enforcedBy: row.enforced_by, + enforcementIsMechanism: isMechanism(row.enforced_by), + })); + + // The gap between what the credentials permit and what the mandate governs + // is the attack surface — the v1 schema says so in its own header. + const governed = mandate.map((row) => row.action.toLowerCase()); + const ungovernedWrites = spec.capability.can_change.filter( + (write) => !governed.some((action) => action.includes(write.toLowerCase()) || write.toLowerCase().includes(action)), + ); + + const multiAgent = + spec.systems.connected.map((c) => c.name).find((n) => MULTI_AGENT.test(n)) ?? + mandate.map((m) => m.action).find((a) => MULTI_AGENT.test(a)); + + return { + specVersion: "v1.0", + name: spec.name, + agentVersion: spec.version ?? "unversioned", + mandate, + ungovernedWrites, + control: spec.human_control, + supervision: spec.supervision ?? {}, + exceptions: spec.exceptions, + escalationDefault: spec.escalation_default !== undefined, + shutdownTested: spec.shutdown.tested, + modelPinned: spec.model?.version_pinned ?? false, + // v1 declares no memory. Not false — absent. See trustkit#10. + memory: undefined, + guarantees: [], + tools: spec.capability.can_change, + flows: [], + goldenTranscripts: false, + failureTranscripts: false, + multiAgentSignal: multiAgent, + // A tested shutdown and a pinned model are v1's evidence of exercise, in + // the way transcripts are v0's. + evidenced: spec.shutdown.tested === true, + }; +} diff --git a/packages/aux-audit/src/index.ts b/packages/aux-audit/src/index.ts index d1d12f3..96725a2 100644 --- a/packages/aux-audit/src/index.ts +++ b/packages/aux-audit/src/index.ts @@ -1,5 +1,7 @@ export { audit, shouldFail, type AuditOptions } from "./score.js"; -export { loadSpec, validateSpec, SpecError } from "./spec.js"; +export { loadSpec, loadEvidence, evidenceFrom, validateSpec, SpecError } from "./spec.js"; +export { validateV1, type AgentSpecV1 } from "./spec-v1.js"; +export { fromV0, fromV1, type Evidence } from "./evidence.js"; export { loadConfig, type AuditConfig } from "./config.js"; export { loadCanon, gapsForHeuristic, heuristicsForStage } from "./canon.js"; export { RULES } from "./rules.js"; diff --git a/packages/aux-audit/src/report/markdown.ts b/packages/aux-audit/src/report/markdown.ts index 5c6a592..7c66340 100644 --- a/packages/aux-audit/src/report/markdown.ts +++ b/packages/aux-audit/src/report/markdown.ts @@ -30,7 +30,8 @@ export function toMarkdown(report: AuditReport): string { const lines: string[] = []; const applicable = report.heuristics.filter((h) => h.applicable); - lines.push(`## aux-audit — ${report.spec.name} \`${report.spec.version}\``); + const version = report.spec.version === "unversioned" ? "" : ` \`${report.spec.version}\``; + lines.push(`## aux-audit — ${report.spec.name}${version}`); lines.push(""); lines.push( `**Score ${report.score}/100 · Grade ${report.grade} · Trust stage: ${report.trust_stage ?? "none earned"}**`, @@ -42,13 +43,32 @@ export function toMarkdown(report: AuditReport): string { lines.push("| Stage | Earned | Blocked by |"); lines.push("|---|---|---|"); for (const stage of report.trust_stages) { - const blockers = stage.shortfall.length > 0 ? stage.shortfall.map((s) => `\`${s}\``).join(", ") : "—"; + if (!stage.assessable) { + lines.push( + `| ${stage.order}. ${stage.name} | — | not assessable: ${stage.depends_on + .map((s) => `\`${s}\``) + .join(", ")} could not be scored |`, + ); + continue; + } + const blockers = + stage.shortfall.length > 0 + ? stage.shortfall.map((s) => `\`${s}\``).join(", ") + : stage.earned + ? "—" + : "an earlier stage — trust is sequential"; lines.push(`| ${stage.order}. ${stage.name} | ${stage.earned ? "✅" : "❌"} | ${blockers} |`); } lines.push(""); lines.push( "> Trust is sequential. A stage is earned only when every heuristic its failure modes name scores *present* or better, and only if every earlier stage is earned.", ); + if (report.trust_stages.some((stage) => !stage.assessable)) { + lines.push(""); + lines.push( + "> A stage marked *not assessable* had no scoreable evidence at all. It is not earned and not failed — nothing was measured, and reporting it either way would be a claim rather than a finding.", + ); + } lines.push(""); lines.push(`### Heuristics (${applicable.length} scored)`); diff --git a/packages/aux-audit/src/rules.ts b/packages/aux-audit/src/rules.ts index ee01dbc..392fc9e 100644 --- a/packages/aux-audit/src/rules.ts +++ b/packages/aux-audit/src/rules.ts @@ -1,7 +1,7 @@ /** - * The rule set: one rule per AUX heuristic. + * The rule set: one rule per AUX heuristic, scoring the normalised evidence. * - * Every rule scores what the spec *declares*, on the 0-3 scale from + * Every rule scores what a spec *declares*, on the 0-3 scale from * schemas/aux-heuristics.yaml: * * 0 absent — not even acknowledged @@ -9,12 +9,13 @@ * 2 present — works in happy path, breaks under load * 3 robust — works under novel, adversarial, or edge cases * - * A spec is a claim, not a running product. aux-audit grades the claim: it can - * prove a mechanism was never declared, never that a declared one works. Every - * rule therefore returns the evidence it used, so a reader can disagree with a - * number by pointing at a line. + * Two spec versions reach these rules through one evidence model. v0 declares + * a single autonomy label and prose guarantees; v1 declares per-action + * authority with an enforcing mechanism. Where a rule can ask the better + * question, it does — and its evidence string names which version answered. */ -import type { AgentSpec, Level } from "./types.js"; +import type { Evidence } from "./evidence.js"; +import type { Level } from "./types.js"; export interface RuleOutcome { level: Level; @@ -24,93 +25,152 @@ export interface RuleOutcome { export interface Rule { id: string; - evaluate(spec: AgentSpec): RuleOutcome; + evaluate(e: Evidence): RuleOutcome; } const clamp = (n: number): Level => Math.max(0, Math.min(3, n)) as Level; +const join = (parts: (string | false | undefined)[]): string => + parts.filter(Boolean).join("; "); -/** First guarantee matching the pattern, for use as quotable evidence. */ -function guarantee(spec: AgentSpec, pattern: RegExp): string | undefined { - return (spec.guarantees ?? []).find((g) => pattern.test(g)); -} +const guarantee = (e: Evidence, pattern: RegExp): string | undefined => + e.guarantees.find((g) => pattern.test(g)); +const flowMatching = (e: Evidence, pattern: RegExp): string | undefined => + e.flows.find((f) => pattern.test(f)); -function flowMatching(spec: AgentSpec, pattern: RegExp): string | undefined { - return (spec.flows ?? []).find((f) => pattern.test(f)); -} +const isV1 = (e: Evidence): boolean => e.specVersion === "v1.0"; -const golden = (spec: AgentSpec) => - (spec.evaluation?.golden_transcripts ?? []).length > 0; -const failures = (spec: AgentSpec) => - (spec.evaluation?.failure_transcripts ?? []).length > 0; +/** Mandate rows that need an enforcing mechanism: everything but `autonomous`. */ +const gated = (e: Evidence) => (e.mandate ?? []).filter((r) => r.authority !== "autonomous"); +const unenforced = (e: Evidence) => gated(e).filter((r) => !r.enforcementIsMechanism); -function join(parts: string[]): string { - return parts.filter(Boolean).join("; "); -} +const control = (e: Evidence, which: "observe" | "interrupt" | "approve" | "override" | "disable") => + e.control?.[which]; +const has = (e: Evidence, which: Parameters[1]): boolean => + control(e, which)?.available === true; +const supervises = (e: Evidence, key: string): boolean => e.supervision?.[key] === true; export const RULES: Rule[] = [ { id: "aux.H01", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const checkpoints = (e.mandate ?? []).filter((r) => r.authority === "human_approval"); + const level = clamp( + (checkpoints.length > 0 ? 2 : 0) + + (supervises(e, "action_receipt") ? 1 : 0) + + (supervises(e, "agent_identity_disclosed") ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + checkpoints.length > 0 + ? `${checkpoints.length} of ${e.mandate?.length} action(s) require approval before execution` + : "no action requires approval — every mandate row is autonomous or barred", + supervises(e, "action_receipt") + ? "each action leaves a receipt" + : "no action receipt", + supervises(e, "agent_identity_disclosed") + ? "the agent discloses that it is an agent" + : "agent identity is not disclosed", + ]), + }; + } const base = - spec.autonomy === "human-in-the-loop" - ? 2 - : spec.autonomy === "human-on-the-loop" - ? 1 - : 0; + e.autonomyLabel === "human-in-the-loop" ? 2 : e.autonomyLabel === "human-on-the-loop" ? 1 : 0; const asks = guarantee( - spec, + e, /\b(ask|confirm|preview|propose|approval|before (send|act|writ|delet|purchas))/i, ); - const flows = (spec.flows ?? []).length > 0; - const level = clamp(base + (asks ? 1 : 0) + (flows ? 1 : 0)); + const flows = e.flows.length > 0; return { - level, + level: clamp(base + (asks ? 1 : 0) + (flows ? 1 : 0)), applicable: true, evidence: join([ - `autonomy is \`${spec.autonomy}\``, - asks ? `guarantee declares a checkpoint: "${asks}"` : "no guarantee declares a pre-action checkpoint", - flows ? `${spec.flows?.length} flow(s) documented` : "no flows documented", + `autonomy is \`${e.autonomyLabel}\``, + asks + ? `guarantee declares a checkpoint: "${asks}"` + : "no guarantee declares a pre-action checkpoint", + flows ? `${e.flows.length} flow(s) documented` : "no flows documented", ]), }; }, }, { id: "aux.H02", - evaluate(spec) { - const tools = (spec.tools ?? []).length > 0; - const flows = (spec.flows ?? []).length > 0; - const level = clamp( - (tools ? 1 : 0) + (flows ? 1 : 0) + (golden(spec) ? 1 : 0), - ); + evaluate(e) { + if (isV1(e)) { + const observe = control(e, "observe"); + const level = clamp( + (has(e, "observe") ? 1 : 0) + + (supervises(e, "provenance_at_decision") ? 1 : 0) + + (observe?.audience === "affected_user" ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + has(e, "observe") + ? `runs are observable${observe?.mechanism ? ` via ${observe.mechanism}` : ""}` + : "runs are not observable", + supervises(e, "provenance_at_decision") + ? "provenance is shown at the decision" + : "no provenance at the point of decision", + observe?.audience === "affected_user" + ? "visible to the affected user, not only operators" + : `observability reaches ${observe?.audience ?? "no one"}`, + ]), + }; + } + const tools = e.tools.length > 0; + const flows = e.flows.length > 0; return { - level, + level: clamp((tools ? 1 : 0) + (flows ? 1 : 0) + (e.goldenTranscripts ? 1 : 0)), applicable: true, evidence: join([ - tools ? `${spec.tools?.length} tool(s) named — calls can be attributed` : "no tools named, so tool calls cannot be shown", + tools + ? `${e.tools.length} tool(s) named — calls can be attributed` + : "no tools named, so tool calls cannot be shown", flows ? "flows documented" : "no flows documented", - golden(spec) ? "golden transcripts available as an evidence trail" : "no golden transcripts", + e.goldenTranscripts + ? "golden transcripts available as an evidence trail" + : "no golden transcripts", ]), }; }, }, { id: "aux.H03", - evaluate(spec) { - const base = spec.autonomy === "autonomous" ? 0 : 2; - const undo = guarantee( - spec, - /\b(undo|revert|cancel|stop|interrupt|rollback|pause)\b/i, - ); - const editable = spec.memory.user_editable; - const level = clamp(base + (undo ? 1 : 0) + (editable ? 1 : 0)); + evaluate(e) { + if (isV1(e)) { + const level = clamp( + (has(e, "interrupt") ? 1 : 0) + + (has(e, "override") ? 1 : 0) + + (supervises(e, "reversal") ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + has(e, "interrupt") ? "a run can be interrupted mid-flight" : "no interrupt control", + has(e, "override") ? "a decision can be overridden" : "no override control", + supervises(e, "reversal") ? "actions are reversible" : "actions are not reversible", + ]), + }; + } + const base = e.autonomyLabel === "autonomous" ? 0 : 2; + const undo = guarantee(e, /\b(undo|revert|cancel|stop|interrupt|rollback|pause)\b/i); + const editable = e.memory?.userEditable === true; return { - level, + level: clamp(base + (undo ? 1 : 0) + (editable ? 1 : 0)), applicable: true, evidence: join([ - spec.autonomy === "autonomous" + e.autonomyLabel === "autonomous" ? "autonomous agent — no inherent interruption point" - : `autonomy \`${spec.autonomy}\` provides a review point`, - undo ? `guarantee declares mid-flight control: "${undo}"` : "no undo/cancel/interrupt guarantee", + : `autonomy \`${e.autonomyLabel}\` provides a review point`, + undo + ? `guarantee declares mid-flight control: "${undo}"` + : "no undo/cancel/interrupt guarantee", editable ? "memory is user-editable" : "memory is not user-editable", ]), }; @@ -118,15 +178,30 @@ export const RULES: Rule[] = [ }, { id: "aux.H04", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const levels = new Set((e.mandate ?? []).map((r) => r.authority)); + const scaled = supervises(e, "consequence_scaled_approval"); + return { + level: clamp((scaled ? 2 : 0) + (levels.size > 1 ? 1 : 0)), + applicable: true, + evidence: join([ + scaled + ? "approval scales with consequence" + : "approval does not scale with consequence — the same gate for every stake", + levels.size > 1 + ? `authority varies across actions (${[...levels].join(", ")})` + : `every action carries the same authority (${[...levels][0] ?? "none"})`, + ]), + }; + } const ladder = guarantee( - spec, + e, /\b(tenure|earn|gradual|progressive|trust level|new user|first[- ]time|unlock|graduat)/i, ); - const onboarding = flowMatching(spec, /onboard|first[-_ ]?run|new[-_ ]?user/i); - const level = clamp((ladder ? 2 : 0) + (onboarding ? 1 : 0)); + const onboarding = flowMatching(e, /onboard|first[-_ ]?run|new[-_ ]?user/i); return { - level, + level: clamp((ladder ? 2 : 0) + (onboarding ? 1 : 0)), applicable: true, evidence: join([ ladder @@ -141,13 +216,41 @@ export const RULES: Rule[] = [ }, { id: "aux.H05", - evaluate(spec) { - const count = (spec.guarantees ?? []).length; + evaluate(e) { + if (isV1(e)) { + const rows = e.mandate ?? []; + const weak = unenforced(e); + const ungoverned = e.ungovernedWrites ?? []; + const level = clamp( + (rows.length > 0 ? 1 : 0) + + (gated(e).length > 0 && weak.length === 0 ? 1 : 0) + + (ungoverned.length === 0 ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + rows.length > 0 + ? `${rows.length} action(s) carry an explicit authority` + : "no mandate rows — authority is undefined", + weak.length === 0 + ? gated(e).length > 0 + ? "every gated action names an enforcing mechanism" + : "no gated actions to enforce" + : `${weak.length} gated action(s) enforced by something that is not a mechanism: ${weak + .map((r) => `"${r.enforcedBy}"`) + .join(", ")}`, + ungoverned.length === 0 + ? "no write capability sits outside the mandate" + : `${ungoverned.length} ${ungoverned.length === 1 ? "capability" : "capabilities"} the credentials grant but no mandate governs: ${ungoverned.join(", ")}`, + ]), + }; + } + const count = e.guarantees.length; const base = count === 0 ? 0 : count === 1 ? 1 : 2; - const explicit = guarantee(spec, /\bwill (never|not|always)\b/i); - const level = clamp(base + (explicit ? 1 : 0)); + const explicit = guarantee(e, /\bwill (never|not|always)\b/i); return { - level, + level: clamp(base + (explicit ? 1 : 0)), applicable: true, evidence: join([ count === 0 @@ -162,20 +265,36 @@ export const RULES: Rule[] = [ }, { id: "aux.H06", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const lowConfidence = (e.exceptions ?? []).find((x) => x.condition === "confidence_low"); + const acts = lowConfidence !== undefined && + ["ask_human", "route", "refuse", "stop"].includes(lowConfidence.response); + return { + level: clamp((lowConfidence ? 2 : 0) + (acts ? 1 : 0)), + applicable: true, + evidence: join([ + lowConfidence + ? `low confidence is a designed exception, answered with \`${lowConfidence.response}\`` + : "low confidence is not among the declared exceptions", + lowConfidence && !acts + ? `\`${lowConfidence.response}\` does not surface the uncertainty to anyone` + : acts && "the response hands the decision to a human rather than proceeding", + ]), + }; + } const confidence = guarantee( - spec, + e, /\b(confidence|uncertain|unsure|verify|cite|citation|source|evidence|caveat|flag when|don't know|do not know)/i, ); - const level = clamp((confidence ? 2 : 0) + (failures(spec) ? 1 : 0)); return { - level, + level: clamp((confidence ? 2 : 0) + (e.failureTranscripts ? 1 : 0)), applicable: true, evidence: join([ confidence ? `guarantee addresses confidence or sourcing: "${confidence}"` : "no guarantee addresses how confidence or uncertainty is communicated", - failures(spec) + e.failureTranscripts ? "failure transcripts exist — uncertainty behaviour is exercised" : "no failure transcripts, so uncertainty behaviour is untested", ]), @@ -184,20 +303,43 @@ export const RULES: Rule[] = [ }, { id: "aux.H07", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const escalations = (e.exceptions ?? []).filter((x) => x.escalation !== undefined); + const named = escalations.filter( + (x) => x.escalation!.recipient.trim().length > 0 && x.escalation!.on_timeout, + ); + const level = clamp( + (named.length > 0 ? 2 : 0) + + (e.escalationDefault || supervises(e, "escalation_handoff") ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + named.length > 0 + ? `${named.length} exception(s) escalate to a named recipient with a timeout` + : escalations.length > 0 + ? "escalations exist but none names both a recipient and what happens on timeout" + : "no exception escalates — the agent absorbs every failure itself", + e.escalationDefault + ? "a default escalation catches anything unlisted" + : "no default escalation for unlisted conditions", + ]), + }; + } const assertive = guarantee( - spec, + e, /\b(escalat|hand off|handoff|hand-off|human review|push back|pushback|refuse|decline|second opinion)/i, ); - const level = clamp((assertive ? 2 : 0) + (failures(spec) ? 1 : 0)); return { - level, + level: clamp((assertive ? 2 : 0) + (e.failureTranscripts ? 1 : 0)), applicable: true, evidence: join([ assertive ? `guarantee declares escalation or pushback: "${assertive}"` : "no guarantee declares when the agent escalates, refuses, or pushes back", - failures(spec) + e.failureTranscripts ? "failure transcripts exercise the escalation path" : "no failure transcripts covering escalation", ]), @@ -206,19 +348,27 @@ export const RULES: Rule[] = [ }, { id: "aux.H08", - evaluate(spec) { - if (!spec.memory.persistent) { + evaluate(e) { + // v1 declares no memory at all — not "no memory", but no field for it. + // Scoring absence of a field as absence of a mechanism would be a lie. + if (e.memory === undefined) { return { level: 0, - applicable: true, + applicable: false, evidence: - "memory.persistent is false — the agent starts every session as a stranger", + "not scoreable — agent-spec v1.0 has no memory field, so a spec cannot declare persistence, scoping, or retention (trustkit#10)", + }; + } + if (!e.memory.persistent) { + return { + level: 0, + applicable: true, + evidence: "memory.persistent is false — the agent starts every session as a stranger", }; } - const scopes = spec.memory.scopes.length; - const level = clamp(2 + (scopes >= 2 ? 1 : 0)); + const scopes = e.memory.scopeCount; return { - level, + level: clamp(2 + (scopes >= 2 ? 1 : 0)), applicable: true, evidence: join([ "memory is persistent", @@ -231,17 +381,8 @@ export const RULES: Rule[] = [ }, { id: "aux.H09", - evaluate(spec) { - const multiAgentTool = (spec.tools ?? []).find((t) => - /agent|delegate|sub[-_]?agent|handoff|hand_off|swarm|crew/i.test(t), - ); - const multiAgentGuarantee = guarantee(spec, /\bagents?\b/i); - const applicable = - spec.surface === "multi-surface" || - multiAgentTool !== undefined || - multiAgentGuarantee !== undefined; - - if (!applicable) { + evaluate(e) { + if (e.multiAgentSignal === undefined) { return { level: 3, applicable: false, @@ -249,21 +390,28 @@ export const RULES: Rule[] = [ "not applicable — the spec describes a single agent (no delegating tools, no multi-agent guarantees, single surface)", }; } - const attribution = guarantee( - spec, - /\b(attribut|which agent|responsib|named agent|on behalf)/i, - ); - const handoffFlow = flowMatching(spec, /handoff|hand[-_]off|delegat|escalat/i); - const level = clamp((attribution ? 2 : 0) + (handoffFlow ? 1 : 0)); + if (isV1(e)) { + const disclosed = supervises(e, "agent_identity_disclosed"); + const handoff = (e.exceptions ?? []).some((x) => x.response === "route"); + return { + level: clamp((disclosed ? 2 : 0) + (handoff ? 1 : 0)), + applicable: true, + evidence: join([ + `multi-agent signal present (${e.multiAgentSignal})`, + disclosed + ? "agent identity is disclosed, so actions can be attributed" + : "identity is not disclosed — the user cannot tell which agent acted", + handoff ? "a routing response documents the handoff" : "no routing exception documented", + ]), + }; + } + const attribution = guarantee(e, /\b(attribut|which agent|responsib|named agent|on behalf)/i); + const handoffFlow = flowMatching(e, /handoff|hand[-_]off|delegat|escalat/i); return { - level, + level: clamp((attribution ? 2 : 0) + (handoffFlow ? 1 : 0)), applicable: true, evidence: join([ - `multi-agent signals present (${ - spec.surface === "multi-surface" - ? "multi-surface" - : (multiAgentTool ?? "agent guarantee") - })`, + `multi-agent signals present (${e.multiAgentSignal})`, attribution ? `guarantee addresses attribution: "${attribution}"` : "no guarantee tells the user which agent did what", @@ -274,22 +422,43 @@ export const RULES: Rule[] = [ }, { id: "aux.H10", - evaluate(spec) { + evaluate(e) { + if (isV1(e)) { + const level = clamp( + (e.modelPinned ? 2 : 0) + + (e.shutdownTested ? 1 : 0) + + ((e.exceptions ?? []).length >= 3 ? 1 : 0), + ); + return { + level, + applicable: true, + evidence: join([ + e.modelPinned + ? "the model version is pinned, so behaviour cannot change without a release" + : "the model version is not pinned — behaviour can change without a release", + e.shutdownTested + ? "the shutdown procedure has been tested" + : "the shutdown procedure is untested, so it is a claim rather than a control", + (e.exceptions ?? []).length >= 3 + ? `${e.exceptions?.length} failure conditions are designed for` + : "fewer than three failure conditions designed for", + ]), + }; + } const contract = guarantee( - spec, + e, /\b(deterministic|same (input|output)|consistent|reproducib|pinned|model version|regression)/i, ); - const level = clamp( - (golden(spec) ? 2 : 0) + (failures(spec) ? 1 : 0) + (contract ? 1 : 0), - ); return { - level, + level: clamp( + (e.goldenTranscripts ? 2 : 0) + (e.failureTranscripts ? 1 : 0) + (contract ? 1 : 0), + ), applicable: true, evidence: join([ - golden(spec) + e.goldenTranscripts ? "golden transcripts pin expected behaviour" : "no golden transcripts — nothing detects silent drift between model versions", - failures(spec) ? "failure transcripts present" : "no failure transcripts", + e.failureTranscripts ? "failure transcripts present" : "no failure transcripts", contract ? `guarantee declares a behavioural contract: "${contract}"` : "no guarantee about behavioural consistency over time", diff --git a/packages/aux-audit/src/score.ts b/packages/aux-audit/src/score.ts index 6fb44a1..6a5e3de 100644 --- a/packages/aux-audit/src/score.ts +++ b/packages/aux-audit/src/score.ts @@ -13,6 +13,7 @@ import { type Canon, } from "./canon.js"; import { RULES } from "./rules.js"; +import type { Evidence } from "./evidence.js"; import type { AgentSpec, AuditReport, @@ -45,11 +46,8 @@ const EARNED_AT = 2; */ const ROBUST_CAP = 2; -function hasEvidence(spec: AgentSpec): boolean { - return ( - (spec.evaluation?.golden_transcripts ?? []).length > 0 || - (spec.evaluation?.failure_transcripts ?? []).length > 0 - ); +function hasEvidence(e: Evidence): boolean { + return e.evidenced; } function grade(score: number): Grade { @@ -89,12 +87,12 @@ function recommend(heuristicName: string, fixPattern: string): string { return `${heuristicName}: adopt the \`${fixPattern}\` pattern.`; } -export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport { +export function audit(evidence: Evidence, options: AuditOptions = {}): AuditReport { const canon = loadCanon(); const ignore = new Set(options.ignore ?? []); const overrides = options.severityOverrides ?? {}; - const evidenced = hasEvidence(spec); + const evidenced = hasEvidence(evidence); const byId = new Map(canon.heuristics.map((h) => [h.id, h])); const heuristics: HeuristicResult[] = []; @@ -116,7 +114,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport }); continue; } - const outcome = rule.evaluate(spec); + const outcome = rule.evaluate(evidence); const capped = outcome.applicable && !evidenced && outcome.level > ROBUST_CAP; heuristics.push({ id: rule.id, @@ -143,11 +141,17 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport let ladderIntact = true; for (const stage of canon.stages) { const depends_on = heuristicsForStage(canon, stage.id); - const shortfall = depends_on.filter((id) => { - const result = levelOf.get(id); - return result !== undefined && result.applicable && result.level < EARNED_AT; - }); - const earnedHere = shortfall.length === 0; + const backing = depends_on + .map((id) => levelOf.get(id)) + .filter((result): result is HeuristicResult => result !== undefined); + // Nothing measured means nothing earned. Reporting a stage as earned + // because its only evidence could not be scored is the failure this whole + // taxonomy exists to name. + const assessable = backing.some((result) => result.applicable); + const shortfall = backing + .filter((result) => result.applicable && result.level < EARNED_AT) + .map((result) => result.id); + const earnedHere = assessable && shortfall.length === 0; const earned = ladderIntact && earnedHere; if (!earned) ladderIntact = false; trust_stages.push({ @@ -155,6 +159,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport name: stage.name, order: stage.order, earned, + assessable, depends_on, shortfall, }); @@ -181,7 +186,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport } for (const stage of trust_stages) { - if (stage.earned || stage.shortfall.length === 0) continue; + if (stage.earned || !stage.assessable || stage.shortfall.length === 0) continue; issues.push({ id: stage.id, type: `${stage.name.toLowerCase().replace(/ trust$/, "").replace(/\s+/g, "_")}_trust_gap`, @@ -197,7 +202,7 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport ); return { - spec: { name: spec.name, version: spec.version }, + spec: { name: evidence.name, version: evidence.agentVersion }, score, grade: grade(score), trust_stage, @@ -209,7 +214,8 @@ export function audit(spec: AgentSpec, options: AuditOptions = {}): AuditReport recommendations: [...new Set(recommendations)], meta: { tool: "aux-audit", - tool_version: options.toolVersion ?? "0.1.0", + spec_version: evidence.specVersion, + tool_version: options.toolVersion ?? "0.2.0", heuristics_version: canon.heuristicsVersion, trust_architecture_version: canon.trustArchitectureVersion, taxonomy_version: canon.taxonomyVersion, diff --git a/packages/aux-audit/src/spec-v1.ts b/packages/aux-audit/src/spec-v1.ts new file mode 100644 index 0000000..53c7ef4 --- /dev/null +++ b/packages/aux-audit/src/spec-v1.ts @@ -0,0 +1,129 @@ +/** + * agent-spec v1.0 — real JSON Schema draft 2020-12, validated with ajv. + * + * v1 replaces v0's single `autonomy` label with a per-action mandate, each + * non-autonomous row naming the mechanism that enforces it. That is the whole + * reason it exists, and it is what makes the audit able to ask a better + * question than "what did you call your autonomy level". + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import ajvModule, { type ErrorObject, type ValidateFunction } from "ajv/dist/2020.js"; + +/** + * ajv ships CommonJS. Under NodeNext the constructor arrives on `.default`, + * but the type is the namespace, so both sides need coaxing. + */ +type AjvCtor = new (options?: Record) => { + compile(schema: object): ValidateFunction; + addFormat(name: string, format: RegExp): unknown; +}; +const Ajv2020 = ((ajvModule as unknown as { default?: unknown }).default ?? + ajvModule) as unknown as AjvCtor; +import { parse } from "yaml"; +import { schemaDir } from "./canon.js"; + +export type Authority = "autonomous" | "human_approval" | "human_only" | "prohibited"; + +export interface MandateRow { + action: string; + authority: Authority; + enforced_by: string; + approver?: string; +} + +export interface Control { + available: boolean; + mechanism?: string; + audience?: "affected_user" | "operator" | "admin_only" | "none"; +} + +export interface Escalation { + recipient: string; + deadline_minutes: number; + on_timeout: "stop" | "fallback" | "refuse" | "page_secondary"; + context_preserved?: boolean; +} + +export interface ExceptionRow { + condition: string; + detail?: string; + response: "retry" | "stop" | "ask_human" | "route" | "fallback" | "log" | "refuse"; + escalation?: Escalation; +} + +export interface AgentSpecV1 { + spec_version: string; + id: string; + name: string; + version?: string; + surface?: string; + purpose: string; + owners: { business: unknown; technical: unknown }; + trigger: { kind: string; detail?: string }; + users?: unknown[]; + systems: { data_sources: string[]; connected: { name: string; auth: unknown }[] }; + capability: { can_read: string[]; can_change: string[] }; + mandate: MandateRow[]; + human_control: Record<"observe" | "interrupt" | "approve" | "override" | "disable", Control>; + exceptions: ExceptionRow[]; + escalation_default?: Escalation; + supervision?: Partial< + Record< + | "agent_identity_disclosed" + | "action_receipt" + | "reversal" + | "consequence_scaled_approval" + | "escalation_handoff" + | "provenance_at_decision", + boolean + > + >; + shutdown: { procedure: string; tested: boolean; last_tested?: string; revokes_access?: boolean }; + model?: { provider?: string; name?: string; version_pinned?: boolean }; + trust_stage?: string; + status?: string; +} + +let compiled: ValidateFunction | undefined; + +function validator(): ValidateFunction { + if (compiled) return compiled; + const schema = parse( + readFileSync(join(schemaDir(), "agent-spec.schema.yaml"), "utf8"), + ) as object; + const ajv = new Ajv2020({ strict: false, allErrors: true }); + // `shutdown.last_tested` is `format: date`. Validate it rather than let ajv + // log that it is ignoring a format on every run. + ajv.addFormat("date", /^\d{4}-\d{2}-\d{2}$/); + const fn = ajv.compile(schema); + compiled = fn; + return fn; +} + +/** ajv's paths are terse; make them read like the field a person would fix. */ +function describe(error: ErrorObject): string { + const where = error.instancePath.replace(/^\//, "").replace(/\//g, ".") || "(root)"; + if (error.keyword === "required") { + const missing = (error.params as { missingProperty: string }).missingProperty; + return where === "(root)" + ? `\`${missing}\` is required` + : `\`${where}.${missing}\` is required`; + } + if (error.keyword === "enum") { + const allowed = (error.params as { allowedValues: unknown[] }).allowedValues; + return `\`${where}\` must be one of: ${allowed.join(", ")}`; + } + if (error.keyword === "additionalProperties") { + const extra = (error.params as { additionalProperty: string }).additionalProperty; + return `\`${where}\` has an unknown field: ${extra}`; + } + return `\`${where}\` ${error.message ?? "is invalid"}`; +} + +export function validateV1(input: unknown): { spec?: AgentSpecV1; problems: string[] } { + const validate = validator(); + if (validate(input)) return { spec: input as AgentSpecV1, problems: [] }; + const problems = (validate.errors ?? []).map(describe); + return { problems: [...new Set(problems)] }; +} diff --git a/packages/aux-audit/src/spec.ts b/packages/aux-audit/src/spec.ts index ef45baf..25e2849 100644 --- a/packages/aux-audit/src/spec.ts +++ b/packages/aux-audit/src/spec.ts @@ -9,6 +9,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { parse } from "yaml"; import { schemaDir } from "./canon.js"; +import { fromV0, fromV1, type Evidence } from "./evidence.js"; +import { validateV1 } from "./spec-v1.js"; import type { AgentSpec, Autonomy, MemorySpec, Surface } from "./types.js"; /** @@ -17,25 +19,6 @@ import type { AgentSpec, Autonomy, MemorySpec, Surface } from "./types.js"; */ const schemaPath = (): string => join(schemaDir(), "agent-spec.v0.yaml"); -/** - * Handed a v1 document, aux-audit 0.1.x used to reject it with four field - * errors — including "`autonomy` is required", demanding the very field v1 - * removes on purpose. That reads as a broken tool rather than a version - * mismatch, so it is detected before validation and reported as itself. - */ -export class SpecVersionError extends Error { - readonly detected = "v1.0"; - constructor() { - super( - "this is an agent-spec v1.0 document; aux-audit 0.1.x reads v0.1.0.\n" + - " Nothing was graded. v1 scoring lands in aux-audit 0.2.0.\n" + - " Migrating a v0 spec: python3 schemas/migrate-v0-to-v1.py your-spec.yaml\n" + - " Background: schemas/MIGRATION.md, trustkit#10", - ); - this.name = "SpecVersionError"; - } -} - /** v1 declares a per-action mandate and a spec_version; v0 declares neither. */ function looksLikeV1(raw: Record): boolean { return "mandate" in raw || "spec_version" in raw; @@ -128,8 +111,6 @@ export function validateSpec(input: unknown): AgentSpec { throw new SpecError(["the spec must be a YAML or JSON mapping"]); } const raw = input as Record; - if (looksLikeV1(raw)) throw new SpecVersionError(); - const { surface, autonomy } = schemaEnums(); for (const key of ["name", "version"] as const) { @@ -219,3 +200,39 @@ export function loadSpec(path: string): AgentSpec { } return validateSpec(parsed); } + + +/** + * Reads either spec version and returns the normalised evidence the rules + * score. Version is detected from the document, never from the filename, so a + * spec is graded as what it is rather than as what it was called. + */ +export function loadEvidence(path: string): Evidence { + let raw: string; + try { + raw = readFileSync(path, "utf8"); + } catch { + throw new SpecError([`cannot read spec file: ${path}`]); + } + let parsed: unknown; + try { + parsed = parse(raw); + } catch (error) { + throw new SpecError([ + `spec is not valid YAML or JSON: ${(error as Error).message}`, + ]); + } + return evidenceFrom(parsed); +} + +export function evidenceFrom(parsed: unknown): Evidence { + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new SpecError(["the spec must be a YAML or JSON mapping"]); + } + if (looksLikeV1(parsed as Record)) { + const { spec, problems } = validateV1(parsed); + if (!spec) throw new SpecError(problems); + return fromV1(spec); + } + return fromV0(validateSpec(parsed)); +} diff --git a/packages/aux-audit/src/types.ts b/packages/aux-audit/src/types.ts index bd03cee..0effe44 100644 --- a/packages/aux-audit/src/types.ts +++ b/packages/aux-audit/src/types.ts @@ -57,6 +57,12 @@ export interface TrustStageResult { name: string; order: number; earned: boolean; + /** + * False when every heuristic backing this stage is inapplicable, so nothing + * was measured. A stage that cannot be assessed is never reported as earned: + * that would be a claim dressed as a finding. + */ + assessable: boolean; /** Heuristic ids that must score >= 2 for this stage, per the gap taxonomy. */ depends_on: string[]; shortfall: string[]; @@ -87,6 +93,8 @@ export interface AuditReport { recommendations: string[]; meta: { tool: string; + /** Which agent-spec format was scored. Scores are not comparable across it. */ + spec_version: "v0.1.0" | "v1.0"; tool_version: string; heuristics_version: string; trust_architecture_version: string; diff --git a/packages/aux-audit/test/audit.test.ts b/packages/aux-audit/test/audit.test.ts index acac779..487058b 100644 --- a/packages/aux-audit/test/audit.test.ts +++ b/packages/aux-audit/test/audit.test.ts @@ -1,13 +1,14 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { audit, shouldFail } from "../src/score.js"; -import { loadSpec } from "../src/spec.js"; +import { loadSpec, loadEvidence, evidenceFrom } from "../src/spec.js"; +import { fromV0 } from "../src/evidence.js"; import { loadCanon, heuristicsForStage } from "../src/canon.js"; import { RULES } from "../src/rules.js"; import { fixture } from "./helpers.js"; -const strong = () => audit(loadSpec(fixture("strong-spec.yaml"))); -const weak = () => audit(loadSpec(fixture("weak-spec.yaml"))); +const strong = () => audit(loadEvidence(fixture("strong-spec.yaml"))); +const weak = () => audit(loadEvidence(fixture("weak-spec.yaml"))); test("every rule maps to a heuristic in the canonical schema", () => { const canon = loadCanon(); @@ -48,13 +49,13 @@ test("aux.H09 is not applicable to a single-agent spec but is to a crew", () => const single = strong().heuristics.find((h) => h.id === "aux.H09")!; assert.equal(single.applicable, false); - const crew = audit(loadSpec(fixture("multi-agent-spec.yaml"))); + const crew = audit(loadEvidence(fixture("multi-agent-spec.yaml"))); const multi = crew.heuristics.find((h) => h.id === "aux.H09")!; assert.equal(multi.applicable, true, "agent.delegate should trigger H09"); }); test("non-applicable heuristics never produce issues", () => { - for (const report of [strong(), weak(), audit(loadSpec(fixture("multi-agent-spec.yaml")))]) { + for (const report of [strong(), weak(), audit(loadEvidence(fixture("multi-agent-spec.yaml")))]) { const skipped = report.heuristics.filter((h) => !h.applicable).map((h) => h.id); for (const id of skipped) { assert.ok(!report.issues.some((issue) => issue.id === id), `${id} should not be reported`); @@ -96,8 +97,7 @@ test("every issue carries evidence a reader can argue with", () => { }); test("config can ignore a heuristic and override a severity", () => { - const spec = loadSpec(fixture("weak-spec.yaml")); - const report = audit(spec, { + const report = audit(loadEvidence(fixture("weak-spec.yaml")), { ignore: ["aux.H04"], severityOverrides: { "aux.H01": "low" }, }); @@ -117,8 +117,8 @@ test("shouldFail respects the threshold", () => { test("the report is deterministic for the same spec", () => { const now = new Date("2026-01-01T00:00:00.000Z"); - const spec = loadSpec(fixture("strong-spec.yaml")); - assert.deepEqual(audit(spec, { now }), audit(spec, { now })); + const ev = loadEvidence(fixture("strong-spec.yaml")); + assert.deepEqual(audit(ev, { now }), audit(ev, { now })); }); test("evolution_stage is permanently out of scope, not pending", () => { @@ -131,9 +131,14 @@ test("evolution_stage is permanently out of scope, not pending", () => { }); test("level 3 requires evidence — a spec with no transcripts is capped at present", () => { - const spec = loadSpec(fixture("strong-spec.yaml")); - const withEvidence = audit(spec); - const withoutEvidence = audit({ ...spec, evaluation: { golden_transcripts: [], failure_transcripts: [] } }); + const ev = loadEvidence(fixture("strong-spec.yaml")); + const withEvidence = audit(ev); + const withoutEvidence = audit({ + ...ev, + goldenTranscripts: false, + failureTranscripts: false, + evidenced: false, + }); assert.ok(withEvidence.heuristics.some((h) => h.applicable && h.level === 3)); assert.ok( @@ -149,7 +154,7 @@ test("level 3 requires evidence — a spec with no transcripts is capped at pres test("the cap cannot be dodged by declaring more prose", () => { const spec = loadSpec(fixture("weak-spec.yaml")); - const wordy = audit({ + const wordy = audit(fromV0({ ...spec, guarantees: [ "will always ask before acting", @@ -161,7 +166,85 @@ test("the cap cannot be dodged by declaring more prose", () => { "responses are deterministic and pinned to a model version", ], flows: ["./flows/onboarding.md", "./flows/handoff.md"], - }); + })); assert.ok(wordy.heuristics.every((h) => !h.applicable || h.level <= 2)); assert.ok(wordy.score <= 67, `prose alone should not reach robust, got ${wordy.score}`); }); + +// --- agent-spec v1 ------------------------------------------------------ + +const v1strong = () => audit(loadEvidence(fixture("v1-strong-spec.yaml"))); +const v1weak = () => audit(loadEvidence(fixture("v1-weak-spec.yaml"))); + +test("v1: the report records which spec format produced the score", () => { + assert.equal(v1strong().meta.spec_version, "v1.0"); + assert.equal(strong().meta.spec_version, "v0.1.0"); +}); + +test("v1: a mandate with enforcement outscores one without", () => { + assert.ok(v1strong().score > v1weak().score); +}); + +test("v1: capability the mandate does not govern is reported as the attack surface", () => { + const h05 = v1weak().heuristics.find((h) => h.id === "aux.H05")!; + assert.match(h05.evidence, /capabilities the credentials grant but no mandate governs/); + assert.match(h05.evidence, /delete_email/); + assert.ok(h05.level < 2, "ungoverned write capability must not score as present"); +}); + +test("v1: a prompt is not an enforcing mechanism", () => { + const spec = { + spec_version: "1.0", + id: "prompt-enforcement-probe", + name: "Prompt Enforcement Probe", + purpose: "Exercise the rule that an enforcing mechanism is not a prompt.", + owners: { business: { name: "A", role: "r" }, technical: { name: "B", role: "r" } }, + trigger: { kind: "event" }, + systems: { data_sources: ["s"], connected: [{ name: "n", auth: "service_account" }] }, + capability: { can_read: [], can_change: ["refund"] }, + mandate: [{ action: "refund", authority: "human_approval", enforced_by: "the system prompt" }], + human_control: { + observe: { available: true }, + interrupt: { available: true }, + approve: { available: true }, + override: { available: true }, + disable: { available: true }, + }, + exceptions: [{ condition: "other", response: "stop" }], + shutdown: { procedure: "Disable the integration.", tested: true }, + }; + const report = audit(evidenceFrom(spec)); + const h05 = report.heuristics.find((h) => h.id === "aux.H05")!; + assert.match(h05.evidence, /enforced by something that is not a mechanism/); + assert.match(h05.evidence, /the system prompt/); +}); + +test("v1: aux.H08 is not scoreable, and says why rather than scoring zero", () => { + const h08 = v1strong().heuristics.find((h) => h.id === "aux.H08")!; + assert.equal(h08.applicable, false); + assert.match(h08.evidence, /no memory field/); + assert.ok(!v1strong().issues.some((i) => i.id === "aux.H08"), "must not be reported as a failing"); +}); + +test("a stage with no scoreable evidence is never reported as earned", () => { + // The bug this guards: aux.H08 is the only heuristic backing aux.T02. Under + // v1 it cannot be scored, and an empty shortfall was reading as "earned" — + // a claim dressed as a finding. + const contextual = v1strong().trust_stages.find((s) => s.id === "aux.T02")!; + assert.equal(contextual.assessable, false); + assert.equal(contextual.earned, false); + assert.ok(!v1strong().issues.some((i) => i.id === "aux.T02"), "not the product's failure to fix"); +}); + +test("an unassessable stage stops the ladder, as a broken one would", () => { + const stages = v1strong().trust_stages; + assert.equal(stages.find((s) => s.id === "aux.T01")!.earned, true); + for (const id of ["aux.T02", "aux.T03", "aux.T04"]) { + assert.equal(stages.find((s) => s.id === id)!.earned, false, `${id} must not be earned`); + } + assert.equal(v1strong().trust_stage, "functional"); +}); + +test("v0 stages stay assessable — the change is version-specific", () => { + assert.ok(strong().trust_stages.every((s) => s.assessable)); +}); diff --git a/packages/aux-audit/test/report.test.ts b/packages/aux-audit/test/report.test.ts index 90b9db5..374ee4f 100644 --- a/packages/aux-audit/test/report.test.ts +++ b/packages/aux-audit/test/report.test.ts @@ -5,13 +5,13 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { audit } from "../src/score.js"; -import { loadSpec } from "../src/spec.js"; +import { loadEvidence } from "../src/spec.js"; import { toJson } from "../src/report/json.js"; import { toMarkdown } from "../src/report/markdown.js"; import { toSarif } from "../src/report/sarif.js"; import { cliPath, fixture } from "./helpers.js"; -const weak = () => audit(loadSpec(fixture("weak-spec.yaml"))); +const weak = () => audit(loadEvidence(fixture("weak-spec.yaml"))); test("json output round-trips and keeps the v0.1 contract fields", () => { const report = weak(); @@ -128,7 +128,7 @@ test("CLI: --config is honoured", () => { test("CLI: --version and --help exit 0", () => { assert.match( execFileSync(process.execPath, [cliPath(), "--version"], { encoding: "utf8" }), - /^0\.1\.0/, + /^0\.2\.0/, ); assert.match( execFileSync(process.execPath, [cliPath(), "--help"], { encoding: "utf8" }), @@ -137,23 +137,32 @@ test("CLI: --version and --help exit 0", () => { }); -test("CLI: a v1 spec exits 2 with a version message, not field errors", () => { - const dir = mkdtempSync(join(tmpdir(), "aux-audit-v1-")); - const specPath = join(dir, "v1.yaml"); - writeFileSync( - specPath, - "spec_version: '1.0'\nid: collections\nname: Collections Agent\nmandate: []\n", +test("CLI: a v1 spec is scored, and the report says which format it was", () => { + const out = execFileSync( + process.execPath, + [cliPath(), "run", fixture("v1-strong-spec.yaml"), "--format", "json"], + { encoding: "utf8" }, ); + const report = JSON.parse(out); + assert.equal(report.meta.spec_version, "v1.0"); + assert.ok(report.score > 0); + assert.equal(report.spec.name, "Accounts Receivable Follow-Up"); +}); + +test("CLI: an invalid v1 spec reports v1 fields, never v0 ones", () => { + const dir = mkdtempSync(join(tmpdir(), "aux-audit-v1-")); + const specPath = join(dir, "broken.yaml"); + writeFileSync(specPath, "spec_version: '1.0'\nid: x\nname: X\nmandate: []\n"); try { execFileSync(process.execPath, [cliPath(), "run", specPath], { stdio: "pipe" }); assert.fail("expected a non-zero exit"); } catch (error) { - const err = error as { status: number; stderr: Buffer; stdout: Buffer }; + const err = error as { status: number; stderr: Buffer }; assert.equal(err.status, 2); const stderr = err.stderr.toString(); - assert.match(stderr, /agent-spec v1\.0 document/); - assert.match(stderr, /MIGRATION\.md/); + assert.match(stderr, /`shutdown` is required/); + // v0's vocabulary must never surface for a v1 document. assert.doesNotMatch(stderr, /`autonomy` is required/); - assert.doesNotMatch(stderr, /Score/); + assert.doesNotMatch(stderr, /`surface` is required/); } }); diff --git a/packages/aux-audit/test/spec.test.ts b/packages/aux-audit/test/spec.test.ts index a9907c1..f46b870 100644 --- a/packages/aux-audit/test/spec.test.ts +++ b/packages/aux-audit/test/spec.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { parse } from "yaml"; -import { loadSpec, validateSpec, SpecError, SpecVersionError } from "../src/spec.js"; +import { loadSpec, loadEvidence, evidenceFrom, validateSpec, SpecError } from "../src/spec.js"; import { fixture, repoSchema } from "./helpers.js"; test("loads a well-formed spec", () => { @@ -105,34 +105,75 @@ test("ISO-8601 durations are validated", () => { }); -test("a v1 document is named as a version mismatch, not a pile of field errors", () => { +test("a v1 document is detected and scored, not rejected", () => { const v1 = { spec_version: "1.0", id: "collections-agent", name: "Collections Agent", - mandate: [{ action: "send reminder", authority: "human_approval", enforced_by: "queue" }], + purpose: "Chase overdue invoices before they age past 90 days.", + owners: { + business: { name: "A. Novak", role: "Head of Credit" }, + technical: { name: "B. Reyes", role: "Staff Engineer" }, + }, + trigger: { kind: "schedule" }, + systems: { data_sources: ["ledger"], connected: [{ name: "erp", auth: "service_account" }] }, + capability: { can_read: ["invoices"], can_change: ["invoice_note"] }, + mandate: [ + { action: "invoice_note", authority: "autonomous", enforced_by: "scoped API token" }, + ], + human_control: { + observe: { available: true }, + interrupt: { available: true }, + approve: { available: false }, + override: { available: true }, + disable: { available: true }, + }, + exceptions: [{ condition: "records_conflict", response: "ask_human" }], + shutdown: { procedure: "Revoke the service account and drain the queue.", tested: true }, }; + const evidence = evidenceFrom(v1); + assert.equal(evidence.specVersion, "v1.0"); + assert.equal(evidence.name, "Collections Agent"); + assert.equal(evidence.mandate?.length, 1); + assert.equal(evidence.mandate?.[0]?.enforcementIsMechanism, true); + // v1 declares no memory, and the absence must not read as "no memory". + assert.equal(evidence.memory, undefined); +}); + +test("a v1 document that breaks the schema reports the real field", () => { assert.throws( - () => validateSpec(v1), + () => evidenceFrom({ spec_version: "1.0", mandate: [] }), (error: unknown) => { - assert.ok(error instanceof SpecVersionError, "should be a version error"); - assert.match(error.message, /agent-spec v1\.0 document/); - assert.match(error.message, /MIGRATION\.md/); - // The old behaviour demanded the very field v1 removes on purpose. - assert.doesNotMatch(error.message, /`autonomy` is required/); - assert.doesNotMatch(error.message, /is not valid/); + assert.ok(error instanceof SpecError); + const joined = error.problems.join("\n"); + assert.match(joined, /`shutdown` is required/); + assert.match(joined, /`mandate` must NOT have fewer than 1 items/); return true; }, ); }); -test("either v1 marker alone is enough to detect it", () => { - for (const marker of [{ mandate: [] }, { spec_version: "1.0" }]) { - assert.throws(() => validateSpec({ name: "x", ...marker }), SpecVersionError); - } +test("v1 rejects the autonomy label the canon forbids", () => { + assert.throws( + () => + evidenceFrom({ + spec_version: "1.0", + mandate: [{ action: "send", authority: "semi_autonomous", enforced_by: "queue" }], + }), + (error: unknown) => { + assert.ok(error instanceof SpecError); + assert.match( + error.problems.join("\n"), + /`mandate\.0\.authority` must be one of: autonomous, human_approval, human_only, prohibited/, + ); + return true; + }, + ); }); -test("a v0 spec is still graded, not mistaken for v1", () => { - const spec = loadSpec(fixture("strong-spec.yaml")); - assert.equal(spec.name, "Support Copilot v2"); +test("a v0 spec still loads as v0", () => { + const evidence = loadEvidence(fixture("strong-spec.yaml")); + assert.equal(evidence.specVersion, "v0.1.0"); + assert.equal(evidence.name, "Support Copilot v2"); + assert.equal(evidence.memory?.persistent, true); });