From 41c78b6b1e445d066a5939d8ab992819d28c9e09 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 06:43:07 +0000 Subject: [PATCH] feat(agent-spec): land v1 as canonical, with a version shim in the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the contradiction #10 opened on: the canonical input format now assigns authority action by action, each non-autonomous row requiring an `enforced_by` mechanism, instead of one per-agent autonomy label the canon tells readers to avoid. schemas/agent-spec.schema.yaml v1.0, real JSON Schema draft 2020-12 schemas/agent-spec.v0.yaml frozen; what aux-audit 0.1.x reads schemas/MIGRATION.md why, the field mapping, the timeline schemas/migrate-v0-to-v1.py converter The shim is the point. Before it, a v1 document handed to the shipped CLI came back with four field errors, including "`autonomy` is required" — demanding the exact field v1 removes on purpose. That reads as a broken tool, which is worse than the inconsistent document it replaced: a reader can reason around a document. Now: aux-audit: this is an agent-spec v1.0 document; aux-audit 0.1.x reads v0.1.0. Nothing was graded. v1 scoring lands in aux-audit 0.2.0. Migrating a v0 spec: python3 schemas/migrate-v0-to-v1.py your-spec.yaml Background: schemas/MIGRATION.md, trustkit#10 Detection is two markers, `mandate` or `spec_version`, checked before validation so no field error is ever produced for a version mismatch. An invalid *v0* spec still gets its field errors — the shim swallows nothing it should not. Four tests cover it, including a CLI-level one asserting the old misleading line is gone. Not in this commit, still #10: scoring v1. aux.H01 and aux.H03 read spec.autonomy today; under v1 they become "does every non-autonomous action name an enforcing mechanism", which is a better question and a different computation. That is aux-audit 0.2.0. Also: yamllint's `colons` rule is disabled. v1 aligns its `$ref` columns deliberately and it reads better that way; leaving warnings on every run would cost more than the rule is worth. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MhpqWEvwKp7NrmJfdXYq4M --- .github/workflows/validate.yml | 6 +- CHANGELOG.md | 6 + README.md | 5 +- docs/audit.md | 2 +- packages/aux-audit/README.md | 4 +- .../aux-audit/schemas/agent-spec.schema.yaml | 433 ++++++++++++++---- packages/aux-audit/schemas/agent-spec.v0.yaml | 101 ++++ packages/aux-audit/scripts/sync-schemas.mjs | 2 + packages/aux-audit/src/cli.ts | 4 +- packages/aux-audit/src/spec.ts | 34 +- packages/aux-audit/test/report.test.ts | 24 +- packages/aux-audit/test/spec.test.ts | 41 +- schemas/MIGRATION.md | 183 ++++++++ schemas/agent-spec.schema.yaml | 433 ++++++++++++++---- schemas/agent-spec.v0.yaml | 101 ++++ schemas/migrate-v0-to-v1.py | 350 ++++++++++++++ 16 files changed, 1522 insertions(+), 207 deletions(-) create mode 100644 packages/aux-audit/schemas/agent-spec.v0.yaml create mode 100644 schemas/MIGRATION.md create mode 100644 schemas/agent-spec.v0.yaml create mode 100755 schemas/migrate-v0-to-v1.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0f86d7d..fff3198 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -8,7 +8,7 @@ # - a taxonomy gap with no pattern, or a pattern missing one of its four files # # Local repro: -# yamllint schemas/ .github/ +# yamllint schemas/ .github/ (colons disabled: agent-spec v1 aligns $ref columns) # lychee --offline --no-progress --include-fragments=full --include-mail=false '**/*.md' # npm --prefix packages/aux-audit ci && npm --prefix packages/aux-audit test @@ -46,11 +46,11 @@ jobs: - name: Lint schemas run: | - yamllint -d "{extends: relaxed, rules: {line-length: {max: 200}}}" schemas/ + yamllint -d "{extends: relaxed, rules: {line-length: {max: 200}, colons: disable}}" schemas/ - name: Lint workflows + Issue templates run: | - yamllint -d "{extends: relaxed, rules: {line-length: {max: 200}}}" .github/ + yamllint -d "{extends: relaxed, rules: {line-length: {max: 200}, colons: disable}}" .github/ links: name: Markdown links diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e00657..4b1ee04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,12 @@ Publishing the GitHub release and tagging `v0.2` is a manual step after merge. ### Notes +- `agent-spec.schema.yaml` is now v1.0: real JSON Schema draft 2020-12, with a + per-action `mandate` requiring `enforced_by`. The previous format is frozen as + `agent-spec.v0.yaml` and is what `aux-audit` 0.1.x still reads; handed a v1 + document, the CLI reports a version mismatch instead of missing fields + ([#10](https://github.com/auxfirst/trustkit/issues/10)). + - `evolution_stage` is reported as `human-assessed` and is always `null` in `aux-audit`. The capability axis is defined in `schemas/aux-evolution-curve.yaml` (E01 Reactive → E04 Personally diff --git a/README.md b/README.md index 43d0484..2a4031c 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,10 @@ Or read **[onboarding/README.md](onboarding/README.md)** — the router with all │ ├── trust-gap-taxonomy.yaml ← named failure modes │ ├── aux-evolution-curve.yaml ← the 4 capability stages (teardowns only) │ ├── trust-contract.yaml ← example Trust Contract -│ ├── agent-spec.schema.yaml ← audit input format (v0.1.0, superseded — #10) +│ ├── agent-spec.schema.yaml ← v1.0, real JSON Schema — per-action mandate +│ ├── agent-spec.v0.yaml ← v0.1.0, frozen; what aux-audit 0.1.x reads +│ ├── MIGRATION.md ← v0 → v1, and why +│ ├── migrate-v0-to-v1.py ← converter; leaves TODOs rather than guessing │ ├── brain-spec.schema.yaml ← Agency Brain input format (JSON Schema) │ └── memory-policy.schema.yaml ← memory governance format │ diff --git a/docs/audit.md b/docs/audit.md index 75845af..dc29a43 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -16,7 +16,7 @@ npx aux-audit run ./agent-spec.yaml ## Contract -- **Input:** an agent spec matching [`schemas/agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) — v0.1.0, superseded; see [#10](https://github.com/auxfirst/trustkit/issues/10). +- **Input:** an agent spec matching [`schemas/agent-spec.v0.yaml`](../schemas/agent-spec.v0.yaml). The canonical [`agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) is now v1.0; aux-audit 0.1.x does not score it yet and says so plainly rather than reporting missing fields — see [MIGRATION.md](../schemas/MIGRATION.md) and [#10](https://github.com/auxfirst/trustkit/issues/10). - **Output:** `score`, `grade`, `trust_stage`, `issues[]`, `recommendations[]`. - **Formats:** Markdown (default), JSON, SARIF 2.1.0. - **Exit codes:** `0` clean · `1` findings at or above `--fail-on` · `2` invalid spec (nothing graded). diff --git a/packages/aux-audit/README.md b/packages/aux-audit/README.md index 664b5b9..bcab9e3 100644 --- a/packages/aux-audit/README.md +++ b/packages/aux-audit/README.md @@ -32,7 +32,9 @@ Change the YAML in `schemas/`, and the audit changes with it. No heuristic text, **aux-audit grades the spec, not the running product.** It can prove a mechanism was never declared. It cannot prove a declared one works. -There is a sharper limit in the current input format, tracked in [trustkit#10](https://github.com/auxfirst/trustkit/issues/10): `agent-spec.schema.yaml` v0.1.0 carries a single per-agent `autonomy` label and free-text `guarantees`, so a score reflects a **declared posture**, not an enforced boundary. The replacement assigns authority action by action and requires a mechanism on every non-autonomous row. +There is a sharper limit in the format this version reads, tracked in [trustkit#10](https://github.com/auxfirst/trustkit/issues/10): `agent-spec.v0.yaml` carries a single per-agent `autonomy` label and free-text `guarantees`, so a score reflects a **declared posture**, not an enforced boundary. + +The replacement, `agent-spec.schema.yaml` v1.0, is published and assigns authority action by action with a required `enforced_by` on every non-autonomous row. **aux-audit 0.1.x does not score it.** Hand it a v1 document and it says so in one line and exits 2, rather than reporting the fields v1 removed on purpose as missing. v1 scoring lands in 0.2.0. Two consequences, both deliberate: diff --git a/packages/aux-audit/schemas/agent-spec.schema.yaml b/packages/aux-audit/schemas/agent-spec.schema.yaml index 7735478..38333be 100644 --- a/packages/aux-audit/schemas/agent-spec.schema.yaml +++ b/packages/aux-audit/schemas/agent-spec.schema.yaml @@ -1,104 +1,345 @@ -# Agent Spec — input format for `aux-audit` -# Version: 0.1.0 · SUPERSEDED, see trustkit#10 +# Agent Spec — input format for an Agent Owner's Manual +# Version: 1.0.0 # License: MIT (see schemas/LICENSE) -# A minimal, opinionated description of an agentic product. -# Fields marked "required" will cause the audit to refuse to run if missing. # -# --------------------------------------------------------------------------- -# Two known defects. Both are real; both are tracked in trustkit#10. +# Real JSON Schema, draft 2020-12. Validates one agent: who owns it, what it is +# authorized to do, what it can technically do, and how a human stops it. # -# 1. This file is NOT JSON Schema. It uses the reserved `$schema` key as a -# container for a bespoke notation ({ type: string, required: true }). Any -# JSON Schema tool will read it wrong. Do not point a validator at it. +# The gap between `capability` and `mandate` is the attack surface. Both are +# required, so the gap is computable rather than assumed. # -# 2. It contradicts the canon it belongs to. `autonomy` below requires one -# label for the whole agent, while trust-architecture.yaml states that -# "autonomy is not a slider you push to the right — it is a spectrum you -# choose per action". `guarantees` is free text, i.e. a promise, while -# enforcement.rule states that "enforcement lives in a mechanism, not in a -# prompt" and this format has no field for the mechanism. -# -# This file predates that doctrine. The replacement assigns authority action -# by action, each row requiring an `enforced_by` mechanism. Until it lands, -# aux-audit reads this format, and a score from it inherits the limitation: -# it grades a declared posture, not an enforced boundary. -# --------------------------------------------------------------------------- +# NOTE: `aux-audit` 0.1.x does not score this format yet — it reads +# agent-spec.v0.yaml and will tell you so if handed a v1 document. v1 scoring +# lands in aux-audit 0.2.0. See schemas/MIGRATION.md and trustkit#10. + +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://auxfirst.com/schemas/agent-spec.schema.yaml +title: Agent Spec +description: >- + A production AI agent's operating specification. Fifteen fields minimum, + derived from the Agent Owner's Manual. +type: object +additionalProperties: false + +required: + - spec_version + - id + - name + - purpose + - owners + - trigger + - systems + - capability + - mandate + - human_control + - exceptions + - shutdown + +properties: -version: "0.1.0" + spec_version: + type: string + const: "1.0" -$schema: - name: { type: string, required: true } - version: { type: string, required: true } + id: + type: string + pattern: "^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$" + description: Stable slug. Never reused after retirement. + + name: + type: string + description: Field 01 — the name humans use for this agent. + + version: + type: string + description: >- + The agent's own release version. Distinct from spec_version, which + versions this document format. Carried over from the v0.1.0 format. surface: - type: enum - values: ["chat", "inline", "ambient", "async", "multi-surface"] - required: true - - autonomy: - # SUPERSEDED (trustkit#10): a per-agent label is the abstraction the AUX - # canon tells readers to avoid. The replacement is a per-action mandate - # with an enforcing mechanism on every non-autonomous row. - type: enum - values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] - required: true - description: > - in-the-loop: every step confirmed. - on-the-loop: batched review, observable. - autonomous: acts without supervision. - - memory: - required: true - fields: - persistent: { type: boolean, required: true } - scopes: { type: list, required: true, item: string } - retention: { type: ISO-8601-duration, required: true } - user_visible: { type: boolean, required: true } - user_editable: { type: boolean, required: true } - - tools: - type: list - item: string - description: "tool/function names available to the agent" - - flows: - type: list - item: path - description: "paths to markdown flow descriptions; used for heuristic scoring" - - guarantees: - # SUPERSEDED (trustkit#10): a guarantee here is a sentence, and the canon - # is explicit that "a recommendation is not a control". The replacement - # pairs each action with the mechanism that enforces it. - type: list - item: string - description: > - explicit contracts the agent makes with the user - (e.g. 'will always ask before sending external email'). - NOTE: this is a stated promise, not a verified mechanism. aux-audit can - confirm the sentence exists; it cannot confirm anything enforces it. - - evaluation: - fields: - golden_transcripts: { type: list, item: path } - failure_transcripts: { type: list, item: path } - -example: | - name: "Support Copilot v2" - version: "2.3.1" - surface: chat - autonomy: human-on-the-loop - memory: - persistent: true - scopes: [preferences, conversation_history, escalation_notes] - retention: P90D - user_visible: true - user_editable: true - tools: [crm.lookup, email.send, ticket.create] - flows: [./flows/handoff.md, ./flows/refund.md] - guarantees: - - "will always ask before sending external email" - - "will always hand off to a human after two failed resolutions" - evaluation: - golden_transcripts: [./transcripts/golden/*.jsonl] - failure_transcripts: [./transcripts/failures/*.jsonl] + type: string + enum: [chat, inline, ambient, async, multi-surface] + description: >- + Where the agent meets its users. Carried over from the v0.1.0 format; + it changes which patterns from trust-contract.yaml apply. + + purpose: + type: string + minLength: 20 + description: >- + Field 02 — why this agent exists, in business terms. If it cannot be + stated without naming the technology, it is not a purpose. + + owners: + type: object + description: >- + Fields 03 and 04. Two owners, not one. Business owner is a named person + in the function the agent serves; technical owner can change how it works. + An agent with only a technical owner is orphaned the day the project + closes. + additionalProperties: false + required: [business, technical] + properties: + business: + $ref: "#/$defs/person" + technical: + $ref: "#/$defs/person" + + trigger: + type: object + description: Field 05 — what starts a run. + additionalProperties: false + required: [kind] + properties: + kind: + type: string + enum: [schedule, event, user_invocation, escalation, continuous] + detail: + type: string + + users: + type: array + description: Field 06 — who interacts with it, and in what role. + items: + type: object + additionalProperties: false + required: [role] + properties: + role: { type: string } + can_invoke: { type: boolean, default: true } + can_approve: { type: boolean, default: false } + + systems: + type: object + description: Fields 07 and 08 — data sources and connected systems. + additionalProperties: false + required: [data_sources, connected] + properties: + data_sources: + type: array + items: { type: string } + connected: + type: array + items: + type: object + additionalProperties: false + required: [name, auth] + properties: + name: { type: string } + auth: + type: string + enum: [service_account, delegated_identity, own_identity, oauth_user] + description: >- + own_identity or delegated_identity is required for + attribution. service_account makes every downstream log wrong + about who acted. + scopes: + type: array + items: { type: string } + + capability: + type: object + description: >- + Fields 09 and 10 — what the agent CAN technically do, granted by its + credentials. Distinct from mandate on purpose. + additionalProperties: false + required: [can_read, can_change] + properties: + can_read: + type: array + items: { type: string } + can_change: + type: array + items: { type: string } + + mandate: + type: array + description: >- + Fields 11, 12 and 13 — assigned authority, action by action. Never an + autonomy score. Every non-autonomous row needs an enforcing mechanism, + or the row is decoration. + minItems: 1 + items: + type: object + additionalProperties: false + required: [action, authority, enforced_by] + properties: + action: + type: string + description: One verb the agent can reach. + authority: + type: string + enum: [autonomous, human_approval, human_only, prohibited] + description: >- + Four values, no fifth. Ambiguity in this field is where trust + collapses. + enforced_by: + type: string + minLength: 3 + description: >- + The mechanism, not the intention. "System prompt" is not a valid + answer for human_only or prohibited. + approver: + type: string + description: Required when authority is human_approval. + heat: + type: object + additionalProperties: false + properties: + reversibility: { $ref: "#/$defs/dial" } + blast_radius: { $ref: "#/$defs/dial" } + exposure: { $ref: "#/$defs/dial" } + commitment: { $ref: "#/$defs/dial" } + authority: { $ref: "#/$defs/dial" } + band: + type: string + enum: [LOW, LOW_MEDIUM, MEDIUM, HIGH, CRITICAL] + description: >- + Set by the single highest dial, never the average. An action is + as hot as its hottest dimension. + autonomy_level: + type: string + enum: [L1, L2, L3, L4, L5] + + human_control: + type: object + description: >- + Five forms of control. A kill switch alone is not oversight — meaningful + control starts at observe. + additionalProperties: false + required: [observe, interrupt, approve, override, disable] + properties: + observe: { $ref: "#/$defs/control" } + interrupt: { $ref: "#/$defs/control" } + approve: { $ref: "#/$defs/control" } + override: { $ref: "#/$defs/control" } + disable: { $ref: "#/$defs/control" } + + exceptions: + type: array + description: >- + Field 14 and the failure design. The happy path is not enough. + minItems: 1 + items: + type: object + additionalProperties: false + required: [condition, response] + properties: + condition: + type: string + enum: + - identifier_missing + - records_conflict + - tool_unavailable + - permissions_insufficient + - confidence_low + - result_disputed + - policy_conflict + - fraud_suspected + - sensitive_data_present + - other + detail: { type: string } + response: + type: string + enum: [retry, stop, ask_human, route, fallback, log, refuse] + escalation: + $ref: "#/$defs/escalation" + + escalation_default: + $ref: "#/$defs/escalation" + + supervision: + type: object + description: The six supervision primitives, as implemented. + additionalProperties: false + properties: + agent_identity_disclosed: { type: boolean } + action_receipt: { type: boolean } + reversal: { type: boolean } + consequence_scaled_approval: { type: boolean } + escalation_handoff: { type: boolean } + provenance_at_decision: { type: boolean } + + shutdown: + type: object + description: >- + Field 15. No shutdown procedure, no deployment. + additionalProperties: false + required: [procedure, tested] + properties: + procedure: { type: string, minLength: 10 } + tested: + type: boolean + description: An untested kill switch is a claim, not a control. + last_tested: { type: string, format: date } + revokes_access: { type: boolean } + + model: + type: object + additionalProperties: false + properties: + provider: { type: string } + name: { type: string } + version_pinned: + type: boolean + description: An unpinned model version means behavior changes without a release. + + trust_stage: + type: string + enum: [aux.T01, aux.T02, aux.T03, aux.T04] + description: >- + The stage demonstrably earned, not the stage aspired to. Ids are + namespaced to match trust-architecture.yaml; the site displays them + unprefixed as T01-T04. + + status: + type: string + enum: [design, shadow, draft, narrow_live, live, retired] + +$defs: + person: + type: object + additionalProperties: false + required: [name] + properties: + name: + type: string + description: A person, not a team. "Platform Team" is not an owner. + role: { type: string } + contact: { type: string } + + dial: + type: integer + minimum: 0 + maximum: 4 + + control: + type: object + additionalProperties: false + required: [available] + properties: + available: { type: boolean } + mechanism: { type: string } + audience: + type: string + enum: [affected_user, operator, admin_only, none] + + escalation: + type: object + additionalProperties: false + required: [recipient, deadline_minutes, on_timeout] + properties: + recipient: + type: string + description: A named human. "The team" is a queue, not a recipient. + deadline_minutes: + type: integer + minimum: 1 + on_timeout: + type: string + enum: [stop, fallback, refuse, page_secondary] + description: >- + What happens when nobody answers. Without this, "ask a human" is a + queue that fills up. + context_preserved: + type: boolean diff --git a/packages/aux-audit/schemas/agent-spec.v0.yaml b/packages/aux-audit/schemas/agent-spec.v0.yaml new file mode 100644 index 0000000..fb44627 --- /dev/null +++ b/packages/aux-audit/schemas/agent-spec.v0.yaml @@ -0,0 +1,101 @@ +# Agent Spec v0.1.0 — DEPRECATED, superseded by agent-spec.schema.yaml (v1.0) +# License: MIT (see schemas/LICENSE) +# +# Frozen. Kept so existing pipelines keep running: no new fields, bug fixes +# only. `aux-audit` 0.1.x reads THIS file, not v1 — see schemas/MIGRATION.md. +# +# Superseded for two doctrinal reasons, not cosmetic ones: +# +# 1. `autonomy` is one enum for the whole agent. trust-architecture.yaml +# states that "autonomy is not a slider you push to the right — it is a +# spectrum you choose per action". v1 replaces the label with a per-action +# mandate, every non-autonomous row requiring an enforcing mechanism. +# +# 2. `guarantees` are free-text promises. The canon's rule is that +# "enforcement lives in a mechanism, not in a prompt", and this format has +# no field for the mechanism at all. +# +# A third, technical: this file is NOT JSON Schema. It uses the reserved +# `$schema` key as a container for a bespoke notation. Do not point a JSON +# Schema validator at it. v1 is real draft 2020-12. +# +# Migrating: python3 schemas/migrate-v0-to-v1.py your-spec.yaml +# Tracked in: https://github.com/auxfirst/trustkit/issues/10 +version: "0.1.0" + +$schema: + name: { type: string, required: true } + version: { type: string, required: true } + + surface: + type: enum + values: ["chat", "inline", "ambient", "async", "multi-surface"] + required: true + + autonomy: + # SUPERSEDED (trustkit#10): a per-agent label is the abstraction the AUX + # canon tells readers to avoid. The replacement is a per-action mandate + # with an enforcing mechanism on every non-autonomous row. + type: enum + values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] + required: true + description: > + in-the-loop: every step confirmed. + on-the-loop: batched review, observable. + autonomous: acts without supervision. + + memory: + required: true + fields: + persistent: { type: boolean, required: true } + scopes: { type: list, required: true, item: string } + retention: { type: ISO-8601-duration, required: true } + user_visible: { type: boolean, required: true } + user_editable: { type: boolean, required: true } + + tools: + type: list + item: string + description: "tool/function names available to the agent" + + flows: + type: list + item: path + description: "paths to markdown flow descriptions; used for heuristic scoring" + + guarantees: + # SUPERSEDED (trustkit#10): a guarantee here is a sentence, and the canon + # is explicit that "a recommendation is not a control". The replacement + # pairs each action with the mechanism that enforces it. + type: list + item: string + description: > + explicit contracts the agent makes with the user + (e.g. 'will always ask before sending external email'). + NOTE: this is a stated promise, not a verified mechanism. aux-audit can + confirm the sentence exists; it cannot confirm anything enforces it. + + evaluation: + fields: + golden_transcripts: { type: list, item: path } + failure_transcripts: { type: list, item: path } + +example: | + name: "Support Copilot v2" + version: "2.3.1" + surface: chat + autonomy: human-on-the-loop + memory: + persistent: true + scopes: [preferences, conversation_history, escalation_notes] + retention: P90D + user_visible: true + user_editable: true + tools: [crm.lookup, email.send, ticket.create] + flows: [./flows/handoff.md, ./flows/refund.md] + guarantees: + - "will always ask before sending external email" + - "will always hand off to a human after two failed resolutions" + evaluation: + golden_transcripts: [./transcripts/golden/*.jsonl] + failure_transcripts: [./transcripts/failures/*.jsonl] diff --git a/packages/aux-audit/scripts/sync-schemas.mjs b/packages/aux-audit/scripts/sync-schemas.mjs index 1201aab..be3f518 100644 --- a/packages/aux-audit/scripts/sync-schemas.mjs +++ b/packages/aux-audit/scripts/sync-schemas.mjs @@ -22,6 +22,8 @@ const FILES = [ "trust-architecture.yaml", "trust-gap-taxonomy.yaml", "agent-spec.schema.yaml", + // What the CLI actually validates against, until v0.2 reads v1. + "agent-spec.v0.yaml", // The schemas are MIT and say so next to themselves, in the tarball too. "LICENSE", ]; diff --git a/packages/aux-audit/src/cli.ts b/packages/aux-audit/src/cli.ts index bb65cef..0d851b7 100644 --- a/packages/aux-audit/src/cli.ts +++ b/packages/aux-audit/src/cli.ts @@ -13,7 +13,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, relative, resolve } from "node:path"; import { audit, shouldFail } from "./score.js"; -import { loadSpec, SpecError } from "./spec.js"; +import { loadSpec, SpecError, SpecVersionError } from "./spec.js"; import { loadConfig } from "./config.js"; import { toJson } from "./report/json.js"; import { toMarkdown } from "./report/markdown.js"; @@ -172,7 +172,7 @@ function main(argv: string[]): number { toolVersion: VERSION, }); } catch (error) { - if (error instanceof SpecError) { + if (error instanceof SpecVersionError || error instanceof SpecError) { process.stderr.write(`aux-audit: ${error.message}\n`); return 2; } diff --git a/packages/aux-audit/src/spec.ts b/packages/aux-audit/src/spec.ts index 842956b..ef45baf 100644 --- a/packages/aux-audit/src/spec.ts +++ b/packages/aux-audit/src/spec.ts @@ -1,5 +1,5 @@ /** - * Loads and validates an agent spec against schemas/agent-spec.schema.yaml. + * Loads and validates an agent spec against schemas/agent-spec.v0.yaml. * * Per the schema header: "Fields marked required will cause the audit to refuse * to run if missing." A spec that fails validation is an error, not a low score @@ -11,7 +11,35 @@ import { parse } from "yaml"; import { schemaDir } from "./canon.js"; import type { AgentSpec, Autonomy, MemorySpec, Surface } from "./types.js"; -const schemaPath = (): string => join(schemaDir(), "agent-spec.schema.yaml"); +/** + * v0.1.0, deliberately. agent-spec.schema.yaml is v1.0 and this release does + * not score it — see the version shim below and schemas/MIGRATION.md. + */ +const schemaPath = (): string => join(schemaDir(), "agent-spec.v0.yaml"); + +/** + * Handed a v1 document, aux-audit 0.1.x used to reject it with four field + * errors — including "`autonomy` is required", demanding the very field v1 + * removes on purpose. That reads as a broken tool rather than a version + * mismatch, so it is detected before validation and reported as itself. + */ +export class SpecVersionError extends Error { + readonly detected = "v1.0"; + constructor() { + super( + "this is an agent-spec v1.0 document; aux-audit 0.1.x reads v0.1.0.\n" + + " Nothing was graded. v1 scoring lands in aux-audit 0.2.0.\n" + + " Migrating a v0 spec: python3 schemas/migrate-v0-to-v1.py your-spec.yaml\n" + + " Background: schemas/MIGRATION.md, trustkit#10", + ); + this.name = "SpecVersionError"; + } +} + +/** v1 declares a per-action mandate and a spec_version; v0 declares neither. */ +function looksLikeV1(raw: Record): boolean { + return "mandate" in raw || "spec_version" in raw; +} export class SpecError extends Error { readonly problems: string[]; @@ -100,6 +128,8 @@ export function validateSpec(input: unknown): AgentSpec { throw new SpecError(["the spec must be a YAML or JSON mapping"]); } const raw = input as Record; + if (looksLikeV1(raw)) throw new SpecVersionError(); + const { surface, autonomy } = schemaEnums(); for (const key of ["name", "version"] as const) { diff --git a/packages/aux-audit/test/report.test.ts b/packages/aux-audit/test/report.test.ts index de456cb..90b9db5 100644 --- a/packages/aux-audit/test/report.test.ts +++ b/packages/aux-audit/test/report.test.ts @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { mkdtempSync, readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { audit } from "../src/score.js"; @@ -135,3 +135,25 @@ test("CLI: --version and --help exit 0", () => { /USAGE/, ); }); + + +test("CLI: a v1 spec exits 2 with a version message, not field errors", () => { + const dir = mkdtempSync(join(tmpdir(), "aux-audit-v1-")); + const specPath = join(dir, "v1.yaml"); + writeFileSync( + specPath, + "spec_version: '1.0'\nid: collections\nname: Collections Agent\nmandate: []\n", + ); + try { + execFileSync(process.execPath, [cliPath(), "run", specPath], { stdio: "pipe" }); + assert.fail("expected a non-zero exit"); + } catch (error) { + const err = error as { status: number; stderr: Buffer; stdout: Buffer }; + assert.equal(err.status, 2); + const stderr = err.stderr.toString(); + assert.match(stderr, /agent-spec v1\.0 document/); + assert.match(stderr, /MIGRATION\.md/); + assert.doesNotMatch(stderr, /`autonomy` is required/); + assert.doesNotMatch(stderr, /Score/); + } +}); diff --git a/packages/aux-audit/test/spec.test.ts b/packages/aux-audit/test/spec.test.ts index 408a52d..a9907c1 100644 --- a/packages/aux-audit/test/spec.test.ts +++ b/packages/aux-audit/test/spec.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { parse } from "yaml"; -import { loadSpec, validateSpec, SpecError } from "../src/spec.js"; +import { loadSpec, validateSpec, SpecError, SpecVersionError } from "../src/spec.js"; import { fixture, repoSchema } from "./helpers.js"; test("loads a well-formed spec", () => { @@ -44,8 +44,8 @@ test("rejects a non-mapping document", () => { assert.throws(() => validateSpec("just a string"), SpecError); }); -test("accepts the example embedded in agent-spec.schema.yaml", () => { - const schema = parse(readFileSync(repoSchema("agent-spec.schema.yaml"), "utf8")) as Record< +test("accepts the example embedded in agent-spec.v0.yaml", () => { + const schema = parse(readFileSync(repoSchema("agent-spec.v0.yaml"), "utf8")) as Record< string, unknown >; @@ -56,7 +56,7 @@ test("accepts the example embedded in agent-spec.schema.yaml", () => { }); test("enum values are read from the schema, not hardcoded", () => { - const schema = parse(readFileSync(repoSchema("agent-spec.schema.yaml"), "utf8")) as Record< + const schema = parse(readFileSync(repoSchema("agent-spec.v0.yaml"), "utf8")) as Record< string, unknown >; @@ -103,3 +103,36 @@ test("ISO-8601 durations are validated", () => { assert.throws(() => validateSpec(withRetention(bad)), SpecError, `${bad} should fail`); } }); + + +test("a v1 document is named as a version mismatch, not a pile of field errors", () => { + const v1 = { + spec_version: "1.0", + id: "collections-agent", + name: "Collections Agent", + mandate: [{ action: "send reminder", authority: "human_approval", enforced_by: "queue" }], + }; + assert.throws( + () => validateSpec(v1), + (error: unknown) => { + assert.ok(error instanceof SpecVersionError, "should be a version error"); + assert.match(error.message, /agent-spec v1\.0 document/); + assert.match(error.message, /MIGRATION\.md/); + // The old behaviour demanded the very field v1 removes on purpose. + assert.doesNotMatch(error.message, /`autonomy` is required/); + assert.doesNotMatch(error.message, /is not valid/); + return true; + }, + ); +}); + +test("either v1 marker alone is enough to detect it", () => { + for (const marker of [{ mandate: [] }, { spec_version: "1.0" }]) { + assert.throws(() => validateSpec({ name: "x", ...marker }), SpecVersionError); + } +}); + +test("a v0 spec is still graded, not mistaken for v1", () => { + const spec = loadSpec(fixture("strong-spec.yaml")); + assert.equal(spec.name, "Support Copilot v2"); +}); diff --git a/schemas/MIGRATION.md b/schemas/MIGRATION.md new file mode 100644 index 0000000..5396618 --- /dev/null +++ b/schemas/MIGRATION.md @@ -0,0 +1,183 @@ +# Agent Spec — migrating from v0.1.0 to v1.0 + +**TL;DR** — v0.1.0 still works and is not going away this year. v1.0 exists +because v0.1.0 contradicts the doctrine this repository publishes. If you have +v0 specs, run the converter, then spend an hour on the TODOs it leaves. The +hour is the migration. + +```bash +python3 schemas/migrate-v0-to-v1.py your-spec.yaml +python3 schemas/validate.py your-agent.agent-spec.yaml +``` + +--- + +## Why v1.0 exists + +v0.1.0 was written as an input format for `aux-audit`, before the Agent Owner's +Manual and the Action Heat Ladder were published. Two of its fields are now at +odds with what auxfirst tells everyone else to do. + +### 1. `autonomy` was one label for the whole agent + +```yaml +# v0.1.0 +autonomy: + type: enum + values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] + required: true +``` + +The [Agent Owner's Manual](https://auxfirst.com/agent-owners-manual.html) is +explicit about this: + +> Avoid vague descriptions such as *semi-autonomous*. Define authority for +> specific actions instead. + +An agent that can read a CRM, draft an email and send an invoice does not have +one autonomy level. It has three, and the third is the one that matters. A +single label averages them, and averaging is what the Action Heat Ladder exists +to stop: + +> An action is as hot as its hottest dimension. No averaging. + +v1.0 replaces the field with a `mandate` array — one row per verb the agent can +reach, each with an authority of `autonomous`, `human_approval`, `human_only` +or `prohibited`. + +### 2. `guarantees` were promises, not controls + +```yaml +# v0.1.0 +guarantees: + - "will always ask before sending external email" +``` + +That sentence is true right up until a prompt injection, a model upgrade, or a +refactor that grants the send scope. The repository's own rule: + +> Enforcement lives in a mechanism, not in a prompt. +> A recommendation is not a control. + +v1.0 requires `enforced_by` on every mandate row. The row above becomes: + +```yaml +# v1.0 +- action: Send external email + authority: human_approval + enforced_by: Send scope withheld from the token until approval is issued + approver: Named support lead +``` + +The schema **rejects** a row without `enforced_by`. That refusal is the feature. + +### 3. `$schema` was used as a container + +v0.1.0 puts a custom notation under the key `$schema`, which is reserved in +JSON Schema. Any JSON Schema tool misreads the file. v1.0 is a real JSON Schema +(draft 2020-12) and validates with `jsonschema`, in editors, and in CI. + +--- + +## What the converter does, and what it refuses to do + +``` +python3 schemas/migrate-v0-to-v1.py support-copilot.yaml +``` + +Writes two files: + +| Output | Why two | +|---|---| +| `.agent-spec.yaml` | The v1.0 spec | +| `.memory-policy.yaml` | v0's `memory` block, moved to its own document — memory governance is its own concern, see `memory-policy.schema.yaml` | + +### Carried over mechanically + +| v0 | v1 | Note | +|---|---|---| +| `name` | `name` | | +| `version` | `version` | Added to v1 for this migration | +| `surface` | `surface` | Added to v1 for this migration | +| `tools` | `mandate[].action` | One row per tool, authority left blank | +| `memory.persistent` | memory policy, `classes[].persistent` | | +| `memory.retention` | memory policy, `ttl_days` | ISO-8601 duration parsed to days | +| `memory.user_visible` | memory policy, `must_be_visible` | | +| `memory.user_editable` | memory policy, `editable` | | +| `memory.scopes` | memory policy, `retained_categories` | **Not** `scope.levels` — see below | + +### Deliberately not guessed + +The converter leaves a `TODO` rather than inventing an answer for: + +- **`purpose`** — v0 had no field for why the agent exists +- **`owners`** — two named people, business and technical. v0 recorded neither +- **`trigger`** — what starts a run +- **`systems`** and **`capability`** — what the credentials actually permit, + which is nearly always broader than the mandate. That difference is the + attack surface +- **every `authority` and `enforced_by`** — this is the migration +- **`human_control`** — the five forms. v0 recorded none +- **`exceptions`** — v0 had no failure design at all +- **`shutdown.tested`** — starts `false`, because an untested kill switch is a + claim rather than a control +- **memory `scope.default`** — v0 recorded *what* was retained but never *how + far it may travel* + +A converted file **will not validate** until these are resolved. That is +intended. A migration that produced a passing file would have invented the +answers, and the answers are the point. + +### One trap worth naming + +v0's `memory.scopes` and v1's `scope.levels` sound alike and are different +axes. `scopes: [preferences, conversation_history]` says **what** is retained. +`levels: [user, team, tenant, global]` says **how far it may travel**. The +converter maps the first to `retained_categories` and leaves the second as a +TODO, because conflating them is how `tg.contextual.context_leak` happens. + +### Not carried over + +`flows` belong to the storyboard artifact and `evaluation.*_transcripts` to the +evaluation suite. Keep the paths; they do not belong in the agent spec. The +converter says so in its report rather than dropping them silently. + +--- + +## Reading your old `autonomy` value + +The converter prints a hint. It is a hypothesis to test, never an answer: + +| v0 value | Likely v1 shape | +|---|---| +| `human-in-the-loop` | Most rows `human_approval`; read-only rows are candidates for `autonomous` | +| `human-on-the-loop` | A mix — `autonomous` for reads and drafts, `human_approval` for anything that leaves the building | +| `autonomous` | Every row needs a deliberate decision. This is where the migration most often surfaces a surprise | + +If filling in the mandate changes your mind about what the agent should be +allowed to do, the migration has already paid for itself. + +--- + +## Timeline + +| Date | What | +|---|---| +| 2026-09-04 | v1.0 published. v0.1.0 renamed to `agent-spec.v0.yaml`, frozen | +| — | `aux-audit` reads v0.1.0 until it learns v1.0 | +| No earlier than 2027-03-01 | `agent-spec.v0.yaml` removed, and only after `aux-audit` reads v1.0 | + +v0.1.0 receives bug fixes only — no new fields. Nothing you have stops working +today. + +--- + +## Worked example + +The `example` block inside `agent-spec.v0.yaml` ("Support Copilot v2") converts +to five mandate rows: three from `tools`, two from `guarantees`. All five need +an authority and a mechanism, and the converter reports ten decisions it could +not make. + +That number is not a criticism of the old spec. It is the difference between +describing an agent and being able to operate one. diff --git a/schemas/agent-spec.schema.yaml b/schemas/agent-spec.schema.yaml index 7735478..38333be 100644 --- a/schemas/agent-spec.schema.yaml +++ b/schemas/agent-spec.schema.yaml @@ -1,104 +1,345 @@ -# Agent Spec — input format for `aux-audit` -# Version: 0.1.0 · SUPERSEDED, see trustkit#10 +# Agent Spec — input format for an Agent Owner's Manual +# Version: 1.0.0 # License: MIT (see schemas/LICENSE) -# A minimal, opinionated description of an agentic product. -# Fields marked "required" will cause the audit to refuse to run if missing. # -# --------------------------------------------------------------------------- -# Two known defects. Both are real; both are tracked in trustkit#10. +# Real JSON Schema, draft 2020-12. Validates one agent: who owns it, what it is +# authorized to do, what it can technically do, and how a human stops it. # -# 1. This file is NOT JSON Schema. It uses the reserved `$schema` key as a -# container for a bespoke notation ({ type: string, required: true }). Any -# JSON Schema tool will read it wrong. Do not point a validator at it. +# The gap between `capability` and `mandate` is the attack surface. Both are +# required, so the gap is computable rather than assumed. # -# 2. It contradicts the canon it belongs to. `autonomy` below requires one -# label for the whole agent, while trust-architecture.yaml states that -# "autonomy is not a slider you push to the right — it is a spectrum you -# choose per action". `guarantees` is free text, i.e. a promise, while -# enforcement.rule states that "enforcement lives in a mechanism, not in a -# prompt" and this format has no field for the mechanism. -# -# This file predates that doctrine. The replacement assigns authority action -# by action, each row requiring an `enforced_by` mechanism. Until it lands, -# aux-audit reads this format, and a score from it inherits the limitation: -# it grades a declared posture, not an enforced boundary. -# --------------------------------------------------------------------------- +# NOTE: `aux-audit` 0.1.x does not score this format yet — it reads +# agent-spec.v0.yaml and will tell you so if handed a v1 document. v1 scoring +# lands in aux-audit 0.2.0. See schemas/MIGRATION.md and trustkit#10. + +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://auxfirst.com/schemas/agent-spec.schema.yaml +title: Agent Spec +description: >- + A production AI agent's operating specification. Fifteen fields minimum, + derived from the Agent Owner's Manual. +type: object +additionalProperties: false + +required: + - spec_version + - id + - name + - purpose + - owners + - trigger + - systems + - capability + - mandate + - human_control + - exceptions + - shutdown + +properties: -version: "0.1.0" + spec_version: + type: string + const: "1.0" -$schema: - name: { type: string, required: true } - version: { type: string, required: true } + id: + type: string + pattern: "^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$" + description: Stable slug. Never reused after retirement. + + name: + type: string + description: Field 01 — the name humans use for this agent. + + version: + type: string + description: >- + The agent's own release version. Distinct from spec_version, which + versions this document format. Carried over from the v0.1.0 format. surface: - type: enum - values: ["chat", "inline", "ambient", "async", "multi-surface"] - required: true - - autonomy: - # SUPERSEDED (trustkit#10): a per-agent label is the abstraction the AUX - # canon tells readers to avoid. The replacement is a per-action mandate - # with an enforcing mechanism on every non-autonomous row. - type: enum - values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] - required: true - description: > - in-the-loop: every step confirmed. - on-the-loop: batched review, observable. - autonomous: acts without supervision. - - memory: - required: true - fields: - persistent: { type: boolean, required: true } - scopes: { type: list, required: true, item: string } - retention: { type: ISO-8601-duration, required: true } - user_visible: { type: boolean, required: true } - user_editable: { type: boolean, required: true } - - tools: - type: list - item: string - description: "tool/function names available to the agent" - - flows: - type: list - item: path - description: "paths to markdown flow descriptions; used for heuristic scoring" - - guarantees: - # SUPERSEDED (trustkit#10): a guarantee here is a sentence, and the canon - # is explicit that "a recommendation is not a control". The replacement - # pairs each action with the mechanism that enforces it. - type: list - item: string - description: > - explicit contracts the agent makes with the user - (e.g. 'will always ask before sending external email'). - NOTE: this is a stated promise, not a verified mechanism. aux-audit can - confirm the sentence exists; it cannot confirm anything enforces it. - - evaluation: - fields: - golden_transcripts: { type: list, item: path } - failure_transcripts: { type: list, item: path } - -example: | - name: "Support Copilot v2" - version: "2.3.1" - surface: chat - autonomy: human-on-the-loop - memory: - persistent: true - scopes: [preferences, conversation_history, escalation_notes] - retention: P90D - user_visible: true - user_editable: true - tools: [crm.lookup, email.send, ticket.create] - flows: [./flows/handoff.md, ./flows/refund.md] - guarantees: - - "will always ask before sending external email" - - "will always hand off to a human after two failed resolutions" - evaluation: - golden_transcripts: [./transcripts/golden/*.jsonl] - failure_transcripts: [./transcripts/failures/*.jsonl] + type: string + enum: [chat, inline, ambient, async, multi-surface] + description: >- + Where the agent meets its users. Carried over from the v0.1.0 format; + it changes which patterns from trust-contract.yaml apply. + + purpose: + type: string + minLength: 20 + description: >- + Field 02 — why this agent exists, in business terms. If it cannot be + stated without naming the technology, it is not a purpose. + + owners: + type: object + description: >- + Fields 03 and 04. Two owners, not one. Business owner is a named person + in the function the agent serves; technical owner can change how it works. + An agent with only a technical owner is orphaned the day the project + closes. + additionalProperties: false + required: [business, technical] + properties: + business: + $ref: "#/$defs/person" + technical: + $ref: "#/$defs/person" + + trigger: + type: object + description: Field 05 — what starts a run. + additionalProperties: false + required: [kind] + properties: + kind: + type: string + enum: [schedule, event, user_invocation, escalation, continuous] + detail: + type: string + + users: + type: array + description: Field 06 — who interacts with it, and in what role. + items: + type: object + additionalProperties: false + required: [role] + properties: + role: { type: string } + can_invoke: { type: boolean, default: true } + can_approve: { type: boolean, default: false } + + systems: + type: object + description: Fields 07 and 08 — data sources and connected systems. + additionalProperties: false + required: [data_sources, connected] + properties: + data_sources: + type: array + items: { type: string } + connected: + type: array + items: + type: object + additionalProperties: false + required: [name, auth] + properties: + name: { type: string } + auth: + type: string + enum: [service_account, delegated_identity, own_identity, oauth_user] + description: >- + own_identity or delegated_identity is required for + attribution. service_account makes every downstream log wrong + about who acted. + scopes: + type: array + items: { type: string } + + capability: + type: object + description: >- + Fields 09 and 10 — what the agent CAN technically do, granted by its + credentials. Distinct from mandate on purpose. + additionalProperties: false + required: [can_read, can_change] + properties: + can_read: + type: array + items: { type: string } + can_change: + type: array + items: { type: string } + + mandate: + type: array + description: >- + Fields 11, 12 and 13 — assigned authority, action by action. Never an + autonomy score. Every non-autonomous row needs an enforcing mechanism, + or the row is decoration. + minItems: 1 + items: + type: object + additionalProperties: false + required: [action, authority, enforced_by] + properties: + action: + type: string + description: One verb the agent can reach. + authority: + type: string + enum: [autonomous, human_approval, human_only, prohibited] + description: >- + Four values, no fifth. Ambiguity in this field is where trust + collapses. + enforced_by: + type: string + minLength: 3 + description: >- + The mechanism, not the intention. "System prompt" is not a valid + answer for human_only or prohibited. + approver: + type: string + description: Required when authority is human_approval. + heat: + type: object + additionalProperties: false + properties: + reversibility: { $ref: "#/$defs/dial" } + blast_radius: { $ref: "#/$defs/dial" } + exposure: { $ref: "#/$defs/dial" } + commitment: { $ref: "#/$defs/dial" } + authority: { $ref: "#/$defs/dial" } + band: + type: string + enum: [LOW, LOW_MEDIUM, MEDIUM, HIGH, CRITICAL] + description: >- + Set by the single highest dial, never the average. An action is + as hot as its hottest dimension. + autonomy_level: + type: string + enum: [L1, L2, L3, L4, L5] + + human_control: + type: object + description: >- + Five forms of control. A kill switch alone is not oversight — meaningful + control starts at observe. + additionalProperties: false + required: [observe, interrupt, approve, override, disable] + properties: + observe: { $ref: "#/$defs/control" } + interrupt: { $ref: "#/$defs/control" } + approve: { $ref: "#/$defs/control" } + override: { $ref: "#/$defs/control" } + disable: { $ref: "#/$defs/control" } + + exceptions: + type: array + description: >- + Field 14 and the failure design. The happy path is not enough. + minItems: 1 + items: + type: object + additionalProperties: false + required: [condition, response] + properties: + condition: + type: string + enum: + - identifier_missing + - records_conflict + - tool_unavailable + - permissions_insufficient + - confidence_low + - result_disputed + - policy_conflict + - fraud_suspected + - sensitive_data_present + - other + detail: { type: string } + response: + type: string + enum: [retry, stop, ask_human, route, fallback, log, refuse] + escalation: + $ref: "#/$defs/escalation" + + escalation_default: + $ref: "#/$defs/escalation" + + supervision: + type: object + description: The six supervision primitives, as implemented. + additionalProperties: false + properties: + agent_identity_disclosed: { type: boolean } + action_receipt: { type: boolean } + reversal: { type: boolean } + consequence_scaled_approval: { type: boolean } + escalation_handoff: { type: boolean } + provenance_at_decision: { type: boolean } + + shutdown: + type: object + description: >- + Field 15. No shutdown procedure, no deployment. + additionalProperties: false + required: [procedure, tested] + properties: + procedure: { type: string, minLength: 10 } + tested: + type: boolean + description: An untested kill switch is a claim, not a control. + last_tested: { type: string, format: date } + revokes_access: { type: boolean } + + model: + type: object + additionalProperties: false + properties: + provider: { type: string } + name: { type: string } + version_pinned: + type: boolean + description: An unpinned model version means behavior changes without a release. + + trust_stage: + type: string + enum: [aux.T01, aux.T02, aux.T03, aux.T04] + description: >- + The stage demonstrably earned, not the stage aspired to. Ids are + namespaced to match trust-architecture.yaml; the site displays them + unprefixed as T01-T04. + + status: + type: string + enum: [design, shadow, draft, narrow_live, live, retired] + +$defs: + person: + type: object + additionalProperties: false + required: [name] + properties: + name: + type: string + description: A person, not a team. "Platform Team" is not an owner. + role: { type: string } + contact: { type: string } + + dial: + type: integer + minimum: 0 + maximum: 4 + + control: + type: object + additionalProperties: false + required: [available] + properties: + available: { type: boolean } + mechanism: { type: string } + audience: + type: string + enum: [affected_user, operator, admin_only, none] + + escalation: + type: object + additionalProperties: false + required: [recipient, deadline_minutes, on_timeout] + properties: + recipient: + type: string + description: A named human. "The team" is a queue, not a recipient. + deadline_minutes: + type: integer + minimum: 1 + on_timeout: + type: string + enum: [stop, fallback, refuse, page_secondary] + description: >- + What happens when nobody answers. Without this, "ask a human" is a + queue that fills up. + context_preserved: + type: boolean diff --git a/schemas/agent-spec.v0.yaml b/schemas/agent-spec.v0.yaml new file mode 100644 index 0000000..fb44627 --- /dev/null +++ b/schemas/agent-spec.v0.yaml @@ -0,0 +1,101 @@ +# Agent Spec v0.1.0 — DEPRECATED, superseded by agent-spec.schema.yaml (v1.0) +# License: MIT (see schemas/LICENSE) +# +# Frozen. Kept so existing pipelines keep running: no new fields, bug fixes +# only. `aux-audit` 0.1.x reads THIS file, not v1 — see schemas/MIGRATION.md. +# +# Superseded for two doctrinal reasons, not cosmetic ones: +# +# 1. `autonomy` is one enum for the whole agent. trust-architecture.yaml +# states that "autonomy is not a slider you push to the right — it is a +# spectrum you choose per action". v1 replaces the label with a per-action +# mandate, every non-autonomous row requiring an enforcing mechanism. +# +# 2. `guarantees` are free-text promises. The canon's rule is that +# "enforcement lives in a mechanism, not in a prompt", and this format has +# no field for the mechanism at all. +# +# A third, technical: this file is NOT JSON Schema. It uses the reserved +# `$schema` key as a container for a bespoke notation. Do not point a JSON +# Schema validator at it. v1 is real draft 2020-12. +# +# Migrating: python3 schemas/migrate-v0-to-v1.py your-spec.yaml +# Tracked in: https://github.com/auxfirst/trustkit/issues/10 +version: "0.1.0" + +$schema: + name: { type: string, required: true } + version: { type: string, required: true } + + surface: + type: enum + values: ["chat", "inline", "ambient", "async", "multi-surface"] + required: true + + autonomy: + # SUPERSEDED (trustkit#10): a per-agent label is the abstraction the AUX + # canon tells readers to avoid. The replacement is a per-action mandate + # with an enforcing mechanism on every non-autonomous row. + type: enum + values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] + required: true + description: > + in-the-loop: every step confirmed. + on-the-loop: batched review, observable. + autonomous: acts without supervision. + + memory: + required: true + fields: + persistent: { type: boolean, required: true } + scopes: { type: list, required: true, item: string } + retention: { type: ISO-8601-duration, required: true } + user_visible: { type: boolean, required: true } + user_editable: { type: boolean, required: true } + + tools: + type: list + item: string + description: "tool/function names available to the agent" + + flows: + type: list + item: path + description: "paths to markdown flow descriptions; used for heuristic scoring" + + guarantees: + # SUPERSEDED (trustkit#10): a guarantee here is a sentence, and the canon + # is explicit that "a recommendation is not a control". The replacement + # pairs each action with the mechanism that enforces it. + type: list + item: string + description: > + explicit contracts the agent makes with the user + (e.g. 'will always ask before sending external email'). + NOTE: this is a stated promise, not a verified mechanism. aux-audit can + confirm the sentence exists; it cannot confirm anything enforces it. + + evaluation: + fields: + golden_transcripts: { type: list, item: path } + failure_transcripts: { type: list, item: path } + +example: | + name: "Support Copilot v2" + version: "2.3.1" + surface: chat + autonomy: human-on-the-loop + memory: + persistent: true + scopes: [preferences, conversation_history, escalation_notes] + retention: P90D + user_visible: true + user_editable: true + tools: [crm.lookup, email.send, ticket.create] + flows: [./flows/handoff.md, ./flows/refund.md] + guarantees: + - "will always ask before sending external email" + - "will always hand off to a human after two failed resolutions" + evaluation: + golden_transcripts: [./transcripts/golden/*.jsonl] + failure_transcripts: [./transcripts/failures/*.jsonl] diff --git a/schemas/migrate-v0-to-v1.py b/schemas/migrate-v0-to-v1.py new file mode 100755 index 0000000..dd87fd4 --- /dev/null +++ b/schemas/migrate-v0-to-v1.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Migrate an agent spec from the v0.1.0 format to v1.0. + + python3 schemas/migrate-v0-to-v1.py my-agent.yaml + python3 schemas/migrate-v0-to-v1.py my-agent.yaml --out-dir ./specs + +Writes two files next to the input (or into --out-dir): + + .agent-spec.yaml the v1.0 spec, with TODO markers + .memory-policy.yaml the memory block, moved to its own document + +and prints a report of every decision the conversion could not make for you. + +This tool deliberately does not guess. Three things in v0 cannot be mechanically +converted, because converting them is the work: + + autonomy A single per-agent label becomes a per-action mandate. There is + no arithmetic that turns "human-on-the-loop" into a list of what + the agent may decide alone. Every tool becomes a mandate row you + have to rule on. + + guarantees Free-text promises become mandate rows with an enforcing + mechanism. "Will always ask before sending external email" is a + sentence; the send scope being withheld until approval is a + control. Only you know which one you actually have. + + heat Scoring an action on reversibility, blast radius, exposure, + commitment and authority requires knowing what the action does in + your systems. + +Exit code 0 if the file converted, 1 on a read or parse failure. A conversion +that leaves TODOs still exits 0 — the TODOs are the point, not an error. + +MIT · auxfirst agency 2026 +""" +import argparse +import os +import re +import sys + +try: + import yaml +except ImportError: + sys.exit("pyyaml is required: pip install pyyaml") + +TODO = "TODO" + +# v0 autonomy label -> the honest v1 reading of it +AUTONOMY_HINT = { + "human-in-the-loop": ( + "every step was confirmed, so most rows are probably human_approval; " + "the read-only ones are candidates for autonomous" + ), + "human-on-the-loop": ( + "review was batched, so rows are probably a mix of autonomous (reads, " + "drafts) and human_approval (anything that leaves the building)" + ), + "autonomous": ( + "nothing was gated, so every row needs a deliberate decision — this is " + "the case where the migration is most likely to surface a surprise" + ), +} + +RETENTION_RE = re.compile(r"^P(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)D)?$") + + +def iso_duration_to_days(value): + """Best-effort ISO-8601 duration to days. Returns None if unparseable.""" + if not isinstance(value, str): + return None + match = RETENTION_RE.match(value.strip()) + if not match: + return None + years, months, days = (int(g) if g else 0 for g in match.groups()) + return years * 365 + months * 30 + days + + +def slugify(text): + slug = re.sub(r"[^a-z0-9]+", "-", str(text).lower()).strip("-") + return slug or "agent" + + +def convert(v0, source_name): + notes = [] + todos = [] + + name = v0.get("name") or source_name + slug = slugify(name) + + spec = { + "spec_version": "1.0", + "id": slug, + "name": name, + } + + if v0.get("version"): + spec["version"] = str(v0["version"]) + if v0.get("surface"): + spec["surface"] = v0["surface"] + + spec["purpose"] = f"{TODO}: why this agent exists, in business terms, in one sentence." + todos.append("purpose — v0 had no equivalent field") + + spec["owners"] = { + "business": {"name": f"{TODO}: a named person in the business function"}, + "technical": {"name": f"{TODO}: a named person who can change how it works"}, + } + todos.append("owners — two named people, not a team. v0 recorded neither") + + spec["trigger"] = {"kind": f"{TODO}", "detail": f"{TODO}: what starts a run"} + todos.append("trigger — one of schedule, event, user_invocation, escalation, continuous") + + spec["systems"] = { + "data_sources": [f"{TODO}: what it reads"], + "connected": [ + { + "name": f"{TODO}: system name", + "auth": f"{TODO}: own_identity, delegated_identity, oauth_user or service_account", + } + ], + } + todos.append("systems — v0 listed tools but not the systems behind them or how the agent authenticates") + + spec["capability"] = { + "can_read": [f"{TODO}: what the credentials permit reading"], + "can_change": [f"{TODO}: what the credentials permit changing"], + } + todos.append( + "capability — what the credentials PERMIT, which is usually broader than " + "the mandate. The difference is the attack surface" + ) + + # --- the mandate, from tools and guarantees --------------------------- + mandate = [] + tools = v0.get("tools") or [] + for tool in tools: + mandate.append( + { + "action": f"Call {tool}", + "authority": f"{TODO}: autonomous | human_approval | human_only | prohibited", + "enforced_by": f"{TODO}: the mechanism, not the intention", + } + ) + guarantees = v0.get("guarantees") or [] + for guarantee in guarantees: + mandate.append( + { + "action": f"{TODO}: name the action behind this promise", + "authority": f"{TODO}", + "enforced_by": f"{TODO}: what makes this true when the prompt is ignored?", + "_from_v0_guarantee": guarantee, + } + ) + if not mandate: + mandate.append( + { + "action": f"{TODO}: one verb the agent can reach", + "authority": f"{TODO}", + "enforced_by": f"{TODO}", + } + ) + spec["mandate"] = mandate + + autonomy = v0.get("autonomy") + if autonomy: + hint = AUTONOMY_HINT.get(autonomy, "no hint available for this value") + notes.append( + f'autonomy: "{autonomy}" was a single label for the whole agent and has ' + f"no v1 equivalent. Reading it: {hint}." + ) + todos.append( + f"mandate — {len(tools)} row(s) from tools and {len(guarantees)} from " + f'guarantees, all needing an authority and a mechanism. The v0 label ' + f'"{autonomy}" is a starting hypothesis, not an answer' + ) + if guarantees: + notes.append( + f"{len(guarantees)} guarantee(s) were carried across as mandate rows with the " + "original text preserved in _from_v0_guarantee. Remove that key once the row " + "is filled in — the schema rejects it." + ) + + spec["human_control"] = { + key: {"available": f"{TODO}: true or false", "mechanism": f"{TODO}"} + for key in ("observe", "interrupt", "approve", "override", "disable") + } + todos.append("human_control — five forms. v0 recorded none of them explicitly") + + spec["exceptions"] = [ + { + "condition": f"{TODO}: one of the nine conditions", + "response": f"{TODO}: retry | stop | ask_human | route | fallback | log | refuse", + "escalation": { + "recipient": f"{TODO}: a named human", + "deadline_minutes": f"{TODO}", + "on_timeout": f"{TODO}: stop | fallback | refuse | page_secondary", + }, + } + ] + todos.append("exceptions — v0 had no failure design at all") + + spec["shutdown"] = { + "procedure": f"{TODO}: how the agent is stopped and its access revoked", + "tested": False, + } + todos.append("shutdown — and tested must become true before this is production") + + if v0.get("evaluation"): + notes.append( + "evaluation.golden_transcripts and failure_transcripts were not carried " + "over. They belong to the evaluation suite artifact, not the agent spec. " + "Keep the paths." + ) + if v0.get("flows"): + notes.append( + f"{len(v0['flows'])} flow path(s) were not carried over. Flows belong to " + "the storyboard artifact." + ) + + # --- memory, moved to its own document -------------------------------- + memory_doc = None + mem = v0.get("memory") or {} + if mem: + ttl = iso_duration_to_days(mem.get("retention")) + if ttl is None and mem.get("retention"): + notes.append( + f'memory.retention "{mem["retention"]}" could not be parsed as an ' + "ISO-8601 duration; ttl_days left as TODO." + ) + memory_doc = { + "schema": "memory-policy", + "version": "1.0.0", + "agent": slug, + "classes": [ + { + "id": "semantic", + "persistent": bool(mem.get("persistent", False)), + "ttl_days": ttl if ttl is not None else f"{TODO}", + "editable": bool(mem.get("user_editable", False)), + "must_be_visible": bool(mem.get("user_visible", False)), + } + ], + # v0 `scopes` are content categories, not scope levels. They are + # different axes and must not be conflated: levels say how far a + # memory may travel, categories say what is retained. + "retained_categories": mem.get("scopes") or [f"{TODO}: what is retained"], + "scope": { + "levels": ["user", "team", "tenant", "global"], + "default": f"{TODO}: how far may these travel? v0 did not record this", + }, + "user_rights": { + "inspect": {"required": bool(mem.get("user_visible", False))}, + "correct": {"required": bool(mem.get("user_editable", False))}, + "forget": {"required": f"{TODO}: v0 had no forget control"}, + }, + } + if not mem.get("user_visible", False): + notes.append( + "memory.user_visible was false. Under memory-policy.yaml the semantic " + "class must be visible — this is a gap to close, recorded rather than " + "silently upgraded." + ) + todos.append("memory policy — forget control, sensitive categories and provenance") + todos.append( + "memory scope.default — v0 recorded content categories but never how far " + "a memory may travel. A memory with no scope level is a context leak " + "waiting to be found (tg.contextual.context_leak)" + ) + + return spec, memory_doc, notes, todos + + +def dump(doc, path, header): + with open(path, "w", encoding="utf-8") as fh: + fh.write(header) + yaml.safe_dump(doc, fh, sort_keys=False, allow_unicode=True, width=88) + + +def main(): + parser = argparse.ArgumentParser(description="Migrate a v0.1.0 agent spec to v1.0.") + parser.add_argument("spec", help="path to the v0 spec") + parser.add_argument("--out-dir", default=None, help="where to write (default: alongside input)") + args = parser.parse_args() + + try: + with open(args.spec, encoding="utf-8") as fh: + v0 = yaml.safe_load(fh) + except Exception as exc: # noqa: BLE001 + sys.exit(f"cannot read {args.spec}: {exc}") + if not isinstance(v0, dict): + sys.exit(f"{args.spec} does not contain a YAML mapping") + + # The v0 example block is a string; a real v0 spec is the mapping itself. + if "example" in v0 and "name" not in v0: + sys.exit( + f"{args.spec} looks like the v0 format DEFINITION, not an agent spec " + "written in it. Pass one of your own specs." + ) + + out_dir = args.out_dir or os.path.dirname(os.path.abspath(args.spec)) + os.makedirs(out_dir, exist_ok=True) + + spec, memory_doc, notes, todos = convert(v0, os.path.basename(args.spec)) + slug = spec["id"] + + spec_path = os.path.join(out_dir, f"{slug}.agent-spec.yaml") + dump( + spec, + spec_path, + "# Migrated from the v0.1.0 agent spec format.\n" + "# Every TODO is a decision the conversion could not make for you.\n" + "# Validate with: python3 schemas/validate.py " + f"{slug}.agent-spec.yaml\n\n", + ) + written = [spec_path] + + if memory_doc: + mem_path = os.path.join(out_dir, f"{slug}.memory-policy.yaml") + dump( + memory_doc, + mem_path, + "# Memory block extracted from the v0.1.0 agent spec.\n" + "# Memory governance lives in its own document — see\n" + "# schemas/memory-policy.schema.yaml.\n\n", + ) + written.append(mem_path) + + print(f"read {args.spec}") + for path in written: + print(f"wrote {path}") + + if notes: + print("\nnotes") + for note in notes: + print(f" · {note}") + + print(f"\n{len(todos)} decision(s) the conversion could not make") + for todo in todos: + print(f" TODO {todo}") + + print( + "\nThe converted file will not validate until the TODOs are resolved. " + "That is deliberate:\nthe schema refuses a mandate row without an enforcing " + "mechanism, and refusing it is\nthe whole point of v1.0." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())