diff --git a/README.md b/README.md index d46f6fb..43d0484 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Or read **[onboarding/README.md](onboarding/README.md)** — the router with all │ ├── trust-gap-taxonomy.yaml ← named failure modes │ ├── aux-evolution-curve.yaml ← the 4 capability stages (teardowns only) │ ├── trust-contract.yaml ← example Trust Contract -│ ├── agent-spec.schema.yaml ← audit input format +│ ├── agent-spec.schema.yaml ← audit input format (v0.1.0, superseded — #10) │ ├── brain-spec.schema.yaml ← Agency Brain input format (JSON Schema) │ └── memory-policy.schema.yaml ← memory governance format │ diff --git a/docs/audit.md b/docs/audit.md index 001f6a2..75845af 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -16,7 +16,7 @@ npx aux-audit run ./agent-spec.yaml ## Contract -- **Input:** an agent spec matching [`schemas/agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml). +- **Input:** an agent spec matching [`schemas/agent-spec.schema.yaml`](../schemas/agent-spec.schema.yaml) — v0.1.0, superseded; see [#10](https://github.com/auxfirst/trustkit/issues/10). - **Output:** `score`, `grade`, `trust_stage`, `issues[]`, `recommendations[]`. - **Formats:** Markdown (default), JSON, SARIF 2.1.0. - **Exit codes:** `0` clean · `1` findings at or above `--fail-on` · `2` invalid spec (nothing graded). diff --git a/packages/aux-audit/README.md b/packages/aux-audit/README.md index 82625c3..664b5b9 100644 --- a/packages/aux-audit/README.md +++ b/packages/aux-audit/README.md @@ -32,6 +32,8 @@ Change the YAML in `schemas/`, and the audit changes with it. No heuristic text, **aux-audit grades the spec, not the running product.** It can prove a mechanism was never declared. It cannot prove a declared one works. +There is a sharper limit in the current input format, tracked in [trustkit#10](https://github.com/auxfirst/trustkit/issues/10): `agent-spec.schema.yaml` v0.1.0 carries a single per-agent `autonomy` label and free-text `guarantees`, so a score reflects a **declared posture**, not an enforced boundary. The replacement assigns authority action by action and requires a mechanism on every non-autonomous row. + Two consequences, both deliberate: - **Level 3 requires evidence.** "Robust" means *works under novel, adversarial, or edge cases* — prose cannot demonstrate that. A spec with no `evaluation.golden_transcripts` or `evaluation.failure_transcripts` is capped at level 2 on every heuristic, however well written. This is what stops the score from being gamed by rewriting guarantees. diff --git a/packages/aux-audit/schemas/agent-spec.schema.yaml b/packages/aux-audit/schemas/agent-spec.schema.yaml index 88f5106..7735478 100644 --- a/packages/aux-audit/schemas/agent-spec.schema.yaml +++ b/packages/aux-audit/schemas/agent-spec.schema.yaml @@ -1,8 +1,28 @@ # Agent Spec — input format for `aux-audit` -# Version: 0.1.0 +# Version: 0.1.0 · SUPERSEDED, see trustkit#10 # License: MIT (see schemas/LICENSE) # A minimal, opinionated description of an agentic product. # Fields marked "required" will cause the audit to refuse to run if missing. +# +# --------------------------------------------------------------------------- +# Two known defects. Both are real; both are tracked in trustkit#10. +# +# 1. This file is NOT JSON Schema. It uses the reserved `$schema` key as a +# container for a bespoke notation ({ type: string, required: true }). Any +# JSON Schema tool will read it wrong. Do not point a validator at it. +# +# 2. It contradicts the canon it belongs to. `autonomy` below requires one +# label for the whole agent, while trust-architecture.yaml states that +# "autonomy is not a slider you push to the right — it is a spectrum you +# choose per action". `guarantees` is free text, i.e. a promise, while +# enforcement.rule states that "enforcement lives in a mechanism, not in a +# prompt" and this format has no field for the mechanism. +# +# This file predates that doctrine. The replacement assigns authority action +# by action, each row requiring an `enforced_by` mechanism. Until it lands, +# aux-audit reads this format, and a score from it inherits the limitation: +# it grades a declared posture, not an enforced boundary. +# --------------------------------------------------------------------------- version: "0.1.0" @@ -16,6 +36,9 @@ $schema: required: true autonomy: + # SUPERSEDED (trustkit#10): a per-agent label is the abstraction the AUX + # canon tells readers to avoid. The replacement is a per-action mandate + # with an enforcing mechanism on every non-autonomous row. type: enum values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] required: true @@ -44,11 +67,16 @@ $schema: description: "paths to markdown flow descriptions; used for heuristic scoring" guarantees: + # SUPERSEDED (trustkit#10): a guarantee here is a sentence, and the canon + # is explicit that "a recommendation is not a control". The replacement + # pairs each action with the mechanism that enforces it. type: list item: string description: > explicit contracts the agent makes with the user (e.g. 'will always ask before sending external email'). + NOTE: this is a stated promise, not a verified mechanism. aux-audit can + confirm the sentence exists; it cannot confirm anything enforces it. evaluation: fields: diff --git a/schemas/agent-spec.schema.yaml b/schemas/agent-spec.schema.yaml index 88f5106..7735478 100644 --- a/schemas/agent-spec.schema.yaml +++ b/schemas/agent-spec.schema.yaml @@ -1,8 +1,28 @@ # Agent Spec — input format for `aux-audit` -# Version: 0.1.0 +# Version: 0.1.0 · SUPERSEDED, see trustkit#10 # License: MIT (see schemas/LICENSE) # A minimal, opinionated description of an agentic product. # Fields marked "required" will cause the audit to refuse to run if missing. +# +# --------------------------------------------------------------------------- +# Two known defects. Both are real; both are tracked in trustkit#10. +# +# 1. This file is NOT JSON Schema. It uses the reserved `$schema` key as a +# container for a bespoke notation ({ type: string, required: true }). Any +# JSON Schema tool will read it wrong. Do not point a validator at it. +# +# 2. It contradicts the canon it belongs to. `autonomy` below requires one +# label for the whole agent, while trust-architecture.yaml states that +# "autonomy is not a slider you push to the right — it is a spectrum you +# choose per action". `guarantees` is free text, i.e. a promise, while +# enforcement.rule states that "enforcement lives in a mechanism, not in a +# prompt" and this format has no field for the mechanism. +# +# This file predates that doctrine. The replacement assigns authority action +# by action, each row requiring an `enforced_by` mechanism. Until it lands, +# aux-audit reads this format, and a score from it inherits the limitation: +# it grades a declared posture, not an enforced boundary. +# --------------------------------------------------------------------------- version: "0.1.0" @@ -16,6 +36,9 @@ $schema: required: true autonomy: + # SUPERSEDED (trustkit#10): a per-agent label is the abstraction the AUX + # canon tells readers to avoid. The replacement is a per-action mandate + # with an enforcing mechanism on every non-autonomous row. type: enum values: ["human-in-the-loop", "human-on-the-loop", "autonomous"] required: true @@ -44,11 +67,16 @@ $schema: description: "paths to markdown flow descriptions; used for heuristic scoring" guarantees: + # SUPERSEDED (trustkit#10): a guarantee here is a sentence, and the canon + # is explicit that "a recommendation is not a control". The replacement + # pairs each action with the mechanism that enforces it. type: list item: string description: > explicit contracts the agent makes with the user (e.g. 'will always ask before sending external email'). + NOTE: this is a stated promise, not a verified mechanism. aux-audit can + confirm the sentence exists; it cannot confirm anything enforces it. evaluation: fields: