diff --git a/packages/core/src/controller.ts b/packages/core/src/controller.ts index e7a86ff..1e4b547 100644 --- a/packages/core/src/controller.ts +++ b/packages/core/src/controller.ts @@ -47,6 +47,13 @@ import type { StructuredPermissionSet, TracePermissionDecision, } from "./types.js"; +import { + directEvidenceArtifactTypes, + directEvidenceTraceEventTypes, + evidenceKindsForCapability, + isUserReportedEvidencePath, + requiresDirectEvidence, +} from "./controller/evidence-validator.js"; const artifactInputsByPhase: Record> = { context_grounding: new Set([ @@ -605,67 +612,6 @@ const potentiallyMutatingPlanCapabilities = new Set([ "subagent.spawn", ]); -function requiresDirectEvidence( - path: string, - submissionType?: string, -): boolean { - return ( - (submissionType === "WorkResult" && - (/^body\.evidenceRefs\[\d+\]$/u.test(path) || - /^body\.actions\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path))) || - /\.claimEvidenceMatrix\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.completionClaims\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.observations\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.inferences\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.toolEventRefs\[\d+\]$/u.test(path) || - /\.verificationRefs\[\d+\]$/u.test(path) || - /\.diagnosisGate\.directEvidenceRefs\[\d+\]$/u.test(path) || - /\.acceptanceCoverage\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.acceptanceResults\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.filesChanged\[\d+\]\.diffRef$/u.test(path) || - /\.testResults\[\d+\]\.(?:commandRef|outputRef)$/u.test(path) || - /\.verificationResults\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || - /\.(?:ruleCompliance|regressionChecks|userFidelity)\[\d+\]\.evidenceRefs\[\d+\]$/u.test( - path, - ) || - (submissionType === "QualityReview" && - /\.hardGates\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path)) - ); -} - -const directEvidenceArtifactTypes = new Set([ - "Evidence", - "ContextDocument", - "TraceEvent", -]); - -const directEvidenceTraceEventTypes = new Set(); - -function evidenceKindsForCapability(capability: string): ReadonlySet { - switch (capability) { - case "repository.read": - return new Set(["repository", "diff", "tool_output"]); - case "repository.write": - case "repository.delete": - return new Set(["diff"]); - case "shell.inspect": - case "shell.execute": - return new Set(["tool_output", "log", "test"]); - case "runtime.inspect": - case "runtime.restart": - return new Set(["runtime", "log", "tool_output"]); - case "browser.inspect": - case "browser.mutate": - return new Set(["browser"]); - case "network.read": - case "external.write": - case "subagent.spawn": - return new Set(["tool_output"]); - default: - return new Set(); - } -} - class PermissionAuthorizationError extends Error { constructor( message: string, @@ -713,50 +659,6 @@ function terminalField(path: string): string { return /\.([A-Za-z][A-Za-z0-9]*)(?:\[\d+\])?$/u.exec(path)?.[1] ?? ""; } -function isUserReportedEvidencePath( - submission: ArtifactSubmission, - path: string, -): boolean { - if (submission.type === "ReleaseAudit") { - const match = - /^body\.claimEvidenceMatrix\[(\d+)\]\.evidenceRefs\[\d+\]$/u.exec(path); - const matrix = - typeof submission.body === "object" && - submission.body !== null && - "claimEvidenceMatrix" in submission.body - ? (submission.body as { claimEvidenceMatrix?: unknown }) - .claimEvidenceMatrix - : null; - return ( - match !== null && - Array.isArray(matrix) && - typeof matrix[Number(match[1])] === "object" && - matrix[Number(match[1])] !== null && - (matrix[Number(match[1])] as { basis?: unknown }).basis === - "user_reported" - ); - } - if (submission.type === "UserReport") { - const match = - /^body\.completionClaims\[(\d+)\]\.evidenceRefs\[\d+\]$/u.exec(path); - const claims = - typeof submission.body === "object" && - submission.body !== null && - "completionClaims" in submission.body - ? (submission.body as { completionClaims?: unknown }).completionClaims - : null; - return ( - match !== null && - Array.isArray(claims) && - typeof claims[Number(match[1])] === "object" && - claims[Number(match[1])] !== null && - (claims[Number(match[1])] as { status?: unknown }).status === - "user_reported" - ); - } - return false; -} - function equalStringSets( left: readonly string[], right: readonly string[], diff --git a/packages/core/src/controller/evidence-validator.test.ts b/packages/core/src/controller/evidence-validator.test.ts new file mode 100644 index 0000000..5665c6f --- /dev/null +++ b/packages/core/src/controller/evidence-validator.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; + +import type { ArtifactSubmission } from "../types.js"; +import { + directEvidenceArtifactTypes, + directEvidenceTraceEventTypes, + evidenceKindsForCapability, + isUserReportedEvidencePath, + requiresDirectEvidence, +} from "./evidence-validator.js"; + +function artifact(type: string, body: unknown = {}): ArtifactSubmission { + return { + id: `artifact-${type}`, + runId: "run-1", + type, + schemaVersion: "1.0", + producer: "test", + body, + }; +} + +describe("requiresDirectEvidence", () => { + it("matches WorkResult evidence reference paths", () => { + expect(requiresDirectEvidence("body.evidenceRefs[0]", "WorkResult")).toBe( + true, + ); + expect( + requiresDirectEvidence("body.actions[2].evidenceRefs[1]", "WorkResult"), + ).toBe(true); + expect(requiresDirectEvidence("body.observations[0].evidenceRefs[0]")).toBe( + true, + ); + expect(requiresDirectEvidence("body.filesChanged[0].diffRef")).toBe(true); + expect(requiresDirectEvidence("body.testResults[0].commandRef")).toBe(true); + }); + + it("matches QualityReview hard gate evidence paths", () => { + expect( + requiresDirectEvidence( + "body.hardGates[0].evidenceRefs[0]", + "QualityReview", + ), + ).toBe(true); + }); + + it("rejects non-evidence paths", () => { + expect(requiresDirectEvidence("body.summary")).toBe(false); + expect(requiresDirectEvidence("body.evidenceRefs[0]", "WorkPlan")).toBe( + false, + ); + }); +}); + +describe("directEvidenceArtifactTypes", () => { + it("includes expected artifact types", () => { + expect(directEvidenceArtifactTypes).toEqual( + new Set(["Evidence", "ContextDocument", "TraceEvent"]), + ); + }); +}); + +describe("directEvidenceTraceEventTypes", () => { + it("starts empty until trace event types are registered", () => { + expect(directEvidenceTraceEventTypes.size).toBe(0); + }); +}); + +describe("evidenceKindsForCapability", () => { + it("maps repository.read to repository, diff, and tool_output", () => { + expect(evidenceKindsForCapability("repository.read")).toEqual( + new Set(["repository", "diff", "tool_output"]), + ); + }); + + it("maps shell capabilities to tool_output, log, and test", () => { + expect(evidenceKindsForCapability("shell.execute")).toEqual( + new Set(["tool_output", "log", "test"]), + ); + }); + + it("returns an empty set for unknown capabilities", () => { + expect(evidenceKindsForCapability("unknown.capability")).toEqual(new Set()); + }); +}); + +describe("isUserReportedEvidencePath", () => { + it("detects user-reported ReleaseAudit claim evidence", () => { + const submission = artifact("ReleaseAudit", { + claimEvidenceMatrix: [ + { basis: "user_reported", evidenceRefs: ["artifact://run-1/msg-1"] }, + ], + }); + expect( + isUserReportedEvidencePath( + submission, + "body.claimEvidenceMatrix[0].evidenceRefs[0]", + ), + ).toBe(true); + }); + + it("detects user-reported UserReport completion claims", () => { + const submission = artifact("UserReport", { + completionClaims: [ + { + status: "user_reported", + evidenceRefs: ["artifact://run-1/msg-1"], + }, + ], + }); + expect( + isUserReportedEvidencePath( + submission, + "body.completionClaims[0].evidenceRefs[0]", + ), + ).toBe(true); + }); + + it("returns false for other artifact types and paths", () => { + const submission = artifact("WorkResult", { + evidenceRefs: ["artifact://run-1/ev-1"], + }); + expect(isUserReportedEvidencePath(submission, "body.evidenceRefs[0]")).toBe( + false, + ); + }); +}); diff --git a/packages/core/src/controller/evidence-validator.ts b/packages/core/src/controller/evidence-validator.ts new file mode 100644 index 0000000..d47e3c2 --- /dev/null +++ b/packages/core/src/controller/evidence-validator.ts @@ -0,0 +1,108 @@ +import type { ArtifactSubmission } from "../types.js"; + +export function requiresDirectEvidence( + path: string, + submissionType?: string, +): boolean { + return ( + (submissionType === "WorkResult" && + (/^body\.evidenceRefs\[\d+\]$/u.test(path) || + /^body\.actions\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path))) || + /\.claimEvidenceMatrix\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.completionClaims\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.observations\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.inferences\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.toolEventRefs\[\d+\]$/u.test(path) || + /\.verificationRefs\[\d+\]$/u.test(path) || + /\.diagnosisGate\.directEvidenceRefs\[\d+\]$/u.test(path) || + /\.acceptanceCoverage\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.acceptanceResults\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.filesChanged\[\d+\]\.diffRef$/u.test(path) || + /\.testResults\[\d+\]\.(?:commandRef|outputRef)$/u.test(path) || + /\.verificationResults\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path) || + /\.(?:ruleCompliance|regressionChecks|userFidelity)\[\d+\]\.evidenceRefs\[\d+\]$/u.test( + path, + ) || + (submissionType === "QualityReview" && + /\.hardGates\[\d+\]\.evidenceRefs\[\d+\]$/u.test(path)) + ); +} + +export const directEvidenceArtifactTypes = new Set([ + "Evidence", + "ContextDocument", + "TraceEvent", +]); + +export const directEvidenceTraceEventTypes = new Set(); + +export function evidenceKindsForCapability( + capability: string, +): ReadonlySet { + switch (capability) { + case "repository.read": + return new Set(["repository", "diff", "tool_output"]); + case "repository.write": + case "repository.delete": + return new Set(["diff"]); + case "shell.inspect": + case "shell.execute": + return new Set(["tool_output", "log", "test"]); + case "runtime.inspect": + case "runtime.restart": + return new Set(["runtime", "log", "tool_output"]); + case "browser.inspect": + case "browser.mutate": + return new Set(["browser"]); + case "network.read": + case "external.write": + case "subagent.spawn": + return new Set(["tool_output"]); + default: + return new Set(); + } +} + +export function isUserReportedEvidencePath( + submission: ArtifactSubmission, + path: string, +): boolean { + if (submission.type === "ReleaseAudit") { + const match = + /^body\.claimEvidenceMatrix\[(\d+)\]\.evidenceRefs\[\d+\]$/u.exec(path); + const matrix = + typeof submission.body === "object" && + submission.body !== null && + "claimEvidenceMatrix" in submission.body + ? (submission.body as { claimEvidenceMatrix?: unknown }) + .claimEvidenceMatrix + : null; + return ( + match !== null && + Array.isArray(matrix) && + typeof matrix[Number(match[1])] === "object" && + matrix[Number(match[1])] !== null && + (matrix[Number(match[1])] as { basis?: unknown }).basis === + "user_reported" + ); + } + if (submission.type === "UserReport") { + const match = + /^body\.completionClaims\[(\d+)\]\.evidenceRefs\[\d+\]$/u.exec(path); + const claims = + typeof submission.body === "object" && + submission.body !== null && + "completionClaims" in submission.body + ? (submission.body as { completionClaims?: unknown }).completionClaims + : null; + return ( + match !== null && + Array.isArray(claims) && + typeof claims[Number(match[1])] === "object" && + claims[Number(match[1])] !== null && + (claims[Number(match[1])] as { status?: unknown }).status === + "user_reported" + ); + } + return false; +}