From 8103ad11a52f3c6daac9f7015c87425f0d03c2e3 Mon Sep 17 00:00:00 2001 From: evilpsycho42 <202909006+evilpsycho42@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:01:44 +0800 Subject: [PATCH 1/4] fix: preflight workflow output schemas --- scripts/e2e/workflow-features.mjs | 8 +- src/prompts.ts | 6 +- src/workflow/output-schema.ts | 119 ++++++++++++++++++++++++++++++ src/workflow/runtime.ts | 11 +++ src/workflow/script-validation.ts | 110 +++++++++++++++++++++++++-- src/workflow/source.ts | 2 +- src/workflow/types.ts | 2 +- test/workflow-integration.test.ts | 30 +++++++- test/workflow.test.ts | 71 +++++++++++++++++- 9 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 src/workflow/output-schema.ts diff --git a/scripts/e2e/workflow-features.mjs b/scripts/e2e/workflow-features.mjs index 6914e55..ec69605 100644 --- a/scripts/e2e/workflow-features.mjs +++ b/scripts/e2e/workflow-features.mjs @@ -118,7 +118,7 @@ const analyzed = await parallel(targets.map(function (f) { return agent("Read the file " + f + " in this repository and analyze it. Report the file path and the number of exported symbols.", { label: "analyze:" + f, phase: "collect", - schema: { type: "object", required: ["file", "exportCount"], properties: { file: { type: "string" }, exportCount: { type: "number" } } } + schema: { type: "object", additionalProperties: false, required: ["file", "exportCount"], properties: { file: { type: "string" }, exportCount: { type: "number" } } } }); }; })); @@ -165,7 +165,7 @@ const flags = await parallel(files.map(function (f) { return function () { return agent("Does the file " + f + " import the 'kleur' package? Answer strictly from its source.", { label: "flag:" + f, - schema: { type: "object", required: ["file", "importsKleur"], properties: { file: { type: "string" }, importsKleur: { type: "boolean" } } } + schema: { type: "object", additionalProperties: false, required: ["file", "importsKleur"], properties: { file: { type: "string" }, importsKleur: { type: "boolean" } } } }); }; })); @@ -184,7 +184,7 @@ const flags = await parallel(files.map(function (f) { return function () { return agent("Does " + f + " import the 'kleur' package? Answer strictly from its source.", { label: "scan:" + f, - schema: { type: "object", required: ["file", "importsKleur"], properties: { file: { type: "string" }, importsKleur: { type: "boolean" } } } + schema: { type: "object", additionalProperties: false, required: ["file", "importsKleur"], properties: { file: { type: "string" }, importsKleur: { type: "boolean" } } } }); }; })); @@ -202,7 +202,7 @@ const ROUTE_PROBE = `export const meta = { name: "route_probe", description: "Cl const target = args.file; const c = await agent("Classify " + target + " as exactly one of: entry (a CLI entry point, e.g. reads process.argv or is declared as a bin), lib (an imported helper module), or test (a test file). Judge strictly from its source and role.", { label: "classify", - schema: { type: "object", required: ["kind"], properties: { kind: { type: "string", enum: ["entry", "lib", "test"] } } } + schema: { type: "object", additionalProperties: false, required: ["kind"], properties: { kind: { type: "string", enum: ["entry", "lib", "test"] } } } }); let follow; if (c && c.kind === "entry") follow = await agent("List the command-line argument(s) " + target + " reads. Plain text only.", { label: "route:entry" }); diff --git a/src/prompts.ts b/src/prompts.ts index 57c36d6..9f97c1e 100644 --- a/src/prompts.ts +++ b/src/prompts.ts @@ -32,7 +32,7 @@ export const WORKFLOW_PROMPT_GUIDELINES = [ "parallel() takes functions, not promises: `await parallel(items.map(item => () => agent('...', { label: '...' })))`. Results come back in input order.", "pipeline(items, ...stages) runs each item through the stages in order while different items run concurrently; each stage receives (previousValue, originalItem, index). Prefer pipeline() for multi-stage work — there is no barrier between stages. Reach for parallel() only when you genuinely need all results together, e.g. dedup or a zero-count early exit.", "Give each agent() a unique short `label` and pick a `subagent_type` (defaults to general-purpose) so it uses that profile's configured backend, model, thinking level, prompt, and pi-backend tool allowlist. Pass `session_key` only when you intentionally want to continue a prior subagent conversation.", - "Pass a JSON Schema as agent()'s `schema` option whenever the script must branch, route, filter, or aggregate on a result: the subagent is forced to return one validated object (agent() resolves to that object instead of text), so `if (r.kind === ...)` / `flags.filter(...)` are reliable. Omit `schema` for prose findings you only read or synthesize.", + "Pass a portable strict JSON Schema as agent()'s `schema` option whenever the script must branch, route, filter, or aggregate on a result: every object must set `additionalProperties: false`, every property must be listed in `required`, and optional values must use a nullable type. Schemas must be static object literals or top-level consts so workflow preflight can validate all of them before any subagent starts. Omit `schema` for prose findings you only read or synthesize.", "When `session_key` is omitted, subagents are fresh one-shot sessions with no parent context. Pi-backed subagents do not receive Agent/workflow; external CLI backends use their own tool surface. Include all needed context and paths in each fresh agent() prompt.", "Failed agent()/parallel()/pipeline() branches resolve to null and are logged unless the workflow is aborted; check for nulls before synthesizing.", ]; @@ -78,9 +78,9 @@ Inline script contract: Each agent() spawns a fresh one-shot subagent unless you pass \`session_key\` to create or continue a resumable child conversation. Set \`subagent_type\` to use a profile's backend, model, thinking, prompt, and pi-backend tool allowlist: ${formatAvailableAgents(profiles)} -agent() options: \`label\` (short unique id), \`phase\` (progress group), \`subagent_type\` (profile above), \`session_key\` (caller-chosen key for a resumable child conversation), and \`schema\` (a JSON Schema). Pass \`schema\` when the script must branch, route, filter, or aggregate on the result: the subagent is forced to return one validated object and agent() resolves to that object instead of free text. Omit \`schema\` for prose findings you only synthesize. Example — classify, then dispatch: +agent() options: \`label\` (short unique id), \`phase\` (progress group), \`subagent_type\` (profile above), \`session_key\` (caller-chosen key for a resumable child conversation), and \`schema\` (a portable strict JSON Schema). Pass \`schema\` when the script must branch, route, filter, or aggregate on the result: the subagent is forced to return one validated object and agent() resolves to that object instead of free text. Every object schema must set \`additionalProperties: false\`, list every property in \`required\`, and represent optional values with nullable types. Define schemas as static object literals or top-level consts so preflight can reject invalid schemas before any subagent starts. Omit \`schema\` for prose findings you only synthesize. Example — classify, then dispatch: \`\`\` -const r = await agent("Classify " + file, { label: "classify", schema: { type: "object", required: ["kind"], properties: { kind: { type: "string", enum: ["entry", "lib", "test"] } } } }); +const r = await agent("Classify " + file, { label: "classify", schema: { type: "object", additionalProperties: false, required: ["kind"], properties: { kind: { type: "string", enum: ["entry", "lib", "test"] } } } }); if (r.kind === "entry") { /* ... */ } \`\`\` diff --git a/src/workflow/output-schema.ts b/src/workflow/output-schema.ts new file mode 100644 index 0000000..27047db --- /dev/null +++ b/src/workflow/output-schema.ts @@ -0,0 +1,119 @@ +const JSON_SCHEMA_TYPES = new Set(["array", "boolean", "integer", "null", "number", "object", "string"]); + +function isRecord(value: unknown): value is Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function appendPath(path: string, key: string): string { + return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? `${path}.${key}` : `${path}[${JSON.stringify(key)}]`; +} + +function fail(path: string, message: string): never { + throw new Error(`${path}: ${message}`); +} + +function assertJsonValue(value: unknown, path: string, seen: WeakSet): void { + if (value === null || typeof value === "string" || typeof value === "boolean") return; + if (typeof value === "number") { + if (!Number.isFinite(value)) fail(path, "must contain only finite JSON numbers"); + return; + } + if (typeof value !== "object") fail(path, "must contain only JSON values"); + if (seen.has(value)) fail(path, "must not contain circular references"); + seen.add(value); + try { + if (Array.isArray(value)) { + value.forEach((item, index) => assertJsonValue(item, `${path}[${index}]`, seen)); + return; + } + if (!isRecord(value)) fail(path, "must contain only plain JSON objects"); + for (const [key, child] of Object.entries(value)) { + assertJsonValue(child, appendPath(path, key), seen); + } + } finally { + seen.delete(value); + } +} + +function schemaTypes(schema: Record, path: string): string[] { + if (schema.type === undefined) return []; + const types = Array.isArray(schema.type) ? schema.type : [schema.type]; + if (types.length === 0 || types.some((type) => typeof type !== "string" || !JSON_SCHEMA_TYPES.has(type))) { + fail(appendPath(path, "type"), "must be a valid JSON Schema type or non-empty array of types"); + } + if (new Set(types).size !== types.length) { + fail(appendPath(path, "type"), "must not contain duplicate types"); + } + return types as string[]; +} + +function schemaRecord(value: unknown, path: string): Record { + if (!isRecord(value)) fail(path, "must be a schema object"); + return value; +} + +function validateSchemaNode(schema: Record, path: string): void { + const types = schemaTypes(schema, path); + const hasObjectShape = types.includes("object") || schema.properties !== undefined; + if (hasObjectShape) { + if (!types.includes("object")) { + fail(appendPath(path, "type"), 'must include "object" when properties are defined'); + } + if (schema.additionalProperties !== false) { + fail(appendPath(path, "additionalProperties"), "must be false for every object schema"); + } + const propertiesPath = appendPath(path, "properties"); + const properties = schemaRecord(schema.properties, propertiesPath); + const propertyNames = Object.keys(properties); + if (!Array.isArray(schema.required) || schema.required.some((name) => typeof name !== "string")) { + fail(appendPath(path, "required"), "must be an array containing every property name"); + } + const required = schema.required as string[]; + if (new Set(required).size !== required.length) { + fail(appendPath(path, "required"), "must not contain duplicate property names"); + } + const missing = propertyNames.filter((name) => !required.includes(name)); + if (missing.length > 0) { + fail(appendPath(path, "required"), `must include every property; missing: ${missing.join(", ")}`); + } + const unknown = required.filter((name) => !Object.hasOwn(properties, name)); + if (unknown.length > 0) { + fail(appendPath(path, "required"), `contains names not present in properties: ${unknown.join(", ")}`); + } + for (const [name, child] of Object.entries(properties)) { + const childPath = appendPath(propertiesPath, name); + validateSchemaNode(schemaRecord(child, childPath), childPath); + } + } + + if (schema.items !== undefined) { + validateSchemaNode(schemaRecord(schema.items, appendPath(path, "items")), appendPath(path, "items")); + } + if (schema.anyOf !== undefined) { + if (!Array.isArray(schema.anyOf) || schema.anyOf.length === 0) { + fail(appendPath(path, "anyOf"), "must be a non-empty array of schema objects"); + } + schema.anyOf.forEach((child, index) => { + const childPath = `${appendPath(path, "anyOf")}[${index}]`; + validateSchemaNode(schemaRecord(child, childPath), childPath); + }); + } + if (schema.$defs !== undefined) { + const definitions = schemaRecord(schema.$defs, appendPath(path, "$defs")); + for (const [name, child] of Object.entries(definitions)) { + const childPath = appendPath(appendPath(path, "$defs"), name); + validateSchemaNode(schemaRecord(child, childPath), childPath); + } + } +} + +export function assertPortableOutputSchema(schema: unknown): void { + assertJsonValue(schema, "$", new WeakSet()); + const root = schemaRecord(schema, "$"); + if (root.type !== "object") { + fail("$.type", 'root schema must have type "object"'); + } + validateSchemaNode(root, "$"); +} diff --git a/src/workflow/runtime.ts b/src/workflow/runtime.ts index ce9e39c..e9c1ed6 100644 --- a/src/workflow/runtime.ts +++ b/src/workflow/runtime.ts @@ -1,3 +1,4 @@ +import { assertPortableOutputSchema } from "./output-schema.ts"; import { parseWorkflowScript } from "./script-validation.ts"; import { fingerprintWorkflowAgentCall } from "./replay-cache.ts"; import { createWorkflowScriptWorker, type ParentToWorkerMessage, type WorkerToParentMessage } from "./script-worker.ts"; @@ -151,6 +152,16 @@ export async function runWorkflow( } const taskPrompt = requireString(prompt, "agent prompt"); const opts = normalizeAgentOptions(agentOptions); + if (opts.schema != null) { + try { + assertPortableOutputSchema(opts.schema); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const fatal = new WorkflowFatalError(`Workflow schema validation failed before launching agent: ${detail}`); + abortRuntime(fatal); + throw fatal; + } + } const assignedPhase = opts.phase ?? state.currentPhase; const subagentType = opts.subagentType ?? defaultSubagentType; diff --git a/src/workflow/script-validation.ts b/src/workflow/script-validation.ts index b59bd7a..5e58be1 100644 --- a/src/workflow/script-validation.ts +++ b/src/workflow/script-validation.ts @@ -1,7 +1,13 @@ import { parse, type Node } from "acorn"; +import { assertPortableOutputSchema } from "./output-schema.ts"; import type { WorkflowMeta, WorkflowMetaPhase } from "./types.ts"; -type AnyNode = Node & { [key: string]: any; start: number; end: number }; +type AnyNode = Node & { + [key: string]: any; + start: number; + end: number; + loc?: { start: { line: number; column: number } }; +}; const NONDETERMINISM_ERROR = "Workflow scripts must be deterministic: Date APIs and Math.random() (including simple aliases) are unavailable"; @@ -12,6 +18,7 @@ export function parseWorkflowScript(script: string): { meta: WorkflowMeta; body: sourceType: "module", allowAwaitOutsideFunction: true, allowReturnOutsideFunction: true, + locations: true, ranges: false, }) as unknown as AnyNode; @@ -40,6 +47,7 @@ export function parseWorkflowScript(script: string): { meta: WorkflowMeta; body: const meta = evaluateLiteral(declarator.init, "meta"); validateMeta(meta); + assertStaticWorkflowSchemas(ast); return { meta, @@ -47,20 +55,27 @@ export function parseWorkflowScript(script: string): { meta: WorkflowMeta; body: }; } -function evaluateLiteral(node: AnyNode, path: string): unknown { +function evaluateLiteral( + node: AnyNode, + path: string, + resolveIdentifier?: (name: string, path: string) => unknown, + allowReservedKeys = false, +): unknown { switch (node.type) { case "ObjectExpression": { - const out: Record = {}; + const out: Record = allowReservedKeys + ? Object.create(null) as Record + : {}; for (const prop of node.properties as AnyNode[]) { if (prop.type === "SpreadElement") throw new Error(`spread not allowed in ${path}`); if (prop.type !== "Property") throw new Error(`only plain properties allowed in ${path}`); if (prop.computed) throw new Error(`computed keys not allowed in ${path}`); if (prop.kind !== "init" || prop.method) throw new Error(`methods/accessors not allowed in ${path}`); const key = propertyKey(prop.key as AnyNode, path); - if (key === "__proto__" || key === "constructor" || key === "prototype") { + if (!allowReservedKeys && (key === "__proto__" || key === "constructor" || key === "prototype")) { throw new Error(`reserved key name not allowed in ${path}: ${key}`); } - out[key] = evaluateLiteral(prop.value as AnyNode, `${path}.${key}`); + out[key] = evaluateLiteral(prop.value as AnyNode, `${path}.${key}`, resolveIdentifier, allowReservedKeys); } return out; } @@ -68,7 +83,7 @@ function evaluateLiteral(node: AnyNode, path: string): unknown { return (node.elements as Array).map((element, index) => { if (!element) throw new Error(`sparse arrays not allowed in ${path}`); if (element.type === "SpreadElement") throw new Error(`spread not allowed in ${path}`); - return evaluateLiteral(element, `${path}[${index}]`); + return evaluateLiteral(element, `${path}[${index}]`, resolveIdentifier, allowReservedKeys); }); case "Literal": return node.value; @@ -80,11 +95,94 @@ function evaluateLiteral(node: AnyNode, path: string): unknown { return -node.argument.value; } throw new Error(`only negative-number unary allowed in ${path}`); + case "Identifier": + if (resolveIdentifier) return resolveIdentifier(node.name, path); + throw new Error(`non-literal node type in ${path}: ${node.type}`); default: throw new Error(`non-literal node type in ${path}: ${node.type}`); } } +function collectTopLevelConstants(ast: AnyNode): Map { + const constants = new Map(); + for (const statement of ast.body as AnyNode[]) { + if (statement.type !== "VariableDeclaration" || statement.kind !== "const") continue; + for (const declarator of statement.declarations as AnyNode[]) { + if (declarator.id?.type === "Identifier" && declarator.init) { + constants.set(declarator.id.name, declarator.init as AnyNode); + } + } + } + return constants; +} + +function resolveStaticObject(node: AnyNode, constants: Map): AnyNode | undefined { + if (node.type === "ObjectExpression") return node; + if (node.type !== "Identifier") return undefined; + const value = constants.get(node.name); + return value?.type === "ObjectExpression" ? value : undefined; +} + +function staticSchemaNode(call: AnyNode, constants: Map): AnyNode | undefined { + const optionsNode = call.arguments?.[1] as AnyNode | undefined; + if (!optionsNode) return undefined; + const options = resolveStaticObject(optionsNode, constants); + if (!options) return undefined; + + let schemaNode: AnyNode | undefined; + for (const prop of options.properties as AnyNode[]) { + if (prop.type !== "Property" || prop.kind !== "init" || prop.method) continue; + if (propertyNameOfPatternProperty(prop) === "schema") { + schemaNode = prop.value as AnyNode; + } + } + return schemaNode; +} + +function evaluateStaticSchema(node: AnyNode, constants: Map): unknown { + if (node.type !== "ObjectExpression" && node.type !== "Identifier") { + throw new Error("schema must be a static object literal or reference a top-level const"); + } + const resolving = new Set(); + const resolveIdentifier = (name: string, path: string): unknown => { + const value = constants.get(name); + if (!value) { + throw new Error(`${path} must be a static object literal or reference a top-level const`); + } + if (resolving.has(name)) { + throw new Error(`${path} contains a circular const reference: ${name}`); + } + resolving.add(name); + try { + return evaluateLiteral(value, path, resolveIdentifier, true); + } finally { + resolving.delete(name); + } + }; + return evaluateLiteral(node, "schema", resolveIdentifier, true); +} + +function assertStaticWorkflowSchemas(ast: AnyNode): void { + const constants = collectTopLevelConstants(ast); + const visit = (node: AnyNode): void => { + if (node.type === "CallExpression" && node.callee?.type === "Identifier" && node.callee.name === "agent") { + const schemaNode = staticSchemaNode(node, constants); + if (schemaNode && !(schemaNode.type === "Literal" && schemaNode.value == null)) { + const location = schemaNode.loc?.start ?? node.loc?.start; + const where = location ? ` at line ${location.line}, column ${location.column + 1}` : ""; + try { + assertPortableOutputSchema(evaluateStaticSchema(schemaNode, constants)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Workflow schema preflight failed for agent()${where}: ${message}`); + } + } + } + for (const child of astChildren(node)) visit(child); + }; + visit(ast); +} + function propertyKey(node: AnyNode, path: string): string { if (node.type === "Identifier") return node.name; if (node.type === "Literal" && (typeof node.value === "string" || typeof node.value === "number")) { diff --git a/src/workflow/source.ts b/src/workflow/source.ts index 0d74980..1ae0644 100644 --- a/src/workflow/source.ts +++ b/src/workflow/source.ts @@ -199,7 +199,7 @@ export async function prepareWorkflowToolSource( plannedPhases = parsed.meta.phases?.map((phase) => ({ ...phase })); } catch (error) { const message = error instanceof Error ? error.message : String(error); - return sourceError(`Workflow script is invalid: ${message}`, { + return sourceError(`Workflow script is invalid; no subagents were started: ${message}`, { name: metaName, error: message, logs: source.warnings, diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 61d4e0d..11a036a 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -23,7 +23,7 @@ export interface WorkflowAgentCall { sessionKey?: string; /** Backend-native session/thread id captured for workflow replay bookkeeping. */ sessionId?: string; - /** JSON Schema for structured output from the child subagent. */ + /** Portable strict JSON Schema for structured output from the child subagent. */ schema?: unknown; } diff --git a/test/workflow-integration.test.ts b/test/workflow-integration.test.ts index cdc071b..aa0aa76 100644 --- a/test/workflow-integration.test.ts +++ b/test/workflow-integration.test.ts @@ -76,6 +76,32 @@ describe("pi-subagent workflow integration", () => { disposeSession(session); }); + it("rejects invalid schemas before starting any workflow subagent", async () => { + const { session, registration, model, modelRegistry } = await createSession(); + const tool = session.getToolDefinition("workflow") as any; + registration.setResponses([fauxAssistantMessage("must remain unused")]); + + const script = `export const meta = { name: 'invalid_schema', description: 'preflight invalid schemas' }; +const schema = { type: 'object', required: ['answer'], properties: { answer: { type: 'string' } } }; +await agent('first', { label: 'first' }); +return await agent('second', { label: 'second', schema });`; + const result = await tool.execute( + "wf-invalid-schema", + { script }, + undefined, + undefined, + makeExecutionContext({ hasUI: false, model, modelRegistry }), + ); + + expect(result.details.status).toBe("error"); + expect(result.details.agentCount).toBe(0); + expect(result.content[0].text).toContain("no subagents were started"); + expect(result.content[0].text).toMatch(/schema preflight.*additionalProperties/i); + expect(registration.getPendingResponseCount()).toBe(1); + + disposeSession(session); + }); + it("continues a pi workflow subagent when agent() reuses session_key", async () => { const { session, registration, model, modelRegistry } = await createSession(); const tool = session.getToolDefinition("workflow") as any; @@ -321,7 +347,7 @@ return [a, b];`; const script = `export const meta = { name: 'solve', description: 'solve a task' }; return await agent('compute the answer', { label: 'solver', - schema: { type: 'object', properties: { answer: { type: 'string' }, confidence: { type: 'number' } }, required: ['answer'] }, + schema: { type: 'object', additionalProperties: false, properties: { answer: { type: 'string' }, confidence: { type: 'number' } }, required: ['answer', 'confidence'] }, });`; const result = await tool.execute( "wf-struct", @@ -368,7 +394,7 @@ console.log(JSON.stringify({ type: 'result', subtype: 'success', is_error: false return await agent('compute the answer', { label: 'solver', subagent_type: 'claude-struct', - schema: { type: 'object', properties: { answer: { type: 'string' }, confidence: { type: 'number' } }, required: ['answer'] }, + schema: { type: 'object', additionalProperties: false, properties: { answer: { type: 'string' }, confidence: { type: 'number' } }, required: ['answer', 'confidence'] }, });`; const result = await tool.execute( "wf-claude-struct", diff --git a/test/workflow.test.ts b/test/workflow.test.ts index 32ce3cd..40ef73c 100644 --- a/test/workflow.test.ts +++ b/test/workflow.test.ts @@ -80,7 +80,38 @@ describe("parseWorkflowScript", () => { }); it("allows Date as a deterministic data field name", () => { - expect(() => parseWorkflowScript(`${META}const schema = { type: 'object', properties: { Date: { type: 'string' } } };\nreturn await agent('x', { schema });`)).not.toThrow(); + expect(() => parseWorkflowScript(`${META}const schema = { type: 'object', additionalProperties: false, required: ['Date'], properties: { Date: { type: 'string' } } };\nreturn await agent('x', { schema });`)).not.toThrow(); + }); + + it("preflights static structured output schemas", () => { + const valid = `${META}const schema = { + type: 'object', additionalProperties: false, required: ['items'], properties: { + items: { type: 'array', items: { type: 'object', additionalProperties: false, required: ['name'], properties: { name: { type: 'string' } } } } + } +};\nreturn await agent('x', { schema });`; + expect(() => parseWorkflowScript(valid)).not.toThrow(); + + const missingAdditionalProperties = `${META}const schema = { type: 'object', required: ['name'], properties: { name: { type: 'string' } } };\nreturn await agent('x', { schema });`; + expect(() => parseWorkflowScript(missingAdditionalProperties)).toThrow(/preflight.*\$\.additionalProperties.*must be false/i); + + const missingRequiredProperty = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['name'], properties: { name: { type: 'string' }, note: { type: ['string', 'null'] } } } });`; + expect(() => parseWorkflowScript(missingRequiredProperty)).toThrow(/\$\.required.*missing: note/i); + + const invalidNestedObject = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['item'], properties: { item: { type: 'object', required: ['name'], properties: { name: { type: 'string' } } } } } });`; + expect(() => parseWorkflowScript(invalidNestedObject)).toThrow(/\$\.properties\.item\.additionalProperties.*must be false/i); + + const dataNamedProperties = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['properties', 'metadata'], properties: { properties: { type: 'string' }, metadata: { type: 'object', additionalProperties: false, required: ['value'], properties: { value: { type: 'string', enum: [{ type: 'object' }] } } } } } });`; + expect(() => parseWorkflowScript(dataNamedProperties)).not.toThrow(); + + const invalidPropertySchema = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['answer'], properties: { answer: 42 } } });`; + expect(() => parseWorkflowScript(invalidPropertySchema)).toThrow(/properties\.answer.*schema object/i); + + const invalidType = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['answer'], properties: { answer: { type: 'wat' } } } });`; + expect(() => parseWorkflowScript(invalidType)).toThrow(/properties\.answer\.type.*valid JSON Schema type/i); + }); + + it("requires schemas to be statically available during preflight", () => { + expect(() => parseWorkflowScript(`${META}return await agent('x', { schema: args.schema });`)).toThrow(/static object literal|top-level const/i); }); it("rejects non-literal meta", () => { @@ -102,6 +133,44 @@ describe("runWorkflow", () => { expect(result.agentCount).toBe(1); }); + it("rejects invalid schemas before launching any agent", async () => { + let calls = 0; + const runAgent: WorkflowAgentRunner = async () => { + calls += 1; + return {}; + }; + await expect( + runWorkflow(`${META}await agent('first'); +return await agent('second', { schema: { type: 'object', required: ['answer'], properties: { answer: { type: 'string' } } } });`, { + cwd: "/tmp", + limiter: new ConcurrencyLimiter(4), + runAgent, + }), + ).rejects.toThrow(/schema preflight.*additionalProperties/i); + expect(calls).toBe(0); + }); + + it("validates dynamic schema options before launching the requested agent", async () => { + let calls = 0; + const runAgent: WorkflowAgentRunner = async () => { + calls += 1; + return {}; + }; + await expect( + runWorkflow(`${META}return await agent('x', args.options);`, { + args: { + options: { + schema: { type: "object", required: ["answer"], properties: { answer: { type: "string" } } }, + }, + }, + cwd: "/tmp", + limiter: new ConcurrencyLimiter(4), + runAgent, + }), + ).rejects.toThrow(/schema validation failed.*additionalProperties/i); + expect(calls).toBe(0); + }); + it("requires at least one agent call", async () => { await expect( runWorkflow(`${META}return 'no agents';`, { From c4109e55df386b4ab249fd84e861d61df153ed34 Mon Sep 17 00:00:00 2001 From: evilpsycho42 <202909006+evilpsycho42@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:20:56 +0800 Subject: [PATCH 2/4] no-mistakes(document): docs: update AGENTS.md for schema preflight contract --- AGENTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b498168..420215e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,12 +45,12 @@ - Opt-in via `createFlowExtension({ workflow })` (compatibility alias: `createSubagentExtension`); defaults to `true`. Set `false` for a subagents-only surface (then only `Agent` registers and no workflow prompt is appended). - The `workflow` tool runs a trusted, model-written JavaScript script in an isolated Worker-hosted `node:vm` context so pi can detect stalls and abort unresponsive scripts. Initial synchronous execution is bounded (5s by default), and post-`await` event-loop stalls are caught by a heartbeat watchdog. This is not a security sandbox; saved workflows are trusted code like extensions, and inline workflows are model-written code executed by the local process. Globals: `agent(prompt, opts)`, `parallel(thunks)`, `pipeline(items, ...stages)`, `phase(title)`, `log(message)`, `args`, `cwd`. The script must start with `export const meta = { name, description }` (a plain literal) and call `agent()` at least once. - Determinism is a cooperative parse-time lint via an `acorn` AST scan: Date APIs and `Math.random()` uses, including simple aliases/destructuring, are rejected for normal model-written scripts. Dynamic authoring, deterministic-by-convention execution. The scan checks determinism ONLY — it intentionally permits ordinary computed member access (`obj[key]`, `arr[i]`, `{ [k]: v }`) except static `Math['random']`, and does not attempt vm-escape hardening. Do not claim malicious JavaScript is sandboxed. -- `agent()` reuses the shared spawn core, so a `subagent_type` selects a real profile and the subagent gets that profile's configured backend, model, thinking level, prompt, optional `session_key`, and (for pi-backed profiles only) tool allow-list — not stubbed guidance. `agent({ schema })` returns a schema-validated object: pi-backed subagents receive a terminating `structured_output` tool via `createAgentSession`'s `customTools` with the profile tool allow-list extended to admit it, while Codex-backed subagents use Codex CLI `--output-schema` and Claude-backed subagents use Claude Code `--json-schema`. The first successful structured result is captured; duplicate successful calls are ignored. +- `agent()` reuses the shared spawn core, so a `subagent_type` selects a real profile and the subagent gets that profile's configured backend, model, thinking level, prompt, optional `session_key`, and (for pi-backed profiles only) tool allow-list — not stubbed guidance. `agent({ schema })` requires a portable strict JSON Schema: every object must have `additionalProperties: false` and list all its properties in `required`; optional values use nullable types. Static schemas (inline object literals and top-level `const` references) are validated at script parse time before any subagent starts; dynamic options are validated at runtime before the requested agent launches. On a valid schema the subagent is forced to return one validated object: pi-backed subagents receive a terminating `structured_output` tool via `createAgentSession`'s `customTools` with the profile tool allow-list extended to admit it, while Codex-backed subagents use Codex CLI `--output-schema` and Claude-backed subagents use Claude Code `--json-schema`. The first successful structured result is captured; duplicate successful calls are ignored. - Concurrency is the SAME global cap as `Agent`: both tools share one `ConcurrencyLimiter`. Normal `Agent` calls and workflow `agent()` calls both queue and drain via `acquire`; the cap limits simultaneously running subagents, not total requested subagents. The `workflow` tool itself does not consume a slot; only its `agent()` calls do. A workflow also has hard caps on total `agent()` calls, retained logs, and orchestration-worker memory (512MB old generation by default; subagent/tool subprocess memory is not included). - Foreground-only still holds: the `workflow` tool blocks until the script completes. No background execution, polling, steering, or scheduling — orchestration is front-loaded into the script, not a reactive coordinator. V3 adds foreground resume-by-replay using a run journal. Per-call model/thinking *override* remains out of contract; profile-based selection via `subagent_type` is the supported path. - Nesting is hard-blocked for pi-backed workflow subagents: they get neither `Agent` nor `workflow`. External CLI backends use their own tool surface; this extension does not try to prevent nested/delegation features inside those CLIs. - Do not put exact concurrency values in the model-facing workflow prompt; say fan-out is bounded and queued. -- Architecture: `src/core/{spawn,concurrency,model,progress,stream}.ts` is the shared core; `src/workflow/{runtime,tool,structured-output}.ts` is the workflow layer; `src/pi-subagent.ts` wires both tools and shares one limiter. Adds an `acorn` dependency (the only runtime dependency). +- Architecture: `src/core/{spawn,concurrency,model,progress,stream}.ts` is the shared core; `src/workflow/{runtime,tool,structured-output,output-schema}.ts` is the workflow layer; `src/pi-subagent.ts` wires both tools and shares one limiter. Adds an `acorn` dependency (the only runtime dependency). - The throttled progress-emit + heartbeat machinery lives ONCE in `progress.ts` as `createProgressEmitter` and is shared by all three backends (`spawn.ts` pi, `codex.ts`, `claude.ts`); do not re-inline per-backend copies. The queued→running and abort emit timing is owned by that emitter. - External-CLI backends bound parent-side child output via `createBoundedBuffer` (`stream.ts`): stderr is capped (`MAX_STDERR_CHARS`) and a single newline-free stdout line over `MAX_STDOUT_LINE_CHARS` aborts/fails the run clearly, so one runaway subagent cannot OOM the host pi process. A clean exit (code 0) with usable final text but no recognized terminal event is accepted rather than failed, so a CLI stream-format change does not turn good runs into failures. @@ -64,7 +64,7 @@ - The `workflow` tool now accepts exactly one source: inline `script` for ad-hoc orchestration, `name` for a saved workflow, or `scriptPath` for a persisted script. `args` is still exposed to the script as the `args` global. - Saved workflow files are plain JavaScript under `~/.pi/agent/workflows/*.js` (global) and trusted `.pi/workflows/*.js` (project-local). There is no per-workflow slash command surface; the agent discovers saved workflows from the prompt roster and invokes `workflow({ name, args })` from natural language. -- Project workflows are loaded only when `ctx.isProjectTrusted()` is true. Saved files are realpath-checked to stay inside an allowed workflow root, must end in `.js`, and are parsed with the same `export const meta = { name, description }` plus determinism-lint validator before every run. Never auto-run on discovery. +- Project workflows are loaded only when `ctx.isProjectTrusted()` is true. Saved files are realpath-checked to stay inside an allowed workflow root, must end in `.js`, and are parsed with the same `export const meta = { name, description }` plus determinism-lint and schema-preflight validators before every run. Never auto-run on discovery. - Workflow identity is `meta.name`; valid saved names match lowercase letters/digits plus `_` or `-`. Project workflows override global workflows with the same name. - The root prompt includes a compact saved-workflow roster (`name`, `description`) when workflows exist. Put both summary and “when to use” routing guidance in `description`; do not include script bodies in the prompt. - Inline workflow runs auto-persist their script under the current persisted session's workflow directory and return `scriptPath`, `runId`, and `journalPath` in tool details. In-memory sessions may run without persistence. From 330f74096785b8372cfd00f016a31af7f701724a Mon Sep 17 00:00:00 2001 From: evilpsycho42 <202909006+evilpsycho42@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:16:17 +0800 Subject: [PATCH 3/4] no-mistakes(review): reject allOf and oneOf in portable strict schema preflight --- src/workflow/output-schema.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/workflow/output-schema.ts b/src/workflow/output-schema.ts index 27047db..ad6bbb6 100644 --- a/src/workflow/output-schema.ts +++ b/src/workflow/output-schema.ts @@ -100,6 +100,12 @@ function validateSchemaNode(schema: Record, path: string): void validateSchemaNode(schemaRecord(child, childPath), childPath); }); } + if (schema.allOf !== undefined) { + fail(appendPath(path, "allOf"), "allOf is not supported in the portable strict schema subset; use anyOf with nullable types instead"); + } + if (schema.oneOf !== undefined) { + fail(appendPath(path, "oneOf"), "oneOf is not supported in the portable strict schema subset; use anyOf with nullable types instead"); + } if (schema.$defs !== undefined) { const definitions = schemaRecord(schema.$defs, appendPath(path, "$defs")); for (const [name, child] of Object.entries(definitions)) { From ad3429665f0ce0db50117b9423762b86b3e73cda Mon Sep 17 00:00:00 2001 From: evilpsycho42 <202909006+evilpsycho42@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:21:34 +0800 Subject: [PATCH 4/4] no-mistakes(document): update schema preflight contract in AGENTS.md and prompts --- test/workflow.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/workflow.test.ts b/test/workflow.test.ts index 40ef73c..85aef73 100644 --- a/test/workflow.test.ts +++ b/test/workflow.test.ts @@ -108,6 +108,15 @@ describe("parseWorkflowScript", () => { const invalidType = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['answer'], properties: { answer: { type: 'wat' } } } });`; expect(() => parseWorkflowScript(invalidType)).toThrow(/properties\.answer\.type.*valid JSON Schema type/i); + + const allOfAtRoot = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['x'], properties: { x: { type: 'string' } }, allOf: [{ type: 'object' }] } });`; + expect(() => parseWorkflowScript(allOfAtRoot)).toThrow(/\$\.allOf.*allOf is not supported/i); + + const oneOfAtRoot = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['x'], properties: { x: { type: 'string' } }, oneOf: [{ type: 'object', additionalProperties: false, required: [], properties: {} }] } });`; + expect(() => parseWorkflowScript(oneOfAtRoot)).toThrow(/\$\.oneOf.*oneOf is not supported/i); + + const allOfNested = `${META}return await agent('x', { schema: { type: 'object', additionalProperties: false, required: ['item'], properties: { item: { type: 'object', additionalProperties: false, required: ['val'], properties: { val: { type: 'string' } }, allOf: [] } } } });`; + expect(() => parseWorkflowScript(allOfNested)).toThrow(/\$\.properties\.item\.allOf.*allOf is not supported/i); }); it("requires schemas to be statically available during preflight", () => {