From 226e699b27f69c1e4b5e225684252d34665f22a2 Mon Sep 17 00:00:00 2001 From: Michael Livshits Date: Sun, 31 May 2026 10:06:31 +0300 Subject: [PATCH] Fix determinism checks for prompt mentions --- src/workflow.ts | 67 +++++++++++++++++++++++++++--- tests/workflow-parser.test.ts | 76 ++++++++++++++++++++++++++++------ tests/workflow-runtime.test.ts | 20 +++++++++ 3 files changed, 146 insertions(+), 17 deletions(-) diff --git a/src/workflow.ts b/src/workflow.ts index 8d66598..b7f03ac 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -57,7 +57,8 @@ interface RuntimeState { type AnyNode = Node & { [key: string]: any; start: number; end: number }; -const DETERMINISM_BLOCKLIST = /\bDate\s*\.\s*now\b|\bMath\s*\.\s*random\b|\bnew\s+Date\s*\(\s*\)/; +const NONDETERMINISM_ERROR = + "Workflow scripts must be deterministic: Date.now()/Math.random()/new Date() are unavailable"; export async function runWorkflow( script: string, @@ -225,10 +226,6 @@ export async function runWorkflow( } export function parseWorkflowScript(script: string): { meta: WorkflowMeta; body: string } { - if (DETERMINISM_BLOCKLIST.test(script)) { - throw new Error("Workflow scripts must be deterministic: Date.now()/Math.random()/new Date() are unavailable"); - } - const ast = parse(script, { ecmaVersion: "latest", sourceType: "module", @@ -237,6 +234,8 @@ export function parseWorkflowScript(script: string): { meta: WorkflowMeta; body: ranges: false, }) as AnyNode; + assertDeterministicAst(ast); + const first = ast.body?.[0] as AnyNode | undefined; if (first?.type !== "ExportNamedDeclaration") { throw new Error("`export const meta = { name, description }` must be the first statement in the script"); @@ -310,6 +309,64 @@ function propertyKey(node: AnyNode, path: string): string { throw new Error(`unsupported key type in ${path}: ${node.type}`); } +function assertDeterministicAst(node: AnyNode): void { + if (isDateNowCall(node) || isMathRandomCall(node) || isNewDateExpression(node)) { + throw new Error(NONDETERMINISM_ERROR); + } + + for (const child of astChildren(node)) assertDeterministicAst(child); +} + +function astChildren(node: AnyNode): AnyNode[] { + const children: AnyNode[] = []; + for (const value of Object.values(node)) { + if (Array.isArray(value)) children.push(...value.filter(isAstNode)); + else if (isAstNode(value)) children.push(value); + } + return children; +} + +function isAstNode(value: unknown): value is AnyNode { + return !!value && typeof value === "object" && typeof (value as AnyNode).type === "string"; +} + +function isDateNowCall(node: AnyNode): boolean { + return node.type === "CallExpression" && isMemberExpression(node.callee, "Date", "now"); +} + +function isMathRandomCall(node: AnyNode): boolean { + return node.type === "CallExpression" && isMemberExpression(node.callee, "Math", "random"); +} + +function isNewDateExpression(node: AnyNode): boolean { + return node.type === "NewExpression" && node.callee?.type === "Identifier" && node.callee.name === "Date"; +} + +function isMemberExpression(node: AnyNode | undefined, objectName: string, propertyName: string): boolean { + if (node?.type !== "MemberExpression" || node.object?.type !== "Identifier" || node.object.name !== objectName) { + return false; + } + return propertyNameOf(node) === propertyName; +} + +function propertyNameOf(node: AnyNode): string | undefined { + if (!node.computed && node.property?.type === "Identifier") return node.property.name; + return staticStringOf(node.property); +} + +function staticStringOf(node: AnyNode | undefined): string | undefined { + if (node?.type === "Literal" && typeof node.value === "string") return node.value; + if (node?.type === "TemplateLiteral" && node.expressions.length === 0) { + return node.quasis.map((quasi: AnyNode) => quasi.value.cooked ?? quasi.value.raw).join(""); + } + if (node?.type === "BinaryExpression" && node.operator === "+") { + const left = staticStringOf(node.left); + const right = staticStringOf(node.right); + if (left !== undefined && right !== undefined) return left + right; + } + return undefined; +} + function validateMeta(meta: unknown): asserts meta is WorkflowMeta { if (!meta || typeof meta !== "object") throw new Error("meta must be an object"); const value = meta as WorkflowMeta; diff --git a/tests/workflow-parser.test.ts b/tests/workflow-parser.test.ts index ca571c8..e060e7b 100644 --- a/tests/workflow-parser.test.ts +++ b/tests/workflow-parser.test.ts @@ -93,16 +93,68 @@ test("parseWorkflowScript rejects template interpolation", () => { }); test("parseWorkflowScript rejects nondeterministic APIs", () => { - assert.throws( - () => parseWorkflowScript("export const meta = { name: 'demo', description: 'desc' }\nreturn Date.now()"), - /must be deterministic/, - ); - assert.throws( - () => parseWorkflowScript("export const meta = { name: 'demo', description: 'desc' }\nreturn Math.random()"), - /must be deterministic/, - ); - assert.throws( - () => parseWorkflowScript("export const meta = { name: 'demo', description: 'desc' }\nreturn new Date()"), - /must be deterministic/, - ); + for (const expression of [ + "Date.now()", + "Date['now']()", + "Date[`now`]()", + "Date['n' + 'ow']()", + "Date?.now()", + "Date.now?.()", + "Math.random()", + "Math['random']()", + "Math[`random`]()", + "Math['ran' + 'dom']()", + "Math?.random()", + "Math.random?.()", + "new Date()", + "new (Date)()", + "`timestamp $" + "{Date.now()}`", + ]) { + assert.throws( + () => parseWorkflowScript(`export const meta = { name: 'demo', description: 'desc' }\nreturn ${expression}`), + /must be deterministic/, + expression, + ); + } +}); + +test("parseWorkflowScript allows deterministic Date and Math APIs", () => { + for (const expression of [ + "Date.parse('2020-01-01T00:00:00Z')", + "Date.UTC(2020, 0, 1)", + "Math.max(1, 2)", + "Math.floor(1.5)", + "({ Date: { now: true }, Math: { random: true } })", + "({ now: () => 1 }).now()", + "({ random: () => 1 }).random()", + ]) { + assert.doesNotThrow( + () => parseWorkflowScript(`export const meta = { name: 'demo', description: 'desc' }\nreturn ${expression}`), + expression, + ); + } +}); + +test("parseWorkflowScript allows nondeterministic API names in text", () => { + const parsed = parseWorkflowScript(`export const meta = { + name: 'mentions_demo', + description: 'Catalog Date.now(), Math.random(), and new Date() usage', + whenToUse: 'When prompts mention Date.now()', + phases: [{ title: 'Find Date.now() mentions', detail: 'Check Math.random() and new Date() too' }] +} + +// Comments may mention Date.now(), Math.random(), and new Date(). +const terms = { + 'Date.now()': 'Date.now()', + 'Math.random()': 'Math.random()', + 'new Date()': 'new Date()' +} +phase('Find Date.now() mentions') +await agent('Catalog Date.now(), Math.random(), and new Date() usage') +await agent(\`Find Date.now(), Math.random(), and new Date() mentions\`) +return { ok: true, terms } +`); + + assert.equal(parsed.meta.description, "Catalog Date.now(), Math.random(), and new Date() usage"); + assert.match(parsed.body, /Catalog Date\.now\(\)/); }); diff --git a/tests/workflow-runtime.test.ts b/tests/workflow-runtime.test.ts index db7300e..3b77df1 100644 --- a/tests/workflow-runtime.test.ts +++ b/tests/workflow-runtime.test.ts @@ -102,3 +102,23 @@ return { ok: true } /phase title must be a string/, ); }); + +test("runWorkflow allows prompts that mention nondeterministic API names", async () => { + const result = await runWorkflow( + `export const meta = { + name: 'prompt_mentions', + description: 'Ask about Date.now(), Math.random(), and new Date() usage' +} + +phase('Catalog mentions') +const scan = await agent('Catalog Date.now(), Math.random(), and new Date() usage', { label: 'scan' }) +return { scan } +`, + { agent: fakeAgent }, + ); + + assert.equal( + (result.result as { scan: string }).scan, + "result:Catalog Date.now(), Math.random(), and new Date() usage", + ); +});