From 4d7e410ee21127aea323cc04d5c10f0e5fe8c475 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sun, 17 May 2026 20:31:37 -0700 Subject: [PATCH 1/3] Handle missing dynamic tool calls --- .../core/execution/src/tool-invoker.test.ts | 20 ++++ packages/core/execution/src/tool-invoker.ts | 30 +++++- packages/core/sdk/src/errors.ts | 5 +- packages/core/sdk/src/executor.test.ts | 47 +++++++++ packages/core/sdk/src/executor.ts | 96 ++++++++++++++++--- 5 files changed, 180 insertions(+), 18 deletions(-) diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index 0f79a80ee..3ce1d2a74 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -641,6 +641,26 @@ describe("tool discovery", () => { }), ); + it.effect("returns missing tool dispatches as ToolResult.fail", () => + Effect.gen(function* () { + const executor = yield* createExecutor(makeTestConfig({ plugins: [] as const })); + const invoker = makeExecutorToolInvoker(executor, { + invokeOptions: { onElicitation: acceptAll }, + }); + + const result = yield* invoker.invoke({ path: "missing.sourceTool", args: {} }); + + expect(result).toEqual({ + ok: false, + error: { + code: "tool_not_found", + message: "Tool not found: missing.sourceTool", + details: { toolId: "missing.sourceTool", suggestions: [] }, + }, + }); + }), + ); + it.effect("preserves nested upstream error bodies through ToolResult.fail", () => Effect.gen(function* () { const executor = yield* createExecutor( diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index 5c4b4f53d..79ef4ea1c 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -8,7 +8,7 @@ import type { InvokeOptions, Source, } from "@executor-js/sdk/core"; -import { isToolResult } from "@executor-js/sdk/core"; +import { isToolResult, ToolResult } from "@executor-js/sdk/core"; import type { SandboxToolInvoker } from "@executor-js/codemode-core"; import { ExecutionToolError } from "./errors"; @@ -34,6 +34,28 @@ const newCorrelationId = (): string => { .padStart(8, "0"); }; +const expectedToolFailure = ( + value: unknown, +): { readonly code: string; readonly message: string; readonly details?: unknown } | null => { + if (Predicate.isTagged(value, "ToolNotFoundError") && "toolId" in value) { + const suggestions = + "suggestions" in value && Array.isArray(value.suggestions) ? value.suggestions : undefined; + return { + code: "tool_not_found", + message: `Tool not found: ${String(value.toolId)}`, + details: { toolId: value.toolId, ...(suggestions ? { suggestions } : {}) }, + }; + } + if (Predicate.isTagged(value, "ToolBlockedError") && "toolId" in value) { + return { + code: "tool_blocked", + message: `Tool blocked by policy: ${String(value.toolId)}`, + details: value, + }; + } + return null; +}; + /** * Extract the source namespace from a tool path. Tool paths look like * "." or ".." — we take the first @@ -70,8 +92,12 @@ export const makeExecutorToolInvoker = ( }); const result = yield* executor.tools.invoke(path as ToolId, args, options.invokeOptions).pipe( - Effect.catchCause((cause): Effect.Effect => { + Effect.catchCause((cause) => { const err = cause.reasons.find(Cause.isFailReason)?.error; + const expected = expectedToolFailure(err); + if (expected) { + return Effect.succeed(ToolResult.fail(expected)); + } if (isElicitationDeclinedError(err)) { return Effect.fail( new ExecutionToolError({ diff --git a/packages/core/sdk/src/errors.ts b/packages/core/sdk/src/errors.ts index 85b8de385..59efd95a5 100644 --- a/packages/core/sdk/src/errors.ts +++ b/packages/core/sdk/src/errors.ts @@ -8,7 +8,10 @@ import { ConnectionId, ToolId, SecretId } from "./ids"; export class ToolNotFoundError extends Schema.TaggedErrorClass()( "ToolNotFoundError", - { toolId: ToolId }, + { + toolId: ToolId, + suggestions: Schema.optional(Schema.Array(ToolId)), + }, ) {} export class ToolInvocationError extends Data.TaggedError("ToolInvocationError")<{ diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index c76b16a8a..fd1c0a959 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -3,6 +3,7 @@ import { Data, Effect } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { scopedExecutorTable, textColumn } from "./core-schema"; +import { ToolNotFoundError } from "./errors"; import { createExecutor } from "./executor"; import { ScopeId } from "./ids"; import { definePlugin } from "./plugin"; @@ -171,6 +172,22 @@ const schemaProbePlugin = definePlugin(() => ({ }), }))(); +const caseSensitiveDynamicPlugin = definePlugin(() => ({ + id: "caseDynamic" as const, + storage: () => ({}), + extension: (ctx) => ({ + registerSource: () => + ctx.core.sources.register({ + id: "case_source", + scope: String(ctx.scopes[0]!.id), + kind: "case", + name: "Case Source", + tools: [{ name: "listdashboards", description: "list dashboards" }], + }), + }), + invokeTool: ({ toolRow }) => Effect.succeed({ invokedToolId: toolRow.id }), +}))(); + describe("createExecutor", () => { it.effect("rolls back plugin and core writes from ctx.transaction failures", () => Effect.gen(function* () { @@ -311,4 +328,34 @@ describe("createExecutor", () => { ); }), ); + + it.effect("resolves dynamic tool ids case-insensitively before invoking plugins", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [caseSensitiveDynamicPlugin] as const, + }); + yield* executor.caseDynamic.registerSource(); + + const result = yield* executor.tools.invoke("case_source.listDashboards", {}); + + expect(result).toEqual({ invokedToolId: "case_source.listdashboards" }); + }), + ); + + it.effect("suggests visible tools for missing dynamic tool ids", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [caseSensitiveDynamicPlugin] as const, + }); + yield* executor.caseDynamic.registerSource(); + + const error = yield* executor.tools + .invoke("case_source.listDashboardsWRONG", {}) + .pipe(Effect.flip); + + expect(error).toBeInstanceOf(ToolNotFoundError); + if (!(error instanceof ToolNotFoundError)) return; + expect(error.suggestions).toEqual(["case_source.listdashboards"]); + }), + ); }); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 7eeb73173..0f40dda61 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -562,6 +562,50 @@ const byScopedId = (b: ConditionBuilder>): Condition => b.and(b("scope_id", "=", scope), b("id", "=", id)) as Condition; +const toolSourceId = (toolId: string): string | null => { + const dot = toolId.indexOf("."); + return dot === -1 ? null : toolId.slice(0, dot); +}; + +const levenshteinDistance = (left: string, right: string): number => { + const previous = Array.from({ length: right.length + 1 }, (_, index) => index); + const current = Array.from({ length: right.length + 1 }, () => 0); + for (let i = 0; i < left.length; i++) { + current[0] = i + 1; + for (let j = 0; j < right.length; j++) { + current[j + 1] = + left[i] === right[j] + ? previous[j]! + : Math.min(previous[j]!, previous[j + 1]!, current[j]!) + 1; + } + for (let j = 0; j < previous.length; j++) previous[j] = current[j]!; + } + return previous[right.length]!; +}; + +const missingToolSuggestionScore = (query: string, candidate: string): number => { + const normalizedQuery = query.toLowerCase(); + const normalizedCandidate = candidate.toLowerCase(); + if (normalizedCandidate === normalizedQuery) return 0; + if (normalizedCandidate.startsWith(normalizedQuery)) return 1; + if (normalizedQuery.startsWith(normalizedCandidate)) return 2; + if (normalizedCandidate.includes(normalizedQuery)) return 3; + const queryLeaf = normalizedQuery.split(".").at(-1) ?? normalizedQuery; + const candidateLeaf = normalizedCandidate.split(".").at(-1) ?? normalizedCandidate; + if (candidateLeaf.startsWith(queryLeaf) || queryLeaf.startsWith(candidateLeaf)) return 4; + return 10 + levenshteinDistance(normalizedQuery, normalizedCandidate); +}; + +const missingToolSuggestions = ( + toolId: string, + rows: readonly { readonly id: string }[], +): readonly ToolId[] => + rows + .map((row) => ({ id: row.id, score: missingToolSuggestionScore(toolId, row.id) })) + .sort((left, right) => left.score - right.score || left.id.localeCompare(right.id)) + .slice(0, 5) + .map((item) => ToolId.make(item.id)); + type CoreTableName = keyof CoreSchema & string; type CoreRow = FumaRow; type CoreWhere<_TName extends CoreTableName> = ( @@ -3191,19 +3235,19 @@ export const createExecutor = ( - effect: Effect.Effect, - ): Effect.Effect => - effect.pipe( + const wrapInvocationError = + (resolvedToolId: string) => + (effect: Effect.Effect): Effect.Effect => + effect.pipe( Effect.mapError( (cause) => new ToolInvocationError({ - toolId: ToolId.make(toolId), + toolId: ToolId.make(resolvedToolId), message: formatInvocationCauseMessage(cause), cause, }), - ), - ); + ), + ); // Resolve the user-authored policy first. A `block` rule // short-circuits both the static and dynamic paths before any @@ -3230,7 +3274,7 @@ export const createExecutor = [] = toolRows; + if (!row) { + suggestionRows = yield* core + .findMany("tool", { + where: scopedWhere(scopeIds), + }) + .pipe(Effect.withSpan("executor.tool.resolve_suggestions")); + const sourceId = toolSourceId(toolId); + if (sourceId) { + const normalizedToolId = toolId.toLowerCase(); + row = findInnermost( + suggestionRows.filter( + (toolRow) => + toolRow.source_id === sourceId && toolRow.id.toLowerCase() === normalizedToolId, + ), + ); + if (row) resolvedToolId = row.id; + } + } if (!row) { return yield* new ToolNotFoundError({ toolId: ToolId.make(toolId), + suggestions: missingToolSuggestions(toolId, suggestionRows), }); } yield* Effect.annotateCurrentSpan({ "executor.tool.dispatch_path": "dynamic", "executor.source_id": row.source_id, "executor.plugin_id": row.plugin_id, + "executor.tool.resolved_id": resolvedToolId, }); const runtime = runtimes.get(row.plugin_id); if (!runtime) { @@ -3287,20 +3353,20 @@ export const createExecutor = Date: Sun, 17 May 2026 21:19:51 -0700 Subject: [PATCH 2/3] Apply policy after dynamic tool canonicalization --- packages/core/sdk/src/executor.test.ts | 31 +++++++++++++++++++++++ packages/core/sdk/src/executor.ts | 34 ++++++++++++++++---------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index fd1c0a959..28746d66e 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -3,6 +3,7 @@ import { Data, Effect } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { scopedExecutorTable, textColumn } from "./core-schema"; +import { ElicitationResponse } from "./elicitation"; import { ToolNotFoundError } from "./errors"; import { createExecutor } from "./executor"; import { ScopeId } from "./ids"; @@ -342,6 +343,36 @@ describe("createExecutor", () => { }), ); + it.effect("applies policies after case-insensitive dynamic tool id resolution", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [caseSensitiveDynamicPlugin] as const, + }); + yield* executor.caseDynamic.registerSource(); + yield* executor.policies.create({ + targetScope: "test-scope", + pattern: "case_source.listdashboards", + action: "require_approval", + }); + const calls = { count: 0 }; + + const result = yield* executor.tools.invoke( + "case_source.listDashboards", + {}, + { + onElicitation: () => + Effect.sync(() => { + calls.count += 1; + return ElicitationResponse.make({ action: "accept" }); + }), + }, + ); + + expect(result).toEqual({ invokedToolId: "case_source.listdashboards" }); + expect(calls.count).toBe(1); + }), + ); + it.effect("suggests visible tools for missing dynamic tool ids", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 0f40dda61..33e5182a4 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3249,22 +3249,21 @@ export const createExecutor = Date: Mon, 18 May 2026 10:29:53 -0700 Subject: [PATCH 3/3] Fix dynamic tool policy CI issues --- packages/core/sdk/src/executor.test.ts | 4 ++-- packages/core/sdk/src/executor.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 28746d66e..76dbefddf 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@effect/vitest"; -import { Data, Effect } from "effect"; +import { Data, Effect, Predicate } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { scopedExecutorTable, textColumn } from "./core-schema"; @@ -385,7 +385,7 @@ describe("createExecutor", () => { .pipe(Effect.flip); expect(error).toBeInstanceOf(ToolNotFoundError); - if (!(error instanceof ToolNotFoundError)) return; + if (!Predicate.isTagged("ToolNotFoundError")(error)) return; expect(error.suggestions).toEqual(["case_source.listdashboards"]); }), ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 33e5182a4..bad0fadc1 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3239,13 +3239,13 @@ export const createExecutor = (effect: Effect.Effect): Effect.Effect => effect.pipe( - Effect.mapError( - (cause) => - new ToolInvocationError({ - toolId: ToolId.make(resolvedToolId), - message: formatInvocationCauseMessage(cause), - cause, - }), + Effect.mapError( + (cause) => + new ToolInvocationError({ + toolId: ToolId.make(resolvedToolId), + message: formatInvocationCauseMessage(cause), + cause, + }), ), );