Fix tool result contracts

Author: RhysSullivan

apps/cloud/src/services/sources-api.node.test.ts

@@ -398,7 +398,7 @@ describe("sources api (HTTP)", () => {
         status: "completed",
         result: {
           ok: true,
-          data: [{ type: "text", text: "cloud-mcp-ok" }],
+          data: { content: [{ type: "text", text: "cloud-mcp-ok" }] },
         },
       });
       expect((yield* server.requests).length).toBeGreaterThanOrEqual(2);

packages/core/execution/src/description.ts

@@ -48,6 +48,7 @@ const formatDescription = (sources: readonly Source[]): string => {
     "- `tools.search()` returns paginated, ranked matches: `{ items, total, hasMore, nextOffset }`. Best-first. Use short intent phrases like `github issues`, `repo details`, or `create calendar event`.",
     '- When you already know the namespace, narrow with `tools.search({ namespace: "github", query: "issues" })`.',
     "- `tools.executor.sources.list()` returns the same paged shape: `{ items: [{ id, toolCount, ... }], total, hasMore, nextOffset }`.",
+    "- Tool calls return a value union: `{ ok: true, data }` for success or `{ ok: false, error: { code, message, status?, details?, retryable? } }` for expected tool/domain failures. Branch on `result.ok`.",
     "- If `hasMore` is true and you didn't find what you need, fetch the next page: `tools.search({ query, offset: nextOffset, limit })`. Same `offset` parameter on `tools.executor.sources.list({ offset, limit })`.",
     "- Always use the namespace prefix when calling tools: `tools.<namespace>.<tool>(args)`. Example: `tools.home_assistant_rest_api.states.getState(...)` — not `tools.states.getState(...)`.",
     "- The `tools` object is a lazy proxy — `Object.keys(tools)` won't work. Use `tools.search()` or `tools.executor.sources.list()` instead.",

packages/core/execution/src/tool-invoker.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "@effect/vitest";
 import { Effect, Fiber, Schema } from "effect";
+import * as ts from "typescript";
 
 import {
   ElicitationResponse,
@@ -19,6 +20,10 @@ const RepoInputSchema = Schema.toStandardSchemaV1(
   Schema.toStandardJSONSchemaV1(Schema.Struct({ owner: Schema.String, repo: Schema.String })),
 );
 
+const RepoDetailsOutputSchema = Schema.toStandardSchemaV1(
+  Schema.toStandardJSONSchemaV1(Schema.Struct({ defaultBranch: Schema.String })),
+);
+
 const ContactInputSchema = Schema.toStandardSchemaV1(
   Schema.toStandardJSONSchemaV1(Schema.Struct({ email: Schema.String })),
 );
@@ -29,6 +34,58 @@ const EmptyInputSchema = Schema.toStandardSchemaV1(
 
 const acceptAll = () => Effect.succeed(ElicitationResponse.make({ action: "accept" }));
 
+type DescribedToolContract = {
+  readonly outputTypeScript: string;
+  readonly typeScriptDefinitions: Record<string, string>;
+};
+
+const typeCheckDescribedInvocation = (
+  described: DescribedToolContract,
+  runtimeResult: unknown,
+  consumerSource: string,
+): readonly string[] => {
+  const fileName = "described-tool-contract.ts";
+  const source = [
+    ...Object.entries(described.typeScriptDefinitions).map(([name, definition]) => {
+      return `type ${name} = ${definition};`;
+    }),
+    `type ToolOutput = ${described.outputTypeScript};`,
+    `const invokedResult: ToolOutput = ${JSON.stringify(runtimeResult)};`,
+    consumerSource,
+  ].join("\n");
+
+  const options: ts.CompilerOptions = {
+    module: ts.ModuleKind.ESNext,
+    noEmit: true,
+    skipLibCheck: true,
+    strict: true,
+    target: ts.ScriptTarget.ES2022,
+  };
+  const host = ts.createCompilerHost(options);
+  const originalGetSourceFile = host.getSourceFile.bind(host);
+  const originalReadFile = host.readFile.bind(host);
+  const originalFileExists = host.fileExists.bind(host);
+
+  host.getSourceFile = (candidate, languageVersion, onError, shouldCreateNewSourceFile) => {
+    if (candidate === fileName) {
+      return ts.createSourceFile(candidate, source, languageVersion, true);
+    }
+    return originalGetSourceFile(candidate, languageVersion, onError, shouldCreateNewSourceFile);
+  };
+  host.readFile = (candidate) => (candidate === fileName ? source : originalReadFile(candidate));
+  host.fileExists = (candidate) => candidate === fileName || originalFileExists(candidate);
+
+  const program = ts.createProgram([fileName], options, host);
+  return ts.getPreEmitDiagnostics(program).map((diagnostic) => {
+    const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n");
+    if (!diagnostic.file || diagnostic.start === undefined) {
+      return message;
+    }
+    const position = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
+    return `${diagnostic.file.fileName}:${position.line + 1}:${position.character + 1} ${message}`;
+  });
+};
+
 // ---------------------------------------------------------------------------
 // Test plugins — each one declares a namespace as a static source with N
 // tools. Handlers return static data; the suite only cares about discovery
@@ -54,6 +111,7 @@ const githubPlugin = definePlugin(() => ({
           name: "getRepositoryDetails",
           description: "Get repository details including the default branch",
           inputSchema: RepoInputSchema,
+          outputSchema: RepoDetailsOutputSchema,
           handler: () => Effect.succeed({ defaultBranch: "main" }),
         },
         {
@@ -342,8 +400,102 @@ describe("tool discovery", () => {
       expect(described.name).toBe("listRepositoryIssues");
       expect(described.description).toBe("List issues for a repository");
       expect(described.inputTypeScript).toBe("{ owner: string; repo: string; }");
-      expect(described.outputTypeScript).toBeUndefined();
-      expect(described.typeScriptDefinitions).toBeUndefined();
+      expect(described.outputTypeScript).toBe(
+        "{ ok: true; data: unknown } | { ok: false; error: ToolError }",
+      );
+      expect(described.typeScriptDefinitions).toEqual({
+        ToolError:
+          "{ code: string; message: string; status?: number; details?: unknown; retryable?: boolean }",
+      });
+    }),
+  );
+
+  it.effect("describes a return type that accepts the sandbox invocation result", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+      const engine = createExecutionEngine({ executor, codeExecutor });
+
+      const execution = yield* engine.execute(
+        [
+          'const details = await tools.describe.tool({ path: "github.getRepositoryDetails" });',
+          "const result = await tools.github.getRepositoryDetails({ owner: 'executor', repo: 'executor' });",
+          "return {",
+          "  outputTypeScript: details.outputTypeScript,",
+          "  typeScriptDefinitions: details.typeScriptDefinitions,",
+          "  result,",
+          "};",
+        ].join("\n"),
+        { onElicitation: acceptAll },
+      );
+
+      expect(execution.error).toBeUndefined();
+      const observed = execution.result as DescribedToolContract & { readonly result: unknown };
+      const diagnostics = typeCheckDescribedInvocation(
+        observed,
+        observed.result,
+        [
+          "function readDefaultBranch(result: ToolOutput): string {",
+          "  if (!result.ok) return result.error.message;",
+          "  return result.data.defaultBranch;",
+          "}",
+          "readDefaultBranch(invokedResult);",
+        ].join("\n"),
+      );
+      expect(diagnostics).toEqual([]);
+    }),
+  );
+
+  it.effect(
+    "describes an error-as-value return type that accepts sandbox invocation failures",
+    () =>
+      Effect.gen(function* () {
+        const executor = yield* createExecutor(
+          makeTestConfig({ plugins: [errorPlugin()] as const }),
+        );
+        const engine = createExecutionEngine({ executor, codeExecutor });
+
+        const execution = yield* engine.execute(
+          [
+            'const details = await tools.describe.tool({ path: "records.queryRows" });',
+            "const result = await tools.records.queryRows({});",
+            "return {",
+            "  outputTypeScript: details.outputTypeScript,",
+            "  typeScriptDefinitions: details.typeScriptDefinitions,",
+            "  result,",
+            "};",
+          ].join("\n"),
+          { onElicitation: acceptAll },
+        );
+
+        expect(execution.error).toBeUndefined();
+        const observed = execution.result as DescribedToolContract & { readonly result: unknown };
+        const diagnostics = typeCheckDescribedInvocation(
+          observed,
+          observed.result,
+          [
+            "function readToolResult(result: ToolOutput): unknown {",
+            "  if (!result.ok) return result.error.message;",
+            "  return result.data;",
+            "}",
+            "readToolResult(invokedResult);",
+          ].join("\n"),
+        );
+        expect(diagnostics).toEqual([]);
+      }),
+  );
+
+  it.effect("describes the ToolResult wrapper through the direct describe helper", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+      const described = yield* describeTool(executor, "github.getRepositoryDetails");
+
+      expect(described.outputTypeScript).toBe(
+        "{ ok: true; data: { defaultBranch: string; } } | { ok: false; error: ToolError }",
+      );
+      expect(described.typeScriptDefinitions).toEqual({
+        ToolError:
+          "{ code: string; message: string; status?: number; details?: unknown; retryable?: boolean }",
+      });
     }),
   );
 

packages/core/execution/src/tool-invoker.ts

@@ -13,6 +13,18 @@ import type { SandboxToolInvoker } from "@executor-js/codemode-core";
 import { ExecutionToolError } from "./errors";
 
 const OPAQUE_DEFECT_MESSAGE = "Internal tool error";
+const TOOL_ERROR_TYPESCRIPT =
+  "{ code: string; message: string; status?: number; details?: unknown; retryable?: boolean }";
+
+const wrapOutputTypeScript = (outputTypeScript?: string): string =>
+  `{ ok: true; data: ${outputTypeScript ?? "unknown"} } | { ok: false; error: ToolError }`;
+
+const withToolResultDefinitions = (
+  definitions?: Record<string, string>,
+): Record<string, string> => ({
+  ...(definitions ?? {}),
+  ToolError: TOOL_ERROR_TYPESCRIPT,
+});
 
 const newCorrelationId = (): string => {
   // 8-hex-char correlation id; enough entropy to disambiguate within a
@@ -497,7 +509,7 @@ export const describeTool = Effect.fn("executor.tools.describe")(function* (
     name: schema.name ?? path,
     description: schema.description,
     inputTypeScript: schema.inputTypeScript,
-    outputTypeScript: schema.outputTypeScript,
-    typeScriptDefinitions: schema.typeScriptDefinitions,
+    outputTypeScript: wrapOutputTypeScript(schema.outputTypeScript),
+    typeScriptDefinitions: withToolResultDefinitions(schema.typeScriptDefinitions),
   };
 });

packages/kernel/runtime-dynamic-worker/src/invocation.test.ts

@@ -4,6 +4,7 @@ import * as Cause from "effect/Cause";
 import * as Data from "effect/Data";
 import * as Effect from "effect/Effect";
 import type { SandboxToolInvoker } from "@executor-js/codemode-core";
+import { ExecutionToolError } from "@executor-js/execution";
 import {
   ToolDispatcher,
   makeDynamicWorkerExecutor,
@@ -257,6 +258,27 @@ describe("makeDynamicWorkerExecutor", () => {
     expect(result.error).toBe("Internal tool error");
   });
 
+  it("preserves public ExecutionToolError messages across the worker bridge", async () => {
+    const executor = makeDynamicWorkerExecutor({ loader });
+    const invoker = {
+      invoke: () =>
+        Effect.fail(
+          new ExecutionToolError({
+            message:
+              "tools.search expects an object: { query?: string; namespace?: string; limit?: number; offset?: number }",
+          }),
+        ),
+    } satisfies SandboxToolInvoker;
+
+    const result = await Effect.runPromise(
+      executor.execute("async () => await tools.search('github')", invoker),
+    );
+
+    expect(result.error).toBe(
+      "tools.search expects an object: { query?: string; namespace?: string; limit?: number; offset?: number }",
+    );
+  });
+
   it("does not expose host error stack details to sandbox error handlers", async () => {
     const executor = makeDynamicWorkerExecutor({ loader });
     const invoker = {

packages/kernel/runtime-dynamic-worker/src/module-template.ts

@@ -127,6 +127,14 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string =>
     "      for (const [k, v] of Object.entries(value)) out[k] = __decodeBinary(v, seen);",
     "      return out;",
     "    };",
+    "    const __publicToolErrorMessage = (error) => {",
+    "      const values = [error && error.primary, ...(Array.isArray(error && error.failures) ? error.failures : [])];",
+    "      for (const value of values) {",
+    "        if (value && value.__type === 'Error' && value.name === 'ExecutionToolError' && typeof value.message === 'string') return value.message;",
+    "      }",
+    "      if (error && typeof error.message === 'string' && error.message.startsWith('Internal tool error')) return error.message;",
+    "      return null;",
+    "    };",
     "    const __makeToolsProxy = (path = []) => new Proxy(() => undefined, {",
     "      get(_target, prop) {",
     "        if (prop === 'then' || typeof prop === 'symbol') return undefined;",
@@ -138,7 +146,7 @@ export const buildExecutorModule = (body: string, timeoutMs: number): string =>
     "        return (async () => {",
     "          const encoded = await __encodeBinary(args[0]);",
     "          const data = await __dispatcher.call(toolPath, encoded);",
-    "          if (!data.ok) throw new Error(data.error && typeof data.error.message === 'string' && data.error.message.startsWith('Internal tool error') ? data.error.message : 'Internal tool error');",
+    "          if (!data.ok) throw new Error(__publicToolErrorMessage(data.error) || 'Internal tool error');",
     "          return __decodeBinary(data.result);",
     "        })();",
     "      },",

packages/plugins/graphql/src/sdk/plugin.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from "@effect/vitest";
 import { Effect, Predicate } from "effect";
+import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
 
 import {
   ConnectionId,
@@ -13,7 +14,7 @@ import {
   SecretId,
   TokenMaterial,
 } from "@executor-js/sdk";
-import { makeTestConfig } from "@executor-js/sdk/testing";
+import { makeTestConfig, serveTestHttpApp } from "@executor-js/sdk/testing";
 import { memorySecretsPlugin } from "@executor-js/sdk/testing";
 
 import { graphqlPlugin } from "./plugin";
@@ -208,6 +209,46 @@ describe("graphqlPlugin real protocol server", () => {
     }),
   );
 
+  it.effect("surfaces non-2xx invocation responses as ToolResult.fail", () =>
+    Effect.gen(function* () {
+      const server = yield* serveTestHttpApp((request) =>
+        Effect.gen(function* () {
+          const webRequest = yield* HttpServerRequest.toWeb(request);
+          const body = yield* Effect.promise(() => webRequest.text());
+          if (body.includes("__schema")) {
+            return HttpServerResponse.jsonUnsafe({ data: introspectionResult });
+          }
+          return HttpServerResponse.text("temporary upstream outage", {
+            status: 503,
+            contentType: "text/plain",
+          });
+        }),
+      );
+      const executor = yield* createExecutor(
+        makeTestConfig({ plugins: [graphqlPlugin()] as const }),
+      );
+
+      yield* executor.graphql.addSource({
+        endpoint: server.url("/graphql"),
+        scope: TEST_SCOPE,
+        namespace: "http_error_graph",
+      });
+
+      const result = yield* executor.tools.invoke("http_error_graph.query.hello", {
+        name: "Ada",
+      });
+
+      expect(result).toMatchObject({
+        ok: false,
+        error: {
+          code: "graphql_http_error",
+          status: 503,
+          message: "GraphQL request failed with HTTP 503",
+        },
+      });
+    }),
+  );
+
   it.effect("invokes OAuth-backed sources with a bearer token", () =>
     Effect.gen(function* () {
       const server = yield* serveGreetingServer;

packages/plugins/graphql/src/sdk/plugin.ts

@@ -1074,6 +1074,18 @@ export const graphqlPlugin = definePlugin((options?: GraphqlPluginOptions) => {
             details: { errors },
           });
         }
+        if (result.status < 200 || result.status >= 300) {
+          return ToolResult.fail({
+            code: "graphql_http_error",
+            status: result.status,
+            message: `GraphQL request failed with HTTP ${result.status}`,
+            details: {
+              status: result.status,
+              data: result.data,
+              errors: result.errors,
+            },
+          });
+        }
         return ToolResult.ok(result.data);
       }),