From c8849b2dee95472333a0f289a477b78be2b65eca Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 20 May 2026 19:20:46 +0000 Subject: [PATCH 1/2] Use explicit Anthropic computer tools --- packages/agent/README.md | 31 ++--- packages/agent/src/agent.ts | 14 +-- packages/agent/src/index.ts | 2 - packages/agent/src/tools.ts | 105 +++++------------ packages/agent/test/agent.test.ts | 36 +++--- .../agent/test/tool-exhaustiveness.test.ts | 66 ++++++----- packages/ai/README.md | 59 +++++----- packages/ai/examples/quickstart.ts | 3 +- .../ai/src/providers/anthropic/actions.ts | 106 ++++++++++++++++++ packages/ai/src/providers/anthropic/index.ts | 27 ++--- packages/ai/src/providers/common.ts | 54 ++++++++- packages/ai/src/providers/gemini/index.ts | 1 + packages/ai/src/providers/openai/index.ts | 1 + packages/ai/src/providers/tzafon/index.ts | 2 + packages/ai/src/providers/yutori/actions.ts | 7 ++ packages/ai/src/providers/yutori/index.ts | 1 + packages/ai/src/runtime-spec.ts | 13 ++- packages/ai/test/runtime-spec.test.ts | 10 +- packages/ai/test/tools.test.ts | 44 +++++++- 19 files changed, 378 insertions(+), 204 deletions(-) create mode 100644 packages/ai/src/providers/anthropic/actions.ts diff --git a/packages/agent/README.md b/packages/agent/README.md index 84bccc0e..f47e48e9 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -85,7 +85,6 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions: - `client` (Kernel SDK client) - CUA model refs (`"provider:model"`) accepted where pi expects a concrete model - `extraTools` to add your own pi tools alongside the built-in browser tools -- `batchTool: true` to let the model run multiple browser actions in one tool call - `computerUseExtra: true` to let the model use a small navigation helper If auth callbacks are omitted, both classes default to CUA env var conventions: @@ -97,27 +96,20 @@ If auth callbacks are omitted, both classes default to CUA env var conventions: ### Tool Defaults -By default, the classes install provider-selected canonical CUA computer tool -executors using runtime specs from `@onkernel/cua-ai`. Use `extraTools` to add -your own pi tools alongside the provider's computer-use tools. This is useful -when the model needs to call application-specific code, such as looking up a -record, writing a database row, or handing off to another service while it also -controls the browser. +By default, the classes install provider-selected CUA computer tool executors +from `@onkernel/cua-ai`. Each provider decides which tool names the model sees; +the matching executor adapter translates returned tool calls into canonical CUA +actions that run against the Kernel browser. -`batchTool: true` adds the `batch_computer_actions` tool. Use it when you want -the model to group several browser actions into one call, for example moving, -clicking, typing, waiting, and then reading a screenshot. The batch tool is -synthesized from the selected provider's normal browser action definitions, so -it only batches actions that provider runtime already supports. +Use `extraTools` to add your own pi tools alongside the provider's +computer-use tools. This is useful when the model needs to call +application-specific code, such as looking up a record, writing a database row, +or handing off to another service while it also controls the browser. `computerUseExtra: true` adds the `computer_use_extra` tool. Use it when you want one compact helper for common browser navigation/read operations: `goto`, `back`, `forward`, and `url`. -The TypeScript API follows pi's camelCase option style (`extraTools`, -`batchTool`, `computerUseExtra`). Names like `batch_computer_actions` and -`computer_use_extra` are the literal tool names the model may see in traces. - ### Model Switching `CuaAgent` follows pi `Agent` semantics: assign `agent.state.model` to a @@ -131,8 +123,8 @@ the next provider request. ### Tool Composition -Use `createCuaComputerTools()` to compose your own tool list from canonical -tool definitions: +Use `createCuaComputerTools()` to compose your own tool list from provider +execution adapters: ```ts import { resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; @@ -143,8 +135,7 @@ const tools = [ ...createCuaComputerTools({ browser, client, - toolDefinitions: runtime.toolDefinitions, - batchTool: true, + toolExecutors: runtime.toolExecutors, }), myCustomTool, ]; diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 20be10fa..6a5dfeb1 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -11,7 +11,6 @@ import { } from "./vendor/pi-agent-core/index"; import { type Api, - CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, type CuaModelRef, getCuaEnvApiKey, @@ -65,8 +64,6 @@ export type CuaAgentOptions = Omit & { initialState: CuaAgentInitialState; /** Add your own pi tools alongside the built-in browser tools. */ extraTools?: AgentTool[]; - /** Expose a batch tool so the model can run multiple browser actions in one call. */ - batchTool?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; }; @@ -90,8 +87,6 @@ export type CuaAgentHarnessOptions< model: CuaRuntimeInput; /** Add your own pi tools alongside the built-in browser tools. */ extraTools?: AgentTool[]; - /** Expose a batch tool so the model can run multiple browser actions in one call. */ - batchTool?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */ @@ -115,7 +110,6 @@ class CuaRuntimeController { client: Kernel; model: CuaRuntimeInput; extraTools?: AgentTool[]; - batchTool?: boolean; computerUseExtra?: boolean; systemPrompt?: unknown; onPayload?: SimpleStreamOptions["onPayload"]; @@ -149,10 +143,9 @@ class CuaRuntimeController { ...createCuaComputerTools({ browser: this.options.browser, client: this.options.client, - toolDefinitions: this.runtimeSpec.toolDefinitions, + toolExecutors: this.runtimeSpec.toolExecutors, coordinateSystem: this.runtimeSpec.coordinateSystem, screenshot: this.runtimeSpec.screenshot, - batchTool: this.options.batchTool, computerUseExtra: this.options.computerUseExtra, }), ...(this.options.extraTools ?? []), @@ -170,7 +163,6 @@ class CuaRuntimeController { keepToolNames(): string[] { return [ ...(this.options.extraTools ?? []).map((tool) => tool.name), - ...(this.options.batchTool ? [CUA_BATCH_TOOL_NAME] : []), ...(this.options.computerUseExtra ? [CUA_NAVIGATION_TOOL_NAME] : []), ]; } @@ -242,7 +234,6 @@ export class CuaAgent extends Agent { streamFn, prepareNextTurn, extraTools, - batchTool, computerUseExtra, ...agentOptions } = options; @@ -251,7 +242,6 @@ export class CuaAgent extends Agent { client, model: initialState.model, extraTools, - batchTool, computerUseExtra, systemPrompt: initialState.systemPrompt, onPayload, @@ -362,7 +352,6 @@ export class CuaAgentHarness< client, model, extraTools, - batchTool, computerUseExtra, systemPrompt, getApiKeyAndHeaders, @@ -375,7 +364,6 @@ export class CuaAgentHarness< client, model, extraTools, - batchTool, computerUseExtra, systemPrompt, onPayload, diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 57a49ccb..b48d3e4e 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -7,8 +7,6 @@ export type { ComputerToolOptions, CuaExecutorTool, NavigationDetails, - SupportedCuaExecutorToolName, } from "./tools"; -export { SUPPORTED_CUA_EXECUTOR_TOOL_NAMES } from "./tools"; export { CuaAgent, CuaAgentHarness } from "./agent"; export type { CuaAgentHarnessOptions, CuaAgentOptions, CuaAgentState } from "./agent"; diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 997615f4..3c916369 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -2,17 +2,13 @@ import type Kernel from "@onkernel/sdk"; import type { ImageContent, TextContent, Tool } from "@earendil-works/pi-ai"; import type { TSchema } from "typebox"; import { - CUA_ACTION_TYPES, - CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, - createCuaBatchToolDefinition, createCuaNavigationToolDefinition, type ComputerToolCoordinateSystem, - type CuaAction, - type CuaActionType, type CuaBatchInput, type CuaNavigationInput, type CuaScreenshotSpec, + type CuaToolExecutorSpec, } from "@onkernel/cua-ai"; import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; import type { AgentTool, AgentToolResult } from "./vendor/pi-agent-core/index"; @@ -20,21 +16,12 @@ import type { AgentTool, AgentToolResult } from "./vendor/pi-agent-core/index"; export interface ComputerToolOptions { browser: KernelBrowser; client: Kernel; - toolDefinitions: Tool[]; + toolExecutors: CuaToolExecutorSpec[]; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; - batchTool?: boolean; computerUseExtra?: boolean; } -const CUA_ACTION_TOOL_NAMES = new Set(CUA_ACTION_TYPES); -export const SUPPORTED_CUA_EXECUTOR_TOOL_NAMES = [ - CUA_BATCH_TOOL_NAME, - CUA_NAVIGATION_TOOL_NAME, - ...CUA_ACTION_TYPES, -] as const; -export type SupportedCuaExecutorToolName = (typeof SUPPORTED_CUA_EXECUTOR_TOOL_NAMES)[number]; - type ToolContent = Array; export interface BatchDetails { @@ -54,43 +41,27 @@ type BatchTool = AgentTool; type NavigationTool = AgentTool; type ActionTool = AgentTool; export type CuaExecutorTool = BatchTool | NavigationTool | ActionTool; +type NavigationExecutorSpec = { kind: "navigation"; definition: Tool }; +type ComputerExecutorSpec = CuaToolExecutorSpec | NavigationExecutorSpec; export function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTool[] { const translator = new InternalComputerTranslator(args); - return withSynthesizedTools(args).map((definition) => createExecutorTool(definition, translator)); + return withNavigationTool(args).map((executor) => createExecutorTool(executor, translator)); } -function withSynthesizedTools(args: ComputerToolOptions): Tool[] { - const definitions = [...args.toolDefinitions]; - const existing = new Set(definitions.map((definition) => definition.name)); - const actionTypes = definitions - .map((definition) => definition.name) - .filter((name): name is CuaActionType => CUA_ACTION_TOOL_NAMES.has(name)); - if (args.batchTool && actionTypes.length > 0 && !existing.has(CUA_BATCH_TOOL_NAME)) { - definitions.push(createCuaBatchToolDefinition(actionTypes)); - } +function withNavigationTool(args: ComputerToolOptions): ComputerExecutorSpec[] { + const executors: ComputerExecutorSpec[] = [...args.toolExecutors]; + const existing = new Set(executors.map((executor) => executor.definition.name)); if (args.computerUseExtra && !existing.has(CUA_NAVIGATION_TOOL_NAME)) { - definitions.push(createCuaNavigationToolDefinition()); + const definition = createCuaNavigationToolDefinition(); + executors.push({ kind: "navigation", definition }); } - return definitions; + return executors; } -function createExecutorTool(definition: Tool, translator: InternalComputerTranslator): CuaExecutorTool { - if (definition.name === CUA_BATCH_TOOL_NAME) { - const tool: BatchTool = { - name: definition.name, - label: definition.name, - description: definition.description, - parameters: definition.parameters, - async execute(_toolCallId: string, params: unknown): Promise> { - const result = await executeBatchTool(translator, asBatchInput(params)); - if (result.isError) throw Object.assign(new Error(result.details.statusText), result); - return { content: result.content, details: result.details }; - }, - }; - return tool; - } - if (definition.name === CUA_NAVIGATION_TOOL_NAME) { +function createExecutorTool(executor: ComputerExecutorSpec, translator: InternalComputerTranslator): CuaExecutorTool { + const { definition } = executor; + if (isNavigationExecutor(executor)) { const tool: NavigationTool = { name: definition.name, label: definition.name, @@ -104,26 +75,23 @@ function createExecutorTool(definition: Tool, translator: InternalComputerTransl }; return tool; } - if (CUA_ACTION_TOOL_NAMES.has(definition.name)) { - const actionType = definition.name as CuaActionType; - const tool: ActionTool = { - name: definition.name, - label: definition.name, - description: definition.description, - parameters: definition.parameters, - executionMode: "sequential", - async execute(_toolCallId: string, params: unknown): Promise> { - const action = { ...(params && typeof params === "object" ? params : {}), type: actionType } as CuaAction; - const result = await executeBatchTool(translator, { actions: [action] }); - if (result.isError) throw Object.assign(new Error(result.details.statusText), result); - return { content: result.content, details: result.details }; - }, - }; - return tool; - } - throw new Error( - `unsupported CUA computer tool definition: ${definition.name}`, - ); + const tool: ActionTool = { + name: definition.name, + label: definition.name, + description: definition.description, + parameters: definition.parameters, + executionMode: "sequential", + async execute(_toolCallId: string, params: unknown): Promise> { + const result = await executeBatchTool(translator, { actions: executor.toActions(params) }); + if (result.isError) throw Object.assign(new Error(result.details.statusText), result); + return { content: result.content, details: result.details }; + }, + }; + return tool; +} + +function isNavigationExecutor(executor: ComputerExecutorSpec): executor is NavigationExecutorSpec { + return "kind" in executor && executor.kind === "navigation"; } async function executeBatchTool(translator: InternalComputerTranslator, params: CuaBatchInput): Promise<{ @@ -190,17 +158,6 @@ async function executeNavigationTool(translator: InternalComputerTranslator, par return { content, details: { action, statusText, ...(url ? { url } : {}), ...(error ? { error: error.message } : {}) }, isError: Boolean(error) }; } -function asBatchInput(value: unknown): CuaBatchInput { - if ( - value && - typeof value === "object" && - Array.isArray((value as { actions?: unknown }).actions) - ) { - return value as CuaBatchInput; - } - throw new Error("invalid batch_computer_actions parameters"); -} - function asNavigationInput(value: unknown): CuaNavigationInput { if ( value && diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 4d965e5e..3d5a7335 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -16,6 +16,7 @@ import { const browser = { session_id: "browser_123" } as KernelBrowser; const client = {} as Kernel; +const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; const tinyPng = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", "base64", @@ -91,7 +92,7 @@ describe("CuaAgent", () => { }, }); - expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolDefinitions.map((item) => item.name), "custom"]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "custom"]); }); it("always keeps provider CUA tools when adding extra tools", () => { @@ -108,16 +109,29 @@ describe("CuaAgent", () => { }, }); - expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolDefinitions.map((item) => item.name), "custom"]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "custom"]); expect(agent.state.systemPrompt).toBe("Use the browser carefully."); }); - it("synthesizes batch and navigation tools when requested", () => { + it("installs provider-defined batch tools", () => { + const runtime = resolveCuaRuntimeSpec("anthropic:claude-opus-4-7"); + const agent = new CuaAgent({ + browser, + client, + initialState: { + model: "anthropic:claude-opus-4-7", + }, + }); + + expect(runtime.toolDefinitions.map((tool) => tool.name)).toContain(ANTHROPIC_BATCH_TOOL_NAME); + expect(agent.state.tools.map((tool) => tool.name)).toEqual(runtime.toolExecutors.map((tool) => tool.definition.name)); + }); + + it("synthesizes navigation tools when requested", () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const agent = new CuaAgent({ browser, client, - batchTool: true, computerUseExtra: true, initialState: { model: "openai:gpt-5.5", @@ -125,8 +139,7 @@ describe("CuaAgent", () => { }); expect(agent.state.tools.map((tool) => tool.name)).toEqual([ - ...runtime.toolDefinitions.map((tool) => tool.name), - "batch_computer_actions", + ...runtime.toolExecutors.map((tool) => tool.definition.name), "computer_use_extra", ]); }); @@ -145,7 +158,7 @@ describe("CuaAgent", () => { expect(agent.state.model.id).toBe(runtime.model.id); expect(agent.state.systemPrompt).toBe(runtime.defaultSystemPrompt); - expect(agent.state.tools).toHaveLength(runtime.toolDefinitions.length); + expect(agent.state.tools).toHaveLength(runtime.toolExecutors.length); }); it("keeps extra tools and caller-owned system prompt when state.model changes", () => { @@ -163,7 +176,7 @@ describe("CuaAgent", () => { agent.state.model = "google:gemini-3-pro-preview"; const runtime = resolveCuaRuntimeSpec("google:gemini-3-pro-preview"); - expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolDefinitions.map((item) => item.name), "custom"]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "custom"]); expect(agent.state.systemPrompt).toBe("custom prompt"); }); @@ -214,7 +227,6 @@ describe("CuaAgent", () => { messages: [{ role: "user", content: "Inspect the page" }], tools: [ { type: "function", function: { name: "click" } }, - { type: "function", function: { name: "batch_computer_actions" } }, { type: "function", function: { name: "computer_use_extra" } }, { type: "function", function: { name: "custom_tool" } }, ], @@ -235,7 +247,6 @@ describe("CuaAgent", () => { client: screenshotClient, streamFn, extraTools: [createCustomTool("custom_tool")], - batchTool: true, computerUseExtra: true, initialState: { model: "yutori:n1.5-latest", @@ -251,7 +262,6 @@ describe("CuaAgent", () => { }; expect(payload.tool_set).toBe("browser_tools_core-20260403"); expect(payload.tools?.map((tool) => tool.function?.name)).toEqual([ - "batch_computer_actions", "computer_use_extra", "custom_tool", ]); @@ -286,7 +296,7 @@ describe("CuaAgentHarness", () => { await harness.setModel("google:gemini-3-pro-preview"); expect(harness.agent.state.model.id).toBe(runtime.model.id); - expect(harness.agent.state.tools).toHaveLength(runtime.toolDefinitions.length); + expect(harness.agent.state.tools).toHaveLength(runtime.toolExecutors.length); }); it("appends extraTools in harness construction", async () => { @@ -301,7 +311,7 @@ describe("CuaAgentHarness", () => { }); expect(harness.agent.state.tools.map((item) => item.name)).toEqual([ - ...runtime.toolDefinitions.map((item) => item.name), + ...runtime.toolExecutors.map((item) => item.definition.name), "custom", ]); }); diff --git a/packages/agent/test/tool-exhaustiveness.test.ts b/packages/agent/test/tool-exhaustiveness.test.ts index 541a7f20..2a75273d 100644 --- a/packages/agent/test/tool-exhaustiveness.test.ts +++ b/packages/agent/test/tool-exhaustiveness.test.ts @@ -1,34 +1,30 @@ -import { Type, type Tool } from "@onkernel/cua-ai"; import { CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; import { describe, expect, it } from "vitest"; -import { SUPPORTED_CUA_EXECUTOR_TOOL_NAMES, createCuaComputerTools, type KernelBrowser } from "../src/index"; +import { createCuaComputerTools, type KernelBrowser } from "../src/index"; const browser = { session_id: "browser_123" } as KernelBrowser; const client = {} as Kernel; +const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; const tinyPng = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", "base64", ); describe("Cua tool executor coverage", () => { - it("covers every canonical tool name exported by cua-ai defaults", () => { - const names = new Set(); + it("covers every tool executor exported by cua-ai defaults", () => { for (const provider of CUA_PROVIDERS) { const model = listCuaModels(provider)[0]; expect(model, `no CUA model for provider ${provider}`).toBeDefined(); - for (const definition of resolveCuaRuntimeSpec(model!.ref).toolDefinitions) { - names.add(definition.name); - } + const runtime = resolveCuaRuntimeSpec(model!.ref); + expect(() => createCuaComputerTools({ browser, client, toolExecutors: runtime.toolExecutors })).not.toThrow(); } - const supported = new Set(SUPPORTED_CUA_EXECUTOR_TOOL_NAMES); - for (const name of names) expect(supported.has(name)).toBe(true); }); - it("instantiates one executor per canonical definition", () => { - const toolDefinitions = resolveCuaRuntimeSpec("openai:gpt-5.5").toolDefinitions; - const tools = createCuaComputerTools({ browser, client, toolDefinitions }); - expect(tools.map((tool) => tool.name).sort()).toEqual(toolDefinitions.map((tool) => tool.name).sort()); + it("instantiates one executor per provider execution adapter", () => { + const toolExecutors = resolveCuaRuntimeSpec("openai:gpt-5.5").toolExecutors; + const tools = createCuaComputerTools({ browser, client, toolExecutors }); + expect(tools.map((tool) => tool.name).sort()).toEqual(toolExecutors.map((tool) => tool.definition.name).sort()); }); it("executes Yutori local canonical action tools", async () => { @@ -46,7 +42,7 @@ describe("Cua tool executor coverage", () => { }, }, } as unknown as Kernel, - toolDefinitions: runtime.toolDefinitions, + toolExecutors: runtime.toolExecutors, coordinateSystem: runtime.coordinateSystem, screenshot: runtime.screenshot, }); @@ -61,20 +57,32 @@ describe("Cua tool executor coverage", () => { expect(result.content.at(-1)).toMatchObject({ type: "image", mimeType: "image/webp" }); }); - it("fails fast on unsupported tool names", () => { - const unsupportedDefinitions: Tool[] = [ - { - name: "unknown_tool", - description: "unsupported", - parameters: Type.Object({}), - }, - ]; - expect(() => - createCuaComputerTools({ - browser, - client, - toolDefinitions: unsupportedDefinitions, - }), - ).toThrow(/unsupported CUA computer tool definition/); + it("executes provider-defined batch tools as canonical CUA batches", async () => { + const batches: unknown[] = []; + const runtime = resolveCuaRuntimeSpec("anthropic:claude-opus-4-7"); + const tools = createCuaComputerTools({ + browser, + client: { + browsers: { + computer: { + batch: async (_id: string, body: { actions: unknown[] }) => { + batches.push(body.actions); + }, + captureScreenshot: async () => new Response(tinyPng), + }, + }, + } as unknown as Kernel, + toolExecutors: runtime.toolExecutors, + coordinateSystem: runtime.coordinateSystem, + }); + const batch = tools.find((tool) => tool.name === ANTHROPIC_BATCH_TOOL_NAME); + expect(batch).toBeDefined(); + + const result = await batch!.execute("call_1", { actions: [{ type: "click", x: 10, y: 20 }] }); + + expect(batches).toEqual([ + [{ type: "click_mouse", click_mouse: { x: 10, y: 20, button: "left" } }], + ]); + expect(result.content.at(-1)).toMatchObject({ type: "image", mimeType: "image/png" }); }); }); diff --git a/packages/ai/README.md b/packages/ai/README.md index 93f4983b..57f42bfe 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -111,7 +111,8 @@ Top-level exports: runtime consumers: - canonical provider id -- CUA tool definitions installed by `CuaAgent`/`CuaAgentHarness` +- provider-facing CUA tool definitions used in model requests +- local execution adapters used by `CuaAgent`/`CuaAgentHarness` - default system prompt text - provider coordinate convention - optional provider screenshot input policy @@ -119,11 +120,12 @@ runtime consumers: Provider namespaces expose `computerTools({ actions? })` for building the provider's default CUA `Tool[]` definitions. These are the tools -that agent runtimes install and execute locally. Most providers send the same -definitions to the model API; providers whose APIs expose tools through -separate request fields can adapt the outgoing payload with runtime middleware. -Omit `actions` for the provider's default computer tool set, or pass an action -subset to narrow the schema for a single `complete()` call: +sent to the model when you call `complete()` or `stream()` directly. The +default set can differ by provider: Anthropic includes its `computer_batch` +tool from the computer-use best-practices reference, while providers such as +OpenAI currently expose individual canonical browser actions. Omit `actions` +for the provider's default computer tool set, or pass an action subset to narrow +the schema for a single `complete()` call: ```ts import { openai } from "@onkernel/cua-ai"; @@ -132,14 +134,19 @@ const allComputerTools = openai.computerTools(); const clickOnlyTools = openai.computerTools({ actions: ["click"] }); ``` -Provider namespaces expose individual canonical action definitions by default. -Some providers are different on the wire: Yutori exposes browser actions -through its documented `tool_set` request field, and Tzafon exposes them -through its native `computer_use` Responses tool. Their payload adapters remove -local canonical CUA action definitions before requests and enable the -provider-native computer-use interface. Caller-provided tools that should -remain on the provider payload can be preserved by payload middleware via -`CuaPayloadContext.keepToolNames`. +When `actions` is provided, it must be a subset of that provider's supported +canonical action set. + +Runtime specs also include `toolExecutors`: provider-owned adapters that use +the same tool-call names as the model-facing tools and translate their +arguments into canonical CUA actions for `@onkernel/cua-agent`. For most +providers, `toolDefinitions` and `toolExecutors` line up one-for-one. Some +providers are different on the wire: Yutori exposes browser actions through its +documented `tool_set` request field, so its runtime spec has no model-facing +`toolDefinitions` but still provides local `toolExecutors` for the canonical +actions emitted after Yutori's native tool calls are normalized. Caller-provided +tools that should remain on the provider payload can be preserved by payload +middleware via `CuaPayloadContext.keepToolNames`. Provider namespaces also expose `COMPUTER_TOOL_COORDINATES`, which describes the coordinates the provider's computer tool calls are expected to emit: @@ -155,7 +162,7 @@ gemini.COMPUTER_TOOL_COORDINATES Current coordinate contracts: - `openai`: pixel coordinates -- `anthropic`: pixel coordinates +- `anthropic`: pixel coordinates, matching Anthropic's computer-use quickstart - `gemini`: normalized coordinates in the 0-999 range ([source](https://ai.google.dev/gemini-api/docs/computer-use)) - `yutori`: normalized coordinates in the 0-1000 range ([source](https://docs.yutori.com/reference/navigator), [SDK helper](https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/coordinates.py)) - `tzafon`: normalized coordinates in the 0-999 range ([source](https://docs.lightcone.ai/guides/coordinates/), [model card](https://huggingface.co/Tzafon/Northstar-CUA-Fast)) @@ -200,8 +207,8 @@ type CuaActionGoto = { }; ``` -`createCuaBatchToolDefinition(actions?)` can synthesize a -`batch_computer_actions` tool whose input is: +`createCuaBatchToolDefinition(actions?, options?)` builds a batch tool schema +whose input is: ```ts type CuaBatchInput = { @@ -209,10 +216,10 @@ type CuaBatchInput = { }; ``` -Agent runtimes can opt into this as local sugar when they want the model to -plan several writes and reads in one call. Read actions such as `screenshot`, -`url`, and `cursor_position` can be interleaved with writes so your executor -can return fresh state in the same order. +Providers can include a batch tool when their model is expected to use one. +Anthropic does this by default with `computer_batch`; Yutori does not. +`createCuaBatchToolExecutor()` is the matching execution adapter for turning +that provider-defined batch input into canonical CUA actions. `createCuaNavigationToolDefinition()` can synthesize a `computer_use_extra` navigation tool whose input is: @@ -226,12 +233,12 @@ type CuaNavigationInput = { Provider namespaces: -- `openai`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, OpenAI CUA action schemas, and prompt helpers -- `anthropic`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases -- `gemini`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases -- `tzafon`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and local `tzafon-responses` stream adapter +- `openai`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, OpenAI CUA action schemas, and prompt helpers +- `anthropic`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases +- `gemini`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases +- `tzafon`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and local `tzafon-responses` stream adapter - `yutori`: native Navigator action sets, native-to-canonical action helpers, - `computerTools`, `COMPUTER_TOOL_COORDINATES`, local + `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, local `yutori-chat-completions` stream adapter, and `yutoriNativeToolSetOnPayload` This package does not execute browser actions. Use `@onkernel/cua-agent` when diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index 1becccd7..b48ac465 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -12,7 +12,8 @@ const modelRef = "openai:gpt-5.5"; const model = getCuaModel(modelRef); const screenshot = await readFile(screenshotPath); -// Other provider examples: +// Other provider examples. Add the provider namespace to the top-level import +// before switching these values. // const apiKey = process.env.ANTHROPIC_API_KEY; // const modelRef = "anthropic:claude-opus-4-7"; // const model = getCuaModel(modelRef); diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts new file mode 100644 index 00000000..4e3c72af --- /dev/null +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -0,0 +1,106 @@ +import type { Tool, TSchema } from "@earendil-works/pi-ai"; +import { + CUA_BATCH_TOOL_DESCRIPTION, + createCuaActionSchema, + createCuaActionToolExecutors, + createCuaActionToolDefinitions, + createCuaBatchToolExecutor, + createCuaBatchToolDefinition, + type ComputerToolsOptions, + type CuaAction, + type CuaActionType, + type CuaToolExecutorSpec, +} from "../common"; + +// Source of truth: Anthropic's computer-use best-practices quickstart +// computer/browser tool action enums. These are the browser actions Anthropic +// currently accepts under CUA's canonical individual tool names. +// https://github.com/anthropics/claude-quickstarts/blob/main/computer-use-best-practices/computer_use/tools/computer.py +// https://github.com/anthropics/claude-quickstarts/blob/main/computer-use-best-practices/computer_use/tools/browser.py +const ANTHROPIC_CANONICAL_ACTION_TYPES = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "goto", + "cursor_position", +] as const satisfies readonly CuaActionType[]; + +type AnthropicCanonicalActionType = (typeof ANTHROPIC_CANONICAL_ACTION_TYPES)[number]; + +const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set(ANTHROPIC_CANONICAL_ACTION_TYPES); +const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; +const ANTHROPIC_BATCH_TOOL_DESCRIPTION = [ + CUA_BATCH_TOOL_DESCRIPTION, + "Coordinates in a batch refer to the screenshot taken before the batch call.", +].join("\n"); + +/** Options for building Anthropic browser computer-use tools. */ +export interface AnthropicComputerToolsOptions extends ComputerToolsOptions { + /** Exclude the batch computer action tool from the returned tools. */ + excludeBatch?: boolean; +} + +/** Canonical CUA action shape supported by Anthropic browser computer-use tools. */ +export type AnthropicAction = Extract; + +function resolveAnthropicActions(actions: readonly CuaActionType[] | undefined): readonly AnthropicCanonicalActionType[] { + const resolved = actions ?? ANTHROPIC_CANONICAL_ACTION_TYPES; + const supported: AnthropicCanonicalActionType[] = []; + const unsupported: CuaActionType[] = []; + for (const action of resolved) { + if (isAnthropicCanonicalAction(action)) supported.push(action); + else unsupported.push(action); + } + if (unsupported.length > 0) throw new Error(`unsupported Anthropic canonical action(s): ${unsupported.join(", ")}`); + return supported; +} + +function isAnthropicCanonicalAction(action: CuaActionType): action is AnthropicCanonicalActionType { + return ANTHROPIC_CANONICAL_ACTION_TYPE_SET.has(action); +} + +/** Build the TypeBox schema for Anthropic-supported canonical browser actions. */ +export function createActionSchema(actions?: readonly CuaActionType[]): TSchema { + return createCuaActionSchema(resolveAnthropicActions(actions)); +} + +/** + * Build Anthropic CUA computer-use tools. + * + * Use this when calling `complete()` or `stream()` directly and you need an + * array of `Tool` objects for Anthropic browser actions. Pass `actions` to + * expose only a supported subset, such as `["click"]`. Anthropic includes a + * batch tool by default; pass `excludeBatch: true` to omit it. + */ +export function computerTools(options: AnthropicComputerToolsOptions = {}): Tool[] { + const actions = resolveAnthropicActions(options.actions); + const tools = createCuaActionToolDefinitions(actions); + if (!options.excludeBatch) { + tools.push(createCuaBatchToolDefinition(actions, { + name: ANTHROPIC_BATCH_TOOL_NAME, + description: ANTHROPIC_BATCH_TOOL_DESCRIPTION, + })); + } + return tools; +} + +/** Build the local execution adapters used by CuaAgent and CuaAgentHarness. */ +export function computerToolExecutors(options: AnthropicComputerToolsOptions = {}): CuaToolExecutorSpec[] { + const actions = resolveAnthropicActions(options.actions); + const executors = createCuaActionToolExecutors(actions); + if (!options.excludeBatch) { + executors.push(createCuaBatchToolExecutor(actions, { + name: ANTHROPIC_BATCH_TOOL_NAME, + description: ANTHROPIC_BATCH_TOOL_DESCRIPTION, + })); + } + return executors; +} diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index d6df4ea7..1cf146f7 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -1,27 +1,22 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { - CUA_ACTION_TYPES as ANTHROPIC_CUA_ACTION_TYPES, + computerToolExecutors, computerTools, - createCuaActionSchema as createActionSchema, -} from "../common"; + createActionSchema, +} from "./actions"; export type { - CuaAction as AnthropicAction, - ComputerToolsOptions, -} from "../common"; + AnthropicAction, + AnthropicComputerToolsOptions, + AnthropicComputerToolsOptions as ComputerToolsOptions, +} from "./actions"; -// Provider-native action vocabulary emitted on `tool_use.input.action`. Latest -// tool version is `computer_20251124`, which extends earlier dated versions: -// computer_20241022: key, type, mouse_move, left_click, left_click_drag, -// right_click, middle_click, double_click, screenshot, -// cursor_position -// computer_20250124: + left_mouse_down, left_mouse_up, scroll, hold_key, -// wait, triple_click -// computer_20251124: + zoom -// Source: https://github.com/anthropics/anthropic-quickstarts/blob/main/computer-use-demo/computer_use_demo/tools/computer.py +// Anthropic's quickstart uses pixel coordinates for both its computer and +// browser tools. +// Source: https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-best-practices export const COMPUTER_TOOL_COORDINATES = { type: "pixel" } as const satisfies ComputerToolCoordinateSystem; -export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use keyboard navigation where possible, and request explicit screenshot or url reads when you need to inspect state.`; +export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use keyboard navigation where possible, and request screenshots when you need to inspect state.`; export function buildAnthropicSystemPrompt(opts: { suffix?: string } = {}): string { return [ANTHROPIC_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 079c7b51..0bb083d6 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -289,6 +289,14 @@ export interface CuaBatchInput { } export type CuaNavigationInput = Static; +/** Tool schema plus execution adapter for a browser computer-use tool. */ +export interface CuaToolExecutorSpec { + /** Tool schema installed by CuaAgent/CuaAgentHarness. The name must match the provider tool call name. */ + definition: Tool; + /** Convert that tool's arguments into canonical CUA actions for browser execution. */ + toActions(args: unknown): CuaAction[]; +} + export const CUA_BATCH_TOOL_NAME = "batch_computer_actions"; export const CUA_NAVIGATION_TOOL_NAME = "computer_use_extra"; @@ -324,6 +332,19 @@ export function computerTools(options: ComputerToolsOptions = {}): Tool[] { return createCuaActionToolDefinitions(options.actions); } +/** Build execution adapters for individual canonical CUA action tools. */ +export function createCuaActionToolExecutors(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): CuaToolExecutorSpec[] { + return createCuaActionToolDefinitions(actions).map((definition) => { + const actionType = definition.name as CuaActionType; + return { + definition, + toActions(args: unknown): CuaAction[] { + return [{ ...(args && typeof args === "object" ? args : {}), type: actionType } as CuaAction]; + }, + }; + }); +} + /** Return the canonical tool name that should execute a normalized CUA action. */ export function canonicalToolCallName(action: CuaAction): CuaActionType { return action.type; @@ -343,14 +364,41 @@ export function normalizeGotoUrl(value: unknown): string | undefined { return /^[a-z][a-z0-9+.-]*:\/\//i.test(url) ? url : `https://${url}`; } -export function createCuaBatchToolDefinition(actions?: readonly CuaActionType[]): Tool { +export function createCuaBatchToolDefinition( + actions?: readonly CuaActionType[], + options: { name?: string; description?: string } = {}, +): Tool { return { - name: CUA_BATCH_TOOL_NAME, - description: CUA_BATCH_TOOL_DESCRIPTION, + name: options.name ?? CUA_BATCH_TOOL_NAME, + description: options.description ?? CUA_BATCH_TOOL_DESCRIPTION, parameters: createCuaBatchSchema(actions), }; } +/** Build an execution adapter for a batch tool whose input is `{ actions }`. */ +export function createCuaBatchToolExecutor( + actions?: readonly CuaActionType[], + options: { name?: string; description?: string } = {}, +): CuaToolExecutorSpec { + const definition = createCuaBatchToolDefinition(actions, options); + return { + definition, + toActions(args: unknown): CuaAction[] { + if (!isBatchInput(args)) throw new Error("invalid batch tool parameters"); + return args.actions; + }, + }; +} + +/** Build the provider's default CUA tool execution adapters. */ +export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaToolExecutorSpec[] { + return createCuaActionToolExecutors(options.actions); +} + +function isBatchInput(value: unknown): value is CuaBatchInput { + return Boolean(value && typeof value === "object" && Array.isArray((value as { actions?: unknown }).actions)); +} + export function createCuaNavigationToolDefinition(): Tool { return { name: CUA_NAVIGATION_TOOL_NAME, diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index d5648a5f..b49d7c9d 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -2,6 +2,7 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, + computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, } from "../common"; diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index cefa0274..04c658aa 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -4,6 +4,7 @@ export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, CUA_NAVIGATION_TOOL_DESCRIPTION as OPENAI_EXTRA_TOOL_DESCRIPTION, CUA_NAVIGATION_TOOL_NAME as OPENAI_EXTRA_TOOL_NAME, + computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, CuaNavigationSchema as OpenAIExtraSchema, diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 277f6f5c..a946add0 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -2,6 +2,7 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, + computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, } from "../common"; @@ -31,6 +32,7 @@ export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] } export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Include screenshot or URL reads when you need updated state.`; +/** Build the default system prompt used with Tzafon CUA models. */ export function buildTzafonSystemPrompt(opts: { suffix?: string } = {}): string { return [TZAFON_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); } diff --git a/packages/ai/src/providers/yutori/actions.ts b/packages/ai/src/providers/yutori/actions.ts index ef624499..93b32ba8 100644 --- a/packages/ai/src/providers/yutori/actions.ts +++ b/packages/ai/src/providers/yutori/actions.ts @@ -1,7 +1,9 @@ import { createCuaActionToolDefinitions, + createCuaActionToolExecutors, normalizeGotoUrl, type CuaAction, + type CuaToolExecutorSpec, type CuaActionType, } from "../common"; @@ -118,6 +120,11 @@ export function computerTools(_options?: unknown) { return createCuaActionToolDefinitions(YUTORI_CANONICAL_ACTION_TYPES); } +/** Build the local execution adapters used by CuaAgent and CuaAgentHarness. */ +export function computerToolExecutors(_options?: unknown): CuaToolExecutorSpec[] { + return createCuaActionToolExecutors(YUTORI_CANONICAL_ACTION_TYPES); +} + export function yutoriToolSetForModel(modelId: string): typeof YUTORI_N15_CORE_TOOL_SET | undefined { return modelId.startsWith("n1.5") ? YUTORI_N15_CORE_TOOL_SET : undefined; } diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index 449e6d1f..f24cbcc8 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,6 +1,7 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { + computerToolExecutors, computerTools, toCanonicalActions, yutoriNativeActionsForModel, diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 24bd3c5c..0473ae77 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -5,7 +5,7 @@ import * as gemini from "./providers/gemini/index"; import * as openai from "./providers/openai/index"; import * as tzafon from "./providers/tzafon/index"; import * as yutori from "./providers/yutori/index"; -import type { ComputerToolCoordinateSystem } from "./providers/common"; +import type { ComputerToolCoordinateSystem, CuaToolExecutorSpec } from "./providers/common"; export interface CuaScreenshotTransformSpec { width: number; @@ -38,8 +38,10 @@ export type CuaPayloadHook = (payload: unknown, model: Model, context?: Cua export interface CuaRuntimeSpec { model: Model; provider: CuaProvider; - /** CUA tool definitions installed by CuaAgent/CuaAgentHarness for local execution. */ + /** Provider-facing CUA tool definitions used for model requests. */ toolDefinitions: Tool[]; + /** Local execution adapters that turn provider tool calls into canonical CUA actions. */ + toolExecutors: CuaToolExecutorSpec[]; /** Provider-tuned baseline prompt for browser control behavior. */ defaultSystemPrompt: string; /** Coordinate convention emitted by provider tool calls. */ @@ -67,6 +69,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe model, provider, toolDefinitions: anthropic.computerTools(), + toolExecutors: anthropic.computerToolExecutors(), defaultSystemPrompt: anthropic.buildAnthropicSystemPrompt(), coordinateSystem: anthropic.COMPUTER_TOOL_COORDINATES, }; @@ -75,6 +78,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe model, provider, toolDefinitions: gemini.computerTools(), + toolExecutors: gemini.computerToolExecutors(), defaultSystemPrompt: gemini.buildGeminiSystemPrompt(), coordinateSystem: gemini.COMPUTER_TOOL_COORDINATES, }; @@ -83,6 +87,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe model, provider, toolDefinitions: tzafon.computerTools(), + toolExecutors: tzafon.computerToolExecutors(), defaultSystemPrompt: tzafon.buildTzafonSystemPrompt(), coordinateSystem: tzafon.COMPUTER_TOOL_COORDINATES, onPayload: tzafon.tzafonComputerUseOnPayload, @@ -91,7 +96,8 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe return { model, provider, - toolDefinitions: yutori.computerTools(), + toolDefinitions: [], + toolExecutors: yutori.computerToolExecutors(), defaultSystemPrompt: yutori.buildYutoriSystemPrompt(), coordinateSystem: yutori.COMPUTER_TOOL_COORDINATES, screenshot: { @@ -106,6 +112,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe model, provider, toolDefinitions: openai.computerTools(), + toolExecutors: openai.computerToolExecutors(), defaultSystemPrompt: openai.OPENAI_COMPUTER_INSTRUCTIONS, coordinateSystem: openai.COMPUTER_TOOL_COORDINATES, onPayload: openai.openaiResponsesStoreOnPayload, diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index 2d1ebd9f..2a88de3b 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index"; +import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index"; describe("resolveCuaRuntimeSpec", () => { it("resolves a runtime spec for every CUA provider", () => { @@ -12,10 +12,14 @@ describe("resolveCuaRuntimeSpec", () => { expect(spec.model.id).toBe(model!.model); expect(typeof spec.defaultSystemPrompt).toBe("string"); expect(spec.coordinateSystem).toBeDefined(); - expect(spec.toolDefinitions.length).toBeGreaterThan(0); - expect(spec.toolDefinitions.map((tool) => tool.name)).not.toContain(CUA_BATCH_TOOL_NAME); + expect(spec.toolExecutors.length).toBeGreaterThan(0); expect(spec.toolDefinitions.map((tool) => tool.name)).not.toContain(CUA_NAVIGATION_TOOL_NAME); + if (provider === "anthropic") { + expect(spec.toolDefinitions.map((tool) => tool.name)).toContain("computer_batch"); + expect(spec.toolExecutors.map((executor) => executor.definition.name)).toContain("computer_batch"); + } if (provider === "yutori") { + expect(spec.toolDefinitions).toEqual([]); expect(spec.defaultSystemPrompt).toBe(""); expect(spec.screenshot).toEqual({ appendToLatestMessage: true, diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index d3d9ddc8..8273477a 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -13,7 +13,8 @@ import { yutori, } from "../src/index"; -const providers = { openai, anthropic, gemini, tzafon }; +const providers = { openai, gemini, tzafon }; +const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; function batchActionVariants(tool: { parameters: any }): any[] { const items = tool.parameters.properties.actions.items; @@ -79,6 +80,47 @@ describe("computer tool definitions", () => { ]); }); + it("exposes Anthropic-supported canonical action tools", () => { + const tools = anthropic.computerTools(); + expect(tools.map((tool) => tool.name)).toEqual([ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "goto", + "cursor_position", + ANTHROPIC_BATCH_TOOL_NAME, + ]); + expect(tools.map((tool) => tool.name)).not.toContain("back"); + expect(tools.map((tool) => tool.name)).not.toContain("forward"); + expect(tools.map((tool) => tool.name)).not.toContain("url"); + }); + + it("narrows Anthropic tools to supported actions", () => { + const tools = anthropic.computerTools({ actions: ["click"], excludeBatch: true }); + expect(tools.map((tool) => tool.name)).toEqual(["click"]); + expect(tools[0]!.parameters.properties.type).toBeUndefined(); + expect(tools[0]!.parameters.required).toEqual(["x", "y"]); + }); + + it("includes an Anthropic batch tool by default", () => { + const tool = anthropic.computerTools({ actions: ["click"] }).find((item) => item.name === ANTHROPIC_BATCH_TOOL_NAME); + expect(tool).toBeDefined(); + expect(batchActionVariants(tool!).map((variant) => variant.properties.type.const)).toEqual(["click"]); + }); + + it("rejects unsupported Anthropic action narrowing", () => { + expect(() => anthropic.computerTools({ actions: ["url"] })).toThrow("unsupported Anthropic canonical action(s): url"); + expect(() => anthropic.createActionSchema(["url"])).toThrow("unsupported Anthropic canonical action(s): url"); + }); + it("exports provider coordinate systems", () => { expect(openai.COMPUTER_TOOL_COORDINATES).toEqual({ type: "pixel" }); expect(anthropic.COMPUTER_TOOL_COORDINATES).toEqual({ type: "pixel" }); From 8098e35da490ea7dd2e644aa5360ed9bd095f638 Mon Sep 17 00:00:00 2001 From: Rafael Garcia Date: Wed, 3 Jun 2026 11:20:26 -0400 Subject: [PATCH 2/2] Introduce CuaProviderModule contract in cua-ai Replace the per-provider switch in resolveCuaRuntimeSpec with a registry lookup over provider modules that conform to a shared CuaProviderModule interface. Adding a provider is now "write the module, add one line"; the `satisfies Record` makes an unconformed or missing provider a compile error. - Define CuaProviderModule in providers/common.ts and move the provider contract types (CuaPayloadHook, CuaScreenshotSpec, CuaRuntimeSpec, etc.) there so common.ts owns the contract and runtime-spec.ts only composes it. - Expose coordinateSystem() as a function on each provider instead of the constant-like COMPUTER_TOOL_COORDINATES field. - Standardize on buildSystemPrompt() everywhere; add the missing buildOpenAISystemPrompt(). - Drop the dead yutoriBuiltinToolsOnPayload alias; yutoriNativeToolSetOnPayload is the single canonical name. Fix yutori/provider.ts to import the payload context type from ../common instead of back-importing runtime-spec. - Preserve Yutori's model-facing toolDefinitions: [] invariant in its module. - Add provider-module.test.ts asserting every provider conforms to the contract; update README to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/ai/README.md | 27 ++-- packages/ai/src/providers/anthropic/index.ts | 14 +- packages/ai/src/providers/common.ts | 66 ++++++++- packages/ai/src/providers/gemini/index.ts | 14 +- packages/ai/src/providers/openai/index.ts | 19 ++- packages/ai/src/providers/tzafon/index.ts | 15 +- packages/ai/src/providers/tzafon/provider.ts | 3 +- packages/ai/src/providers/yutori/index.ts | 21 ++- packages/ai/src/providers/yutori/provider.ts | 7 +- packages/ai/src/runtime-spec.ts | 139 ++++--------------- packages/ai/test/provider-module.test.ts | 46 ++++++ packages/ai/test/tools.test.ts | 10 +- packages/ai/test/yutori-payload.test.ts | 8 +- 13 files changed, 240 insertions(+), 149 deletions(-) create mode 100644 packages/ai/test/provider-module.test.ts diff --git a/packages/ai/README.md b/packages/ai/README.md index 57f42bfe..3163d82f 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -148,14 +148,14 @@ actions emitted after Yutori's native tool calls are normalized. Caller-provided tools that should remain on the provider payload can be preserved by payload middleware via `CuaPayloadContext.keepToolNames`. -Provider namespaces also expose `COMPUTER_TOOL_COORDINATES`, which describes -the coordinates the provider's computer tool calls are expected to emit: +Provider namespaces also expose `coordinateSystem()`, which returns the +coordinates the provider's computer tool calls are expected to emit: ```ts -openai.COMPUTER_TOOL_COORDINATES +openai.coordinateSystem() // { type: "pixel" } -gemini.COMPUTER_TOOL_COORDINATES +gemini.coordinateSystem() // { type: "normalized", range: [0, 999] } ``` @@ -233,13 +233,18 @@ type CuaNavigationInput = { Provider namespaces: -- `openai`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, OpenAI CUA action schemas, and prompt helpers -- `anthropic`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases -- `gemini`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases -- `tzafon`: `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and local `tzafon-responses` stream adapter -- `yutori`: native Navigator action sets, native-to-canonical action helpers, - `computerTools`, `computerToolExecutors`, `COMPUTER_TOOL_COORDINATES`, local - `yutori-chat-completions` stream adapter, and `yutoriNativeToolSetOnPayload` +Each provider namespace exposes `computerTools`, `computerToolExecutors`, +`coordinateSystem`, a `buildSystemPrompt` helper, and a +`providerModule` object wiring those functions to the uniform +`CuaProviderModule` contract that `resolveCuaRuntimeSpec` looks up: + +- `openai`: `buildOpenAISystemPrompt`, OpenAI CUA action schemas +- `anthropic`: `buildAnthropicSystemPrompt`, CUA action schema aliases +- `gemini`: `buildGeminiSystemPrompt`, CUA action schema aliases +- `tzafon`: `buildTzafonSystemPrompt`, local `tzafon-responses` stream adapter +- `yutori`: `buildYutoriSystemPrompt`, native Navigator action sets, + native-to-canonical action helpers, local `yutori-chat-completions` stream + adapter, and `yutoriNativeToolSetOnPayload` This package does not execute browser actions. Use `@onkernel/cua-agent` when you want model tool calls executed against a Kernel browser. diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 1cf146f7..4d41dac1 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -1,4 +1,5 @@ -import type { ComputerToolCoordinateSystem } from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors, computerTools } from "./actions"; export { computerToolExecutors, @@ -14,10 +15,19 @@ export type { // Anthropic's quickstart uses pixel coordinates for both its computer and // browser tools. // Source: https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-best-practices -export const COMPUTER_TOOL_COORDINATES = { type: "pixel" } as const satisfies ComputerToolCoordinateSystem; +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "pixel" }; +} export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use keyboard navigation where possible, and request screenshots when you need to inspect state.`; export function buildAnthropicSystemPrompt(opts: { suffix?: string } = {}): string { return [ANTHROPIC_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } + +export const providerModule = { + toolDefinitions: computerTools, + toolExecutors: computerToolExecutors, + coordinateSystem, + buildSystemPrompt: buildAnthropicSystemPrompt, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 0bb083d6..e2efa1df 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -1,4 +1,5 @@ -import { Type, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai"; +import { Type, type Api, type Model, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai"; +import type { CuaModelRef, CuaProvider } from "../models"; export const CUA_ACTION_TYPES = [ "click", @@ -406,3 +407,66 @@ export function createCuaNavigationToolDefinition(): Tool { parameters: CuaNavigationSchema, }; } + +export interface CuaScreenshotTransformSpec { + width: number; + height: number; + format: "png" | "jpeg" | "webp"; + quality?: number; +} + +export interface CuaScreenshotSpec { + /** Append a provider-prepared screenshot to the latest user/tool message before each request. */ + appendToLatestMessage?: boolean; + /** Optional image transform applied to Kernel screenshots before they are sent to the provider. */ + transform?: CuaScreenshotTransformSpec; +} + +export interface CuaPayloadContext { + /** Tool names that should remain in the outbound provider payload even if the provider strips local CUA executors. */ + keepToolNames?: readonly string[]; +} + +export type CuaPayloadHook = (payload: unknown, model: Model, context?: CuaPayloadContext) => unknown | Promise; + +/** + * Runtime configuration for a supported CUA model. + * + * Use this to pair a model with the agent tool definitions, baseline prompt, + * coordinate convention, screenshot policy, and request payload middleware + * expected by its provider. + */ +export interface CuaRuntimeSpec { + model: Model; + provider: CuaProvider; + /** Provider-facing CUA tool definitions used for model requests. */ + toolDefinitions: Tool[]; + /** Local execution adapters that turn provider tool calls into canonical CUA actions. */ + toolExecutors: CuaToolExecutorSpec[]; + /** Provider-tuned baseline prompt for browser control behavior. */ + defaultSystemPrompt: string; + /** Coordinate convention emitted by provider tool calls. */ + coordinateSystem: ComputerToolCoordinateSystem; + /** Optional provider screenshot input policy used by CuaAgent/CuaAgentHarness. */ + screenshot?: CuaScreenshotSpec; + /** Optional provider middleware for request payload adaptation. */ + onPayload?: CuaPayloadHook; +} + +export type CuaRuntimeSpecInput = CuaModelRef | Model; + +/** Uniform provider contract resolved by the CUA runtime registry. */ +export interface CuaProviderModule { + /** Model-facing CUA tool definitions sent in provider requests. */ + toolDefinitions(options?: ComputerToolsOptions): Tool[]; + /** Local execution adapters (provider tool-call name -> canonical CUA actions). */ + toolExecutors(options?: ComputerToolsOptions): CuaToolExecutorSpec[]; + /** Coordinate convention emitted by this provider's tool calls. */ + coordinateSystem(): ComputerToolCoordinateSystem; + /** Provider-tuned baseline browser-control system prompt. */ + buildSystemPrompt(opts?: { suffix?: string }): string; + /** Optional request-payload middleware for provider protocol quirks. */ + onPayload?: CuaPayloadHook; + /** Optional provider screenshot input policy. */ + screenshot?: CuaScreenshotSpec; +} diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index b49d7c9d..0770fc37 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -1,4 +1,5 @@ -import type { ComputerToolCoordinateSystem } from "../common"; +import { computerToolExecutors, computerTools } from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, @@ -18,10 +19,19 @@ export type { // Coordinates are normalized to 0-999 regardless of input image size. // Source: https://github.com/google/computer-use-preview/blob/main/agent.py // Docs: https://ai.google.dev/gemini-api/docs/computer-use -export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] } as const satisfies ComputerToolCoordinateSystem; +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "normalized", range: [0, 999] }; +} export const GEMINI_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Use the provider coordinate system for tool calls, and request screenshots or URL reads when state changes.`; export function buildGeminiSystemPrompt(opts: { suffix?: string } = {}): string { return [GEMINI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); } + +export const providerModule = { + toolDefinitions: computerTools, + toolExecutors: computerToolExecutors, + coordinateSystem, + buildSystemPrompt: buildGeminiSystemPrompt, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 04c658aa..1b1a7a1f 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -1,4 +1,5 @@ -import type { ComputerToolCoordinateSystem } from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors, computerTools } from "../common"; export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, @@ -18,10 +19,16 @@ export type { // Provider-native action vocabulary emitted on `computer_call.action.type`: // click, double_click, drag, move, scroll, type, keypress, wait, screenshot // Source: https://github.com/openai/openai-cua-sample-app/blob/main/packages/runner-core/src/responses-loop.ts -export const COMPUTER_TOOL_COORDINATES = { type: "pixel" } as const satisfies ComputerToolCoordinateSystem; +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "pixel" }; +} export const OPENAI_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use the available tools for browser interaction and request explicit url, cursor_position, or screenshot reads when you need updated state.`; +export function buildOpenAISystemPrompt(opts: { suffix?: string } = {}): string { + return [OPENAI_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); +} + export function openaiResponsesStoreOnPayload(payload: unknown): unknown | undefined { if (!payload || typeof payload !== "object") return undefined; const current = payload as Record; @@ -31,3 +38,11 @@ export function openaiResponsesStoreOnPayload(payload: unknown): unknown | undef store: true, }; } + +export const providerModule = { + toolDefinitions: computerTools, + toolExecutors: computerToolExecutors, + coordinateSystem, + buildSystemPrompt: buildOpenAISystemPrompt, + onPayload: openaiResponsesStoreOnPayload, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index a946add0..0445f92e 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -1,4 +1,5 @@ -import type { ComputerToolCoordinateSystem } from "../common"; +import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, type CuaProviderModule } from "../common"; +import { tzafonComputerUseOnPayload } from "./provider"; export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, @@ -28,7 +29,9 @@ export { // https://huggingface.co/Tzafon/Northstar-CUA-Fast // https://docs.lightcone.ai/guides/cua-protocol/ // https://docs.lightcone.ai/guides/coordinates/ -export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] } as const satisfies ComputerToolCoordinateSystem; +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "normalized", range: [0, 999] }; +} export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Include screenshot or URL reads when you need updated state.`; @@ -36,3 +39,11 @@ export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser throu export function buildTzafonSystemPrompt(opts: { suffix?: string } = {}): string { return [TZAFON_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); } + +export const providerModule = { + toolDefinitions: computerTools, + toolExecutors: computerToolExecutors, + coordinateSystem, + buildSystemPrompt: buildTzafonSystemPrompt, + onPayload: tzafonComputerUseOnPayload, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index e4c2b93a..dc78f349 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -13,8 +13,7 @@ import { type ToolCall, } from "@earendil-works/pi-ai"; import Lightcone from "@tzafon/lightcone"; -import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction } from "../common"; -import type { CuaPayloadContext } from "../../runtime-spec"; +import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction, type CuaPayloadContext } from "../common"; export const TZAFON_RESPONSES_API = "tzafon-responses"; const TZAFON_COMPUTER_USE_TOOL = { diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index f24cbcc8..aa07e3c1 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,4 +1,6 @@ -import type { ComputerToolCoordinateSystem } from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors } from "./actions"; +import { yutoriNativeToolSetOnPayload } from "./provider"; export { computerToolExecutors, @@ -19,7 +21,6 @@ export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori, - yutoriBuiltinToolsOnPayload, yutoriNativeToolSetOnPayload, } from "./provider"; @@ -38,7 +39,9 @@ export { // https://docs.yutori.com/reference/n1-5 // https://docs.yutori.com/llm-quickstart.md // https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/coordinates.py -export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 1000] } as const satisfies ComputerToolCoordinateSystem; +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "normalized", range: [0, 1000] }; +} // Yutori's Navigator quickstart recommends putting extra instructions in the // first user message instead of supplying a custom system prompt. @@ -48,3 +51,15 @@ export const YUTORI_INSTRUCTIONS_RAW = ""; export function buildYutoriSystemPrompt(opts: { suffix?: string } = {}): string { return [YUTORI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); } + +export const providerModule = { + toolDefinitions: () => [], + toolExecutors: computerToolExecutors, + coordinateSystem, + buildSystemPrompt: buildYutoriSystemPrompt, + onPayload: yutoriNativeToolSetOnPayload, + screenshot: { + appendToLatestMessage: true, + transform: { width: 1280, height: 800, format: "webp", quality: 90 }, + }, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index fec1f701..35d8adaf 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -19,8 +19,7 @@ import { yutoriToolSetForModel, YUTORI_N15_EXPANDED_ACTION_TYPES, } from "./actions"; -import { canonicalToolCallArguments, canonicalToolCallName } from "../common"; -import type { CuaPayloadContext } from "../../runtime-spec"; +import { canonicalToolCallArguments, canonicalToolCallName, type CuaPayloadContext } from "../common"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; @@ -41,10 +40,6 @@ export const streamSimpleYutori: StreamFunction streamYutori(model, context, options); -export function yutoriBuiltinToolsOnPayload(payload: unknown, model?: Model, context?: CuaPayloadContext): unknown | undefined { - return yutoriNativeToolSetOnPayload(payload, model, context); -} - export function yutoriNativeToolSetOnPayload(payload: unknown, model?: Model, context?: CuaPayloadContext): unknown | undefined { if (!payload || typeof payload !== "object") return undefined; const current = payload as { tools?: unknown }; diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 0473ae77..d8469ace 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -1,58 +1,23 @@ -import type { Api, Model, Tool } from "@earendil-works/pi-ai"; -import { type CuaModelRef, type CuaProvider, getCuaModel, providerForModel } from "./models"; -import * as anthropic from "./providers/anthropic/index"; -import * as gemini from "./providers/gemini/index"; -import * as openai from "./providers/openai/index"; -import * as tzafon from "./providers/tzafon/index"; -import * as yutori from "./providers/yutori/index"; -import type { ComputerToolCoordinateSystem, CuaToolExecutorSpec } from "./providers/common"; +import type { CuaProvider } from "./models"; +import { getCuaModel, providerForModel } from "./models"; +import { providerModule as anthropic } from "./providers/anthropic/index"; +import { providerModule as gemini } from "./providers/gemini/index"; +import { providerModule as openai } from "./providers/openai/index"; +import { providerModule as tzafon } from "./providers/tzafon/index"; +import { providerModule as yutori } from "./providers/yutori/index"; +import type { + CuaProviderModule, + CuaRuntimeSpec, + CuaRuntimeSpecInput, +} from "./providers/common"; -export interface CuaScreenshotTransformSpec { - width: number; - height: number; - format: "png" | "jpeg" | "webp"; - quality?: number; -} - -export interface CuaScreenshotSpec { - /** Append a provider-prepared screenshot to the latest user/tool message before each request. */ - appendToLatestMessage?: boolean; - /** Optional image transform applied to Kernel screenshots before they are sent to the provider. */ - transform?: CuaScreenshotTransformSpec; -} - -export interface CuaPayloadContext { - /** Tool names that should remain in the outbound provider payload even if the provider strips local CUA executors. */ - keepToolNames?: readonly string[]; -} - -export type CuaPayloadHook = (payload: unknown, model: Model, context?: CuaPayloadContext) => unknown | Promise; - -/** - * Runtime configuration for a supported CUA model. - * - * Use this to pair a model with the agent tool definitions, baseline prompt, - * coordinate convention, screenshot policy, and request payload middleware - * expected by its provider. - */ -export interface CuaRuntimeSpec { - model: Model; - provider: CuaProvider; - /** Provider-facing CUA tool definitions used for model requests. */ - toolDefinitions: Tool[]; - /** Local execution adapters that turn provider tool calls into canonical CUA actions. */ - toolExecutors: CuaToolExecutorSpec[]; - /** Provider-tuned baseline prompt for browser control behavior. */ - defaultSystemPrompt: string; - /** Coordinate convention emitted by provider tool calls. */ - coordinateSystem: ComputerToolCoordinateSystem; - /** Optional provider screenshot input policy used by CuaAgent/CuaAgentHarness. */ - screenshot?: CuaScreenshotSpec; - /** Optional provider middleware for request payload adaptation. */ - onPayload?: CuaPayloadHook; -} - -export type CuaRuntimeSpecInput = CuaModelRef | Model; +const PROVIDERS = { + openai, + anthropic, + google: gemini, + tzafon, + yutori, +} satisfies Record; /** * Resolve provider defaults from either a CUA model ref or a concrete model. @@ -63,59 +28,15 @@ export type CuaRuntimeSpecInput = CuaModelRef | Model; export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpec { const model = typeof input === "string" ? getCuaModel(input) : input; const provider = providerForModel(model); - switch (provider) { - case "anthropic": - return { - model, - provider, - toolDefinitions: anthropic.computerTools(), - toolExecutors: anthropic.computerToolExecutors(), - defaultSystemPrompt: anthropic.buildAnthropicSystemPrompt(), - coordinateSystem: anthropic.COMPUTER_TOOL_COORDINATES, - }; - case "google": - return { - model, - provider, - toolDefinitions: gemini.computerTools(), - toolExecutors: gemini.computerToolExecutors(), - defaultSystemPrompt: gemini.buildGeminiSystemPrompt(), - coordinateSystem: gemini.COMPUTER_TOOL_COORDINATES, - }; - case "tzafon": - return { - model, - provider, - toolDefinitions: tzafon.computerTools(), - toolExecutors: tzafon.computerToolExecutors(), - defaultSystemPrompt: tzafon.buildTzafonSystemPrompt(), - coordinateSystem: tzafon.COMPUTER_TOOL_COORDINATES, - onPayload: tzafon.tzafonComputerUseOnPayload, - }; - case "yutori": - return { - model, - provider, - toolDefinitions: [], - toolExecutors: yutori.computerToolExecutors(), - defaultSystemPrompt: yutori.buildYutoriSystemPrompt(), - coordinateSystem: yutori.COMPUTER_TOOL_COORDINATES, - screenshot: { - appendToLatestMessage: true, - transform: { width: 1280, height: 800, format: "webp", quality: 90 }, - }, - onPayload: yutori.yutoriBuiltinToolsOnPayload, - }; - case "openai": - default: - return { - model, - provider, - toolDefinitions: openai.computerTools(), - toolExecutors: openai.computerToolExecutors(), - defaultSystemPrompt: openai.OPENAI_COMPUTER_INSTRUCTIONS, - coordinateSystem: openai.COMPUTER_TOOL_COORDINATES, - onPayload: openai.openaiResponsesStoreOnPayload, - }; - } + const mod: CuaProviderModule = PROVIDERS[provider]; + return { + model, + provider, + toolDefinitions: mod.toolDefinitions(), + toolExecutors: mod.toolExecutors(), + defaultSystemPrompt: mod.buildSystemPrompt(), + coordinateSystem: mod.coordinateSystem(), + screenshot: mod.screenshot, + onPayload: mod.onPayload, + }; } diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts new file mode 100644 index 00000000..b78a1458 --- /dev/null +++ b/packages/ai/test/provider-module.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { anthropic, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; +import type { CuaProviderModule } from "../src/providers/common"; + +const MODULES: Record = { + openai, + anthropic, + google: gemini, + tzafon, + yutori, +}; + +describe("provider modules satisfy the uniform contract", () => { + for (const provider of CUA_PROVIDERS) { + it(`${provider} conforms to CuaProviderModule`, () => { + const mod = MODULES[provider].providerModule; + + expect(mod.toolDefinitions).toBeTypeOf("function"); + expect(mod.toolExecutors).toBeTypeOf("function"); + expect(mod.coordinateSystem).toBeTypeOf("function"); + expect(mod.buildSystemPrompt).toBeTypeOf("function"); + + expect(Array.isArray(mod.toolDefinitions())).toBe(true); + + const executors = mod.toolExecutors(); + expect(Array.isArray(executors)).toBe(true); + expect(executors.length).toBeGreaterThan(0); + + const coordinates = mod.coordinateSystem(); + if (coordinates.type === "pixel") { + expect(coordinates).toEqual({ type: "pixel" }); + } else { + expect(coordinates.type).toBe("normalized"); + expect(coordinates.range).toHaveLength(2); + expect(coordinates.range[0]).toBeTypeOf("number"); + expect(coordinates.range[1]).toBeTypeOf("number"); + } + + expect(mod.buildSystemPrompt()).toBeTypeOf("string"); + }); + } + + it("yutori sends no model-facing tool definitions", () => { + expect(yutori.providerModule.toolDefinitions()).toEqual([]); + }); +}); diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index 8273477a..b5b9f5fd 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -122,10 +122,10 @@ describe("computer tool definitions", () => { }); it("exports provider coordinate systems", () => { - expect(openai.COMPUTER_TOOL_COORDINATES).toEqual({ type: "pixel" }); - expect(anthropic.COMPUTER_TOOL_COORDINATES).toEqual({ type: "pixel" }); - expect(gemini.COMPUTER_TOOL_COORDINATES).toEqual({ type: "normalized", range: [0, 999] }); - expect(yutori.COMPUTER_TOOL_COORDINATES).toEqual({ type: "normalized", range: [0, 1000] }); - expect(tzafon.COMPUTER_TOOL_COORDINATES).toEqual({ type: "normalized", range: [0, 999] }); + expect(openai.coordinateSystem()).toEqual({ type: "pixel" }); + expect(anthropic.coordinateSystem()).toEqual({ type: "pixel" }); + expect(gemini.coordinateSystem()).toEqual({ type: "normalized", range: [0, 999] }); + expect(yutori.coordinateSystem()).toEqual({ type: "normalized", range: [0, 1000] }); + expect(tzafon.coordinateSystem()).toEqual({ type: "normalized", range: [0, 999] }); }); }); diff --git a/packages/ai/test/yutori-payload.test.ts b/packages/ai/test/yutori-payload.test.ts index d3af6493..5f399ad0 100644 --- a/packages/ai/test/yutori-payload.test.ts +++ b/packages/ai/test/yutori-payload.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { yutori } from "../src/index"; -describe("yutoriBuiltinToolsOnPayload", () => { +describe("yutoriNativeToolSetOnPayload", () => { it("removes local Yutori action tool definitions from payload.tools", () => { const payload = { tools: [ @@ -12,7 +12,7 @@ describe("yutoriBuiltinToolsOnPayload", () => { { type: "function", function: { name: "custom_tool" } }, ], }; - const next = yutori.yutoriBuiltinToolsOnPayload(payload) as { tools?: Array<{ function?: { name?: string } }> }; + const next = yutori.yutoriNativeToolSetOnPayload(payload) as { tools?: Array<{ function?: { name?: string } }> }; expect(next.tools?.map((tool) => tool.function?.name)).toEqual([ "batch_computer_actions", "computer_use_extra", @@ -40,7 +40,7 @@ describe("yutoriBuiltinToolsOnPayload", () => { }); it("returns undefined for non-object payloads", () => { - expect(yutori.yutoriBuiltinToolsOnPayload(undefined)).toBeUndefined(); - expect(yutori.yutoriBuiltinToolsOnPayload("x")).toBeUndefined(); + expect(yutori.yutoriNativeToolSetOnPayload(undefined)).toBeUndefined(); + expect(yutori.yutoriNativeToolSetOnPayload("x")).toBeUndefined(); }); });