From c3154f671b466ca0eb1ea9fbcefdceb57bf23088 Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 16:49:20 +0000 Subject: [PATCH 01/34] Add action-plane modes (os/dom/hybrid) and Anthropic native computer/browser tools --- docs/architecture.md | 42 ++ packages/agent/src/agent.ts | 42 +- packages/agent/src/index.ts | 3 + packages/agent/src/tools.ts | 31 +- packages/agent/src/translator/cdp.ts | 135 +++++ packages/agent/src/translator/dom.ts | 560 ++++++++++++++++++ packages/agent/src/translator/translator.ts | 61 +- packages/agent/src/translator/types.ts | 3 +- packages/agent/test/translator-dom.test.ts | 88 +++ packages/ai/src/actions/dom.ts | 331 +++++++++++ packages/ai/src/actions/index.ts | 47 ++ packages/ai/src/actions/os.ts | 297 ++++++++++ packages/ai/src/modes.ts | 173 ++++++ packages/ai/src/native-tools.ts | 115 ++++ packages/ai/src/providers.ts | 28 + .../ai/src/providers/anthropic/actions.ts | 36 +- packages/ai/src/providers/anthropic/index.ts | 27 +- packages/ai/src/providers/anthropic/native.ts | 310 ++++++++++ packages/ai/src/providers/common.ts | 325 ++-------- packages/ai/src/providers/gemini/index.ts | 17 +- packages/ai/src/providers/openai/index.ts | 12 +- packages/ai/src/providers/tzafon/index.ts | 19 +- packages/ai/src/providers/yutori/index.ts | 12 +- packages/ai/src/runtime-spec.ts | 60 +- packages/ai/test/modes.test.ts | 122 ++++ packages/ai/test/native-tools.test.ts | 163 +++++ packages/ai/test/provider-module.test.ts | 4 +- packages/cli/src/cli-harness.ts | 25 + packages/cli/src/cli.ts | 21 + packages/cli/src/harness.ts | 17 +- 30 files changed, 2804 insertions(+), 322 deletions(-) create mode 100644 packages/agent/src/translator/cdp.ts create mode 100644 packages/agent/src/translator/dom.ts create mode 100644 packages/agent/test/translator-dom.test.ts create mode 100644 packages/ai/src/actions/dom.ts create mode 100644 packages/ai/src/actions/index.ts create mode 100644 packages/ai/src/actions/os.ts create mode 100644 packages/ai/src/modes.ts create mode 100644 packages/ai/src/native-tools.ts create mode 100644 packages/ai/src/providers/anthropic/native.ts create mode 100644 packages/ai/test/modes.test.ts create mode 100644 packages/ai/test/native-tools.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 81f7e201..a965f545 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -92,6 +92,48 @@ provider conditionals. A new provider difference is a new or extended `CuaRuntimeSpec`/`CuaPayloadContext` field plus provider code in `@onkernel/cua-ai` — never a branch in `@onkernel/cua-agent`. +## Action planes and modes + +The canonical action vocabulary is split into two planes, delineated in code +under `packages/ai/src/actions/`: + +- **OS plane** (`actions/os.ts`) — real OS-level input against the browser + VM: mouse, keyboard, display capture, executed through Kernel's + `browsers.computer` REST API. Coordinates are pixels in the OS screenshot + frame. +- **DOM plane** (`actions/dom.ts`, ids prefixed `page_`) — CDP-driven page + tools: accessibility snapshots with element refs, element-targeted + interaction, navigation, tabs, viewport screenshots. Executed by + `packages/agent/src/translator/dom.ts` over a raw CDP websocket + (`translator/cdp.ts`) to the browser's `cdp_ws_url` — no Playwright. + Coordinates, where used, are viewport pixels. + +A `CuaMode` selects which plane(s) the model sees: + +| mode | tools | coordinate frame | +| --- | --- | --- | +| `os` (default) | OS actions under their canonical ids (`click`, `screenshot`, …) | OS screenshot pixels | +| `dom` | DOM actions with the `page_` prefix stripped (`snapshot`, `click`, …) plus `wait` | none for refs; viewport pixels where coordinates are allowed | +| `hybrid` | both planes, one tool per capability: OS actions as `computer_*`, DOM reads/element-writes as `page_*` (ref-only) | OS screenshot pixels — the single live frame | + +Hybrid deduplicates capabilities: navigation and tabs live on the DOM plane, +pointer/keyboard input and the (only) screenshot live on the OS plane, and +DOM tools take element refs only so exactly one coordinate frame exists. +Element refs are snapshot-scoped (`e12`); a stale ref resolves to an error +string that tells the model to re-snapshot. + +**Native tools.** `resolveCuaRuntimeSpec(model, { nativeTool })` drives an +Anthropic model through its provider-defined tool schema instead of the +canonical function tools: `computer_20260601` pairs with `os` mode and +`browser_20260701` with `dom` mode (mismatches throw, mirroring the API's +own rejection of mixed frames). The spec routes the model to a CUA-owned api +id; the registered `anthropic` provider dispatches it to pi's builtin +`anthropic-messages` transport with the tool's `anthropic-beta` header, an +`onPayload` hook swaps the placeholder tool for the native declaration, and +`providers/anthropic/native.ts` maps incoming `tool_use` inputs onto the same +canonical actions the mode uses — so canonical vs native is purely a wire +format difference over one execution path. + ## Layers `cua` is a thin TypeScript monorepo on top of the diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 1c289c56..ad0db291 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -14,7 +14,9 @@ import { CUA_NAVIGATION_TOOL_NAME, CUA_PLAYWRIGHT_TOOL_NAME, cuaModels, + type CuaMode, type CuaModelRef, + type CuaNativeToolSpec, type CuaRuntimeSpec, type CuaSimpleStreamOptions, getCuaEnvApiKey, @@ -70,6 +72,12 @@ export type CuaAgentOptions = Omit & { initialState: CuaAgentInitialState; /** Add your own pi tools alongside the built-in browser tools. */ extraTools?: AgentTool[]; + /** Which canonical action plane(s) to expose: "os" (default), "dom", or "hybrid". */ + mode?: CuaMode; + /** Drive the model through a provider-native tool declaration (validated against `mode`). */ + nativeTool?: CuaNativeToolSpec; + /** Expose `page_evaluate` in dom/hybrid modes. Default false. */ + javascriptExec?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ @@ -104,6 +112,12 @@ export type CuaAgentHarnessOptions< models?: Models; /** Add your own pi tools alongside the built-in browser tools. */ extraTools?: AgentTool[]; + /** Which canonical action plane(s) to expose: "os" (default), "dom", or "hybrid". */ + mode?: CuaMode; + /** Drive the model through a provider-native tool declaration (validated against `mode`). */ + nativeTool?: CuaNativeToolSpec; + /** Expose `page_evaluate` in dom/hybrid modes. Default false. */ + javascriptExec?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ @@ -128,15 +142,26 @@ class CuaRuntimeController { client: Kernel; model: CuaRuntimeInput; extraTools?: AgentTool[]; + mode?: CuaMode; + nativeTool?: CuaNativeToolSpec; + javascriptExec?: boolean; computerUseExtra?: boolean; playwright?: boolean; onPayload?: SimpleStreamOptions["onPayload"]; }, ) { - this.runtimeSpec = resolveCuaRuntimeSpec(options.model); + this.runtimeSpec = this.resolveSpec(options.model); this.translator = this.createTranslator(); } + private resolveSpec(model: CuaRuntimeInput): CuaRuntimeSpec { + return resolveCuaRuntimeSpec(model, { + mode: this.options.mode, + nativeTool: this.options.nativeTool, + javascriptExec: this.options.javascriptExec, + }); + } + get model(): Model { return this.runtimeSpec.model; } @@ -146,7 +171,7 @@ class CuaRuntimeController { } setModel(model: CuaRuntimeInput): void { - this.runtimeSpec = resolveCuaRuntimeSpec(model); + this.runtimeSpec = this.resolveSpec(model); this.translator = this.createTranslator(); } @@ -155,6 +180,7 @@ class CuaRuntimeController { ...buildCuaComputerTools( { toolExecutors: this.runtimeSpec.toolExecutors, + mode: this.runtimeSpec.mode, computerUseExtra: this.options.computerUseExtra, playwright: this.options.playwright, }, @@ -221,6 +247,9 @@ export class CuaAgent extends Agent { streamFn, prepareNextTurn, extraTools, + mode, + nativeTool, + javascriptExec, computerUseExtra, playwright, ...agentOptions @@ -230,6 +259,9 @@ export class CuaAgent extends Agent { client, model: initialState.model, extraTools, + mode, + nativeTool, + javascriptExec, computerUseExtra, playwright, onPayload, @@ -347,6 +379,9 @@ export class CuaAgentHarness< model, models, extraTools, + mode, + nativeTool, + javascriptExec, computerUseExtra, playwright, systemPrompt, @@ -359,6 +394,9 @@ export class CuaAgentHarness< client, model, extraTools, + mode, + nativeTool, + javascriptExec, computerUseExtra, playwright, onPayload, diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 75bea96f..165f05d8 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -2,6 +2,9 @@ export * from "@earendil-works/pi-agent-core"; export { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node"; export type { KernelBrowser } from "./translator/translator"; +export { CdpConnection } from "./translator/cdp"; +export { DomExecutor } from "./translator/dom"; +export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { BatchDetails, diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index a326116a..eeef177e 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -7,6 +7,7 @@ import { createCuaPlaywrightToolDefinition, type ComputerToolCoordinateSystem, type CuaBatchInput, + type CuaMode, type CuaNavigationInput, type CuaPlaywrightInput, type CuaScreenshotSpec, @@ -22,6 +23,8 @@ export interface ComputerToolOptions { toolExecutors: CuaToolExecutorSpec[]; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; + /** Action plane(s) in play; controls whether the post-action fallback capture is the OS display or the viewport. Default "os". */ + mode?: CuaMode; computerUseExtra?: boolean; playwright?: boolean; } @@ -30,7 +33,12 @@ type ToolContent = Array; export interface BatchDetails { statusText: string; - readResults: Array<{ type: "url"; url: string } | { type: "screenshot"; bytes: number } | { type: "cursor_position"; x: number; y: number }>; + readResults: Array< + | { type: "url"; url: string } + | { type: "screenshot"; bytes: number } + | { type: "cursor_position"; x: number; y: number } + | { type: "dom_text"; label: string; bytes: number } + >; } export interface NavigationDetails { @@ -78,10 +86,10 @@ export function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTo /** Build executor tools against an existing translator (internal; not part of the package surface). */ export function buildCuaComputerTools( - args: Pick, + args: Pick, translator: InternalComputerTranslator, ): CuaExecutorTool[] { - return withExtraTools(args).map((executor) => createExecutorTool(executor, translator)); + return withExtraTools(args).map((executor) => createExecutorTool(executor, translator, args.mode ?? "os")); } function withExtraTools(args: Pick): ComputerExecutorSpec[] { @@ -96,7 +104,7 @@ function withExtraTools(args: Pick> { - return executeBatchTool(translator, { actions: executor.toActions(params) }); + return executeBatchTool(translator, { actions: executor.toActions(params) }, mode); }, }; return tool; @@ -144,7 +152,11 @@ function isPlaywrightExecutor(executor: ComputerExecutorSpec): executor is Playw return "kind" in executor && executor.kind === "playwright"; } -async function executeBatchTool(translator: InternalComputerTranslator, params: CuaBatchInput): Promise> { +async function executeBatchTool( + translator: InternalComputerTranslator, + params: CuaBatchInput, + mode: CuaMode = "os", +): Promise> { const content: ToolContent = []; const readResults: BatchDetails["readResults"] = []; try { @@ -156,13 +168,18 @@ async function executeBatchTool(translator: InternalComputerTranslator, params: } else if (read.type === "cursor_position") { readResults.push({ type: "cursor_position", x: read.x, y: read.y }); content.push({ type: "text", text: `cursor_position(): ${read.x},${read.y}` }); + } else if (read.type === "dom_text") { + readResults.push({ type: "dom_text", label: read.label, bytes: read.text.length }); + content.push({ type: "text", text: read.text }); } else { readResults.push({ type: "screenshot", bytes: read.data.length }); content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType }); } } if (content.length === 0) { - const screenshot = await translator.screenshot(); + // Post-action grounding capture: the OS display in os/hybrid mode, + // the browser viewport in dom mode (the only frame the model sees). + const screenshot = mode === "dom" ? await translator.dom().screenshot() : await translator.screenshot(); readResults.push({ type: "screenshot", bytes: screenshot.data.length }); content.push({ type: "image", data: screenshot.data.toString("base64"), mimeType: screenshot.mimeType }); } diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts new file mode 100644 index 00000000..7d0bf168 --- /dev/null +++ b/packages/agent/src/translator/cdp.ts @@ -0,0 +1,135 @@ +/** + * Minimal raw-CDP client for the DOM action plane. + * + * Connects to a Kernel browser's `cdp_ws_url` over a plain WebSocket and + * speaks the DevTools JSON-RPC protocol directly — no Playwright and no + * driver dependency. Only what the DOM executor needs: command dispatch on + * the browser connection and on attached page sessions. + */ + +interface PendingCommand { + resolve(result: unknown): void; + reject(error: Error): void; +} + +interface CdpEventMessage { + method: string; + params: Record; + sessionId?: string; +} + +export interface CdpTargetInfo { + targetId: string; + type: string; + title: string; + url: string; + attached?: boolean; +} + +export type CdpEventListener = (event: CdpEventMessage) => void; + +export class CdpConnection { + private socket?: WebSocket; + private opening?: Promise; + private nextId = 1; + private readonly pending = new Map(); + private readonly listeners = new Set(); + private readonly sessionsByTarget = new Map(); + + constructor(private readonly wsUrl: string) {} + + onEvent(listener: CdpEventListener): void { + this.listeners.add(listener); + } + + async send>(method: string, params: Record = {}, sessionId?: string): Promise { + const socket = await this.connect(); + const id = this.nextId++; + const message = JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }); + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve: resolve as (result: unknown) => void, reject }); + socket.send(message); + }); + } + + /** List page targets (tabs) on the browser connection. */ + async pageTargets(): Promise { + const { targetInfos } = await this.send<{ targetInfos: CdpTargetInfo[] }>("Target.getTargets"); + return targetInfos.filter((target) => target.type === "page"); + } + + /** Attach to a page target (flat protocol) and cache the session. */ + async attachToTarget(targetId: string): Promise { + const cached = this.sessionsByTarget.get(targetId); + if (cached) return cached; + const { sessionId } = await this.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true }); + this.sessionsByTarget.set(targetId, sessionId); + return sessionId; + } + + async createTarget(url: string): Promise { + const { targetId } = await this.send<{ targetId: string }>("Target.createTarget", { url }); + return targetId; + } + + close(): void { + this.socket?.close(); + this.socket = undefined; + this.opening = undefined; + this.sessionsByTarget.clear(); + const error = new Error("CDP connection closed"); + for (const pending of this.pending.values()) pending.reject(error); + this.pending.clear(); + } + + private connect(): Promise { + if (this.socket && this.socket.readyState === WebSocket.OPEN) return Promise.resolve(this.socket); + this.opening ??= new Promise((resolve, reject) => { + const socket = new WebSocket(this.wsUrl); + socket.addEventListener("open", () => { + this.socket = socket; + resolve(socket); + }); + socket.addEventListener("error", () => { + this.opening = undefined; + reject(new Error(`CDP connection to ${this.wsUrl} failed`)); + }); + socket.addEventListener("close", () => { + this.socket = undefined; + this.opening = undefined; + this.sessionsByTarget.clear(); + }); + socket.addEventListener("message", (event) => { + this.handleMessage(typeof event.data === "string" ? event.data : String(event.data)); + }); + }); + return this.opening; + } + + private handleMessage(data: string): void { + let message: { id?: number; result?: unknown; error?: { message?: string }; method?: string; params?: Record; sessionId?: string }; + try { + message = JSON.parse(data); + } catch { + return; + } + if (typeof message.id === "number") { + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + if (message.error) pending.reject(new Error(message.error.message ?? "CDP command failed")); + else pending.resolve(message.result ?? {}); + return; + } + if (typeof message.method === "string") { + if (message.method === "Target.detachedFromTarget") { + const detached = message.params?.sessionId; + for (const [targetId, sessionId] of this.sessionsByTarget) { + if (sessionId === detached) this.sessionsByTarget.delete(targetId); + } + } + const event: CdpEventMessage = { method: message.method, params: message.params ?? {}, sessionId: message.sessionId }; + for (const listener of this.listeners) listener(event); + } + } +} diff --git a/packages/agent/src/translator/dom.ts b/packages/agent/src/translator/dom.ts new file mode 100644 index 00000000..ff5cc7ed --- /dev/null +++ b/packages/agent/src/translator/dom.ts @@ -0,0 +1,560 @@ +import { + normalizeGotoUrl, + type CuaActionPageClick, + type CuaActionPageDrag, + type CuaActionPageFill, + type CuaActionPageFind, + type CuaActionPageHover, + type CuaActionPageKey, + type CuaActionPageNavigate, + type CuaActionPageScroll, + type CuaActionPageScrollTo, + type CuaActionPageSnapshot, + type CuaDomAction, +} from "@onkernel/cua-ai"; +import { CdpConnection } from "./cdp"; +import type { BatchReadResult } from "./types"; + +const SNAPSHOT_CHAR_LIMIT = 50_000; +const DEFAULT_SNAPSHOT_DEPTH = 15; +const FIND_MATCH_LIMIT = 20; +const SCROLL_NOTCH_PX = 120; + +const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; + +interface AXNode { + nodeId: string; + ignored?: boolean; + role?: { value?: string }; + name?: { value?: string }; + backendDOMNodeId?: number; + parentId?: string; + childIds?: string[]; +} + +interface RefEntry { + backendNodeId: number; + targetId: string; + generation: number; +} + +/** + * Executes DOM-plane canonical actions over CDP. + * + * Element refs are snapshot-scoped: each snapshot/find mints `e` ids + * mapped to CDP backend node ids for the target's current generation. A + * navigation bumps the generation, and refs from earlier generations resolve + * to a stale-ref error whose message tells the model how to recover. + */ +export class DomExecutor { + private readonly refs = new Map(); + private readonly generations = new Map(); + private refCounter = 0; + private activeTargetId?: string; + + constructor(private readonly cdp: CdpConnection) {} + + async execute(action: CuaDomAction): Promise { + switch (action.type) { + case "page_snapshot": + return [{ type: "dom_text", label: "snapshot", text: await this.snapshot(action) }]; + case "page_text": + return [{ type: "dom_text", label: "text", text: await this.pageText(tabOf(action)) }]; + case "page_find": + return [{ type: "dom_text", label: "find", text: await this.find(action) }]; + case "page_click": + await this.click(action); + return []; + case "page_hover": + await this.hover(action); + return []; + case "page_drag": + await this.drag(action); + return []; + case "page_fill": + await this.fill(action); + return []; + case "page_scroll_to": + await this.scrollTo(action); + return []; + case "page_scroll": + await this.scroll(action); + return []; + case "page_type": { + const session = await this.session(tabOf(action)); + await this.cdp.send("Input.insertText", { text: action.text }, session); + return []; + } + case "page_key": + await this.key(action); + return []; + case "page_navigate": + return [{ type: "dom_text", label: "navigate", text: await this.navigate(action) }]; + case "page_list_tabs": + return [{ type: "dom_text", label: "tabs", text: await this.listTabs() }]; + case "page_new_tab": + return [{ type: "dom_text", label: "new_tab", text: await this.newTab() }]; + case "page_screenshot": + return [{ type: "screenshot", ...(await this.screenshot(action.region, action.tab_id)) }]; + case "page_evaluate": + return [{ type: "dom_text", label: "evaluate", text: await this.evaluate(action.code, tabOf(action)) }]; + } + } + + async screenshot(region?: [number, number, number, number], tabId?: string): Promise<{ data: Buffer; mimeType: string }> { + const session = await this.session(tabId); + const clip = region + ? { clip: { x: region[0], y: region[1], width: Math.max(1, region[2] - region[0]), height: Math.max(1, region[3] - region[1]), scale: 1 } } + : {}; + const { data } = await this.cdp.send<{ data: string }>("Page.captureScreenshot", { format: "png", ...clip }, session); + return { data: Buffer.from(data, "base64"), mimeType: "image/png" }; + } + + private async snapshot(action: CuaActionPageSnapshot): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); + const byId = new Map(nodes.map((node) => [node.nodeId, node])); + const roots = nodes.filter((node) => !node.parentId); + let rootIds = roots.map((node) => node.nodeId); + if (action.ref) { + const entry = this.resolveRef(action.ref, targetId); + const rootNode = nodes.find((node) => node.backendDOMNodeId === entry.backendNodeId); + if (!rootNode) throw new Error(`ref ${action.ref} is stale or not on the current page. ${STALE_REF_HINT}`); + rootIds = [rootNode.nodeId]; + } + + const generation = this.generation(targetId); + const lines: string[] = []; + const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; + const interactiveOnly = action.filter === "interactive"; + const walk = (nodeId: string, depth: number): void => { + const node = byId.get(nodeId); + if (!node) return; + if (depth <= maxDepth && !node.ignored) { + const line = this.renderNode(node, targetId, generation, depth, interactiveOnly); + if (line) lines.push(line); + } + if (depth < maxDepth) { + for (const childId of node.childIds ?? []) walk(childId, depth + 1); + } + }; + for (const rootId of rootIds) walk(rootId, 0); + + let text = lines.join("\n"); + if (text.length > SNAPSHOT_CHAR_LIMIT) { + text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; + } + return text || "(empty accessibility tree)"; + } + + private renderNode(node: AXNode, targetId: string, generation: number, depth: number, interactiveOnly: boolean): string | undefined { + const role = node.role?.value ?? ""; + const name = node.name?.value ?? ""; + const interactive = INTERACTIVE_ROLES.has(role); + if (interactiveOnly && !interactive) return undefined; + if (!interactiveOnly && !name && !interactive && SKIPPED_ROLES.has(role)) return undefined; + let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; + if (node.backendDOMNodeId !== undefined && interactive) { + line += ` [${this.mintRef(node.backendDOMNodeId, targetId, generation)}]`; + } + return line; + } + + private async find(action: CuaActionPageFind): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); + const queryTokens = tokenize(action.query); + const scored = nodes + .filter((node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? ""))) + .map((node) => ({ node, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })) + .filter((entry) => entry.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, FIND_MATCH_LIMIT); + if (scored.length === 0) return `No elements matched ${JSON.stringify(action.query)}. Try snapshot for the full tree.`; + return scored + .map(({ node }) => { + const role = node.role?.value ?? "node"; + const name = node.name?.value ? ` ${JSON.stringify(node.name.value)}` : ""; + return `${role}${name} [${this.mintRef(node.backendDOMNodeId!, targetId, this.generation(targetId))}]`; + }) + .join("\n"); + } + + private async click(action: CuaActionPageClick): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const point = await this.resolvePoint(action, targetId, session); + const modifiers = modifierBits(action.modifiers); + const button = action.button ?? "left"; + const clickCount = action.num_clicks ?? 1; + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y, modifiers }, session); + await this.cdp.send( + "Input.dispatchMouseEvent", + { type: "mousePressed", x: point.x, y: point.y, button, clickCount, modifiers }, + session, + ); + await this.cdp.send( + "Input.dispatchMouseEvent", + { type: "mouseReleased", x: point.x, y: point.y, button, clickCount, modifiers }, + session, + ); + } + + private async hover(action: CuaActionPageHover): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const point = await this.resolvePoint(action, targetId, session); + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y }, session); + } + + private async drag(action: CuaActionPageDrag): Promise { + const session = await this.session(tabOf(action)); + await this.cdp.send("Input.dispatchMouseEvent", { type: "mousePressed", x: action.from.x, y: action.from.y, button: "left", clickCount: 1 }, session); + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: action.to.x, y: action.to.y, button: "left" }, session); + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseReleased", x: action.to.x, y: action.to.y, button: "left", clickCount: 1 }, session); + } + + private async fill(action: CuaActionPageFill): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const entry = this.resolveRef(action.ref, targetId); + const objectId = await this.resolveObject(entry, action.ref, session); + const { exceptionDetails } = await this.cdp.send<{ exceptionDetails?: { exception?: { description?: string } } }>( + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: FILL_FUNCTION, + arguments: [{ value: action.value }], + }, + session, + ); + if (exceptionDetails) { + throw new Error(`page_fill failed: ${exceptionDetails.exception?.description ?? "element rejected the value"}`); + } + } + + private async scrollTo(action: CuaActionPageScrollTo): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const entry = this.resolveRef(action.ref, targetId); + await this.scrollIntoView(entry, action.ref, session); + } + + private async scroll(action: CuaActionPageScroll): Promise { + const session = await this.session(tabOf(action)); + const notches = action.amount ?? 3; + const delta = Math.trunc(notches) * SCROLL_NOTCH_PX; + const deltaX = action.direction === "left" ? -delta : action.direction === "right" ? delta : 0; + const deltaY = action.direction === "up" ? -delta : action.direction === "down" ? delta : 0; + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseWheel", x: action.x, y: action.y, deltaX, deltaY }, session); + } + + private async key(action: CuaActionPageKey): Promise { + const session = await this.session(tabOf(action)); + const repeat = Math.min(Math.max(1, Math.trunc(action.repeat ?? 1)), 100); + const chords = action.text.trim().split(/\s+/).filter(Boolean); + for (let iteration = 0; iteration < repeat; iteration += 1) { + for (const chord of chords) { + await this.dispatchChord(chord, session); + } + } + } + + private async dispatchChord(chord: string, session: string): Promise { + const parts = chord.split("+").filter(Boolean); + const keyPart = parts[parts.length - 1] ?? ""; + const modifierParts = parts.slice(0, -1); + const modifiers = modifierBits(modifierParts); + const key = resolveKey(keyPart); + const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, modifiers }; + await this.cdp.send("Input.dispatchKeyEvent", { type: key.text ? "keyDown" : "rawKeyDown", ...base, ...(key.text ? { text: key.text } : {}) }, session); + await this.cdp.send("Input.dispatchKeyEvent", { type: "keyUp", ...base }, session); + } + + private async navigate(action: CuaActionPageNavigate): Promise { + const targetId = await this.resolveTarget(action.tab_id); + const session = await this.attach(targetId); + const direction = action.url.trim().toLowerCase(); + if (direction === "back" || direction === "forward") { + const history = await this.cdp.send<{ currentIndex: number; entries: Array<{ id: number; url: string }> }>( + "Page.getNavigationHistory", + {}, + session, + ); + const entry = history.entries[history.currentIndex + (direction === "back" ? -1 : 1)]; + if (!entry) throw new Error(`cannot go ${direction}: no history entry`); + await this.cdp.send("Page.navigateToHistoryEntry", { entryId: entry.id }, session); + this.invalidateRefs(targetId); + return `Navigated ${direction}.\n${await this.tabContext(targetId)}`; + } + const url = normalizeGotoUrl(action.url); + if (!url) throw new Error("invalid url"); + const { errorText } = await this.cdp.send<{ errorText?: string }>("Page.navigate", { url }, session); + if (errorText) throw new Error(`navigation to ${url} failed: ${errorText}`); + this.invalidateRefs(targetId); + return `Navigated to ${url}.\n${await this.tabContext(targetId)}`; + } + + private async listTabs(): Promise { + const targets = await this.cdp.pageTargets(); + if (targets.length === 0) return "No open tabs."; + return targets.map((target) => `tab_id ${shortTabId(target.targetId)}: ${JSON.stringify(target.title)} (${target.url})`).join("\n"); + } + + private async newTab(): Promise { + const targetId = await this.cdp.createTarget("about:blank"); + return `Opened tab_id ${shortTabId(targetId)}.\n${await this.tabContext(targetId)}`; + } + + private async evaluate(code: string, tabId?: string): Promise { + const session = await this.session(tabId); + const { result, exceptionDetails } = await this.cdp.send<{ + result: { value?: unknown; description?: string; type?: string }; + exceptionDetails?: { exception?: { description?: string } }; + }>("Runtime.evaluate", { expression: code, returnByValue: true, awaitPromise: true }, session); + if (exceptionDetails) throw new Error(`page_evaluate failed: ${exceptionDetails.exception?.description ?? "evaluation threw"}`); + if (result.value === undefined) return result.description ?? String(result.type ?? "undefined"); + return typeof result.value === "string" ? result.value : JSON.stringify(result.value); + } + + private async pageText(tabId?: string): Promise { + return this.evaluate("document.body ? document.body.innerText : ''", tabId); + } + + private async tabContext(executedOn: string): Promise { + const targets = await this.cdp.pageTargets(); + const lines = targets.map((target) => ` • tab_id ${shortTabId(target.targetId)}: ${JSON.stringify(target.title)} (${target.url})`); + return [`Tab Context:`, `- Executed on tab_id: ${shortTabId(executedOn)}`, `- Available tabs:`, ...lines].join("\n"); + } + + private async resolvePoint( + action: CuaActionPageClick | CuaActionPageHover, + targetId: string, + session: string, + ): Promise<{ x: number; y: number }> { + if (action.ref !== undefined) { + const entry = this.resolveRef(action.ref, targetId); + await this.scrollIntoView(entry, action.ref, session); + const { model } = await this.cdp.send<{ model: { content: number[] } }>( + "DOM.getBoxModel", + { backendNodeId: entry.backendNodeId }, + session, + ); + const quad = model.content; + return { x: (quad[0]! + quad[4]!) / 2, y: (quad[1]! + quad[5]!) / 2 }; + } + if (typeof action.x === "number" && typeof action.y === "number") return { x: action.x, y: action.y }; + throw new Error("page target required: pass a ref or viewport coordinates"); + } + + private async scrollIntoView(entry: RefEntry, ref: string, session: string): Promise { + try { + await this.cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: entry.backendNodeId }, session); + } catch (err) { + throw new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, { cause: err }); + } + } + + private async resolveObject(entry: RefEntry, ref: string, session: string): Promise { + try { + const { object } = await this.cdp.send<{ object: { objectId: string } }>( + "DOM.resolveNode", + { backendNodeId: entry.backendNodeId }, + session, + ); + return object.objectId; + } catch (err) { + throw new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, { cause: err }); + } + } + + private mintRef(backendNodeId: number, targetId: string, generation: number): string { + this.refCounter += 1; + const ref = `e${this.refCounter}`; + this.refs.set(ref, { backendNodeId, targetId, generation }); + return ref; + } + + private resolveRef(ref: string, targetId: string): RefEntry { + const entry = this.refs.get(ref); + if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(targetId)) { + throw new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`); + } + return entry; + } + + private generation(targetId: string): number { + return this.generations.get(targetId) ?? 0; + } + + private invalidateRefs(targetId: string): void { + this.generations.set(targetId, this.generation(targetId) + 1); + } + + private async session(tabId?: string): Promise { + return this.attach(await this.resolveTarget(tabId)); + } + + private async attach(targetId: string): Promise { + return this.cdp.attachToTarget(targetId); + } + + private async resolveTarget(tabId?: string): Promise { + const targets = await this.cdp.pageTargets(); + if (targets.length === 0) throw new Error("no open browser tabs"); + if (tabId) { + const match = targets.find((target) => shortTabId(target.targetId) === tabId || target.targetId === tabId); + if (!match) throw new Error(`unknown tab_id "${tabId}". Call list_tabs for current tabs.`); + this.activeTargetId = match.targetId; + return match.targetId; + } + if (this.activeTargetId && targets.some((target) => target.targetId === this.activeTargetId)) { + return this.activeTargetId; + } + this.activeTargetId = targets[0]!.targetId; + return this.activeTargetId; + } +} + +function tabOf(action: { tab_id?: string }): string | undefined { + return action.tab_id; +} + +function shortTabId(targetId: string): string { + return targetId.slice(0, 10).toUpperCase(); +} + +function tokenize(value: string): string[] { + return value + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((token) => token.length > 1); +} + +function overlapScore(query: string[], candidate: string[]): number { + if (query.length === 0 || candidate.length === 0) return 0; + const set = new Set(candidate); + return query.reduce((score, token) => score + (set.has(token) ? 1 : 0), 0); +} + +function modifierBits(modifiers: readonly string[] | undefined): number { + let bits = 0; + for (const modifier of modifiers ?? []) { + switch (modifier.trim().toLowerCase()) { + case "alt": + case "option": + bits |= 1; + break; + case "ctrl": + case "control": + bits |= 2; + break; + case "meta": + case "cmd": + case "command": + case "super": + bits |= 4; + break; + case "shift": + bits |= 8; + break; + } + } + return bits; +} + +interface ResolvedKey { + key: string; + code: string; + keyCode: number; + text?: string; +} + +const NAMED_KEYS: Record = { + enter: { key: "Enter", code: "Enter", keyCode: 13, text: "\r" }, + return: { key: "Enter", code: "Enter", keyCode: 13, text: "\r" }, + tab: { key: "Tab", code: "Tab", keyCode: 9 }, + escape: { key: "Escape", code: "Escape", keyCode: 27 }, + esc: { key: "Escape", code: "Escape", keyCode: 27 }, + backspace: { key: "Backspace", code: "Backspace", keyCode: 8 }, + delete: { key: "Delete", code: "Delete", keyCode: 46 }, + space: { key: " ", code: "Space", keyCode: 32, text: " " }, + up: { key: "ArrowUp", code: "ArrowUp", keyCode: 38 }, + down: { key: "ArrowDown", code: "ArrowDown", keyCode: 40 }, + left: { key: "ArrowLeft", code: "ArrowLeft", keyCode: 37 }, + right: { key: "ArrowRight", code: "ArrowRight", keyCode: 39 }, + arrowup: { key: "ArrowUp", code: "ArrowUp", keyCode: 38 }, + arrowdown: { key: "ArrowDown", code: "ArrowDown", keyCode: 40 }, + arrowleft: { key: "ArrowLeft", code: "ArrowLeft", keyCode: 37 }, + arrowright: { key: "ArrowRight", code: "ArrowRight", keyCode: 39 }, + home: { key: "Home", code: "Home", keyCode: 36 }, + end: { key: "End", code: "End", keyCode: 35 }, + pageup: { key: "PageUp", code: "PageUp", keyCode: 33 }, + page_up: { key: "PageUp", code: "PageUp", keyCode: 33 }, + pagedown: { key: "PageDown", code: "PageDown", keyCode: 34 }, + page_down: { key: "PageDown", code: "PageDown", keyCode: 34 }, +}; + +function resolveKey(value: string): ResolvedKey { + const named = NAMED_KEYS[value.toLowerCase()]; + if (named) return named; + if (value.length === 1) { + const upper = value.toUpperCase(); + const code = /[a-z]/i.test(value) ? `Key${upper}` : /[0-9]/.test(value) ? `Digit${value}` : ""; + return { key: value, code, keyCode: upper.charCodeAt(0), text: value }; + } + return { key: value, code: value, keyCode: 0 }; +} + +const FILL_FUNCTION = `function(value) { + const el = this; + const tag = el.tagName ? el.tagName.toLowerCase() : ""; + if (tag === "select") { + const str = String(value); + const options = Array.from(el.options); + const match = options.find((o) => o.value === str) ?? options.find((o) => (o.label || o.textContent || "").trim() === str); + if (!match) throw new Error("no option matches " + JSON.stringify(str)); + el.value = match.value; + } else if (el.type === "checkbox" || el.type === "radio") { + el.checked = Boolean(value); + } else if (tag === "input" || tag === "textarea") { + const proto = Object.getPrototypeOf(el); + const setter = Object.getOwnPropertyDescriptor(proto, "value"); + if (setter && setter.set) setter.set.call(el, String(value)); + else el.value = String(value); + } else if (el.isContentEditable) { + el.textContent = String(value); + } else { + throw new Error("element is not a form control"); + } + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); +}`; + +const INTERACTIVE_ROLES: ReadonlySet = new Set([ + "button", + "link", + "textbox", + "searchbox", + "checkbox", + "radio", + "combobox", + "listbox", + "option", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "slider", + "spinbutton", + "switch", + "tab", + "textarea", +]); + +const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); + +export function createDomExecutor(cdpWsUrl: string): DomExecutor { + return new DomExecutor(new CdpConnection(cdpWsUrl)); +} diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 26d309c4..7205c999 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -1,6 +1,7 @@ import type Kernel from "@onkernel/sdk"; import type { BrowserCreateResponse, BrowserRetrieveResponse } from "@onkernel/sdk/resources/browsers"; import { + isCuaDomAction, normalizeGotoUrl, type ComputerToolCoordinateSystem, type CuaAction, @@ -13,11 +14,14 @@ import { type CuaActionScroll, type CuaActionTypeText, type CuaActionWait, + type CuaActionZoom, + type CuaDomAction, type CuaDragMouseButton, type CuaMouseButton, type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import sharp from "sharp"; +import { createDomExecutor, type DomExecutor } from "./dom"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult } from "./types"; @@ -28,6 +32,8 @@ export interface InternalComputerTranslatorOptions { client: Kernel; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; + /** DOM executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ + createDomExecutor?: (cdpWsUrl: string) => DomExecutor; } export class InternalComputerTranslator { @@ -36,6 +42,9 @@ export class InternalComputerTranslator { private readonly coordinateSystem: ComputerToolCoordinateSystem; private readonly screenshotSpec?: CuaScreenshotSpec; private readonly viewport: { width: number; height: number }; + private readonly cdpWsUrl?: string; + private readonly domExecutorFactory: (cdpWsUrl: string) => DomExecutor; + private domExecutor?: DomExecutor; constructor(opts: InternalComputerTranslatorOptions) { this.sessionId = opts.browser.session_id; @@ -43,6 +52,17 @@ export class InternalComputerTranslator { this.coordinateSystem = opts.coordinateSystem ?? { type: "pixel" }; this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; + this.cdpWsUrl = opts.browser.cdp_ws_url; + this.domExecutorFactory = opts.createDomExecutor ?? createDomExecutor; + } + + /** The DOM-plane executor, connected lazily over the browser's CDP websocket. */ + dom(): DomExecutor { + if (!this.domExecutor) { + if (!this.cdpWsUrl) throw new Error("browser has no cdp_ws_url; DOM actions are unavailable"); + this.domExecutor = this.domExecutorFactory(this.cdpWsUrl); + } + return this.domExecutor; } async screenshotRaw(): Promise { @@ -106,11 +126,20 @@ export class InternalComputerTranslator { }; for (const action of actions) { + if (isCuaDomAction(action)) { + await flush(); + result.readResults.push(...(await this.dom().execute(action))); + continue; + } switch (action.type) { case "screenshot": await flush(); result.readResults.push({ type: "screenshot", ...(await this.screenshot()) }); break; + case "zoom": + await flush(); + result.readResults.push({ type: "screenshot", ...(await this.zoom(action)) }); + break; case "url": await flush(); result.readResults.push({ type: "url", url: await this.currentUrl() }); @@ -133,6 +162,17 @@ export class InternalComputerTranslator { pending.push(keypress(["Alt", "Right"])); break; default: + // Native computer mappings may omit click coordinates, meaning + // "at the current cursor position" — resolve before batching. + if ( + (action.type === "click" || action.type === "mouse_down" || action.type === "mouse_up") && + (action.x === undefined || action.y === undefined) + ) { + await flush(); + const position = await this.currentMousePosition(); + pending.push(this.toSdkAction({ ...action, x: action.x ?? position.x, y: action.y ?? position.y })); + break; + } pending.push(this.toSdkAction(action)); break; } @@ -142,12 +182,27 @@ export class InternalComputerTranslator { return result; } + /** Crop the OS screenshot to a region; coordinates stay in the full-screenshot frame. */ + async zoom(action: CuaActionZoom): Promise<{ data: Buffer; mimeType: string }> { + const screenshot = await this.screenshot(); + const [x0, y0, x1, y1] = action.region; + const left = Math.max(0, Math.trunc(Math.min(x0, x1))); + const top = Math.max(0, Math.trunc(Math.min(y0, y1))); + const width = Math.max(1, Math.trunc(Math.abs(x1 - x0))); + const height = Math.max(1, Math.trunc(Math.abs(y1 - y0))); + const data = await sharp(screenshot.data).extract({ left, top, width, height }).png().toBuffer(); + return { data, mimeType: "image/png" }; + } + private toSdkAction( - action: Exclude, + action: Exclude, ): KernelBatchAction { switch (action.type) { case "click": - return this.clickAction(action, { button: mouseButton(action.button) }); + return this.clickAction(action, { + button: mouseButton(action.button), + ...(action.num_clicks !== undefined && action.num_clicks > 1 ? { num_clicks: Math.trunc(action.num_clicks) } : {}), + }); case "double_click": return this.clickAction(action, { num_clicks: 2 }); case "mouse_down": @@ -175,7 +230,7 @@ export class InternalComputerTranslator { action: CuaActionClick | CuaActionDoubleClick | CuaActionMouseDown | CuaActionMouseUp, extra: { button?: CuaMouseButton; num_clicks?: number; click_type?: "down" | "up" }, ): KernelBatchAction { - const point = this.toViewportPoint(action.x, action.y); + const point = this.toViewportPoint(action.x ?? 0, action.y ?? 0); return { type: "click_mouse", click_mouse: { diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index e9aaca37..2ff31967 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -1,7 +1,8 @@ export type BatchReadResult = | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } - | { type: "cursor_position"; x: number; y: number }; + | { type: "cursor_position"; x: number; y: number } + | { type: "dom_text"; label: string; text: string }; export interface BatchExecutionResult { readResults: BatchReadResult[]; diff --git a/packages/agent/test/translator-dom.test.ts b/packages/agent/test/translator-dom.test.ts new file mode 100644 index 00000000..cba6fc4c --- /dev/null +++ b/packages/agent/test/translator-dom.test.ts @@ -0,0 +1,88 @@ +import type Kernel from "@onkernel/sdk"; +import sharp from "sharp"; +import { describe, expect, it } from "vitest"; +import type { CuaDomAction } from "@onkernel/cua-ai"; +import type { DomExecutor } from "../src/translator/dom"; +import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; +import type { BatchReadResult } from "../src/translator/types"; + +const browser = { session_id: "browser_123", cdp_ws_url: "wss://example.test/cdp" } as KernelBrowser; + +function createClient() { + const batches: unknown[] = []; + const client = { + browsers: { + computer: { + batch: async (_id: string, body: { actions: unknown[] }) => { + batches.push(body.actions); + }, + captureScreenshot: async () => new Response(new Uint8Array(await sharp({ create: { width: 100, height: 80, channels: 3, background: "#fff" } }).png().toBuffer())), + getMousePosition: async () => ({ x: 42, y: 24 }), + }, + }, + } as unknown as Kernel; + return { batches, client }; +} + +function createFakeDom() { + const executed: CuaDomAction[] = []; + const dom = { + execute: async (action: CuaDomAction): Promise => { + executed.push(action); + if (action.type === "page_text") return [{ type: "dom_text", label: "text", text: "hello" }]; + return []; + }, + screenshot: async () => ({ data: Buffer.from("png"), mimeType: "image/png" }), + } as unknown as DomExecutor; + return { executed, dom }; +} + +describe("InternalComputerTranslator DOM plane", () => { + it("dispatches DOM actions to the DOM executor, flushing pending OS input first", async () => { + const { batches, client } = createClient(); + const { executed, dom } = createFakeDom(); + const translator = new InternalComputerTranslator({ browser, client, createDomExecutor: () => dom }); + + const result = await translator.executeBatch([ + { type: "click", x: 1, y: 2 }, + { type: "page_text" }, + { type: "page_click", ref: "e3" }, + ]); + + expect(batches).toHaveLength(1); + expect(executed.map((action) => action.type)).toEqual(["page_text", "page_click"]); + expect(result.readResults).toEqual([{ type: "dom_text", label: "text", text: "hello" }]); + }); + + it("errors on DOM actions when the browser has no cdp_ws_url", async () => { + const { client } = createClient(); + const translator = new InternalComputerTranslator({ browser: { session_id: "b" } as KernelBrowser, client }); + await expect(translator.executeBatch([{ type: "page_text" }])).rejects.toThrow(/cdp_ws_url/); + }); +}); + +describe("InternalComputerTranslator OS additions", () => { + it("crops the OS screenshot for zoom, staying in the screenshot frame", async () => { + const { client } = createClient(); + const translator = new InternalComputerTranslator({ browser, client }); + const result = await translator.executeBatch([{ type: "zoom", region: [10, 10, 60, 40] }]); + const read = result.readResults[0]!; + if (read.type !== "screenshot") throw new Error("expected screenshot read result"); + const metadata = await sharp(read.data).metadata(); + expect(metadata.width).toBe(50); + expect(metadata.height).toBe(30); + }); + + it("passes num_clicks through and resolves missing click coordinates from the cursor", async () => { + const { batches, client } = createClient(); + const translator = new InternalComputerTranslator({ browser, client }); + await translator.executeBatch([ + { type: "click", x: 1, y: 2, num_clicks: 3 }, + { type: "click" }, + ]); + expect(batches.flat()).toEqual([ + { type: "click_mouse", click_mouse: { x: 1, y: 2, button: "left", num_clicks: 3 } }, + { type: "click_mouse", click_mouse: { x: 42, y: 24, button: "left" } }, + ]); + }); +}); diff --git a/packages/ai/src/actions/dom.ts b/packages/ai/src/actions/dom.ts new file mode 100644 index 00000000..5d7f859e --- /dev/null +++ b/packages/ai/src/actions/dom.ts @@ -0,0 +1,331 @@ +import { Type, type TSchema } from "@earendil-works/pi-ai"; + +/** + * DOM-plane canonical actions. + * + * These execute over CDP against the browser itself: accessibility-tree + * reads with element references, element-targeted interaction, navigation, + * tabs, and viewport screenshots. Where a DOM action takes coordinates + * (`page_click`, `page_hover`, `page_drag`, `page_scroll`), they are pixels + * in the browser viewport — a different frame from the OS-plane actions in + * `./os`. Modes that expose both planes (hybrid) therefore restrict DOM + * actions to element references so exactly one coordinate frame is live. + * + * Element references (`ref`) are snapshot-scoped opaque ids (`e12`) minted + * by `page_snapshot` / `page_find`; a stale ref is an error instructing the + * model to re-snapshot. + */ +export const CUA_DOM_ACTION_TYPES = [ + "page_snapshot", + "page_text", + "page_find", + "page_click", + "page_hover", + "page_drag", + "page_fill", + "page_scroll_to", + "page_scroll", + "page_type", + "page_key", + "page_navigate", + "page_list_tabs", + "page_new_tab", + "page_screenshot", + "page_evaluate", +] as const; + +export type CuaDomActionType = (typeof CUA_DOM_ACTION_TYPES)[number]; + +/** + * The default DOM-mode toolset: everything except `page_evaluate`, which + * runs arbitrary JavaScript in the page and must be enabled explicitly + * (`javascriptExec: true`). + */ +export const CUA_DEFAULT_DOM_ACTION_TYPES = CUA_DOM_ACTION_TYPES.filter( + (action): action is Exclude => action !== "page_evaluate", +); + +export interface CuaActionPageSnapshot { + type: "page_snapshot"; + filter?: "all" | "interactive"; + ref?: string; + depth?: number; + tab_id?: string; +} + +export interface CuaActionPageText { + type: "page_text"; + tab_id?: string; +} + +export interface CuaActionPageFind { + type: "page_find"; + query: string; + tab_id?: string; +} + +export interface CuaActionPageClick { + type: "page_click"; + ref?: string; + x?: number; + y?: number; + button?: "left" | "right" | "middle"; + num_clicks?: number; + modifiers?: string[]; + tab_id?: string; +} + +export interface CuaActionPageHover { + type: "page_hover"; + ref?: string; + x?: number; + y?: number; + tab_id?: string; +} + +export interface CuaActionPageDrag { + type: "page_drag"; + from: { x: number; y: number }; + to: { x: number; y: number }; + tab_id?: string; +} + +export interface CuaActionPageFill { + type: "page_fill"; + ref: string; + value: string | number | boolean; + tab_id?: string; +} + +export interface CuaActionPageScrollTo { + type: "page_scroll_to"; + ref: string; + tab_id?: string; +} + +export interface CuaActionPageScroll { + type: "page_scroll"; + x: number; + y: number; + direction: "up" | "down" | "left" | "right"; + amount?: number; + tab_id?: string; +} + +export interface CuaActionPageType { + type: "page_type"; + text: string; + tab_id?: string; +} + +export interface CuaActionPageKey { + type: "page_key"; + text: string; + repeat?: number; + tab_id?: string; +} + +export interface CuaActionPageNavigate { + type: "page_navigate"; + /** A URL, or the sentinels "back" / "forward" for history navigation. */ + url: string; + tab_id?: string; +} + +export interface CuaActionPageListTabs { + type: "page_list_tabs"; +} + +export interface CuaActionPageNewTab { + type: "page_new_tab"; +} + +export interface CuaActionPageScreenshot { + type: "page_screenshot"; + /** Optional crop region, [x0, y0, x1, y1] in viewport pixels. */ + region?: [number, number, number, number]; + tab_id?: string; +} + +export interface CuaActionPageEvaluate { + type: "page_evaluate"; + code: string; + tab_id?: string; +} + +export type CuaDomAction = + | CuaActionPageSnapshot + | CuaActionPageText + | CuaActionPageFind + | CuaActionPageClick + | CuaActionPageHover + | CuaActionPageDrag + | CuaActionPageFill + | CuaActionPageScrollTo + | CuaActionPageScroll + | CuaActionPageType + | CuaActionPageKey + | CuaActionPageNavigate + | CuaActionPageListTabs + | CuaActionPageNewTab + | CuaActionPageScreenshot + | CuaActionPageEvaluate; + +/** Options for building DOM action schemas. */ +export interface CuaDomSchemaOptions { + /** + * Whether coordinate targeting is allowed on `page_click` / `page_hover` + * and whether `page_drag` / `page_scroll` are expressible at all. DOM + * mode allows viewport coordinates (they are the only frame); hybrid mode + * must disallow them so the OS screenshot frame stays the single live + * coordinate frame. + */ + coordinates: boolean; +} + +const TabId = () => Type.Optional(Type.String({ description: "Tab to act on. Defaults to the active tab." })); + +const RefProperty = () => Type.String({ description: "Element reference from page_snapshot or page_find, e.g. \"e12\"." }); + +export function createCuaDomActionSchemaByType(options: CuaDomSchemaOptions): Record { + const clickTarget: Record = options.coordinates + ? { + ref: Type.Optional(RefProperty()), + x: Type.Optional(Type.Number({ description: "Viewport x in pixels. Prefer ref targeting when available." })), + y: Type.Optional(Type.Number({ description: "Viewport y in pixels. Prefer ref targeting when available." })), + } + : { ref: RefProperty() }; + + return { + page_snapshot: Type.Object( + { + type: Type.Literal("page_snapshot"), + filter: Type.Optional(Type.Union([Type.Literal("all"), Type.Literal("interactive")])), + ref: Type.Optional(Type.String({ description: "Restrict the snapshot to the subtree rooted at this element reference." })), + depth: Type.Optional(Type.Number({ description: "Maximum tree depth (default 15)." })), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_text: Type.Object( + { + type: Type.Literal("page_text"), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_find: Type.Object( + { + type: Type.Literal("page_find"), + query: Type.String({ description: "Natural-language element description, e.g. \"the cookie consent accept button\"." }), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_click: Type.Object( + { + type: Type.Literal("page_click"), + ...clickTarget, + button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")])), + num_clicks: Type.Optional(Type.Number()), + modifiers: Type.Optional(Type.Array(Type.String())), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_hover: Type.Object( + { + type: Type.Literal("page_hover"), + ...clickTarget, + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_drag: Type.Object( + { + type: Type.Literal("page_drag"), + from: Type.Object({ x: Type.Number(), y: Type.Number() }, { additionalProperties: false }), + to: Type.Object({ x: Type.Number(), y: Type.Number() }, { additionalProperties: false }), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_fill: Type.Object( + { + type: Type.Literal("page_fill"), + ref: RefProperty(), + value: Type.Union([Type.String(), Type.Number(), Type.Boolean()], { + description: "Value to set. Use a boolean for checkboxes, an option value or label for selects.", + }), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_scroll_to: Type.Object( + { + type: Type.Literal("page_scroll_to"), + ref: RefProperty(), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_scroll: Type.Object( + { + type: Type.Literal("page_scroll"), + x: Type.Number({ description: "Viewport x anchor in pixels." }), + y: Type.Number({ description: "Viewport y anchor in pixels." }), + direction: Type.Union([Type.Literal("up"), Type.Literal("down"), Type.Literal("left"), Type.Literal("right")]), + amount: Type.Optional(Type.Number({ description: "Scroll-wheel notches (default 3)." })), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_type: Type.Object( + { + type: Type.Literal("page_type"), + text: Type.String(), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_key: Type.Object( + { + type: Type.Literal("page_key"), + text: Type.String({ description: "Key or chord, e.g. \"Return\", \"ctrl+a\"." }), + repeat: Type.Optional(Type.Number()), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_navigate: Type.Object( + { + type: Type.Literal("page_navigate"), + url: Type.String({ description: "URL to navigate to, or \"back\" / \"forward\" for history navigation." }), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_list_tabs: Type.Object({ type: Type.Literal("page_list_tabs") }, { additionalProperties: false }), + page_new_tab: Type.Object({ type: Type.Literal("page_new_tab") }, { additionalProperties: false }), + page_screenshot: Type.Object( + { + type: Type.Literal("page_screenshot"), + region: Type.Optional( + Type.Tuple([Type.Number(), Type.Number(), Type.Number(), Type.Number()], { + description: "Optional crop region, [x0, y0, x1, y1] in viewport pixels.", + }), + ), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + page_evaluate: Type.Object( + { + type: Type.Literal("page_evaluate"), + code: Type.String({ description: "JavaScript to evaluate in the page context. The value of the last expression is returned." }), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), + }; +} diff --git a/packages/ai/src/actions/index.ts b/packages/ai/src/actions/index.ts new file mode 100644 index 00000000..7bb01ca3 --- /dev/null +++ b/packages/ai/src/actions/index.ts @@ -0,0 +1,47 @@ +import type { TSchema } from "@earendil-works/pi-ai"; +import { CUA_DOM_ACTION_TYPES, createCuaDomActionSchemaByType, type CuaDomAction, type CuaDomActionType, type CuaDomSchemaOptions } from "./dom"; +import { CUA_OS_ACTION_SCHEMA_BY_TYPE, CUA_OS_ACTION_TYPES, type CuaOsAction, type CuaOsActionType } from "./os"; + +export * from "./dom"; +export * from "./os"; + +/** Any canonical CUA action type, across the OS and DOM planes. */ +export type CuaActionType = CuaOsActionType | CuaDomActionType; + +/** Any canonical CUA action, across the OS and DOM planes. */ +export type CuaAction = CuaOsAction | CuaDomAction; + +/** Every canonical action type: the OS plane followed by the DOM plane. */ +export const CUA_ALL_ACTION_TYPES: readonly CuaActionType[] = [...CUA_OS_ACTION_TYPES, ...CUA_DOM_ACTION_TYPES]; + +const OS_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_OS_ACTION_TYPES); +const DOM_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_DOM_ACTION_TYPES); + +/** Whether a canonical action type belongs to the OS plane. */ +export function isCuaOsActionType(action: CuaActionType): action is CuaOsActionType { + return OS_ACTION_TYPE_SET.has(action); +} + +/** Whether a canonical action type belongs to the DOM plane. */ +export function isCuaDomActionType(action: CuaActionType): action is CuaDomActionType { + return DOM_ACTION_TYPE_SET.has(action); +} + +/** Whether a canonical action belongs to the DOM plane. */ +export function isCuaDomAction(action: CuaAction): action is CuaDomAction { + return DOM_ACTION_TYPE_SET.has(action.type); +} + +/** Options for building canonical action schemas. */ +export interface CuaActionSchemaOptions { + /** DOM-plane schema variants; see {@link CuaDomSchemaOptions}. Defaults to coordinates allowed. */ + dom?: CuaDomSchemaOptions; +} + +/** Build the full action-type → schema map for a schema-options combination. */ +export function cuaActionSchemaByType(options: CuaActionSchemaOptions = {}): Record { + return { + ...CUA_OS_ACTION_SCHEMA_BY_TYPE, + ...createCuaDomActionSchemaByType(options.dom ?? { coordinates: true }), + }; +} diff --git a/packages/ai/src/actions/os.ts b/packages/ai/src/actions/os.ts new file mode 100644 index 00000000..8fc6cf47 --- /dev/null +++ b/packages/ai/src/actions/os.ts @@ -0,0 +1,297 @@ +import { Type, type Static, type TSchema } from "@earendil-works/pi-ai"; + +/** + * OS-plane canonical actions. + * + * These execute as real OS-level input against the Kernel browser VM (mouse, + * keyboard, display capture) — never CDP. All coordinates are pixels in the + * OS screenshot frame. The DOM-plane vocabulary lives in `./dom` and is + * executed over CDP; the two planes never share a coordinate frame. + */ +export const CUA_OS_ACTION_TYPES = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "zoom", + "goto", + "back", + "forward", + "url", + "cursor_position", +] as const; + +export type CuaOsActionType = (typeof CUA_OS_ACTION_TYPES)[number]; + +/** + * The default OS-mode toolset. This is the pre-modes canonical action list: + * every OS action except `zoom`, which is only exposed by default in hybrid + * mode and by Anthropic's native computer tool (`enable_zoom`). + */ +export const CUA_DEFAULT_OS_ACTION_TYPES = CUA_OS_ACTION_TYPES.filter( + (action): action is Exclude => action !== "zoom", +); + +/** + * Mouse buttons accepted by click, mouse_down, and mouse_up actions. The + * executor coerces anything outside this set to "left". + */ +export type CuaMouseButton = "left" | "right" | "middle" | "back" | "forward"; + +/** + * Mouse buttons accepted by drag actions. The executor coerces anything + * outside this set to "left". + */ +export type CuaDragMouseButton = "left" | "right" | "middle"; + +export interface CuaActionClick { + type: "click"; + /** OS screenshot pixels. Omitted (native mappings only) means the current cursor position. */ + x?: number; + y?: number; + button?: CuaMouseButton; + hold_keys?: string[]; + num_clicks?: number; +} + +export interface CuaActionDoubleClick { + type: "double_click"; + x: number; + y: number; + hold_keys?: string[]; +} + +export interface CuaActionMouseDown { + type: "mouse_down"; + x?: number; + y?: number; + button?: CuaMouseButton; + hold_keys?: string[]; +} + +export interface CuaActionMouseUp { + type: "mouse_up"; + x?: number; + y?: number; + button?: CuaMouseButton; + hold_keys?: string[]; +} + +export interface CuaActionTypeText { + type: "type"; + text: string; +} + +export interface CuaActionKeypress { + type: "keypress"; + keys: string[]; + duration?: number; +} + +export interface CuaActionScroll { + type: "scroll"; + x?: number; + y?: number; + scroll_x?: number; + scroll_y?: number; + hold_keys?: string[]; +} + +export interface CuaActionMove { + type: "move"; + x: number; + y: number; +} + +export interface CuaActionDrag { + type: "drag"; + path: Array<{ x: number; y: number }>; + button?: CuaDragMouseButton; + hold_keys?: string[]; +} + +export interface CuaActionWait { + type: "wait"; + ms?: number; +} + +export interface CuaActionScreenshot { + type: "screenshot"; +} + +/** Crop of the most recent OS screenshot; region is [x0, y0, x1, y1] in OS screenshot pixels. */ +export interface CuaActionZoom { + type: "zoom"; + region: [number, number, number, number]; +} + +export interface CuaActionGoto { + type: "goto"; + url: string; +} + +export interface CuaActionBack { + type: "back"; +} + +export interface CuaActionForward { + type: "forward"; +} + +export interface CuaActionUrl { + type: "url"; +} + +export interface CuaActionCursorPosition { + type: "cursor_position"; +} + +export type CuaOsAction = + | CuaActionClick + | CuaActionDoubleClick + | CuaActionMouseDown + | CuaActionMouseUp + | CuaActionTypeText + | CuaActionKeypress + | CuaActionScroll + | CuaActionMove + | CuaActionDrag + | CuaActionWait + | CuaActionScreenshot + | CuaActionZoom + | CuaActionGoto + | CuaActionBack + | CuaActionForward + | CuaActionUrl + | CuaActionCursorPosition; + +const PointSchema = Type.Object( + { + x: Type.Number(), + y: Type.Number(), + }, + { additionalProperties: false }, +); + +export const CUA_OS_ACTION_SCHEMA_BY_TYPE = { + click: Type.Object( + { + type: Type.Literal("click"), + x: Type.Number(), + y: Type.Number(), + button: Type.Optional(Type.String()), + hold_keys: Type.Optional(Type.Array(Type.String())), + num_clicks: Type.Optional(Type.Number()), + }, + { additionalProperties: false }, + ), + double_click: Type.Object( + { + type: Type.Literal("double_click"), + x: Type.Number(), + y: Type.Number(), + hold_keys: Type.Optional(Type.Array(Type.String())), + }, + { additionalProperties: false }, + ), + mouse_down: Type.Object( + { + type: Type.Literal("mouse_down"), + x: Type.Optional(Type.Number()), + y: Type.Optional(Type.Number()), + button: Type.Optional(Type.String()), + hold_keys: Type.Optional(Type.Array(Type.String())), + }, + { additionalProperties: false }, + ), + mouse_up: Type.Object( + { + type: Type.Literal("mouse_up"), + x: Type.Optional(Type.Number()), + y: Type.Optional(Type.Number()), + button: Type.Optional(Type.String()), + hold_keys: Type.Optional(Type.Array(Type.String())), + }, + { additionalProperties: false }, + ), + type: Type.Object( + { + type: Type.Literal("type"), + text: Type.String(), + }, + { additionalProperties: false }, + ), + keypress: Type.Object( + { + type: Type.Literal("keypress"), + keys: Type.Array(Type.String()), + duration: Type.Optional(Type.Number()), + }, + { additionalProperties: false }, + ), + scroll: Type.Object( + { + type: Type.Literal("scroll"), + x: Type.Optional(Type.Number()), + y: Type.Optional(Type.Number()), + scroll_x: Type.Optional(Type.Number()), + scroll_y: Type.Optional(Type.Number()), + hold_keys: Type.Optional(Type.Array(Type.String())), + }, + { additionalProperties: false }, + ), + move: Type.Object( + { + type: Type.Literal("move"), + x: Type.Number(), + y: Type.Number(), + }, + { additionalProperties: false }, + ), + drag: Type.Object( + { + type: Type.Literal("drag"), + path: Type.Array(PointSchema, { minItems: 2 }), + button: Type.Optional(Type.String()), + hold_keys: Type.Optional(Type.Array(Type.String())), + }, + { additionalProperties: false }, + ), + wait: Type.Object( + { + type: Type.Literal("wait"), + ms: Type.Optional(Type.Number()), + }, + { additionalProperties: false }, + ), + screenshot: Type.Object({ type: Type.Literal("screenshot") }, { additionalProperties: false }), + zoom: Type.Object( + { + type: Type.Literal("zoom"), + region: Type.Tuple([Type.Number(), Type.Number(), Type.Number(), Type.Number()], { + description: "[x0, y0, x1, y1] crop region in OS screenshot pixels.", + }), + }, + { additionalProperties: false }, + ), + goto: Type.Object( + { + type: Type.Literal("goto"), + url: Type.String(), + }, + { additionalProperties: false }, + ), + back: Type.Object({ type: Type.Literal("back") }, { additionalProperties: false }), + forward: Type.Object({ type: Type.Literal("forward") }, { additionalProperties: false }), + url: Type.Object({ type: Type.Literal("url") }, { additionalProperties: false }), + cursor_position: Type.Object({ type: Type.Literal("cursor_position") }, { additionalProperties: false }), +} satisfies Record; + +export type CuaZoomRegion = Static<(typeof CUA_OS_ACTION_SCHEMA_BY_TYPE)["zoom"]>["region"]; diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts new file mode 100644 index 00000000..df35488c --- /dev/null +++ b/packages/ai/src/modes.ts @@ -0,0 +1,173 @@ +import { + CUA_DEFAULT_DOM_ACTION_TYPES, + CUA_DEFAULT_OS_ACTION_TYPES, + isCuaOsActionType, + type CuaActionSchemaOptions, + type CuaActionType, + type CuaDomActionType, + type CuaOsActionType, +} from "./actions/index"; + +/** + * Which canonical action plane(s) a CUA agent exposes to the model. + * + * - `os` — OS-level input only (mouse/keyboard/display against the VM). + * Today's default; coordinates are OS screenshot pixels. + * - `dom` — DOM-level tools only, driven over CDP: accessibility snapshots + * with element refs, element-targeted interaction, navigation, tabs, and + * viewport screenshots. Coordinates, where used, are viewport pixels. + * - `hybrid` — both planes, deduplicated to one tool per capability. OS + * tools are prefixed `computer_`, DOM tools keep their `page_` prefix, + * DOM tools accept element refs only, and the OS screenshot frame is the + * single live coordinate frame. + */ +export type CuaMode = "os" | "dom" | "hybrid"; + +/** Options for resolving a mode's action set. */ +export interface CuaModeOptions { + /** Expose `page_evaluate` (arbitrary JavaScript in the page). Default false. */ + javascriptExec?: boolean; +} + +/** + * OS actions exposed in hybrid mode: navigation reads/writes are excluded + * because they live on the DOM plane (`page_navigate`, `page_list_tabs`), + * and `zoom` is included since the OS screenshot is hybrid's only capture. + */ +export const CUA_HYBRID_OS_ACTION_TYPES: readonly CuaOsActionType[] = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "zoom", + "cursor_position", +]; + +/** + * DOM actions exposed in hybrid mode: reads and element-targeted writes + * only. Pointer/keyboard capabilities (`page_click` by coordinate, + * `page_type`, `page_key`, `page_scroll`, `page_hover`, `page_drag`) and + * `page_screenshot` are excluded — real OS input and the OS screenshot cover + * those, keeping one tool per capability and one coordinate frame. + */ +export const CUA_HYBRID_DOM_ACTION_TYPES: readonly CuaDomActionType[] = [ + "page_snapshot", + "page_text", + "page_find", + "page_click", + "page_fill", + "page_scroll_to", + "page_navigate", + "page_list_tabs", + "page_new_tab", +]; + +/** Resolve the default canonical action set for a mode. */ +export function defaultActionsForMode(mode: CuaMode, options: CuaModeOptions = {}): readonly CuaActionType[] { + switch (mode) { + case "os": + return CUA_DEFAULT_OS_ACTION_TYPES; + case "dom": + return [...CUA_DEFAULT_DOM_ACTION_TYPES, ...(options.javascriptExec ? (["page_evaluate"] as const) : []), "wait"]; + case "hybrid": + return [ + ...CUA_HYBRID_OS_ACTION_TYPES, + ...CUA_HYBRID_DOM_ACTION_TYPES, + ...(options.javascriptExec ? (["page_evaluate"] as const) : []), + ]; + } +} + +/** Resolve the schema-building options for a mode; see {@link CuaActionSchemaOptions}. */ +export function schemaOptionsForMode(mode: CuaMode): CuaActionSchemaOptions { + // Hybrid restricts DOM actions to element refs so the OS screenshot frame + // is the single live coordinate frame. DOM mode has no OS frame, so + // viewport coordinates are allowed there. + return { dom: { coordinates: mode !== "hybrid" } }; +} + +/** + * The model-facing tool name for a canonical action in a mode. + * + * - `os`: canonical action ids as-is (`click`, `screenshot`, …). + * - `dom`: DOM ids with the `page_` prefix stripped (`snapshot`, `click`, …); + * the prefix only exists to disambiguate planes, and dom mode has one. + * - `hybrid`: OS ids prefixed `computer_`, DOM ids kept as `page_*`. + */ +export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): string { + switch (mode) { + case "os": + if (!isCuaOsActionType(action)) throw new Error(`DOM action "${action}" is not available in os mode`); + return action; + case "dom": + return isCuaOsActionType(action) ? action : action.slice("page_".length); + case "hybrid": + return isCuaOsActionType(action) ? `computer_${action}` : action; + } +} + +const DOM_ACTION_DESCRIPTIONS: Record = { + page_snapshot: + "Return an accessibility-tree snapshot of the page with element references like [e12]. " + + "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale.", + page_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", + page_find: "Find elements matching a natural-language description and return them with element references, like a filtered snapshot.", + page_click: "Click an element. Prefer targeting by element reference from a snapshot.", + page_hover: "Move the pointer over an element without clicking.", + page_drag: "Drag from one viewport coordinate to another.", + page_fill: "Set the value of a form element (input, textarea, select, checkbox) by element reference.", + page_scroll_to: "Scroll an element into view by element reference.", + page_scroll: "Scroll the page at a viewport position by wheel notches.", + page_type: "Type a literal string at the current focus.", + page_key: "Press a key or chord, e.g. \"Return\" or \"ctrl+a\".", + page_navigate: "Navigate the page to a URL, or \"back\" / \"forward\" in history.", + page_list_tabs: "List open tabs with each tab's id, title, and URL.", + page_new_tab: "Open a new empty tab and return its tab id.", + page_screenshot: "Capture the current browser viewport.", + page_evaluate: "Execute JavaScript in the page context and return the value of the last expression.", +}; + +// Hybrid exposes both planes, so tool descriptions carry the arbitration +// rules the model needs: which plane is preferred for a capability and why +// (real OS input vs CDP), plus the single-coordinate-frame statement. +const HYBRID_OS_DESCRIPTION_OVERRIDES: Partial> = { + click: + "Click at a coordinate in OS screenshot pixels using real OS-level input. " + + "Preferred over page_click when the target is visible in the screenshot — OS input is indistinguishable from a human user.", + screenshot: "Capture the display. This is the only screenshot tool; all coordinates refer to this image's pixels.", + zoom: "Return a cropped view of the current display for closer inspection. Coordinates in later actions still refer to the full screenshot, not the crop.", + scroll: "Scroll with the OS-level mouse wheel at a coordinate in OS screenshot pixels.", + type: "Type a literal string with OS-level keyboard input at the current focus.", + keypress: "Press keys with OS-level keyboard input.", +}; + +const HYBRID_DOM_DESCRIPTION_OVERRIDES: Partial> = { + page_click: + "Click an element by reference from a page_snapshot. Dispatched via CDP, which protected sites may detect — " + + "prefer computer_click when the element is visible in the screenshot; use page_click for elements that are hard to hit by coordinate.", + page_snapshot: + "Return an accessibility-tree snapshot of the page with element references like [e12]. " + + "This is the high-fidelity way to read page structure — prefer it over screenshots for reading and locating elements. " + + "Refs are only valid until the page changes; re-snapshot when told a ref is stale.", +}; + +/** The model-facing tool description for a canonical action in a mode. */ +export function cuaToolDescriptionForAction(action: CuaActionType, mode: CuaMode): string { + if (isCuaOsActionType(action)) { + if (mode === "hybrid") { + return HYBRID_OS_DESCRIPTION_OVERRIDES[action] ?? `Execute one ${action} computer action using real OS-level input.`; + } + return `Execute one ${action} computer action.`; + } + if (mode === "hybrid") { + return HYBRID_DOM_DESCRIPTION_OVERRIDES[action] ?? DOM_ACTION_DESCRIPTIONS[action]; + } + return DOM_ACTION_DESCRIPTIONS[action]; +} diff --git a/packages/ai/src/native-tools.ts b/packages/ai/src/native-tools.ts new file mode 100644 index 00000000..6263efdc --- /dev/null +++ b/packages/ai/src/native-tools.ts @@ -0,0 +1,115 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import type { CuaMode } from "./modes"; + +/** + * Anthropic's native computer-use tool (`anthropic-beta: + * computer-use-2026-06-01`). Server-defined: the declaration below is sent + * verbatim in `tools[]` and Anthropic fixes the input schema. Maps to CUA's + * `os` mode — actions arrive as OS-plane input in screenshot-pixel + * coordinates. + */ +export interface AnthropicComputerNativeTool { + type: "computer_20260601"; + /** Tool name in the request; defaults to "computer". */ + name?: string; + /** Adds a `zoom` action returning a cropped screenshot region. Default false. */ + enable_zoom?: boolean; + /** X11 display number for multi-display environments. */ + display_number?: number; + /** Prompt-caching breakpoint. */ + cache_control?: { type: "ephemeral" }; +} + +/** + * Anthropic's native browser tool (`anthropic-beta: browser-use-2026-07-01`, + * proposed — the tool version and schema may change before release). Maps to + * CUA's `dom` mode — DOM reads by element reference plus pointer actions in + * viewport-pixel coordinates. + */ +export interface AnthropicBrowserNativeTool { + type: "browser_20260701"; + /** Tool name in the request; defaults to "browser". */ + name?: string; + /** Adds a `javascript_exec` action running arbitrary JS in the page. Default false. */ + enable_javascript_exec?: boolean; + /** Prompt-caching breakpoint. */ + cache_control?: { type: "ephemeral" }; +} + +/** + * A provider-native computer-use tool declaration. + * + * Pass one to `resolveCuaRuntimeSpec` (or `CuaAgent`/`CuaAgentHarness`) to + * drive the model through its provider-defined tool schema instead of CUA's + * canonical function tools. Each native tool pairs with exactly one + * {@link CuaMode}; execution is identical either way — native tool calls are + * translated to the same canonical actions the mode uses. + */ +export type CuaNativeToolSpec = AnthropicComputerNativeTool | AnthropicBrowserNativeTool; + +export type CuaNativeToolType = CuaNativeToolSpec["type"]; + +interface NativeToolInfo { + mode: Extract; + provider: "anthropic"; + betaHeader: string; + defaultName: string; +} + +const NATIVE_TOOL_INFO: Record = { + computer_20260601: { mode: "os", provider: "anthropic", betaHeader: "computer-use-2026-06-01", defaultName: "computer" }, + browser_20260701: { mode: "dom", provider: "anthropic", betaHeader: "browser-use-2026-07-01", defaultName: "browser" }, +}; + +/** The {@link CuaMode} a native tool requires. */ +export function modeForNativeTool(spec: CuaNativeToolSpec): CuaMode { + return NATIVE_TOOL_INFO[spec.type].mode; +} + +/** The `anthropic-beta` header value a native tool requires. */ +export function betaHeaderForNativeTool(spec: CuaNativeToolSpec): string { + return NATIVE_TOOL_INFO[spec.type].betaHeader; +} + +/** The tool name a native tool declares, defaulting per tool type. */ +export function nativeToolName(spec: CuaNativeToolSpec): string { + return spec.name ?? NATIVE_TOOL_INFO[spec.type].defaultName; +} + +/** A validated native tool: the spec plus everything derived from it. */ +export interface ResolvedCuaNativeTool { + spec: CuaNativeToolSpec; + /** The declaration sent verbatim in the provider `tools[]` array. */ + declaration: Record; + /** Tool name the model's tool_use blocks arrive under. */ + name: string; + /** Required `anthropic-beta` header value. */ + betaHeader: string; + mode: CuaMode; +} + +/** + * Validate a native tool spec against the resolved model and mode, and + * derive the request-facing pieces. Throws when the model's provider does + * not serve the tool or the mode conflicts with the tool's plane (mirroring + * the API, which rejects e.g. `browser_20260701` outside a one-frame + * browser-only request). + */ +export function resolveNativeTool(spec: CuaNativeToolSpec, model: Model, mode: CuaMode): ResolvedCuaNativeTool { + const info = NATIVE_TOOL_INFO[spec.type]; + if (!info) throw new Error(`unknown native tool type "${(spec as { type: string }).type}"`); + if (model.provider !== info.provider) { + throw new Error(`native tool "${spec.type}" requires an ${info.provider} model; got provider "${model.provider}"`); + } + if (mode !== info.mode) { + throw new Error(`native tool "${spec.type}" requires mode "${info.mode}"; got "${mode}"`); + } + const name = nativeToolName(spec); + return { + spec, + declaration: { ...spec, name }, + name, + betaHeader: info.betaHeader, + mode, + }; +} diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index 29492684..9a5d1c18 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -13,6 +13,7 @@ import { import { builtinModels } from "@earendil-works/pi-ai/providers/all"; import { cuaApiKeyEnvVarsForProvider } from "./api-keys"; import { cuaOverrideModels } from "./models"; +import { ANTHROPIC_NATIVE_API_BETA_HEADERS, withAnthropicBetaHeader } from "./providers/anthropic/native"; import { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; @@ -25,6 +26,10 @@ import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from ". * {@link getCuaModel} routes OpenAI models to, threading * `previous_response_id`; every other api falls through to pi's builtin * provider. + * - `anthropic` intercepts the native computer/browser tool apis that + * `resolveCuaRuntimeSpec` routes models with a `nativeTool` to, dispatching + * them to pi's builtin `anthropic-messages` transport with the tool's + * `anthropic-beta` header merged in. * - `google` resolves its API key from `GOOGLE_API_KEY` or `GEMINI_API_KEY` * (pi's builtin only reads `GEMINI_API_KEY`). * - `tzafon` and `yutori` are CUA-only providers pi does not ship. @@ -36,6 +41,8 @@ export function createCuaModels(options?: CreateModelsOptions): MutableModels { const models = builtinModels(options); const openai = models.getProvider("openai"); if (openai) models.setProvider(withOpenAICuaResponses(openai)); + const anthropic = models.getProvider("anthropic"); + if (anthropic) models.setProvider(withAnthropicNativeTools(anthropic)); const google = models.getProvider("google"); if (google) models.setProvider(withGoogleEnvKeys(google)); models.setProvider(tzafonProvider()); @@ -74,6 +81,27 @@ function withOpenAICuaResponses(base: Provider): Provider { }; } +// Native-tool runs route Anthropic models to a CUA-owned api id (see +// resolveCuaRuntimeSpec) so the required `anthropic-beta` header can be +// injected here; the request otherwise flows through pi's builtin +// anthropic-messages transport. +function withAnthropicNativeTools(base: Provider): Provider { + const toBuiltin = (model: Model): Model => ({ ...model, api: "anthropic-messages" as Model["api"] }); + return { + ...base, + stream: (model: Model, context: Context, options?: StreamOptions) => { + const beta = ANTHROPIC_NATIVE_API_BETA_HEADERS[model.api]; + return beta ? base.stream(toBuiltin(model), context, withAnthropicBetaHeader(options, beta)) : base.stream(model, context, options); + }, + streamSimple: (model: Model, context: Context, options?: SimpleStreamOptions) => { + const beta = ANTHROPIC_NATIVE_API_BETA_HEADERS[model.api]; + return beta + ? base.streamSimple(toBuiltin(model), context, withAnthropicBetaHeader(options, beta)) + : base.streamSimple(model, context, options); + }, + }; +} + function withGoogleEnvKeys(base: Provider): Provider { return { ...base, auth: { ...base.auth, apiKey: envApiKeyAuth("Google API key", cuaApiKeyEnvVarsForProvider("google")) } }; } diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts index af0a1fc4..38b18b4c 100644 --- a/packages/ai/src/providers/anthropic/actions.ts +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -2,14 +2,18 @@ import type { Tool, TSchema } from "@earendil-works/pi-ai"; import { CUA_BATCH_TOOL_DESCRIPTION, CUA_BATCH_TOOL_NAME, + CUA_DOM_ACTION_TYPES, createCuaActionSchema, createCuaActionToolExecutors, createCuaActionToolDefinitions, createCuaBatchToolExecutor, createCuaBatchToolDefinition, + defaultActionsForMode, + isCuaDomActionType, type ComputerToolsOptions, type CuaAction, type CuaActionType, + type CuaMode, type CuaToolExecutorSpec, } from "../common"; @@ -34,13 +38,14 @@ export const ANTHROPIC_CUA_ACTION_TYPES = [ "drag", "wait", "screenshot", + "zoom", "goto", "cursor_position", ] as const satisfies readonly CuaActionType[]; -type AnthropicCanonicalActionType = (typeof ANTHROPIC_CUA_ACTION_TYPES)[number]; +type AnthropicCanonicalActionType = (typeof ANTHROPIC_CUA_ACTION_TYPES)[number] | (typeof CUA_DOM_ACTION_TYPES)[number]; -const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set(ANTHROPIC_CUA_ACTION_TYPES); +const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set([...ANTHROPIC_CUA_ACTION_TYPES, ...CUA_DOM_ACTION_TYPES]); /** Name of the batch tool included by default in Anthropic computer-use tools. */ export const ANTHROPIC_BATCH_TOOL_NAME = CUA_BATCH_TOOL_NAME; @@ -59,8 +64,15 @@ export interface AnthropicComputerToolsOptions extends ComputerToolsOptions { /** Canonical CUA action shape supported by Anthropic browser computer-use tools. */ export type AnthropicAction = Extract; -function resolveAnthropicActions(actions: readonly CuaActionType[] | undefined): readonly AnthropicCanonicalActionType[] { - const resolved = actions ?? ANTHROPIC_CUA_ACTION_TYPES; +function resolveAnthropicActions(options: AnthropicComputerToolsOptions): readonly AnthropicCanonicalActionType[] { + const mode = options.mode ?? "os"; + const resolved = + options.actions ?? + (mode === "os" + ? ANTHROPIC_CUA_ACTION_TYPES.filter((action) => action !== "zoom") + : defaultActionsForMode(mode, { javascriptExec: options.javascriptExec }).filter( + (action) => isCuaDomActionType(action) || isAnthropicCanonicalAction(action), + )); const supported: AnthropicCanonicalActionType[] = []; const unsupported: CuaActionType[] = []; for (const action of resolved) { @@ -76,8 +88,8 @@ function isAnthropicCanonicalAction(action: CuaActionType): action is AnthropicC } /** Build the TypeBox schema for Anthropic-supported canonical browser actions. */ -export function createActionSchema(actions?: readonly CuaActionType[]): TSchema { - return createCuaActionSchema(resolveAnthropicActions(actions)); +export function createActionSchema(actions?: readonly CuaActionType[], mode: CuaMode = "os"): TSchema { + return createCuaActionSchema(resolveAnthropicActions({ actions, mode }), mode); } /** @@ -89,12 +101,14 @@ export function createActionSchema(actions?: readonly CuaActionType[]): TSchema * batch tool by default; pass `excludeBatch: true` to omit it. */ export function computerTools(options: AnthropicComputerToolsOptions = {}): Tool[] { - const actions = resolveAnthropicActions(options.actions); - const tools = createCuaActionToolDefinitions(actions); + const mode = options.mode ?? "os"; + const actions = resolveAnthropicActions(options); + const tools = createCuaActionToolDefinitions(actions, mode); if (!options.excludeBatch) { tools.push(createCuaBatchToolDefinition(actions, { name: ANTHROPIC_BATCH_TOOL_NAME, description: ANTHROPIC_BATCH_TOOL_DESCRIPTION, + mode, })); } return tools; @@ -102,12 +116,14 @@ export function computerTools(options: AnthropicComputerToolsOptions = {}): Tool /** Build the local execution adapters used by CuaAgent and CuaAgentHarness. */ export function computerToolExecutors(options: AnthropicComputerToolsOptions = {}): CuaToolExecutorSpec[] { - const actions = resolveAnthropicActions(options.actions); - const executors = createCuaActionToolExecutors(actions); + const mode = options.mode ?? "os"; + const actions = resolveAnthropicActions(options); + const executors = createCuaActionToolExecutors(actions, mode); if (!options.excludeBatch) { executors.push(createCuaBatchToolExecutor(actions, { name: ANTHROPIC_BATCH_TOOL_NAME, description: ANTHROPIC_BATCH_TOOL_DESCRIPTION, + mode, })); } return executors; diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index f51b1ae8..98b3dcd1 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -1,5 +1,5 @@ import type { Api, Model } from "@earendil-works/pi-ai"; -import type { ComputerToolCoordinateSystem, CuaPayloadHook, CuaProviderModule } from "../common"; +import type { ComputerToolCoordinateSystem, CuaMode, CuaPayloadHook, CuaProviderModule } from "../common"; import { computerToolExecutors, computerTools } from "./actions"; export { @@ -14,6 +14,17 @@ export type { AnthropicComputerToolsOptions, AnthropicComputerToolsOptions as ComputerToolsOptions, } from "./actions"; +export { + ANTHROPIC_NATIVE_API_BETA_HEADERS, + ANTHROPIC_NATIVE_BROWSER_MESSAGES_API, + ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API, + createNativeToolOnPayload, + mapNativeBrowserInput, + mapNativeComputerInput, + nativeApiForToolType, + nativeToolExecutors, + withAnthropicBetaHeader, +} from "./native"; // Anthropic's quickstart uses pixel coordinates for both its computer and // browser tools. @@ -24,8 +35,18 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use keyboard navigation where possible, and request screenshots when you need to inspect state.`; -export function buildAnthropicSystemPrompt(opts: { suffix?: string } = {}): string { - return [ANTHROPIC_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); +export const ANTHROPIC_DOM_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; + +export const ANTHROPIC_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and page_* tools read and act on the page itself by element reference. Prefer page_snapshot/page_find for reading and locating, and computer_* input for interaction; use page_* interaction for elements that are hard to hit by coordinate.`; + +export function buildAnthropicSystemPrompt(opts: { suffix?: string; mode?: CuaMode } = {}): string { + const base = + opts.mode === "dom" + ? ANTHROPIC_DOM_INSTRUCTIONS + : opts.mode === "hybrid" + ? ANTHROPIC_HYBRID_INSTRUCTIONS + : ANTHROPIC_COMPUTER_INSTRUCTIONS; + return [base, opts.suffix].filter(Boolean).join("\n\n"); } export const anthropicAdaptiveThinkingOnPayload: CuaPayloadHook = (payload, model) => { diff --git a/packages/ai/src/providers/anthropic/native.ts b/packages/ai/src/providers/anthropic/native.ts new file mode 100644 index 00000000..ba4e27ff --- /dev/null +++ b/packages/ai/src/providers/anthropic/native.ts @@ -0,0 +1,310 @@ +import { Type, type Api, type Model, type StreamOptions, type Tool } from "@earendil-works/pi-ai"; +import type { CuaAction, CuaMouseButton } from "../../actions/index"; +import type { ResolvedCuaNativeTool } from "../../native-tools"; +import type { CuaPayloadContext, CuaPayloadHook, CuaToolExecutorSpec } from "../common"; + +/** + * pi-ai api ids CUA routes Anthropic models to when a native tool is + * configured. The registered provider dispatches these to pi's builtin + * `anthropic-messages` transport with the tool's `anthropic-beta` header + * merged in (see `createCuaModels`). + */ +export const ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API = "anthropic-cua-native-computer-messages"; +export const ANTHROPIC_NATIVE_BROWSER_MESSAGES_API = "anthropic-cua-native-browser-messages"; + +export const ANTHROPIC_NATIVE_API_BETA_HEADERS: Record = { + [ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API]: "computer-use-2026-06-01", + [ANTHROPIC_NATIVE_BROWSER_MESSAGES_API]: "browser-use-2026-07-01", +}; + +export function nativeApiForToolType(type: ResolvedCuaNativeTool["spec"]["type"]): string { + return type === "computer_20260601" ? ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API : ANTHROPIC_NATIVE_BROWSER_MESSAGES_API; +} + +/** Merge a native tool's `anthropic-beta` header into stream options. */ +export function withAnthropicBetaHeader(options: T | undefined, beta: string): T { + const headers = { ...(options?.headers ?? {}) }; + headers["anthropic-beta"] = headers["anthropic-beta"] ? `${headers["anthropic-beta"]},${beta}` : beta; + return { ...(options ?? {}), headers } as T; +} + +// The native tool's input schema is Anthropic-defined and validated +// server-side; the local placeholder schema stays permissive and the +// executor validates during mapping. +const NativeActionSchema = Type.Object({ action: Type.String() }, { additionalProperties: true }); + +/** + * Build the single execution adapter for a native Anthropic tool: tool calls + * arrive under the native tool's name with an `action`-discriminated input, + * and map onto the same canonical actions the tool's mode uses. + */ +export function nativeToolExecutors(resolved: ResolvedCuaNativeTool): CuaToolExecutorSpec[] { + const definition: Tool = { + name: resolved.name, + description: `Anthropic native ${resolved.spec.type} tool.`, + parameters: NativeActionSchema, + }; + const toActions = + resolved.spec.type === "computer_20260601" + ? (args: unknown) => mapNativeComputerInput(asNativeInput(args)) + : (args: unknown) => mapNativeBrowserInput(asNativeInput(args)); + return [{ definition, toActions }]; +} + +/** + * Payload hook for native tool requests: replaces the local placeholder + * function tool (matched by name) with the Anthropic-defined declaration. + * Other tools in the payload (e.g. `playwright_execute`, caller extras) are + * left in place. + */ +export function createNativeToolOnPayload(resolved: ResolvedCuaNativeTool): CuaPayloadHook { + return (payload: unknown, _model: Model, _context?: CuaPayloadContext) => { + if (!payload || typeof payload !== "object") return undefined; + const current = payload as { tools?: unknown }; + if (!Array.isArray(current.tools)) return undefined; + const tools = current.tools.map((tool) => + tool && typeof tool === "object" && (tool as { name?: unknown }).name === resolved.name ? resolved.declaration : tool, + ); + return { ...(payload as Record), tools }; + }; +} + +interface NativeInput { + action: string; + [key: string]: unknown; +} + +function asNativeInput(args: unknown): NativeInput { + if (args && typeof args === "object" && typeof (args as { action?: unknown }).action === "string") { + return args as NativeInput; + } + throw new Error("invalid native tool parameters: expected an object with an \"action\" field"); +} + +const MAX_KEY_REPEAT = 100; + +/** Map one `computer_20260601` tool input onto canonical OS-plane actions. */ +export function mapNativeComputerInput(input: NativeInput): CuaAction[] { + switch (input.action) { + case "screenshot": + return [{ type: "screenshot" }]; + case "left_click": + return [click(input, "left")]; + case "right_click": + return [click(input, "right")]; + case "middle_click": + return [click(input, "middle")]; + case "double_click": + return [{ ...click(input, "left"), num_clicks: 2 }]; + case "triple_click": + return [{ ...click(input, "left"), num_clicks: 3 }]; + case "left_click_drag": { + const start = coordinate(input.start_coordinate, "start_coordinate"); + const end = coordinate(input.coordinate, "coordinate"); + return [{ type: "drag", path: [start, end], ...holdKeys(input.text) }]; + } + case "mouse_move": + return [{ type: "move", ...coordinate(input.coordinate, "coordinate") }]; + case "left_mouse_down": + return [{ type: "mouse_down" }]; + case "left_mouse_up": + return [{ type: "mouse_up" }]; + case "scroll": { + const point = input.coordinate === undefined ? {} : coordinate(input.coordinate, "coordinate"); + return [{ type: "scroll", ...point, ...scrollDeltas(input.scroll_direction, input.scroll_amount), ...holdKeys(input.text) }]; + } + case "type": + return [{ type: "type", text: text(input) }]; + case "key": { + const repeat = clampRepeat(input.repeat); + return Array.from({ length: repeat }, () => ({ type: "keypress" as const, keys: [text(input)] })); + } + case "hold_key": + return [{ type: "keypress", keys: [text(input)], duration: durationSeconds(input) }]; + case "wait": + return [{ type: "wait", ms: durationSeconds(input) * 1000 }]; + case "cursor_position": + return [{ type: "cursor_position" }]; + case "zoom": + return [{ type: "zoom", region: region(input.region) }]; + default: + throw new Error(`unsupported computer_20260601 action "${input.action}"`); + } +} + +/** Map one `browser_20260701` tool input onto canonical DOM-plane actions. */ +export function mapNativeBrowserInput(input: NativeInput): CuaAction[] { + const tab = tabId(input); + switch (input.action) { + case "navigate": + return [{ type: "page_navigate", url: requireString(input.url, "url"), ...tab }]; + case "list_tabs": + return [{ type: "page_list_tabs" }]; + case "new_tab": + return [{ type: "page_new_tab" }]; + case "read_page": + return [ + { + type: "page_snapshot", + ...(input.filter === "interactive" || input.filter === "all" ? { filter: input.filter } : {}), + ...(typeof input.depth === "number" ? { depth: input.depth } : {}), + ...(typeof input.ref === "string" ? { ref: input.ref } : {}), + ...tab, + }, + ]; + case "get_page_text": + return [{ type: "page_text", ...tab }]; + case "find": + return [{ type: "page_find", query: requireString(input.query, "query"), ...tab }]; + case "form_input": + return [{ type: "page_fill", ref: refTarget(input.target), value: fillValue(input.value), ...tab }]; + case "scroll_to": + return [{ type: "page_scroll_to", ref: refTarget(input.target), ...tab }]; + case "screenshot": + return [{ type: "page_screenshot", ...tab }]; + case "zoom": + return [{ type: "page_screenshot", region: region(input.region), ...tab }]; + case "left_click": + return [{ type: "page_click", ...pageTarget(input.target), ...modifiers(input.modifiers), ...tab }]; + case "right_click": + return [{ type: "page_click", ...pageTarget(input.target), button: "right", ...modifiers(input.modifiers), ...tab }]; + case "double_click": + return [{ type: "page_click", ...pageTarget(input.target), num_clicks: 2, ...modifiers(input.modifiers), ...tab }]; + case "triple_click": + return [{ type: "page_click", ...pageTarget(input.target), num_clicks: 3, ...modifiers(input.modifiers), ...tab }]; + case "hover": + return [{ type: "page_hover", ...pageTarget(input.target), ...tab }]; + case "left_click_drag": + return [{ type: "page_drag", from: coordinateTarget(input.from, "from"), to: coordinateTarget(input.target, "target"), ...tab }]; + case "scroll": + return [ + { + type: "page_scroll", + ...coordinateTarget(input.target, "target"), + direction: scrollDirection(input.scroll_direction), + ...(typeof input.scroll_amount === "number" ? { amount: input.scroll_amount } : {}), + ...tab, + }, + ]; + case "type": + return [{ type: "page_type", text: text(input), ...tab }]; + case "key": { + const repeat = clampRepeat(input.repeat); + return Array.from({ length: repeat }, () => ({ type: "page_key" as const, text: text(input), ...tab })); + } + case "wait": + return [{ type: "wait", ms: durationSeconds(input) * 1000 }]; + case "javascript_exec": + return [{ type: "page_evaluate", code: text(input), ...tab }]; + default: + throw new Error(`unsupported browser_20260701 action "${input.action}"`); + } +} + +function click(input: NativeInput, button: CuaMouseButton): CuaAction & { type: "click" } { + const point = input.coordinate === undefined ? {} : coordinate(input.coordinate, "coordinate"); + return { type: "click", ...point, button, ...holdKeys(input.text) }; +} + +function coordinate(value: unknown, field: string): { x: number; y: number } { + if (Array.isArray(value) && value.length === 2 && typeof value[0] === "number" && typeof value[1] === "number") { + return { x: value[0], y: value[1] }; + } + throw new Error(`invalid ${field}: expected [x, y]`); +} + +function region(value: unknown): [number, number, number, number] { + if (Array.isArray(value) && value.length === 4 && value.every((entry) => typeof entry === "number")) { + return value as [number, number, number, number]; + } + throw new Error("invalid region: expected [x0, y0, x1, y1]"); +} + +function holdKeys(value: unknown): { hold_keys?: string[] } { + return typeof value === "string" && value.trim() ? { hold_keys: value.split("+").map((key) => key.trim()) } : {}; +} + +function modifiers(value: unknown): { modifiers?: string[] } { + return typeof value === "string" && value.trim() ? { modifiers: value.split("+").map((key) => key.trim()) } : {}; +} + +function scrollDeltas(direction: unknown, amount: unknown): { scroll_x?: number; scroll_y?: number } { + const notches = typeof amount === "number" && Number.isFinite(amount) ? amount : 3; + const delta = Math.trunc(notches) * 120; + switch (direction) { + case "up": + return { scroll_y: -delta }; + case "down": + return { scroll_y: delta }; + case "left": + return { scroll_x: -delta }; + case "right": + return { scroll_x: delta }; + default: + throw new Error(`invalid scroll_direction "${String(direction)}"`); + } +} + +function scrollDirection(value: unknown): "up" | "down" | "left" | "right" { + if (value === "up" || value === "down" || value === "left" || value === "right") return value; + throw new Error(`invalid scroll_direction "${String(value)}"`); +} + +function text(input: NativeInput): string { + return requireString(input.text, "text"); +} + +function requireString(value: unknown, field: string): string { + if (typeof value !== "string") throw new Error(`invalid ${field}: expected a string`); + return value; +} + +function durationSeconds(input: NativeInput): number { + const value = input.duration; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error("invalid duration: expected a non-negative number"); + return Math.min(value, 100); +} + +function clampRepeat(value: unknown): number { + if (value === undefined) return 1; + if (typeof value !== "number" || !Number.isInteger(value) || value < 1) throw new Error("invalid repeat: expected an integer ≥ 1"); + return Math.min(value, MAX_KEY_REPEAT); +} + +interface RefOrCoordinateTarget { + ref?: string; + x?: number; + y?: number; +} + +function pageTarget(value: unknown): RefOrCoordinateTarget { + const target = value as { type?: unknown; ref?: unknown; x?: unknown; y?: unknown } | undefined; + if (target && typeof target === "object") { + if (target.type === "ref" && typeof target.ref === "string") return { ref: target.ref }; + if (target.type === "coordinate" && typeof target.x === "number" && typeof target.y === "number") { + return { x: target.x, y: target.y }; + } + } + throw new Error("invalid target: expected {type: \"ref\", ref} or {type: \"coordinate\", x, y}"); +} + +function refTarget(value: unknown): string { + const target = pageTarget(value); + if (target.ref === undefined) throw new Error("invalid target: this action requires a ref target"); + return target.ref; +} + +function coordinateTarget(value: unknown, field: string): { x: number; y: number } { + const target = pageTarget(value); + if (target.x === undefined || target.y === undefined) throw new Error(`invalid ${field}: expected a coordinate target`); + return { x: target.x, y: target.y }; +} + +function fillValue(value: unknown): string | number | boolean { + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value; + throw new Error("invalid value: expected string, number, or boolean"); +} + +function tabId(input: NativeInput): { tab_id?: string } { + return typeof input.tab_id === "string" ? { tab_id: input.tab_id } : {}; +} diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 31723d0f..59a2faf3 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -9,291 +9,50 @@ import { type TSchema, type Tool, } from "@earendil-works/pi-ai"; +import { CUA_DEFAULT_OS_ACTION_TYPES, cuaActionSchemaByType, type CuaAction, type CuaActionType } from "../actions/index"; +import { cuaToolDescriptionForAction, cuaToolNameForAction, defaultActionsForMode, schemaOptionsForMode, type CuaMode } from "../modes"; +import type { ResolvedCuaNativeTool } from "../native-tools"; import type { CuaModelRef, CuaProvider } from "../models"; -export const CUA_ACTION_TYPES = [ - "click", - "double_click", - "mouse_down", - "mouse_up", - "type", - "keypress", - "scroll", - "move", - "drag", - "wait", - "screenshot", - "goto", - "back", - "forward", - "url", - "cursor_position", -] as const; - -export type CuaActionType = (typeof CUA_ACTION_TYPES)[number]; +export * from "../actions/index"; +export * from "../modes"; +export * from "../native-tools"; /** - * Mouse buttons accepted by click, mouse_down, and mouse_up actions. The - * executor coerces anything outside this set to "left". + * The default os-mode action set: every OS-plane action except `zoom`. + * The full canonical vocabulary is split by plane into + * {@link CUA_OS_ACTION_TYPES} and {@link CUA_DOM_ACTION_TYPES}. */ -export type CuaMouseButton = "left" | "right" | "middle" | "back" | "forward"; - -/** - * Mouse buttons accepted by drag actions. The executor coerces anything - * outside this set to "left". - */ -export type CuaDragMouseButton = "left" | "right" | "middle"; - -export interface CuaActionClick { - type: "click"; - x: number; - y: number; - button?: CuaMouseButton; - hold_keys?: string[]; -} - -export interface CuaActionDoubleClick { - type: "double_click"; - x: number; - y: number; - hold_keys?: string[]; -} - -export interface CuaActionMouseDown { - type: "mouse_down"; - x: number; - y: number; - button?: CuaMouseButton; - hold_keys?: string[]; -} - -export interface CuaActionMouseUp { - type: "mouse_up"; - x: number; - y: number; - button?: CuaMouseButton; - hold_keys?: string[]; -} - -export interface CuaActionTypeText { - type: "type"; - text: string; -} - -export interface CuaActionKeypress { - type: "keypress"; - keys: string[]; - duration?: number; -} - -export interface CuaActionScroll { - type: "scroll"; - x?: number; - y?: number; - scroll_x?: number; - scroll_y?: number; - hold_keys?: string[]; -} - -export interface CuaActionMove { - type: "move"; - x: number; - y: number; -} - -export interface CuaActionDrag { - type: "drag"; - path: Array<{ x: number; y: number }>; - button?: CuaDragMouseButton; - hold_keys?: string[]; -} - -export interface CuaActionWait { - type: "wait"; - ms?: number; -} - -export interface CuaActionScreenshot { - type: "screenshot"; -} - -export interface CuaActionGoto { - type: "goto"; - url: string; -} - -export interface CuaActionBack { - type: "back"; -} - -export interface CuaActionForward { - type: "forward"; -} - -export interface CuaActionUrl { - type: "url"; -} - -export interface CuaActionCursorPosition { - type: "cursor_position"; -} - -export type CuaAction = - | CuaActionClick - | CuaActionDoubleClick - | CuaActionMouseDown - | CuaActionMouseUp - | CuaActionTypeText - | CuaActionKeypress - | CuaActionScroll - | CuaActionMove - | CuaActionDrag - | CuaActionWait - | CuaActionScreenshot - | CuaActionGoto - | CuaActionBack - | CuaActionForward - | CuaActionUrl - | CuaActionCursorPosition; - -const PointSchema = Type.Object( - { - x: Type.Number(), - y: Type.Number(), - }, - { additionalProperties: false }, -); - -const CUA_ACTION_SCHEMA_BY_TYPE = { - click: Type.Object( - { - type: Type.Literal("click"), - x: Type.Number(), - y: Type.Number(), - button: Type.Optional(Type.String()), - hold_keys: Type.Optional(Type.Array(Type.String())), - }, - { additionalProperties: false }, - ), - double_click: Type.Object( - { - type: Type.Literal("double_click"), - x: Type.Number(), - y: Type.Number(), - hold_keys: Type.Optional(Type.Array(Type.String())), - }, - { additionalProperties: false }, - ), - mouse_down: Type.Object( - { - type: Type.Literal("mouse_down"), - x: Type.Number(), - y: Type.Number(), - button: Type.Optional(Type.String()), - hold_keys: Type.Optional(Type.Array(Type.String())), - }, - { additionalProperties: false }, - ), - mouse_up: Type.Object( - { - type: Type.Literal("mouse_up"), - x: Type.Number(), - y: Type.Number(), - button: Type.Optional(Type.String()), - hold_keys: Type.Optional(Type.Array(Type.String())), - }, - { additionalProperties: false }, - ), - type: Type.Object( - { - type: Type.Literal("type"), - text: Type.String(), - }, - { additionalProperties: false }, - ), - keypress: Type.Object( - { - type: Type.Literal("keypress"), - keys: Type.Array(Type.String()), - duration: Type.Optional(Type.Number()), - }, - { additionalProperties: false }, - ), - scroll: Type.Object( - { - type: Type.Literal("scroll"), - x: Type.Optional(Type.Number()), - y: Type.Optional(Type.Number()), - scroll_x: Type.Optional(Type.Number()), - scroll_y: Type.Optional(Type.Number()), - hold_keys: Type.Optional(Type.Array(Type.String())), - }, - { additionalProperties: false }, - ), - move: Type.Object( - { - type: Type.Literal("move"), - x: Type.Number(), - y: Type.Number(), - }, - { additionalProperties: false }, - ), - drag: Type.Object( - { - type: Type.Literal("drag"), - path: Type.Array(PointSchema, { minItems: 2 }), - button: Type.Optional(Type.String()), - hold_keys: Type.Optional(Type.Array(Type.String())), - }, - { additionalProperties: false }, - ), - wait: Type.Object( - { - type: Type.Literal("wait"), - ms: Type.Optional(Type.Number()), - }, - { additionalProperties: false }, - ), - screenshot: Type.Object({ type: Type.Literal("screenshot") }, { additionalProperties: false }), - goto: Type.Object( - { - type: Type.Literal("goto"), - url: Type.String(), - }, - { additionalProperties: false }, - ), - back: Type.Object({ type: Type.Literal("back") }, { additionalProperties: false }), - forward: Type.Object({ type: Type.Literal("forward") }, { additionalProperties: false }), - url: Type.Object({ type: Type.Literal("url") }, { additionalProperties: false }), - cursor_position: Type.Object({ type: Type.Literal("cursor_position") }, { additionalProperties: false }), -} satisfies Record; +export const CUA_ACTION_TYPES = CUA_DEFAULT_OS_ACTION_TYPES; type ObjectSchemaWithProperties = TSchema & { properties: Record }; -function createCuaActionArgumentSchema(action: CuaActionType): TSchema { - const { type: _type, ...properties } = (CUA_ACTION_SCHEMA_BY_TYPE[action] as ObjectSchemaWithProperties).properties; +function createCuaActionArgumentSchema(action: CuaActionType, mode: CuaMode): TSchema { + const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); + const { type: _type, ...properties } = (schemaByType[action] as ObjectSchemaWithProperties).properties; return Type.Object(properties, { additionalProperties: false }); } -export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): TSchema { +export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "os"): TSchema { if (actions.length === 0) throw new Error("actions must include at least one CUA action type"); - if (actions.length === 1) return CUA_ACTION_SCHEMA_BY_TYPE[actions[0]!]; - return Type.Union(actions.map((action) => CUA_ACTION_SCHEMA_BY_TYPE[action])); + const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); + if (actions.length === 1) return schemaByType[actions[0]!]; + return Type.Union(actions.map((action) => schemaByType[action])); } -export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): Tool[] { +export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "os"): Tool[] { return actions.map((action) => ({ - name: action, - description: `Execute one ${action} computer action.`, - parameters: createCuaActionArgumentSchema(action), + name: cuaToolNameForAction(action, mode), + description: cuaToolDescriptionForAction(action, mode), + parameters: createCuaActionArgumentSchema(action, mode), })); } export const CuaActionSchema = createCuaActionSchema(); -export function createCuaBatchSchema(actions?: readonly CuaActionType[]): TSchema { +export function createCuaBatchSchema(actions?: readonly CuaActionType[], mode: CuaMode = "os"): TSchema { return Type.Object({ - actions: Type.Array(createCuaActionSchema(actions), { description: "Ordered computer actions to execute." }), + actions: Type.Array(createCuaActionSchema(actions, mode), { description: "Ordered computer actions to execute." }), }); } @@ -358,6 +117,10 @@ export const CUA_PLAYWRIGHT_TOOL_DESCRIPTION = [ export interface ComputerToolsOptions { actions?: readonly CuaActionType[]; + /** Which action plane(s) to expose. Default "os". */ + mode?: CuaMode; + /** Expose `page_evaluate` in dom/hybrid modes. Default false. */ + javascriptExec?: boolean; } export type ComputerToolCoordinateSystem = @@ -377,13 +140,25 @@ export type ComputerToolCoordinateSystem = * smaller set, such as `["click"]`. */ export function computerTools(options: ComputerToolsOptions = {}): Tool[] { - return createCuaActionToolDefinitions(options.actions); + return createCuaActionToolDefinitions(resolveModeActions(options), options.mode ?? "os"); +} + +/** Resolve the action list for a tools-options object: explicit list, or the mode's default set. */ +export function resolveModeActions(options: ComputerToolsOptions = {}): readonly CuaActionType[] { + return options.actions ?? defaultActionsForMode(options.mode ?? "os", { javascriptExec: options.javascriptExec }); +} + +/** Guard for providers whose computer-use vocabulary only covers the OS plane. */ +export function assertOsModeOnly(provider: CuaProvider, options: ComputerToolsOptions = {}): void { + const mode = options.mode ?? "os"; + if (mode !== "os") throw new Error(`provider "${provider}" does not support mode "${mode}" (os only)`); } /** Build execution adapters for individual canonical CUA action tools. */ -export function createCuaActionToolExecutors(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): CuaToolExecutorSpec[] { - return createCuaActionToolDefinitions(actions).map((definition) => { - const actionType = definition.name as CuaActionType; +export function createCuaActionToolExecutors(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "os"): CuaToolExecutorSpec[] { + const definitions = createCuaActionToolDefinitions(actions, mode); + return definitions.map((definition, index) => { + const actionType = actions[index]!; return { definition, toActions(args: unknown): CuaAction[] { @@ -414,19 +189,19 @@ export function normalizeGotoUrl(value: unknown): string | undefined { export function createCuaBatchToolDefinition( actions?: readonly CuaActionType[], - options: { name?: string; description?: string } = {}, + options: { name?: string; description?: string; mode?: CuaMode } = {}, ): Tool { return { name: options.name ?? CUA_BATCH_TOOL_NAME, description: options.description ?? CUA_BATCH_TOOL_DESCRIPTION, - parameters: createCuaBatchSchema(actions), + parameters: createCuaBatchSchema(actions, options.mode ?? "os"), }; } /** Build an execution adapter for a batch tool whose input is `{ actions }`. */ export function createCuaBatchToolExecutor( actions?: readonly CuaActionType[], - options: { name?: string; description?: string } = {}, + options: { name?: string; description?: string; mode?: CuaMode } = {}, ): CuaToolExecutorSpec { const definition = createCuaBatchToolDefinition(actions, options); return { @@ -440,7 +215,7 @@ export function createCuaBatchToolExecutor( /** Build the provider's default CUA tool execution adapters. */ export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaToolExecutorSpec[] { - return createCuaActionToolExecutors(options.actions); + return createCuaActionToolExecutors(resolveModeActions(options), options.mode ?? "os"); } function isBatchInput(value: unknown): value is CuaBatchInput { @@ -556,6 +331,10 @@ export function responseThreadingDelta(messages: readonly Message[]): ResponseTh export interface CuaRuntimeSpec { model: Model; provider: CuaProvider; + /** Which canonical action plane(s) this runtime exposes. */ + mode: CuaMode; + /** Present when the model is driven through a provider-native tool declaration. */ + nativeTool?: ResolvedCuaNativeTool; /** Provider-facing CUA tool definitions used for model requests. */ toolDefinitions: Tool[]; /** Local execution adapters that turn provider tool calls into canonical CUA actions. */ @@ -581,7 +360,7 @@ export interface CuaProviderModule { /** Coordinate convention emitted by this provider's tool calls. */ coordinateSystem(): ComputerToolCoordinateSystem; /** Provider-tuned baseline browser-control system prompt. */ - buildSystemPrompt(opts?: { suffix?: string }): string; + buildSystemPrompt(opts?: { suffix?: string; mode?: CuaMode }): string; /** Optional request-payload middleware for provider protocol quirks. */ onPayload?: CuaPayloadHook; /** Optional provider screenshot input policy. */ diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index 1a337c7e..e1eb49ae 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -1,5 +1,5 @@ -import { computerToolExecutors, computerTools } from "../common"; -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { assertOsModeOnly, computerToolExecutors, computerTools } from "../common"; +import type { ComputerToolCoordinateSystem, ComputerToolsOptions, CuaProviderModule } from "../common"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, @@ -30,8 +30,17 @@ export function buildGeminiSystemPrompt(opts: { suffix?: string } = {}): string } export const providerModule = { - toolDefinitions: computerTools, - toolExecutors: computerToolExecutors, + // Gemini's computer-use coordinate convention is normalized 0-999, which + // only maps onto the OS plane today; DOM-plane viewport coordinates are + // unvalidated for it. + toolDefinitions: (options?: ComputerToolsOptions) => { + assertOsModeOnly("google", options); + return computerTools(options); + }, + toolExecutors: (options?: ComputerToolsOptions) => { + assertOsModeOnly("google", options); + return computerToolExecutors(options); + }, coordinateSystem, buildSystemPrompt: buildGeminiSystemPrompt, } satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 21c293a8..a8c6df24 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import type { ComputerToolCoordinateSystem, CuaMode, CuaProviderModule } from "../common"; import { computerToolExecutors, computerTools } from "../common"; export { @@ -31,8 +31,14 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { export const OPENAI_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use the available tools for browser interaction and request explicit url, cursor_position, or screenshot reads when you need updated state.`; -export function buildOpenAISystemPrompt(opts: { suffix?: string } = {}): string { - return [OPENAI_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); +export const OPENAI_DOM_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; + +export const OPENAI_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and page_* tools read and act on the page itself by element reference. Prefer page_snapshot/page_find for reading and locating, and computer_* input for interaction; use page_* interaction for elements that are hard to hit by coordinate.`; + +export function buildOpenAISystemPrompt(opts: { suffix?: string; mode?: CuaMode } = {}): string { + const base = + opts.mode === "dom" ? OPENAI_DOM_INSTRUCTIONS : opts.mode === "hybrid" ? OPENAI_HYBRID_INSTRUCTIONS : OPENAI_COMPUTER_INSTRUCTIONS; + return [base, opts.suffix].filter(Boolean).join("\n\n"); } export const providerModule = { diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 4a5e029e..2dc5ae03 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -1,4 +1,11 @@ -import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, type CuaProviderModule } from "../common"; +import { + assertOsModeOnly, + computerToolExecutors, + computerTools, + type ComputerToolCoordinateSystem, + type ComputerToolsOptions, + type CuaProviderModule, +} from "../common"; import { tzafonComputerUseOnPayload } from "./provider"; export { @@ -44,8 +51,14 @@ export function buildTzafonSystemPrompt(opts: { suffix?: string } = {}): string } export const providerModule = { - toolDefinitions: computerTools, - toolExecutors: computerToolExecutors, + toolDefinitions: (options?: ComputerToolsOptions) => { + assertOsModeOnly("tzafon", options); + return computerTools(options); + }, + toolExecutors: (options?: ComputerToolsOptions) => { + assertOsModeOnly("tzafon", options); + return computerToolExecutors(options); + }, coordinateSystem, buildSystemPrompt: buildTzafonSystemPrompt, onPayload: tzafonComputerUseOnPayload, diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index da05b4c3..5f2244b6 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { assertOsModeOnly, type ComputerToolCoordinateSystem, type ComputerToolsOptions, type CuaProviderModule } from "../common"; import { computerToolExecutors } from "./actions"; import { yutoriCuaOnPayload } from "./provider"; @@ -63,8 +63,14 @@ export function buildYutoriSystemPrompt(opts: { suffix?: string } = {}): string } export const providerModule = { - toolDefinitions: () => [], - toolExecutors: computerToolExecutors, + toolDefinitions: (options?: ComputerToolsOptions) => { + assertOsModeOnly("yutori", options); + return []; + }, + toolExecutors: (options?: ComputerToolsOptions) => { + assertOsModeOnly("yutori", options); + return computerToolExecutors(options); + }, coordinateSystem, buildSystemPrompt: buildYutoriSystemPrompt, onPayload: yutoriCuaOnPayload, diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 178de193..37367b4a 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -1,12 +1,16 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; import type { CuaProvider } from "./models"; import { getCuaModel, providerForModel, routeCuaApi } from "./models"; +import { modeForNativeTool, resolveNativeTool, type CuaNativeToolSpec } from "./native-tools"; import { providerModule as anthropic } from "./providers/anthropic/index"; +import { createNativeToolOnPayload, nativeApiForToolType, nativeToolExecutors } from "./providers/anthropic/native"; import { providerModule as gemini } from "./providers/gemini/index"; import { providerModule as openai } from "./providers/openai/index"; import { providerModule as tzafon } from "./providers/tzafon/index"; import { providerModule as yutori } from "./providers/yutori/index"; import type { ComputerToolsOptions, + CuaPayloadHook, CuaProviderModule, CuaRuntimeSpec, CuaRuntimeSpecInput, @@ -20,26 +24,68 @@ const PROVIDERS = { yutori, } satisfies Record; +/** Options accepted by {@link resolveCuaRuntimeSpec}. */ +export interface CuaRuntimeSpecOptions extends ComputerToolsOptions { + /** + * Drive the model through a provider-native tool declaration instead of + * CUA's canonical function tools. The native tool determines (and is + * validated against) the mode: `computer_20260601` requires `"os"`, + * `browser_20260701` requires `"dom"`. When `mode` is omitted it is + * inferred from the native tool. + */ + nativeTool?: CuaNativeToolSpec; +} + /** * Resolve provider defaults from either a CUA model ref or a concrete model. * * Use the returned spec to build computer-use requests without hard-coding - * model-provider rules in your application. Pass `options` (e.g. - * `{ actions: ["click"] }`) to narrow the resolved tool definitions and - * executors to a supported subset. + * model-provider rules in your application. Pass `options` to select the + * action plane(s) (`mode`), narrow the exposed actions (`actions`), or drive + * an Anthropic model through its native tool schema (`nativeTool`). */ -export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options?: ComputerToolsOptions): CuaRuntimeSpec { +export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options: CuaRuntimeSpecOptions = {}): CuaRuntimeSpec { const model = typeof input === "string" ? getCuaModel(input) : routeCuaApi(input); const provider = providerForModel(model); const mod: CuaProviderModule = PROVIDERS[provider]; + const mode = options.mode ?? (options.nativeTool ? modeForNativeTool(options.nativeTool) : "os"); + + if (options.nativeTool) { + const nativeTool = resolveNativeTool(options.nativeTool, model, mode); + const nativeModel: Model = { ...model, api: nativeApiForToolType(nativeTool.spec.type) as Model["api"] }; + const executors = nativeToolExecutors(nativeTool); + return { + model: nativeModel, + provider, + mode, + nativeTool, + toolDefinitions: executors.map((executor) => executor.definition), + toolExecutors: executors, + defaultSystemPrompt: mod.buildSystemPrompt({ mode }), + coordinateSystem: mod.coordinateSystem(), + screenshot: mod.screenshot, + onPayload: composePayloadHooks(createNativeToolOnPayload(nativeTool), mod.onPayload), + }; + } + + const toolsOptions: ComputerToolsOptions = { ...options, mode }; return { model, provider, - toolDefinitions: mod.toolDefinitions(options), - toolExecutors: mod.toolExecutors(options), - defaultSystemPrompt: mod.buildSystemPrompt(), + mode, + toolDefinitions: mod.toolDefinitions(toolsOptions), + toolExecutors: mod.toolExecutors(toolsOptions), + defaultSystemPrompt: mod.buildSystemPrompt({ mode }), coordinateSystem: mod.coordinateSystem(), screenshot: mod.screenshot, onPayload: mod.onPayload, }; } + +function composePayloadHooks(first: CuaPayloadHook, second: CuaPayloadHook | undefined): CuaPayloadHook { + if (!second) return first; + return async (payload, model, context) => { + const afterFirst = (await first(payload, model, context)) ?? payload; + return (await second(afterFirst, model, context)) ?? afterFirst; + }; +} diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts new file mode 100644 index 00000000..357ad161 --- /dev/null +++ b/packages/ai/test/modes.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "vitest"; +import { + CUA_ACTION_TYPES, + CUA_DEFAULT_DOM_ACTION_TYPES, + CUA_HYBRID_DOM_ACTION_TYPES, + CUA_HYBRID_OS_ACTION_TYPES, + anthropic, + computerTools, + cuaToolNameForAction, + defaultActionsForMode, + openai, + resolveCuaRuntimeSpec, +} from "../src/index"; + +describe("mode action sets", () => { + it("os mode defaults to the legacy action set", () => { + expect(defaultActionsForMode("os")).toEqual(CUA_ACTION_TYPES); + }); + + it("dom mode defaults to DOM actions plus wait, without page_evaluate", () => { + const actions = defaultActionsForMode("dom"); + expect(actions).toContain("page_snapshot"); + expect(actions).toContain("wait"); + expect(actions).not.toContain("page_evaluate"); + expect(actions).not.toContain("click"); + }); + + it("dom mode exposes page_evaluate only with javascriptExec", () => { + expect(defaultActionsForMode("dom", { javascriptExec: true })).toContain("page_evaluate"); + }); + + it("hybrid mode dedupes to one tool per capability", () => { + const actions = defaultActionsForMode("hybrid"); + // Navigation lives on the DOM plane. + expect(actions).not.toContain("goto"); + expect(actions).not.toContain("url"); + expect(actions).toContain("page_navigate"); + // One screenshot: the OS display. + expect(actions).toContain("screenshot"); + expect(actions).toContain("zoom"); + expect(actions).not.toContain("page_screenshot"); + // Pointer/keyboard stays OS-level. + expect(actions).not.toContain("page_type"); + expect(actions).not.toContain("page_scroll"); + expect(actions).toEqual([...CUA_HYBRID_OS_ACTION_TYPES, ...CUA_HYBRID_DOM_ACTION_TYPES]); + }); +}); + +describe("mode tool naming", () => { + it("os mode keeps canonical action ids", () => { + expect(cuaToolNameForAction("click", "os")).toBe("click"); + }); + + it("dom mode strips the page_ prefix", () => { + expect(cuaToolNameForAction("page_snapshot", "dom")).toBe("snapshot"); + expect(cuaToolNameForAction("page_click", "dom")).toBe("click"); + expect(cuaToolNameForAction("wait", "dom")).toBe("wait"); + }); + + it("hybrid mode prefixes OS actions and keeps page_ names", () => { + expect(cuaToolNameForAction("click", "hybrid")).toBe("computer_click"); + expect(cuaToolNameForAction("page_click", "hybrid")).toBe("page_click"); + }); + + it("os mode rejects DOM actions", () => { + expect(() => cuaToolNameForAction("page_click", "os")).toThrow(/not available in os mode/); + }); +}); + +describe("mode tool schemas", () => { + it("dom mode click accepts refs or viewport coordinates", () => { + const tools = computerTools({ mode: "dom" }); + const click = tools.find((tool) => tool.name === "click")!; + expect(click.parameters.properties.ref).toBeDefined(); + expect(click.parameters.properties.x).toBeDefined(); + }); + + it("hybrid mode page_click is ref-only, keeping one coordinate frame", () => { + const tools = computerTools({ mode: "hybrid" }); + const pageClick = tools.find((tool) => tool.name === "page_click")!; + expect(pageClick.parameters.properties.ref).toBeDefined(); + expect(pageClick.parameters.properties.x).toBeUndefined(); + expect(pageClick.parameters.required).toContain("ref"); + }); + + it("dom mode exposes every default DOM action under its unprefixed name", () => { + const tools = computerTools({ mode: "dom" }); + const names = tools.map((tool) => tool.name); + for (const action of CUA_DEFAULT_DOM_ACTION_TYPES) { + expect(names).toContain(action.slice("page_".length)); + } + }); +}); + +describe("mode runtime specs", () => { + it("resolves dom mode for anthropic with mode-specific prompt and tools", () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "dom" }); + expect(spec.mode).toBe("dom"); + expect(spec.toolDefinitions.map((tool) => tool.name)).toContain("snapshot"); + expect(spec.defaultSystemPrompt).toBe(anthropic.buildAnthropicSystemPrompt({ mode: "dom" })); + }); + + it("resolves hybrid mode for openai", () => { + const spec = resolveCuaRuntimeSpec("openai:gpt-5.5", { mode: "hybrid" }); + expect(spec.mode).toBe("hybrid"); + const names = spec.toolDefinitions.map((tool) => tool.name); + expect(names).toContain("computer_click"); + expect(names).toContain("page_snapshot"); + expect(spec.defaultSystemPrompt).toBe(openai.buildOpenAISystemPrompt({ mode: "hybrid" })); + }); + + it("rejects non-os modes for os-only providers", () => { + expect(() => resolveCuaRuntimeSpec("yutori:n1.5-latest", { mode: "dom" })).toThrow(/os only/); + expect(() => resolveCuaRuntimeSpec("google:gemini-3-flash-preview", { mode: "hybrid" })).toThrow(/os only/); + }); + + it("keeps os mode byte-compatible with the pre-modes default", () => { + const before = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5"); + const after = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "os" }); + expect(after.toolDefinitions.map((tool) => tool.name)).toEqual(before.toolDefinitions.map((tool) => tool.name)); + }); +}); diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts new file mode 100644 index 00000000..f551ae83 --- /dev/null +++ b/packages/ai/test/native-tools.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import { + anthropic, + betaHeaderForNativeTool, + modeForNativeTool, + resolveCuaRuntimeSpec, + type CuaAction, +} from "../src/index"; +import { + ANTHROPIC_NATIVE_BROWSER_MESSAGES_API, + ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API, + mapNativeBrowserInput, + mapNativeComputerInput, +} from "../src/providers/anthropic/native"; + +describe("native tool validation", () => { + it("infers mode from the native tool", () => { + expect(modeForNativeTool({ type: "computer_20260601" })).toBe("os"); + expect(modeForNativeTool({ type: "browser_20260701" })).toBe("dom"); + }); + + it("carries the beta header per tool", () => { + expect(betaHeaderForNativeTool({ type: "computer_20260601" })).toBe("computer-use-2026-06-01"); + expect(betaHeaderForNativeTool({ type: "browser_20260701" })).toBe("browser-use-2026-07-01"); + }); + + it("rejects a native tool with a conflicting mode", () => { + expect(() => resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "dom", nativeTool: { type: "computer_20260601" } })).toThrow( + /requires mode "os"/, + ); + expect(() => + resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "hybrid", nativeTool: { type: "browser_20260701" } }), + ).toThrow(/requires mode "dom"/); + }); + + it("rejects native tools on non-anthropic models", () => { + expect(() => resolveCuaRuntimeSpec("openai:gpt-5.5", { nativeTool: { type: "computer_20260601" } })).toThrow( + /requires an anthropic model/, + ); + }); +}); + +describe("native runtime specs", () => { + it("routes computer_20260601 to the native api with a single placeholder tool", () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260601", enable_zoom: true } }); + expect(spec.mode).toBe("os"); + expect(spec.model.api).toBe(ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API); + expect(spec.nativeTool?.betaHeader).toBe("computer-use-2026-06-01"); + expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["computer"]); + }); + + it("routes browser_20260701 to the native api under the default name", () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "browser_20260701" } }); + expect(spec.mode).toBe("dom"); + expect(spec.model.api).toBe(ANTHROPIC_NATIVE_BROWSER_MESSAGES_API); + expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["browser"]); + }); + + it("swaps the placeholder tool for the native declaration in the payload", async () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260601", enable_zoom: true } }); + const payload = { + tools: [ + { name: "computer", description: "placeholder", input_schema: {} }, + { name: "playwright_execute", description: "keep me", input_schema: {} }, + ], + }; + const next = (await spec.onPayload?.(payload, spec.model)) as { tools: Array> }; + expect(next.tools[0]).toEqual({ type: "computer_20260601", name: "computer", enable_zoom: true }); + expect(next.tools[1]!.name).toBe("playwright_execute"); + }); +}); + +describe("computer_20260601 action mapping", () => { + it("maps clicks with coordinates, buttons, and modifier chords", () => { + expect(mapNativeComputerInput({ action: "left_click", coordinate: [10, 20], text: "ctrl+shift" })).toEqual([ + { type: "click", x: 10, y: 20, button: "left", hold_keys: ["ctrl", "shift"] }, + ]); + expect(mapNativeComputerInput({ action: "triple_click", coordinate: [1, 2] })).toEqual([ + { type: "click", x: 1, y: 2, button: "left", num_clicks: 3 }, + ]); + }); + + it("maps clicks without coordinates to the current cursor position", () => { + expect(mapNativeComputerInput({ action: "left_click" })).toEqual([{ type: "click", button: "left" }]); + }); + + it("expands key repeat into repeated keypresses", () => { + expect(mapNativeComputerInput({ action: "key", text: "Down", repeat: 3 })).toEqual([ + { type: "keypress", keys: ["Down"] }, + { type: "keypress", keys: ["Down"] }, + { type: "keypress", keys: ["Down"] }, + ]); + }); + + it("maps scroll direction and wheel notches to deltas", () => { + expect(mapNativeComputerInput({ action: "scroll", coordinate: [5, 6], scroll_direction: "down", scroll_amount: 2 })).toEqual([ + { type: "scroll", x: 5, y: 6, scroll_y: 240 }, + ]); + }); + + it("maps drag, wait, and zoom", () => { + expect(mapNativeComputerInput({ action: "left_click_drag", start_coordinate: [1, 2], coordinate: [3, 4] })).toEqual([ + { type: "drag", path: [{ x: 1, y: 2 }, { x: 3, y: 4 }] }, + ]); + expect(mapNativeComputerInput({ action: "wait", duration: 2 })).toEqual([{ type: "wait", ms: 2000 }]); + expect(mapNativeComputerInput({ action: "zoom", region: [0, 0, 10, 10] })).toEqual([{ type: "zoom", region: [0, 0, 10, 10] }]); + }); + + it("rejects unknown actions", () => { + expect(() => mapNativeComputerInput({ action: "warp" })).toThrow(/unsupported computer_20260601 action/); + }); +}); + +describe("browser_20260701 action mapping", () => { + it("maps DOM reads", () => { + expect(mapNativeBrowserInput({ action: "read_page", filter: "interactive", depth: 5 })).toEqual([ + { type: "page_snapshot", filter: "interactive", depth: 5 }, + ]); + expect(mapNativeBrowserInput({ action: "find", query: "search bar" })).toEqual([{ type: "page_find", query: "search bar" }]); + expect(mapNativeBrowserInput({ action: "get_page_text", tab_id: "T1" })).toEqual([{ type: "page_text", tab_id: "T1" }]); + }); + + it("maps ref and coordinate click targets", () => { + expect(mapNativeBrowserInput({ action: "left_click", target: { type: "ref", ref: "e7" } })).toEqual([ + { type: "page_click", ref: "e7" }, + ]); + expect(mapNativeBrowserInput({ action: "left_click", target: { type: "coordinate", x: 4, y: 5 }, modifiers: "shift" })).toEqual([ + { type: "page_click", x: 4, y: 5, modifiers: ["shift"] }, + ]); + }); + + it("requires ref targets on form_input and scroll_to", () => { + expect(mapNativeBrowserInput({ action: "form_input", target: { type: "ref", ref: "e7" }, value: "hi" })).toEqual([ + { type: "page_fill", ref: "e7", value: "hi" }, + ]); + expect(() => mapNativeBrowserInput({ action: "scroll_to", target: { type: "coordinate", x: 1, y: 2 } })).toThrow(/requires a ref/); + }); + + it("maps navigation, tabs, zoom, and javascript_exec", () => { + expect(mapNativeBrowserInput({ action: "navigate", url: "back" })).toEqual([{ type: "page_navigate", url: "back" }]); + expect(mapNativeBrowserInput({ action: "list_tabs" })).toEqual([{ type: "page_list_tabs" }]); + expect(mapNativeBrowserInput({ action: "zoom", region: [1, 2, 3, 4] })).toEqual([ + { type: "page_screenshot", region: [1, 2, 3, 4] }, + ]); + expect(mapNativeBrowserInput({ action: "javascript_exec", text: "document.title" })).toEqual([ + { type: "page_evaluate", code: "document.title" }, + ]); + }); +}); + +describe("native tool executors", () => { + it("translate native tool calls through the runtime spec executors", () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "browser_20260701" } }); + const executor = spec.toolExecutors[0]!; + const actions: CuaAction[] = executor.toActions({ action: "left_click", target: { type: "ref", ref: "e3" } }); + expect(actions).toEqual([{ type: "page_click", ref: "e3" }]); + }); + + it("exports the anthropic namespace surface", () => { + expect(anthropic.mapNativeComputerInput).toBeTypeOf("function"); + expect(anthropic.ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API).toBe(ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API); + }); +}); diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index 17b17a16..97c45713 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; +import { anthropic, CUA_OS_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; import type { CuaProviderModule } from "../src/providers/common"; const MODULES: Record = { @@ -62,7 +62,7 @@ describe("provider namespaces export a uniform surface", () => { expect(Array.isArray(actionTypes), `${prefix}_CUA_ACTION_TYPES must be exported`).toBe(true); expect((actionTypes as unknown[]).length).toBeGreaterThan(0); for (const action of actionTypes as string[]) { - expect(CUA_ACTION_TYPES).toContain(action); + expect(CUA_OS_ACTION_TYPES).toContain(action); } expect(namespace[`${prefix}_COMPUTER_INSTRUCTIONS`], `${prefix}_COMPUTER_INSTRUCTIONS must be exported`).toBeTypeOf( diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index dc130079..8310a94f 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -7,7 +7,9 @@ import { type Skill, } from "@onkernel/cua-agent"; import { + type CuaMode, type CuaModelRef, + type CuaNativeToolSpec, parseCuaModelRef, requireCuaEnvApiKey, } from "@onkernel/cua-ai"; @@ -177,6 +179,9 @@ export interface HarnessCliFlags { jsonlIncludeDeltas: boolean; jsonlIncludeImages: boolean; playwright: boolean; + mode?: string; + nativeTool?: string; + jsExec?: boolean; model?: string; thinking?: string; browserProfile?: string; @@ -414,6 +419,9 @@ async function setupHarnessRuntime( skills, contextFiles, thinkingLevel, + mode: parseMode(flags.mode), + nativeTool: parseNativeTool(flags.nativeTool, flags.jsExec), + javascriptExec: flags.jsExec, playwright: flags.playwright, modelBaseUrl: baseUrlOverride, }); @@ -445,6 +453,23 @@ function providerBaseUrlOverride(provider: string): string | undefined { return value && value.length > 0 ? value : undefined; } +function parseMode(raw: string | undefined): CuaMode | undefined { + if (raw === undefined) return undefined; + const value = raw.trim().toLowerCase(); + if (value === "os" || value === "dom" || value === "hybrid") return value; + throw new Error(`invalid --mode value "${raw}"; expected one of: os | dom | hybrid`); +} + +function parseNativeTool(raw: string | undefined, jsExec: boolean | undefined): CuaNativeToolSpec | undefined { + if (raw === undefined) return undefined; + const value = raw.trim().toLowerCase(); + // enable_zoom follows Anthropic's own recommendation for fine-grained + // visual targeting; the executor implements the zoom crop locally. + if (value === "computer_20260601") return { type: "computer_20260601", enable_zoom: true }; + if (value === "browser_20260701") return { type: "browser_20260701", ...(jsExec ? { enable_javascript_exec: true } : {}) }; + throw new Error(`invalid --native-tool value "${raw}"; expected one of: computer_20260601 | browser_20260701`); +} + function mapThinkingLevel(raw: string | undefined): "off" | "minimal" | "low" | "medium" | "high" | "xhigh" { const v = (raw ?? "low").trim().toLowerCase(); switch (v) { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 5c20070d..a1eed50c 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -47,6 +47,15 @@ Options: --max-steps Max turns for action subcommands (default 3) --playwright Add the playwright_execute tool so the model can run Playwright code against the browser session + --mode Action plane(s) to expose: os (default) | dom | hybrid + os: OS-level input only. dom: CDP page tools (snapshot, + find, click-by-ref, navigate, tabs). hybrid: both, + deduplicated (computer_* + page_* tools). + --native-tool Drive an Anthropic model through its native tool schema: + computer_20260601 (requires --mode os) or + browser_20260701 (requires --mode dom) + --js-exec Expose page_evaluate (arbitrary JS in the page) in + dom/hybrid modes --out Output file for screenshot subcommand -o, --output Output format for --print: text (default) | jsonl --jsonl-include-deltas Include assistant_text_delta events (default off) @@ -101,6 +110,9 @@ interface CliFlags { jsonlIncludeDeltas: boolean; jsonlIncludeImages: boolean; playwright: boolean; + mode?: string; + nativeTool?: string; + jsExec: boolean; model?: string; thinking?: string; browserProfile?: string; @@ -150,6 +162,9 @@ function parseCliArgs(argv: string[]): CliFlags { "jsonl-include-deltas": { type: "boolean", default: false }, "jsonl-include-images": { type: "boolean", default: false }, playwright: { type: "boolean", default: false }, + mode: { type: "string" }, + "native-tool": { type: "string" }, + "js-exec": { type: "boolean", default: false }, }, allowPositionals: true, strict: true, @@ -197,6 +212,9 @@ function parseCliArgs(argv: string[]): CliFlags { jsonlIncludeDeltas: !!parsed.values["jsonl-include-deltas"], jsonlIncludeImages: !!parsed.values["jsonl-include-images"], playwright: !!parsed.values.playwright, + mode: parsed.values.mode as string | undefined, + nativeTool: parsed.values["native-tool"] as string | undefined, + jsExec: !!parsed.values["js-exec"], positionals: parsed.positionals, }; } @@ -213,6 +231,9 @@ function toHarnessFlags(flags: CliFlags): HarnessCliFlags { jsonlIncludeDeltas: flags.jsonlIncludeDeltas, jsonlIncludeImages: flags.jsonlIncludeImages, playwright: flags.playwright, + mode: flags.mode, + nativeTool: flags.nativeTool, + jsExec: flags.jsExec, model: flags.model, thinking: flags.thinking, browserProfile: flags.browserProfile, diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 9ee9e4bd..8770f304 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -10,7 +10,9 @@ import { } from "@onkernel/cua-agent"; import { type Api, + type CuaMode, type CuaModelRef, + type CuaNativeToolSpec, type Model, type Models, getCuaModel, @@ -31,6 +33,12 @@ export interface BuildCuaHarnessOptions { /** Context files (AGENTS.md, CLAUDE.md, …) appended to the system prompt. */ contextFiles?: ContextFile[]; thinkingLevel?: ThinkingLevel; + /** Which canonical action plane(s) to expose: "os" (default), "dom", or "hybrid". */ + mode?: CuaMode; + /** Drive the model through a provider-native tool declaration (validated against `mode`). */ + nativeTool?: CuaNativeToolSpec; + /** Expose `page_evaluate` in dom/hybrid modes. */ + javascriptExec?: boolean; /** Expose the playwright_execute tool that runs Playwright code against the browser session. */ playwright?: boolean; /** Override the default coding-tools extraTools (bash/read/edit/write/grep/find/ls). */ @@ -62,11 +70,18 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { browser: opts.browser, client: opts.client, extraTools, + mode: opts.mode, + nativeTool: opts.nativeTool, + javascriptExec: opts.javascriptExec, playwright: opts.playwright, resources: { skills }, thinkingLevel: opts.thinkingLevel, systemPrompt: ({ model: activeModel, resources }) => { - const runtime = resolveCuaRuntimeSpec(activeModel); + const runtime = resolveCuaRuntimeSpec(activeModel, { + mode: opts.mode, + nativeTool: opts.nativeTool, + javascriptExec: opts.javascriptExec, + }); return composeSystemPrompt(runtime.defaultSystemPrompt, resources.skills ?? [], contextFiles); }, models: opts.models, From ec915599053e1b4e13d88f28fbe75297eb6a768b Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 17:01:58 +0000 Subject: [PATCH 02/34] Add setMode/getMode, /mode slash command, and live-API schema fix - CuaAgent.setMode / CuaAgentHarness.setMode switch action planes at runtime (rejected when it conflicts with a configured nativeTool) - /mode TUI slash command with autocomplete - region schemas use bounded arrays instead of Type.Tuple: tuples emit draft-07 `items: [...]`, which Anthropic's draft 2020-12 validation rejects (found via live API) - anthropic-native-smoke example covering the mode / native-tool matrix --- .../agent/examples/anthropic-native-smoke.ts | 61 +++++++++++++++++++ packages/agent/src/agent.ts | 48 ++++++++++++++- packages/agent/test/agent.test.ts | 51 ++++++++++++++++ packages/ai/src/actions/dom.ts | 6 +- packages/ai/src/actions/os.ts | 10 ++- packages/cli/src/harness.ts | 8 ++- packages/cli/src/tui/main.ts | 18 ++++++ packages/cli/src/tui/slash-commands.ts | 24 +++++++- 8 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 packages/agent/examples/anthropic-native-smoke.ts diff --git a/packages/agent/examples/anthropic-native-smoke.ts b/packages/agent/examples/anthropic-native-smoke.ts new file mode 100644 index 00000000..a819f144 --- /dev/null +++ b/packages/agent/examples/anthropic-native-smoke.ts @@ -0,0 +1,61 @@ +// Smoke-test the mode/native-tool matrix against a live Kernel browser: +// +// MODEL_REF=anthropic:claude-opus-4-8 CONFIG=native-computer tsx examples/anthropic-native-smoke.ts +// +// CONFIG selects the runtime shape: +// os (default) canonical OS-plane tools +// dom canonical DOM-plane tools over CDP +// hybrid both planes, deduplicated +// native-computer Anthropic computer_20260601 (requires the computer-use beta) +// native-browser Anthropic browser_20260701 (requires the browser-use beta) +import Kernel from "@onkernel/sdk"; +import { requireCuaEnvApiKeyForModel, type CuaModelRef, type CuaMode, type CuaNativeToolSpec } from "@onkernel/cua-ai"; +import { CuaAgent } from "../src/index"; +import { logAgentEvent, logAssistant } from "./shared/logging"; + +const modelRef = (process.env.MODEL_REF as CuaModelRef | undefined) ?? "anthropic:claude-opus-4-8"; +const config = process.env.CONFIG ?? "os"; + +const CONFIGS: Record = { + os: { mode: "os" }, + dom: { mode: "dom" }, + hybrid: { mode: "hybrid" }, + "native-computer": { nativeTool: { type: "computer_20260601", enable_zoom: true } }, + "native-browser": { nativeTool: { type: "browser_20260701" } }, +}; + +const PROMPT = [ + "Navigate to https://example.com, read the page, and tell me:", + "1) the main heading text", + "2) the text of the link on the page", + "Then follow that link and tell me the title of the page you land on.", +].join("\n"); + +async function main(): Promise { + const runtime = CONFIGS[config]; + if (!runtime) throw new Error(`unknown CONFIG "${config}" (expected: ${Object.keys(CONFIGS).join(" | ")})`); + const kernelApiKey = process.env.KERNEL_API_KEY; + if (!kernelApiKey) throw new Error("KERNEL_API_KEY is required"); + requireCuaEnvApiKeyForModel(modelRef); + const client = new Kernel({ apiKey: kernelApiKey }); + const browser = await client.browsers.create({ stealth: true }); + + try { + const agent = new CuaAgent({ + browser, + client, + ...runtime, + initialState: { model: modelRef }, + }); + agent.subscribe(logAgentEvent); + + console.log(`running config=${config} model=${modelRef} live_view=${browser.browser_live_view_url}`); + await agent.prompt(PROMPT); + const assistant = [...agent.state.messages].reverse().find((message) => message.role === "assistant"); + logAssistant(assistant?.role === "assistant" ? assistant : undefined); + } finally { + await client.browsers.deleteByID(browser.session_id); + } +} + +void main(); diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index ad0db291..8119cf0c 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -135,6 +135,7 @@ export type CuaAgentHarnessOptions< class CuaRuntimeController { private runtimeSpec: CuaRuntimeSpec; private translator: InternalComputerTranslator; + private currentMode?: CuaMode; constructor( private readonly options: { @@ -150,13 +151,14 @@ class CuaRuntimeController { onPayload?: SimpleStreamOptions["onPayload"]; }, ) { + this.currentMode = options.mode; this.runtimeSpec = this.resolveSpec(options.model); this.translator = this.createTranslator(); } - private resolveSpec(model: CuaRuntimeInput): CuaRuntimeSpec { + private resolveSpec(model: CuaRuntimeInput, mode: CuaMode | undefined = this.currentMode): CuaRuntimeSpec { return resolveCuaRuntimeSpec(model, { - mode: this.options.mode, + mode, nativeTool: this.options.nativeTool, javascriptExec: this.options.javascriptExec, }); @@ -166,6 +168,16 @@ class CuaRuntimeController { return this.runtimeSpec.model; } + get mode(): CuaMode { + return this.runtimeSpec.mode; + } + + setMode(mode: CuaMode): void { + this.runtimeSpec = this.resolveSpec(this.runtimeSpec.model, mode); + this.currentMode = mode; + this.translator = this.createTranslator(); + } + get systemPrompt(): string { return this.runtimeSpec.defaultSystemPrompt; } @@ -345,6 +357,22 @@ export class CuaAgent extends Agent { return this.stateProxy; } + /** Switch the action plane(s) exposed to the model; takes effect next turn. */ + setMode(mode: CuaMode): void { + this.runtime.setMode(mode); + this.runtimeDirty = true; + const state = super.state; + state.tools = this.runtime.tools(); + if (this.ownsSystemPrompt) { + state.systemPrompt = this.runtime.systemPrompt; + } + } + + /** The action plane(s) currently exposed to the model. */ + getMode(): CuaMode { + return this.runtime.mode; + } + private applyRuntime(model: CuaRuntimeInput): void { this.runtime.setModel(model); this.runtimeDirty = true; @@ -439,6 +467,22 @@ export class CuaAgentHarness< await super.setActiveTools(toolNames); this.requestedActiveToolNames = [...toolNames]; } + + /** + * Switch the action plane(s) exposed to the model and refresh CUA-owned + * tools. Throws when the harness was configured with a `nativeTool` whose + * plane conflicts with the requested mode. + */ + async setMode(mode: CuaMode): Promise { + this.runtime.setMode(mode); + const tools = this.runtime.tools(); + await super.setTools(tools, tools.map((tool) => tool.name)); + } + + /** The action plane(s) currently exposed to the model. */ + getMode(): CuaMode { + return this.runtime.mode; + } } function composeOnPayload(first: AgentOptions["onPayload"], second: AgentOptions["onPayload"]): AgentOptions["onPayload"] { diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 686231c0..26946be8 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -178,6 +178,40 @@ describe("CuaAgent", () => { expect(agent.state.tools).toHaveLength(runtime.toolExecutors.length); }); + it("switches action planes through setMode", () => { + const agent = new CuaAgent({ + browser, + client, + initialState: { + model: "anthropic:claude-opus-4-5", + }, + }); + expect(agent.getMode()).toBe("os"); + expect(agent.state.tools.map((tool) => tool.name)).toContain("click"); + + agent.setMode("dom"); + + expect(agent.getMode()).toBe("dom"); + const names = agent.state.tools.map((tool) => tool.name); + expect(names).toContain("snapshot"); + expect(names).not.toContain("move"); + expect(agent.state.systemPrompt).toBe(resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "dom" }).defaultSystemPrompt); + }); + + it("rejects setMode conflicting with a configured native tool", () => { + const agent = new CuaAgent({ + browser, + client, + nativeTool: { type: "browser_20260701" }, + initialState: { + model: "anthropic:claude-opus-4-5", + }, + }); + expect(agent.getMode()).toBe("dom"); + expect(() => agent.setMode("os")).toThrow(/requires mode "dom"/); + expect(agent.getMode()).toBe("dom"); + }); + it("keeps extra tools and caller-owned system prompt when state.model changes", () => { const tool = createCustomTool(); const agent = new CuaAgent({ @@ -392,6 +426,23 @@ describe("CuaAgentHarness", () => { expect(harness.getTools()).toHaveLength(runtime.toolExecutors.length); }); + it("switches action planes through setMode", async () => { + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "anthropic:claude-opus-4-5", + }); + expect(harness.getMode()).toBe("os"); + + await harness.setMode("hybrid"); + + expect(harness.getMode()).toBe("hybrid"); + const names = harness.getTools().map((tool) => tool.name); + expect(names).toContain("computer_click"); + expect(names).toContain("page_snapshot"); + }); + it("appends extraTools in harness construction", async () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const tool = createCustomTool(); diff --git a/packages/ai/src/actions/dom.ts b/packages/ai/src/actions/dom.ts index 5d7f859e..f62ddacc 100644 --- a/packages/ai/src/actions/dom.ts +++ b/packages/ai/src/actions/dom.ts @@ -310,8 +310,12 @@ export function createCuaDomActionSchemaByType(options: CuaDomSchemaOptions): Re page_screenshot: Type.Object( { type: Type.Literal("page_screenshot"), + // Not Type.Tuple: tuples emit draft-07 `items: [...]`, which Anthropic's + // draft 2020-12 schema validation rejects. region: Type.Optional( - Type.Tuple([Type.Number(), Type.Number(), Type.Number(), Type.Number()], { + Type.Array(Type.Number(), { + minItems: 4, + maxItems: 4, description: "Optional crop region, [x0, y0, x1, y1] in viewport pixels.", }), ), diff --git a/packages/ai/src/actions/os.ts b/packages/ai/src/actions/os.ts index 8fc6cf47..cc4c20ac 100644 --- a/packages/ai/src/actions/os.ts +++ b/packages/ai/src/actions/os.ts @@ -1,4 +1,4 @@ -import { Type, type Static, type TSchema } from "@earendil-works/pi-ai"; +import { Type, type TSchema } from "@earendil-works/pi-ai"; /** * OS-plane canonical actions. @@ -275,7 +275,11 @@ export const CUA_OS_ACTION_SCHEMA_BY_TYPE = { zoom: Type.Object( { type: Type.Literal("zoom"), - region: Type.Tuple([Type.Number(), Type.Number(), Type.Number(), Type.Number()], { + // Not Type.Tuple: tuples emit draft-07 `items: [...]`, which Anthropic's + // draft 2020-12 schema validation rejects. + region: Type.Array(Type.Number(), { + minItems: 4, + maxItems: 4, description: "[x0, y0, x1, y1] crop region in OS screenshot pixels.", }), }, @@ -294,4 +298,4 @@ export const CUA_OS_ACTION_SCHEMA_BY_TYPE = { cursor_position: Type.Object({ type: Type.Literal("cursor_position") }, { additionalProperties: false }), } satisfies Record; -export type CuaZoomRegion = Static<(typeof CUA_OS_ACTION_SCHEMA_BY_TYPE)["zoom"]>["region"]; +export type CuaZoomRegion = CuaActionZoom["region"]; diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 8770f304..83406a0e 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -63,7 +63,10 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { const model: CuaModelRef | Model = opts.modelBaseUrl ? { ...getCuaModel(opts.model), baseUrl: opts.modelBaseUrl } : opts.model; - return new CuaAgentHarness({ + // The system-prompt callback re-resolves per turn and must see the live + // mode after /mode switches, so it reads it from the harness (late-bound). + let harness: CuaAgentHarness | undefined; + harness = new CuaAgentHarness({ env: new NodeExecutionEnv({ cwd: opts.cwd }), session: opts.session, model, @@ -78,7 +81,7 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { thinkingLevel: opts.thinkingLevel, systemPrompt: ({ model: activeModel, resources }) => { const runtime = resolveCuaRuntimeSpec(activeModel, { - mode: opts.mode, + mode: harness?.getMode() ?? opts.mode, nativeTool: opts.nativeTool, javascriptExec: opts.javascriptExec, }); @@ -86,6 +89,7 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { }, models: opts.models, }); + return harness; } function composeSystemPrompt(base: string, skills: Skill[], contextFiles: ContextFile[]): string { diff --git a/packages/cli/src/tui/main.ts b/packages/cli/src/tui/main.ts index 67111d39..6c6a9a7d 100644 --- a/packages/cli/src/tui/main.ts +++ b/packages/cli/src/tui/main.ts @@ -303,6 +303,10 @@ export async function runInteractive(opts: InteractiveOptions): Promise await applyModelCommand(opts, footer, status, messages, parsed.argument); return; } + if (parsed?.command === "mode") { + await applyModeCommand(opts, messages, parsed.argument); + return; + } if (parsed?.command === "thinking") { await applyThinkingCommand(opts, footer, messages, parsed.argument); return; @@ -487,6 +491,20 @@ async function applyModelCommand( } } +async function applyModeCommand(opts: InteractiveOptions, messages: MessageList, argument: string): Promise { + const value = argument.trim().toLowerCase(); + if (value !== "os" && value !== "dom" && value !== "hybrid") { + messages.addError("usage: /mode "); + return; + } + try { + await opts.harness.setMode(value); + messages.addNotice(`mode → ${value}`); + } catch (err) { + messages.addError((err as Error).message); + } +} + async function applyThinkingCommand( opts: InteractiveOptions, footer: TelemetryFooter, diff --git a/packages/cli/src/tui/slash-commands.ts b/packages/cli/src/tui/slash-commands.ts index 8d4ac475..34169c88 100644 --- a/packages/cli/src/tui/slash-commands.ts +++ b/packages/cli/src/tui/slash-commands.ts @@ -27,6 +27,13 @@ export function buildAutocompleteProvider( getArgumentCompletions: (prefix: string) => modelCompletions(prefix), }); + commands.push({ + name: "mode", + description: "Switch the action plane(s): os | dom | hybrid", + argumentHint: "", + getArgumentCompletions: (prefix: string) => modeCompletions(prefix), + }); + commands.push({ name: "thinking", description: "Set the reasoning level for future turns", @@ -58,6 +65,18 @@ function modelCompletions(prefix: string): AutocompleteItem[] { return filtered.map((m) => ({ value: m.ref, label: m.ref, description: m.name })); } +const MODES: ReadonlyArray<{ value: string; description: string }> = [ + { value: "os", description: "OS-level input only (default)" }, + { value: "dom", description: "CDP page tools: snapshot, find, click-by-ref, navigate, tabs" }, + { value: "hybrid", description: "Both planes: computer_* input + ref-only page_* tools" }, +]; + +function modeCompletions(prefix: string): AutocompleteItem[] { + const trimmed = prefix.trim().toLowerCase(); + const filtered = trimmed ? MODES.filter((m) => m.value.startsWith(trimmed)) : MODES; + return filtered.map((m) => ({ value: m.value, label: m.value, description: m.description })); +} + const THINKING_LEVELS: ReadonlyArray<{ value: string; description: string }> = [ { value: "off", description: "Disable reasoning" }, { value: "minimal", description: "Minimal reasoning" }, @@ -75,6 +94,7 @@ function thinkingCompletions(prefix: string): AutocompleteItem[] { export type ParsedSlashCommand = | { command: "model"; argument: string } + | { command: "mode"; argument: string } | { command: "thinking"; argument: string } | { command: "compact"; argument: string } | { command: "skill"; name: string; remainder: string }; @@ -91,11 +111,11 @@ export function parseSlashCommand(text: string): ParsedSlashCommand | undefined const [, name, rest] = skillMatch; return { command: "skill", name: name ?? "", remainder: (rest ?? "").trim() }; } - const builtinMatch = trimmed.match(/^\/(model|thinking|compact)\s*(.*)$/); + const builtinMatch = trimmed.match(/^\/(model|mode|thinking|compact)\s*(.*)$/); if (builtinMatch) { const [, name, rest] = builtinMatch; return { - command: name as "model" | "thinking" | "compact", + command: name as "model" | "mode" | "thinking" | "compact", argument: (rest ?? "").trim(), }; } From 2be36717027ef1751054fd3756d77e4551324323 Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 18:42:12 +0000 Subject: [PATCH 03/34] Rename modes to computer/browser, matching Anthropic's tool naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit os → computer, dom → browser (hybrid unchanged). The mode names now pair directly with the native tools: computer_20260601 ⇢ mode computer, browser_20260701 ⇢ mode browser. Plane files, action-set constants, types, the page executor, CLI flag values, and the /mode command follow suit. Hybrid tool prefixes stay computer_* / page_*. Re-verified live: canonical browser mode and native browser_20260701 against a Kernel browser after the rename. --- docs/architecture.md | 20 ++-- .../agent/examples/anthropic-native-smoke.ts | 18 ++-- packages/agent/src/agent.ts | 8 +- packages/agent/src/index.ts | 2 +- packages/agent/src/tools.ts | 16 ++-- packages/agent/src/translator/cdp.ts | 4 +- .../agent/src/translator/{dom.ts => page.ts} | 26 ++--- packages/agent/src/translator/translator.ts | 34 +++---- packages/agent/src/translator/types.ts | 2 +- packages/agent/test/agent.test.ts | 16 ++-- ...or-dom.test.ts => translator-page.test.ts} | 16 ++-- .../ai/src/actions/{dom.ts => browser.ts} | 28 +++--- .../ai/src/actions/{os.ts => computer.ts} | 18 ++-- packages/ai/src/actions/index.ts | 44 ++++----- packages/ai/src/modes.ts | 94 ++++++++++--------- packages/ai/src/native-tools.ts | 10 +- .../ai/src/providers/anthropic/actions.ts | 20 ++-- packages/ai/src/providers/anthropic/index.ts | 6 +- packages/ai/src/providers/anthropic/native.ts | 4 +- packages/ai/src/providers/common.ts | 36 +++---- packages/ai/src/providers/gemini/index.ts | 8 +- packages/ai/src/providers/openai/index.ts | 4 +- packages/ai/src/providers/tzafon/index.ts | 6 +- packages/ai/src/providers/yutori/index.ts | 6 +- packages/ai/src/runtime-spec.ts | 6 +- packages/ai/test/modes.test.ts | 66 ++++++------- packages/ai/test/native-tools.test.ts | 14 +-- packages/ai/test/provider-module.test.ts | 4 +- packages/cli/src/cli-harness.ts | 4 +- packages/cli/src/cli.ts | 14 +-- packages/cli/src/harness.ts | 4 +- packages/cli/src/tui/main.ts | 4 +- packages/cli/src/tui/slash-commands.ts | 8 +- 33 files changed, 287 insertions(+), 283 deletions(-) rename packages/agent/src/translator/{dom.ts => page.ts} (95%) rename packages/agent/test/{translator-dom.test.ts => translator-page.test.ts} (86%) rename packages/ai/src/actions/{dom.ts => browser.ts} (90%) rename packages/ai/src/actions/{os.ts => computer.ts} (92%) diff --git a/docs/architecture.md b/docs/architecture.md index a965f545..8dfb0d72 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -97,14 +97,14 @@ provider conditionals. A new provider difference is a new or extended The canonical action vocabulary is split into two planes, delineated in code under `packages/ai/src/actions/`: -- **OS plane** (`actions/os.ts`) — real OS-level input against the browser +- **Computer plane** (`actions/computer.ts`) — real OS-level input against the browser VM: mouse, keyboard, display capture, executed through Kernel's `browsers.computer` REST API. Coordinates are pixels in the OS screenshot frame. -- **DOM plane** (`actions/dom.ts`, ids prefixed `page_`) — CDP-driven page +- **Browser plane** (`actions/browser.ts`, ids prefixed `page_`) — CDP-driven page tools: accessibility snapshots with element refs, element-targeted interaction, navigation, tabs, viewport screenshots. Executed by - `packages/agent/src/translator/dom.ts` over a raw CDP websocket + `packages/agent/src/translator/page.ts` over a raw CDP websocket (`translator/cdp.ts`) to the browser's `cdp_ws_url` — no Playwright. Coordinates, where used, are viewport pixels. @@ -112,20 +112,20 @@ A `CuaMode` selects which plane(s) the model sees: | mode | tools | coordinate frame | | --- | --- | --- | -| `os` (default) | OS actions under their canonical ids (`click`, `screenshot`, …) | OS screenshot pixels | -| `dom` | DOM actions with the `page_` prefix stripped (`snapshot`, `click`, …) plus `wait` | none for refs; viewport pixels where coordinates are allowed | -| `hybrid` | both planes, one tool per capability: OS actions as `computer_*`, DOM reads/element-writes as `page_*` (ref-only) | OS screenshot pixels — the single live frame | +| `computer` (default) | computer actions under their canonical ids (`click`, `screenshot`, …) | OS screenshot pixels | +| `browser` | browser actions with the `page_` prefix stripped (`snapshot`, `click`, …) plus `wait` | none for refs; viewport pixels where coordinates are allowed | +| `hybrid` | both planes, one tool per capability: computer actions as `computer_*`, browser reads/element-writes as `page_*` (ref-only) | OS screenshot pixels — the single live frame | -Hybrid deduplicates capabilities: navigation and tabs live on the DOM plane, +Hybrid deduplicates capabilities: navigation and tabs live on the browser plane, pointer/keyboard input and the (only) screenshot live on the OS plane, and -DOM tools take element refs only so exactly one coordinate frame exists. +page tools take element refs only so exactly one coordinate frame exists. Element refs are snapshot-scoped (`e12`); a stale ref resolves to an error string that tells the model to re-snapshot. **Native tools.** `resolveCuaRuntimeSpec(model, { nativeTool })` drives an Anthropic model through its provider-defined tool schema instead of the -canonical function tools: `computer_20260601` pairs with `os` mode and -`browser_20260701` with `dom` mode (mismatches throw, mirroring the API's +canonical function tools: `computer_20260601` pairs with `computer` mode and +`browser_20260701` with `browser` mode (mismatches throw, mirroring the API's own rejection of mixed frames). The spec routes the model to a CUA-owned api id; the registered `anthropic` provider dispatches it to pi's builtin `anthropic-messages` transport with the tool's `anthropic-beta` header, an diff --git a/packages/agent/examples/anthropic-native-smoke.ts b/packages/agent/examples/anthropic-native-smoke.ts index a819f144..3a89eb61 100644 --- a/packages/agent/examples/anthropic-native-smoke.ts +++ b/packages/agent/examples/anthropic-native-smoke.ts @@ -1,24 +1,24 @@ // Smoke-test the mode/native-tool matrix against a live Kernel browser: // -// MODEL_REF=anthropic:claude-opus-4-8 CONFIG=native-computer tsx examples/anthropic-native-smoke.ts +// MODEL_REF=anthropic:claude-opus-4-8 CONFIG=native-browser tsx examples/anthropic-native-smoke.ts // // CONFIG selects the runtime shape: -// os (default) canonical OS-plane tools -// dom canonical DOM-plane tools over CDP -// hybrid both planes, deduplicated -// native-computer Anthropic computer_20260601 (requires the computer-use beta) -// native-browser Anthropic browser_20260701 (requires the browser-use beta) +// computer (default) canonical computer-plane (OS input) tools +// browser canonical browser-plane (CDP page) tools +// hybrid both planes, deduplicated +// native-computer Anthropic computer_20260601 (requires the computer-use beta) +// native-browser Anthropic browser_20260701 (requires the browser-use beta) import Kernel from "@onkernel/sdk"; import { requireCuaEnvApiKeyForModel, type CuaModelRef, type CuaMode, type CuaNativeToolSpec } from "@onkernel/cua-ai"; import { CuaAgent } from "../src/index"; import { logAgentEvent, logAssistant } from "./shared/logging"; const modelRef = (process.env.MODEL_REF as CuaModelRef | undefined) ?? "anthropic:claude-opus-4-8"; -const config = process.env.CONFIG ?? "os"; +const config = process.env.CONFIG ?? "computer"; const CONFIGS: Record = { - os: { mode: "os" }, - dom: { mode: "dom" }, + computer: { mode: "computer" }, + browser: { mode: "browser" }, hybrid: { mode: "hybrid" }, "native-computer": { nativeTool: { type: "computer_20260601", enable_zoom: true } }, "native-browser": { nativeTool: { type: "browser_20260701" } }, diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 8119cf0c..ba3c5607 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -72,11 +72,11 @@ export type CuaAgentOptions = Omit & { initialState: CuaAgentInitialState; /** Add your own pi tools alongside the built-in browser tools. */ extraTools?: AgentTool[]; - /** Which canonical action plane(s) to expose: "os" (default), "dom", or "hybrid". */ + /** Which canonical action plane(s) to expose: "computer" (default), "browser", or "hybrid". */ mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `page_evaluate` in dom/hybrid modes. Default false. */ + /** Expose `page_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; @@ -112,11 +112,11 @@ export type CuaAgentHarnessOptions< models?: Models; /** Add your own pi tools alongside the built-in browser tools. */ extraTools?: AgentTool[]; - /** Which canonical action plane(s) to expose: "os" (default), "dom", or "hybrid". */ + /** Which canonical action plane(s) to expose: "computer" (default), "browser", or "hybrid". */ mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `page_evaluate` in dom/hybrid modes. Default false. */ + /** Expose `page_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 165f05d8..e8e8e7c4 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -3,7 +3,7 @@ export { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node"; export type { KernelBrowser } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; -export { DomExecutor } from "./translator/dom"; +export { PageExecutor } from "./translator/page"; export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index eeef177e..f47bc6e7 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -23,7 +23,7 @@ export interface ComputerToolOptions { toolExecutors: CuaToolExecutorSpec[]; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; - /** Action plane(s) in play; controls whether the post-action fallback capture is the OS display or the viewport. Default "os". */ + /** Action plane(s) in play; controls whether the post-action fallback capture is the OS display or the viewport. Default "computer". */ mode?: CuaMode; computerUseExtra?: boolean; playwright?: boolean; @@ -37,7 +37,7 @@ export interface BatchDetails { | { type: "url"; url: string } | { type: "screenshot"; bytes: number } | { type: "cursor_position"; x: number; y: number } - | { type: "dom_text"; label: string; bytes: number } + | { type: "page_text"; label: string; bytes: number } >; } @@ -89,7 +89,7 @@ export function buildCuaComputerTools( args: Pick, translator: InternalComputerTranslator, ): CuaExecutorTool[] { - return withExtraTools(args).map((executor) => createExecutorTool(executor, translator, args.mode ?? "os")); + return withExtraTools(args).map((executor) => createExecutorTool(executor, translator, args.mode ?? "computer")); } function withExtraTools(args: Pick): ComputerExecutorSpec[] { @@ -155,7 +155,7 @@ function isPlaywrightExecutor(executor: ComputerExecutorSpec): executor is Playw async function executeBatchTool( translator: InternalComputerTranslator, params: CuaBatchInput, - mode: CuaMode = "os", + mode: CuaMode = "computer", ): Promise> { const content: ToolContent = []; const readResults: BatchDetails["readResults"] = []; @@ -168,8 +168,8 @@ async function executeBatchTool( } else if (read.type === "cursor_position") { readResults.push({ type: "cursor_position", x: read.x, y: read.y }); content.push({ type: "text", text: `cursor_position(): ${read.x},${read.y}` }); - } else if (read.type === "dom_text") { - readResults.push({ type: "dom_text", label: read.label, bytes: read.text.length }); + } else if (read.type === "page_text") { + readResults.push({ type: "page_text", label: read.label, bytes: read.text.length }); content.push({ type: "text", text: read.text }); } else { readResults.push({ type: "screenshot", bytes: read.data.length }); @@ -178,8 +178,8 @@ async function executeBatchTool( } if (content.length === 0) { // Post-action grounding capture: the OS display in os/hybrid mode, - // the browser viewport in dom mode (the only frame the model sees). - const screenshot = mode === "dom" ? await translator.dom().screenshot() : await translator.screenshot(); + // the browser viewport in browser mode (the only frame the model sees). + const screenshot = mode === "browser" ? await translator.page().screenshot() : await translator.screenshot(); readResults.push({ type: "screenshot", bytes: screenshot.data.length }); content.push({ type: "image", data: screenshot.data.toString("base64"), mimeType: screenshot.mimeType }); } diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index 7d0bf168..b369444f 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -1,9 +1,9 @@ /** - * Minimal raw-CDP client for the DOM action plane. + * Minimal raw-CDP client for the browser action plane. * * Connects to a Kernel browser's `cdp_ws_url` over a plain WebSocket and * speaks the DevTools JSON-RPC protocol directly — no Playwright and no - * driver dependency. Only what the DOM executor needs: command dispatch on + * driver dependency. Only what the page executor needs: command dispatch on * the browser connection and on attached page sessions. */ diff --git a/packages/agent/src/translator/dom.ts b/packages/agent/src/translator/page.ts similarity index 95% rename from packages/agent/src/translator/dom.ts rename to packages/agent/src/translator/page.ts index ff5cc7ed..b9086c3a 100644 --- a/packages/agent/src/translator/dom.ts +++ b/packages/agent/src/translator/page.ts @@ -10,7 +10,7 @@ import { type CuaActionPageScroll, type CuaActionPageScrollTo, type CuaActionPageSnapshot, - type CuaDomAction, + type CuaBrowserAction, } from "@onkernel/cua-ai"; import { CdpConnection } from "./cdp"; import type { BatchReadResult } from "./types"; @@ -39,14 +39,14 @@ interface RefEntry { } /** - * Executes DOM-plane canonical actions over CDP. + * Executes browser-plane canonical actions over CDP. * * Element refs are snapshot-scoped: each snapshot/find mints `e` ids * mapped to CDP backend node ids for the target's current generation. A * navigation bumps the generation, and refs from earlier generations resolve * to a stale-ref error whose message tells the model how to recover. */ -export class DomExecutor { +export class PageExecutor { private readonly refs = new Map(); private readonly generations = new Map(); private refCounter = 0; @@ -54,14 +54,14 @@ export class DomExecutor { constructor(private readonly cdp: CdpConnection) {} - async execute(action: CuaDomAction): Promise { + async execute(action: CuaBrowserAction): Promise { switch (action.type) { case "page_snapshot": - return [{ type: "dom_text", label: "snapshot", text: await this.snapshot(action) }]; + return [{ type: "page_text", label: "snapshot", text: await this.snapshot(action) }]; case "page_text": - return [{ type: "dom_text", label: "text", text: await this.pageText(tabOf(action)) }]; + return [{ type: "page_text", label: "text", text: await this.pageText(tabOf(action)) }]; case "page_find": - return [{ type: "dom_text", label: "find", text: await this.find(action) }]; + return [{ type: "page_text", label: "find", text: await this.find(action) }]; case "page_click": await this.click(action); return []; @@ -89,15 +89,15 @@ export class DomExecutor { await this.key(action); return []; case "page_navigate": - return [{ type: "dom_text", label: "navigate", text: await this.navigate(action) }]; + return [{ type: "page_text", label: "navigate", text: await this.navigate(action) }]; case "page_list_tabs": - return [{ type: "dom_text", label: "tabs", text: await this.listTabs() }]; + return [{ type: "page_text", label: "tabs", text: await this.listTabs() }]; case "page_new_tab": - return [{ type: "dom_text", label: "new_tab", text: await this.newTab() }]; + return [{ type: "page_text", label: "new_tab", text: await this.newTab() }]; case "page_screenshot": return [{ type: "screenshot", ...(await this.screenshot(action.region, action.tab_id)) }]; case "page_evaluate": - return [{ type: "dom_text", label: "evaluate", text: await this.evaluate(action.code, tabOf(action)) }]; + return [{ type: "page_text", label: "evaluate", text: await this.evaluate(action.code, tabOf(action)) }]; } } @@ -555,6 +555,6 @@ const INTERACTIVE_ROLES: ReadonlySet = new Set([ const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); -export function createDomExecutor(cdpWsUrl: string): DomExecutor { - return new DomExecutor(new CdpConnection(cdpWsUrl)); +export function createPageExecutor(cdpWsUrl: string): PageExecutor { + return new PageExecutor(new CdpConnection(cdpWsUrl)); } diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 7205c999..48b497cd 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -1,7 +1,7 @@ import type Kernel from "@onkernel/sdk"; import type { BrowserCreateResponse, BrowserRetrieveResponse } from "@onkernel/sdk/resources/browsers"; import { - isCuaDomAction, + isCuaBrowserAction, normalizeGotoUrl, type ComputerToolCoordinateSystem, type CuaAction, @@ -15,13 +15,13 @@ import { type CuaActionTypeText, type CuaActionWait, type CuaActionZoom, - type CuaDomAction, + type CuaBrowserAction, type CuaDragMouseButton, type CuaMouseButton, type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import sharp from "sharp"; -import { createDomExecutor, type DomExecutor } from "./dom"; +import { createPageExecutor, type PageExecutor } from "./page"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult } from "./types"; @@ -32,8 +32,8 @@ export interface InternalComputerTranslatorOptions { client: Kernel; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; - /** DOM executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ - createDomExecutor?: (cdpWsUrl: string) => DomExecutor; + /** Page executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ + createPageExecutor?: (cdpWsUrl: string) => PageExecutor; } export class InternalComputerTranslator { @@ -43,8 +43,8 @@ export class InternalComputerTranslator { private readonly screenshotSpec?: CuaScreenshotSpec; private readonly viewport: { width: number; height: number }; private readonly cdpWsUrl?: string; - private readonly domExecutorFactory: (cdpWsUrl: string) => DomExecutor; - private domExecutor?: DomExecutor; + private readonly pageExecutorFactory: (cdpWsUrl: string) => PageExecutor; + private pageExecutor?: PageExecutor; constructor(opts: InternalComputerTranslatorOptions) { this.sessionId = opts.browser.session_id; @@ -53,16 +53,16 @@ export class InternalComputerTranslator { this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; this.cdpWsUrl = opts.browser.cdp_ws_url; - this.domExecutorFactory = opts.createDomExecutor ?? createDomExecutor; + this.pageExecutorFactory = opts.createPageExecutor ?? createPageExecutor; } - /** The DOM-plane executor, connected lazily over the browser's CDP websocket. */ - dom(): DomExecutor { - if (!this.domExecutor) { - if (!this.cdpWsUrl) throw new Error("browser has no cdp_ws_url; DOM actions are unavailable"); - this.domExecutor = this.domExecutorFactory(this.cdpWsUrl); + /** The browser-plane executor, connected lazily over the browser's CDP websocket. */ + page(): PageExecutor { + if (!this.pageExecutor) { + if (!this.cdpWsUrl) throw new Error("browser has no cdp_ws_url; browser actions are unavailable"); + this.pageExecutor = this.pageExecutorFactory(this.cdpWsUrl); } - return this.domExecutor; + return this.pageExecutor; } async screenshotRaw(): Promise { @@ -126,9 +126,9 @@ export class InternalComputerTranslator { }; for (const action of actions) { - if (isCuaDomAction(action)) { + if (isCuaBrowserAction(action)) { await flush(); - result.readResults.push(...(await this.dom().execute(action))); + result.readResults.push(...(await this.page().execute(action))); continue; } switch (action.type) { @@ -195,7 +195,7 @@ export class InternalComputerTranslator { } private toSdkAction( - action: Exclude, + action: Exclude, ): KernelBatchAction { switch (action.type) { case "click": diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 2ff31967..4958fd53 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -2,7 +2,7 @@ export type BatchReadResult = | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number } - | { type: "dom_text"; label: string; text: string }; + | { type: "page_text"; label: string; text: string }; export interface BatchExecutionResult { readResults: BatchReadResult[]; diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 26946be8..da022560 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -186,16 +186,16 @@ describe("CuaAgent", () => { model: "anthropic:claude-opus-4-5", }, }); - expect(agent.getMode()).toBe("os"); + expect(agent.getMode()).toBe("computer"); expect(agent.state.tools.map((tool) => tool.name)).toContain("click"); - agent.setMode("dom"); + agent.setMode("browser"); - expect(agent.getMode()).toBe("dom"); + expect(agent.getMode()).toBe("browser"); const names = agent.state.tools.map((tool) => tool.name); expect(names).toContain("snapshot"); expect(names).not.toContain("move"); - expect(agent.state.systemPrompt).toBe(resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "dom" }).defaultSystemPrompt); + expect(agent.state.systemPrompt).toBe(resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "browser" }).defaultSystemPrompt); }); it("rejects setMode conflicting with a configured native tool", () => { @@ -207,9 +207,9 @@ describe("CuaAgent", () => { model: "anthropic:claude-opus-4-5", }, }); - expect(agent.getMode()).toBe("dom"); - expect(() => agent.setMode("os")).toThrow(/requires mode "dom"/); - expect(agent.getMode()).toBe("dom"); + expect(agent.getMode()).toBe("browser"); + expect(() => agent.setMode("computer")).toThrow(/requires mode "browser"/); + expect(agent.getMode()).toBe("browser"); }); it("keeps extra tools and caller-owned system prompt when state.model changes", () => { @@ -433,7 +433,7 @@ describe("CuaAgentHarness", () => { client, model: "anthropic:claude-opus-4-5", }); - expect(harness.getMode()).toBe("os"); + expect(harness.getMode()).toBe("computer"); await harness.setMode("hybrid"); diff --git a/packages/agent/test/translator-dom.test.ts b/packages/agent/test/translator-page.test.ts similarity index 86% rename from packages/agent/test/translator-dom.test.ts rename to packages/agent/test/translator-page.test.ts index cba6fc4c..d2f165e7 100644 --- a/packages/agent/test/translator-dom.test.ts +++ b/packages/agent/test/translator-page.test.ts @@ -1,8 +1,8 @@ import type Kernel from "@onkernel/sdk"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; -import type { CuaDomAction } from "@onkernel/cua-ai"; -import type { DomExecutor } from "../src/translator/dom"; +import type { CuaBrowserAction } from "@onkernel/cua-ai"; +import type { PageExecutor } from "../src/translator/page"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -25,15 +25,15 @@ function createClient() { } function createFakeDom() { - const executed: CuaDomAction[] = []; + const executed: CuaBrowserAction[] = []; const dom = { - execute: async (action: CuaDomAction): Promise => { + execute: async (action: CuaBrowserAction): Promise => { executed.push(action); - if (action.type === "page_text") return [{ type: "dom_text", label: "text", text: "hello" }]; + if (action.type === "page_text") return [{ type: "page_text", label: "text", text: "hello" }]; return []; }, screenshot: async () => ({ data: Buffer.from("png"), mimeType: "image/png" }), - } as unknown as DomExecutor; + } as unknown as PageExecutor; return { executed, dom }; } @@ -41,7 +41,7 @@ describe("InternalComputerTranslator DOM plane", () => { it("dispatches DOM actions to the DOM executor, flushing pending OS input first", async () => { const { batches, client } = createClient(); const { executed, dom } = createFakeDom(); - const translator = new InternalComputerTranslator({ browser, client, createDomExecutor: () => dom }); + const translator = new InternalComputerTranslator({ browser, client, createPageExecutor: () => dom }); const result = await translator.executeBatch([ { type: "click", x: 1, y: 2 }, @@ -51,7 +51,7 @@ describe("InternalComputerTranslator DOM plane", () => { expect(batches).toHaveLength(1); expect(executed.map((action) => action.type)).toEqual(["page_text", "page_click"]); - expect(result.readResults).toEqual([{ type: "dom_text", label: "text", text: "hello" }]); + expect(result.readResults).toEqual([{ type: "page_text", label: "text", text: "hello" }]); }); it("errors on DOM actions when the browser has no cdp_ws_url", async () => { diff --git a/packages/ai/src/actions/dom.ts b/packages/ai/src/actions/browser.ts similarity index 90% rename from packages/ai/src/actions/dom.ts rename to packages/ai/src/actions/browser.ts index f62ddacc..fab54f36 100644 --- a/packages/ai/src/actions/dom.ts +++ b/packages/ai/src/actions/browser.ts @@ -1,21 +1,21 @@ import { Type, type TSchema } from "@earendil-works/pi-ai"; /** - * DOM-plane canonical actions. + * Browser-plane canonical actions. * * These execute over CDP against the browser itself: accessibility-tree * reads with element references, element-targeted interaction, navigation, - * tabs, and viewport screenshots. Where a DOM action takes coordinates + * tabs, and viewport screenshots. Where a browser action takes coordinates * (`page_click`, `page_hover`, `page_drag`, `page_scroll`), they are pixels - * in the browser viewport — a different frame from the OS-plane actions in - * `./os`. Modes that expose both planes (hybrid) therefore restrict DOM + * in the browser viewport — a different frame from the computer-plane actions in + * `./computer`. Modes that expose both planes (hybrid) therefore restrict browser * actions to element references so exactly one coordinate frame is live. * * Element references (`ref`) are snapshot-scoped opaque ids (`e12`) minted * by `page_snapshot` / `page_find`; a stale ref is an error instructing the * model to re-snapshot. */ -export const CUA_DOM_ACTION_TYPES = [ +export const CUA_BROWSER_ACTION_TYPES = [ "page_snapshot", "page_text", "page_find", @@ -34,15 +34,15 @@ export const CUA_DOM_ACTION_TYPES = [ "page_evaluate", ] as const; -export type CuaDomActionType = (typeof CUA_DOM_ACTION_TYPES)[number]; +export type CuaBrowserActionType = (typeof CUA_BROWSER_ACTION_TYPES)[number]; /** - * The default DOM-mode toolset: everything except `page_evaluate`, which + * The default browser-mode toolset: everything except `page_evaluate`, which * runs arbitrary JavaScript in the page and must be enabled explicitly * (`javascriptExec: true`). */ -export const CUA_DEFAULT_DOM_ACTION_TYPES = CUA_DOM_ACTION_TYPES.filter( - (action): action is Exclude => action !== "page_evaluate", +export const CUA_DEFAULT_BROWSER_ACTION_TYPES = CUA_BROWSER_ACTION_TYPES.filter( + (action): action is Exclude => action !== "page_evaluate", ); export interface CuaActionPageSnapshot { @@ -153,7 +153,7 @@ export interface CuaActionPageEvaluate { tab_id?: string; } -export type CuaDomAction = +export type CuaBrowserAction = | CuaActionPageSnapshot | CuaActionPageText | CuaActionPageFind @@ -171,11 +171,11 @@ export type CuaDomAction = | CuaActionPageScreenshot | CuaActionPageEvaluate; -/** Options for building DOM action schemas. */ -export interface CuaDomSchemaOptions { +/** Options for building browser action schemas. */ +export interface CuaBrowserSchemaOptions { /** * Whether coordinate targeting is allowed on `page_click` / `page_hover` - * and whether `page_drag` / `page_scroll` are expressible at all. DOM + * and whether `page_drag` / `page_scroll` are expressible at all. Browser * mode allows viewport coordinates (they are the only frame); hybrid mode * must disallow them so the OS screenshot frame stays the single live * coordinate frame. @@ -187,7 +187,7 @@ const TabId = () => Type.Optional(Type.String({ description: "Tab to act on. Def const RefProperty = () => Type.String({ description: "Element reference from page_snapshot or page_find, e.g. \"e12\"." }); -export function createCuaDomActionSchemaByType(options: CuaDomSchemaOptions): Record { +export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOptions): Record { const clickTarget: Record = options.coordinates ? { ref: Type.Optional(RefProperty()), diff --git a/packages/ai/src/actions/os.ts b/packages/ai/src/actions/computer.ts similarity index 92% rename from packages/ai/src/actions/os.ts rename to packages/ai/src/actions/computer.ts index cc4c20ac..c6c55b8d 100644 --- a/packages/ai/src/actions/os.ts +++ b/packages/ai/src/actions/computer.ts @@ -1,14 +1,14 @@ import { Type, type TSchema } from "@earendil-works/pi-ai"; /** - * OS-plane canonical actions. + * Computer-plane canonical actions. * * These execute as real OS-level input against the Kernel browser VM (mouse, * keyboard, display capture) — never CDP. All coordinates are pixels in the - * OS screenshot frame. The DOM-plane vocabulary lives in `./dom` and is + * OS screenshot frame. The browser-plane vocabulary lives in `./browser` and is * executed over CDP; the two planes never share a coordinate frame. */ -export const CUA_OS_ACTION_TYPES = [ +export const CUA_COMPUTER_ACTION_TYPES = [ "click", "double_click", "mouse_down", @@ -28,15 +28,15 @@ export const CUA_OS_ACTION_TYPES = [ "cursor_position", ] as const; -export type CuaOsActionType = (typeof CUA_OS_ACTION_TYPES)[number]; +export type CuaComputerActionType = (typeof CUA_COMPUTER_ACTION_TYPES)[number]; /** * The default OS-mode toolset. This is the pre-modes canonical action list: * every OS action except `zoom`, which is only exposed by default in hybrid * mode and by Anthropic's native computer tool (`enable_zoom`). */ -export const CUA_DEFAULT_OS_ACTION_TYPES = CUA_OS_ACTION_TYPES.filter( - (action): action is Exclude => action !== "zoom", +export const CUA_DEFAULT_COMPUTER_ACTION_TYPES = CUA_COMPUTER_ACTION_TYPES.filter( + (action): action is Exclude => action !== "zoom", ); /** @@ -153,7 +153,7 @@ export interface CuaActionCursorPosition { type: "cursor_position"; } -export type CuaOsAction = +export type CuaComputerAction = | CuaActionClick | CuaActionDoubleClick | CuaActionMouseDown @@ -180,7 +180,7 @@ const PointSchema = Type.Object( { additionalProperties: false }, ); -export const CUA_OS_ACTION_SCHEMA_BY_TYPE = { +export const CUA_COMPUTER_ACTION_SCHEMA_BY_TYPE = { click: Type.Object( { type: Type.Literal("click"), @@ -296,6 +296,6 @@ export const CUA_OS_ACTION_SCHEMA_BY_TYPE = { forward: Type.Object({ type: Type.Literal("forward") }, { additionalProperties: false }), url: Type.Object({ type: Type.Literal("url") }, { additionalProperties: false }), cursor_position: Type.Object({ type: Type.Literal("cursor_position") }, { additionalProperties: false }), -} satisfies Record; +} satisfies Record; export type CuaZoomRegion = CuaActionZoom["region"]; diff --git a/packages/ai/src/actions/index.ts b/packages/ai/src/actions/index.ts index 7bb01ca3..6fec9fab 100644 --- a/packages/ai/src/actions/index.ts +++ b/packages/ai/src/actions/index.ts @@ -1,47 +1,47 @@ import type { TSchema } from "@earendil-works/pi-ai"; -import { CUA_DOM_ACTION_TYPES, createCuaDomActionSchemaByType, type CuaDomAction, type CuaDomActionType, type CuaDomSchemaOptions } from "./dom"; -import { CUA_OS_ACTION_SCHEMA_BY_TYPE, CUA_OS_ACTION_TYPES, type CuaOsAction, type CuaOsActionType } from "./os"; +import { CUA_BROWSER_ACTION_TYPES, createCuaBrowserActionSchemaByType, type CuaBrowserAction, type CuaBrowserActionType, type CuaBrowserSchemaOptions } from "./browser"; +import { CUA_COMPUTER_ACTION_SCHEMA_BY_TYPE, CUA_COMPUTER_ACTION_TYPES, type CuaComputerAction, type CuaComputerActionType } from "./computer"; -export * from "./dom"; -export * from "./os"; +export * from "./browser"; +export * from "./computer"; -/** Any canonical CUA action type, across the OS and DOM planes. */ -export type CuaActionType = CuaOsActionType | CuaDomActionType; +/** Any canonical CUA action type, across the OS and browser planes. */ +export type CuaActionType = CuaComputerActionType | CuaBrowserActionType; -/** Any canonical CUA action, across the OS and DOM planes. */ -export type CuaAction = CuaOsAction | CuaDomAction; +/** Any canonical CUA action, across the OS and browser planes. */ +export type CuaAction = CuaComputerAction | CuaBrowserAction; -/** Every canonical action type: the OS plane followed by the DOM plane. */ -export const CUA_ALL_ACTION_TYPES: readonly CuaActionType[] = [...CUA_OS_ACTION_TYPES, ...CUA_DOM_ACTION_TYPES]; +/** Every canonical action type: the computer plane followed by the browser plane. */ +export const CUA_ALL_ACTION_TYPES: readonly CuaActionType[] = [...CUA_COMPUTER_ACTION_TYPES, ...CUA_BROWSER_ACTION_TYPES]; -const OS_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_OS_ACTION_TYPES); -const DOM_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_DOM_ACTION_TYPES); +const OS_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_COMPUTER_ACTION_TYPES); +const DOM_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_BROWSER_ACTION_TYPES); -/** Whether a canonical action type belongs to the OS plane. */ -export function isCuaOsActionType(action: CuaActionType): action is CuaOsActionType { +/** Whether a canonical action type belongs to the computer plane. */ +export function isCuaComputerActionType(action: CuaActionType): action is CuaComputerActionType { return OS_ACTION_TYPE_SET.has(action); } -/** Whether a canonical action type belongs to the DOM plane. */ -export function isCuaDomActionType(action: CuaActionType): action is CuaDomActionType { +/** Whether a canonical action type belongs to the browser plane. */ +export function isCuaBrowserActionType(action: CuaActionType): action is CuaBrowserActionType { return DOM_ACTION_TYPE_SET.has(action); } -/** Whether a canonical action belongs to the DOM plane. */ -export function isCuaDomAction(action: CuaAction): action is CuaDomAction { +/** Whether a canonical action belongs to the browser plane. */ +export function isCuaBrowserAction(action: CuaAction): action is CuaBrowserAction { return DOM_ACTION_TYPE_SET.has(action.type); } /** Options for building canonical action schemas. */ export interface CuaActionSchemaOptions { - /** DOM-plane schema variants; see {@link CuaDomSchemaOptions}. Defaults to coordinates allowed. */ - dom?: CuaDomSchemaOptions; + /** browser-plane schema variants; see {@link CuaBrowserSchemaOptions}. Defaults to coordinates allowed. */ + browser?: CuaBrowserSchemaOptions; } /** Build the full action-type → schema map for a schema-options combination. */ export function cuaActionSchemaByType(options: CuaActionSchemaOptions = {}): Record { return { - ...CUA_OS_ACTION_SCHEMA_BY_TYPE, - ...createCuaDomActionSchemaByType(options.dom ?? { coordinates: true }), + ...CUA_COMPUTER_ACTION_SCHEMA_BY_TYPE, + ...createCuaBrowserActionSchemaByType(options.browser ?? { coordinates: true }), }; } diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index df35488c..e4c7d0b5 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -1,27 +1,29 @@ import { - CUA_DEFAULT_DOM_ACTION_TYPES, - CUA_DEFAULT_OS_ACTION_TYPES, - isCuaOsActionType, + CUA_DEFAULT_BROWSER_ACTION_TYPES, + CUA_DEFAULT_COMPUTER_ACTION_TYPES, + isCuaComputerActionType, type CuaActionSchemaOptions, type CuaActionType, - type CuaDomActionType, - type CuaOsActionType, + type CuaBrowserActionType, + type CuaComputerActionType, } from "./actions/index"; /** * Which canonical action plane(s) a CUA agent exposes to the model. * - * - `os` — OS-level input only (mouse/keyboard/display against the VM). - * Today's default; coordinates are OS screenshot pixels. - * - `dom` — DOM-level tools only, driven over CDP: accessibility snapshots + * - `computer` — OS-level input only (mouse/keyboard/display against the + * VM). Today's default; coordinates are OS screenshot pixels. Pairs with + * Anthropic's native `computer_20260601` tool. + * - `browser` — page tools only, driven over CDP: accessibility snapshots * with element refs, element-targeted interaction, navigation, tabs, and * viewport screenshots. Coordinates, where used, are viewport pixels. - * - `hybrid` — both planes, deduplicated to one tool per capability. OS - * tools are prefixed `computer_`, DOM tools keep their `page_` prefix, - * DOM tools accept element refs only, and the OS screenshot frame is the - * single live coordinate frame. + * Pairs with Anthropic's native `browser_20260701` tool. + * - `hybrid` — both planes, deduplicated to one tool per capability. + * Computer tools are prefixed `computer_`, browser tools keep their + * `page_` prefix and accept element refs only, and the OS screenshot + * frame is the single live coordinate frame. */ -export type CuaMode = "os" | "dom" | "hybrid"; +export type CuaMode = "computer" | "browser" | "hybrid"; /** Options for resolving a mode's action set. */ export interface CuaModeOptions { @@ -30,11 +32,12 @@ export interface CuaModeOptions { } /** - * OS actions exposed in hybrid mode: navigation reads/writes are excluded - * because they live on the DOM plane (`page_navigate`, `page_list_tabs`), - * and `zoom` is included since the OS screenshot is hybrid's only capture. + * Computer actions exposed in hybrid mode: navigation reads/writes are + * excluded because they live on the browser plane (`page_navigate`, + * `page_list_tabs`), and `zoom` is included since the OS screenshot is + * hybrid's only capture. */ -export const CUA_HYBRID_OS_ACTION_TYPES: readonly CuaOsActionType[] = [ +export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] = [ "click", "double_click", "mouse_down", @@ -51,13 +54,13 @@ export const CUA_HYBRID_OS_ACTION_TYPES: readonly CuaOsActionType[] = [ ]; /** - * DOM actions exposed in hybrid mode: reads and element-targeted writes + * Browser actions exposed in hybrid mode: reads and element-targeted writes * only. Pointer/keyboard capabilities (`page_click` by coordinate, * `page_type`, `page_key`, `page_scroll`, `page_hover`, `page_drag`) and * `page_screenshot` are excluded — real OS input and the OS screenshot cover * those, keeping one tool per capability and one coordinate frame. */ -export const CUA_HYBRID_DOM_ACTION_TYPES: readonly CuaDomActionType[] = [ +export const CUA_HYBRID_BROWSER_ACTION_TYPES: readonly CuaBrowserActionType[] = [ "page_snapshot", "page_text", "page_find", @@ -72,14 +75,14 @@ export const CUA_HYBRID_DOM_ACTION_TYPES: readonly CuaDomActionType[] = [ /** Resolve the default canonical action set for a mode. */ export function defaultActionsForMode(mode: CuaMode, options: CuaModeOptions = {}): readonly CuaActionType[] { switch (mode) { - case "os": - return CUA_DEFAULT_OS_ACTION_TYPES; - case "dom": - return [...CUA_DEFAULT_DOM_ACTION_TYPES, ...(options.javascriptExec ? (["page_evaluate"] as const) : []), "wait"]; + case "computer": + return CUA_DEFAULT_COMPUTER_ACTION_TYPES; + case "browser": + return [...CUA_DEFAULT_BROWSER_ACTION_TYPES, ...(options.javascriptExec ? (["page_evaluate"] as const) : []), "wait"]; case "hybrid": return [ - ...CUA_HYBRID_OS_ACTION_TYPES, - ...CUA_HYBRID_DOM_ACTION_TYPES, + ...CUA_HYBRID_COMPUTER_ACTION_TYPES, + ...CUA_HYBRID_BROWSER_ACTION_TYPES, ...(options.javascriptExec ? (["page_evaluate"] as const) : []), ]; } @@ -87,33 +90,34 @@ export function defaultActionsForMode(mode: CuaMode, options: CuaModeOptions = { /** Resolve the schema-building options for a mode; see {@link CuaActionSchemaOptions}. */ export function schemaOptionsForMode(mode: CuaMode): CuaActionSchemaOptions { - // Hybrid restricts DOM actions to element refs so the OS screenshot frame - // is the single live coordinate frame. DOM mode has no OS frame, so - // viewport coordinates are allowed there. - return { dom: { coordinates: mode !== "hybrid" } }; + // Hybrid restricts browser actions to element refs so the OS screenshot + // frame is the single live coordinate frame. Browser mode has no OS frame, + // so viewport coordinates are allowed there. + return { browser: { coordinates: mode !== "hybrid" } }; } /** * The model-facing tool name for a canonical action in a mode. * - * - `os`: canonical action ids as-is (`click`, `screenshot`, …). - * - `dom`: DOM ids with the `page_` prefix stripped (`snapshot`, `click`, …); - * the prefix only exists to disambiguate planes, and dom mode has one. - * - `hybrid`: OS ids prefixed `computer_`, DOM ids kept as `page_*`. + * - `computer`: canonical action ids as-is (`click`, `screenshot`, …). + * - `browser`: browser ids with the `page_` prefix stripped (`snapshot`, + * `click`, …); the prefix only exists to disambiguate planes, and + * browser mode has one. + * - `hybrid`: computer ids prefixed `computer_`, browser ids kept as `page_*`. */ export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): string { switch (mode) { - case "os": - if (!isCuaOsActionType(action)) throw new Error(`DOM action "${action}" is not available in os mode`); + case "computer": + if (!isCuaComputerActionType(action)) throw new Error(`browser action "${action}" is not available in computer mode`); return action; - case "dom": - return isCuaOsActionType(action) ? action : action.slice("page_".length); + case "browser": + return isCuaComputerActionType(action) ? action : action.slice("page_".length); case "hybrid": - return isCuaOsActionType(action) ? `computer_${action}` : action; + return isCuaComputerActionType(action) ? `computer_${action}` : action; } } -const DOM_ACTION_DESCRIPTIONS: Record = { +const BROWSER_ACTION_DESCRIPTIONS: Record = { page_snapshot: "Return an accessibility-tree snapshot of the page with element references like [e12]. " + "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale.", @@ -137,7 +141,7 @@ const DOM_ACTION_DESCRIPTIONS: Record = { // Hybrid exposes both planes, so tool descriptions carry the arbitration // rules the model needs: which plane is preferred for a capability and why // (real OS input vs CDP), plus the single-coordinate-frame statement. -const HYBRID_OS_DESCRIPTION_OVERRIDES: Partial> = { +const HYBRID_COMPUTER_DESCRIPTION_OVERRIDES: Partial> = { click: "Click at a coordinate in OS screenshot pixels using real OS-level input. " + "Preferred over page_click when the target is visible in the screenshot — OS input is indistinguishable from a human user.", @@ -148,7 +152,7 @@ const HYBRID_OS_DESCRIPTION_OVERRIDES: Partial> keypress: "Press keys with OS-level keyboard input.", }; -const HYBRID_DOM_DESCRIPTION_OVERRIDES: Partial> = { +const HYBRID_BROWSER_DESCRIPTION_OVERRIDES: Partial> = { page_click: "Click an element by reference from a page_snapshot. Dispatched via CDP, which protected sites may detect — " + "prefer computer_click when the element is visible in the screenshot; use page_click for elements that are hard to hit by coordinate.", @@ -160,14 +164,14 @@ const HYBRID_DOM_DESCRIPTION_OVERRIDES: Partial /** The model-facing tool description for a canonical action in a mode. */ export function cuaToolDescriptionForAction(action: CuaActionType, mode: CuaMode): string { - if (isCuaOsActionType(action)) { + if (isCuaComputerActionType(action)) { if (mode === "hybrid") { - return HYBRID_OS_DESCRIPTION_OVERRIDES[action] ?? `Execute one ${action} computer action using real OS-level input.`; + return HYBRID_COMPUTER_DESCRIPTION_OVERRIDES[action] ?? `Execute one ${action} computer action using real OS-level input.`; } return `Execute one ${action} computer action.`; } if (mode === "hybrid") { - return HYBRID_DOM_DESCRIPTION_OVERRIDES[action] ?? DOM_ACTION_DESCRIPTIONS[action]; + return HYBRID_BROWSER_DESCRIPTION_OVERRIDES[action] ?? BROWSER_ACTION_DESCRIPTIONS[action]; } - return DOM_ACTION_DESCRIPTIONS[action]; + return BROWSER_ACTION_DESCRIPTIONS[action]; } diff --git a/packages/ai/src/native-tools.ts b/packages/ai/src/native-tools.ts index 6263efdc..8846eb7f 100644 --- a/packages/ai/src/native-tools.ts +++ b/packages/ai/src/native-tools.ts @@ -5,7 +5,7 @@ import type { CuaMode } from "./modes"; * Anthropic's native computer-use tool (`anthropic-beta: * computer-use-2026-06-01`). Server-defined: the declaration below is sent * verbatim in `tools[]` and Anthropic fixes the input schema. Maps to CUA's - * `os` mode — actions arrive as OS-plane input in screenshot-pixel + * `computer` mode — actions arrive as OS-level input in screenshot-pixel * coordinates. */ export interface AnthropicComputerNativeTool { @@ -23,7 +23,7 @@ export interface AnthropicComputerNativeTool { /** * Anthropic's native browser tool (`anthropic-beta: browser-use-2026-07-01`, * proposed — the tool version and schema may change before release). Maps to - * CUA's `dom` mode — DOM reads by element reference plus pointer actions in + * CUA's `browser` mode — page reads by element reference plus pointer actions in * viewport-pixel coordinates. */ export interface AnthropicBrowserNativeTool { @@ -50,15 +50,15 @@ export type CuaNativeToolSpec = AnthropicComputerNativeTool | AnthropicBrowserNa export type CuaNativeToolType = CuaNativeToolSpec["type"]; interface NativeToolInfo { - mode: Extract; + mode: Extract; provider: "anthropic"; betaHeader: string; defaultName: string; } const NATIVE_TOOL_INFO: Record = { - computer_20260601: { mode: "os", provider: "anthropic", betaHeader: "computer-use-2026-06-01", defaultName: "computer" }, - browser_20260701: { mode: "dom", provider: "anthropic", betaHeader: "browser-use-2026-07-01", defaultName: "browser" }, + computer_20260601: { mode: "computer", provider: "anthropic", betaHeader: "computer-use-2026-06-01", defaultName: "computer" }, + browser_20260701: { mode: "browser", provider: "anthropic", betaHeader: "browser-use-2026-07-01", defaultName: "browser" }, }; /** The {@link CuaMode} a native tool requires. */ diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts index 38b18b4c..2058fa06 100644 --- a/packages/ai/src/providers/anthropic/actions.ts +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -2,14 +2,14 @@ import type { Tool, TSchema } from "@earendil-works/pi-ai"; import { CUA_BATCH_TOOL_DESCRIPTION, CUA_BATCH_TOOL_NAME, - CUA_DOM_ACTION_TYPES, + CUA_BROWSER_ACTION_TYPES, createCuaActionSchema, createCuaActionToolExecutors, createCuaActionToolDefinitions, createCuaBatchToolExecutor, createCuaBatchToolDefinition, defaultActionsForMode, - isCuaDomActionType, + isCuaBrowserActionType, type ComputerToolsOptions, type CuaAction, type CuaActionType, @@ -43,9 +43,9 @@ export const ANTHROPIC_CUA_ACTION_TYPES = [ "cursor_position", ] as const satisfies readonly CuaActionType[]; -type AnthropicCanonicalActionType = (typeof ANTHROPIC_CUA_ACTION_TYPES)[number] | (typeof CUA_DOM_ACTION_TYPES)[number]; +type AnthropicCanonicalActionType = (typeof ANTHROPIC_CUA_ACTION_TYPES)[number] | (typeof CUA_BROWSER_ACTION_TYPES)[number]; -const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set([...ANTHROPIC_CUA_ACTION_TYPES, ...CUA_DOM_ACTION_TYPES]); +const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set([...ANTHROPIC_CUA_ACTION_TYPES, ...CUA_BROWSER_ACTION_TYPES]); /** Name of the batch tool included by default in Anthropic computer-use tools. */ export const ANTHROPIC_BATCH_TOOL_NAME = CUA_BATCH_TOOL_NAME; @@ -65,13 +65,13 @@ export interface AnthropicComputerToolsOptions extends ComputerToolsOptions { export type AnthropicAction = Extract; function resolveAnthropicActions(options: AnthropicComputerToolsOptions): readonly AnthropicCanonicalActionType[] { - const mode = options.mode ?? "os"; + const mode = options.mode ?? "computer"; const resolved = options.actions ?? - (mode === "os" + (mode === "computer" ? ANTHROPIC_CUA_ACTION_TYPES.filter((action) => action !== "zoom") : defaultActionsForMode(mode, { javascriptExec: options.javascriptExec }).filter( - (action) => isCuaDomActionType(action) || isAnthropicCanonicalAction(action), + (action) => isCuaBrowserActionType(action) || isAnthropicCanonicalAction(action), )); const supported: AnthropicCanonicalActionType[] = []; const unsupported: CuaActionType[] = []; @@ -88,7 +88,7 @@ function isAnthropicCanonicalAction(action: CuaActionType): action is AnthropicC } /** Build the TypeBox schema for Anthropic-supported canonical browser actions. */ -export function createActionSchema(actions?: readonly CuaActionType[], mode: CuaMode = "os"): TSchema { +export function createActionSchema(actions?: readonly CuaActionType[], mode: CuaMode = "computer"): TSchema { return createCuaActionSchema(resolveAnthropicActions({ actions, mode }), mode); } @@ -101,7 +101,7 @@ export function createActionSchema(actions?: readonly CuaActionType[], mode: Cua * batch tool by default; pass `excludeBatch: true` to omit it. */ export function computerTools(options: AnthropicComputerToolsOptions = {}): Tool[] { - const mode = options.mode ?? "os"; + const mode = options.mode ?? "computer"; const actions = resolveAnthropicActions(options); const tools = createCuaActionToolDefinitions(actions, mode); if (!options.excludeBatch) { @@ -116,7 +116,7 @@ export function computerTools(options: AnthropicComputerToolsOptions = {}): Tool /** Build the local execution adapters used by CuaAgent and CuaAgentHarness. */ export function computerToolExecutors(options: AnthropicComputerToolsOptions = {}): CuaToolExecutorSpec[] { - const mode = options.mode ?? "os"; + const mode = options.mode ?? "computer"; const actions = resolveAnthropicActions(options); const executors = createCuaActionToolExecutors(actions, mode); if (!options.excludeBatch) { diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 98b3dcd1..e16642a9 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -35,14 +35,14 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use keyboard navigation where possible, and request screenshots when you need to inspect state.`; -export const ANTHROPIC_DOM_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; +export const ANTHROPIC_BROWSER_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; export const ANTHROPIC_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and page_* tools read and act on the page itself by element reference. Prefer page_snapshot/page_find for reading and locating, and computer_* input for interaction; use page_* interaction for elements that are hard to hit by coordinate.`; export function buildAnthropicSystemPrompt(opts: { suffix?: string; mode?: CuaMode } = {}): string { const base = - opts.mode === "dom" - ? ANTHROPIC_DOM_INSTRUCTIONS + opts.mode === "browser" + ? ANTHROPIC_BROWSER_INSTRUCTIONS : opts.mode === "hybrid" ? ANTHROPIC_HYBRID_INSTRUCTIONS : ANTHROPIC_COMPUTER_INSTRUCTIONS; diff --git a/packages/ai/src/providers/anthropic/native.ts b/packages/ai/src/providers/anthropic/native.ts index ba4e27ff..31b4ff77 100644 --- a/packages/ai/src/providers/anthropic/native.ts +++ b/packages/ai/src/providers/anthropic/native.ts @@ -83,7 +83,7 @@ function asNativeInput(args: unknown): NativeInput { const MAX_KEY_REPEAT = 100; -/** Map one `computer_20260601` tool input onto canonical OS-plane actions. */ +/** Map one `computer_20260601` tool input onto canonical computer-plane actions. */ export function mapNativeComputerInput(input: NativeInput): CuaAction[] { switch (input.action) { case "screenshot": @@ -132,7 +132,7 @@ export function mapNativeComputerInput(input: NativeInput): CuaAction[] { } } -/** Map one `browser_20260701` tool input onto canonical DOM-plane actions. */ +/** Map one `browser_20260701` tool input onto canonical browser-plane actions. */ export function mapNativeBrowserInput(input: NativeInput): CuaAction[] { const tab = tabId(input); switch (input.action) { diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 59a2faf3..81eeb4c2 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -9,7 +9,7 @@ import { type TSchema, type Tool, } from "@earendil-works/pi-ai"; -import { CUA_DEFAULT_OS_ACTION_TYPES, cuaActionSchemaByType, type CuaAction, type CuaActionType } from "../actions/index"; +import { CUA_DEFAULT_COMPUTER_ACTION_TYPES, cuaActionSchemaByType, type CuaAction, type CuaActionType } from "../actions/index"; import { cuaToolDescriptionForAction, cuaToolNameForAction, defaultActionsForMode, schemaOptionsForMode, type CuaMode } from "../modes"; import type { ResolvedCuaNativeTool } from "../native-tools"; import type { CuaModelRef, CuaProvider } from "../models"; @@ -19,11 +19,11 @@ export * from "../modes"; export * from "../native-tools"; /** - * The default os-mode action set: every OS-plane action except `zoom`. + * The default os-mode action set: every computer-plane action except `zoom`. * The full canonical vocabulary is split by plane into - * {@link CUA_OS_ACTION_TYPES} and {@link CUA_DOM_ACTION_TYPES}. + * {@link CUA_COMPUTER_ACTION_TYPES} and {@link CUA_BROWSER_ACTION_TYPES}. */ -export const CUA_ACTION_TYPES = CUA_DEFAULT_OS_ACTION_TYPES; +export const CUA_ACTION_TYPES = CUA_DEFAULT_COMPUTER_ACTION_TYPES; type ObjectSchemaWithProperties = TSchema & { properties: Record }; @@ -33,14 +33,14 @@ function createCuaActionArgumentSchema(action: CuaActionType, mode: CuaMode): TS return Type.Object(properties, { additionalProperties: false }); } -export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "os"): TSchema { +export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): TSchema { if (actions.length === 0) throw new Error("actions must include at least one CUA action type"); const schemaByType = cuaActionSchemaByType(schemaOptionsForMode(mode)); if (actions.length === 1) return schemaByType[actions[0]!]; return Type.Union(actions.map((action) => schemaByType[action])); } -export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "os"): Tool[] { +export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): Tool[] { return actions.map((action) => ({ name: cuaToolNameForAction(action, mode), description: cuaToolDescriptionForAction(action, mode), @@ -50,7 +50,7 @@ export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] export const CuaActionSchema = createCuaActionSchema(); -export function createCuaBatchSchema(actions?: readonly CuaActionType[], mode: CuaMode = "os"): TSchema { +export function createCuaBatchSchema(actions?: readonly CuaActionType[], mode: CuaMode = "computer"): TSchema { return Type.Object({ actions: Type.Array(createCuaActionSchema(actions, mode), { description: "Ordered computer actions to execute." }), }); @@ -117,9 +117,9 @@ export const CUA_PLAYWRIGHT_TOOL_DESCRIPTION = [ export interface ComputerToolsOptions { actions?: readonly CuaActionType[]; - /** Which action plane(s) to expose. Default "os". */ + /** Which action plane(s) to expose. Default "computer". */ mode?: CuaMode; - /** Expose `page_evaluate` in dom/hybrid modes. Default false. */ + /** Expose `page_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; } @@ -140,22 +140,22 @@ export type ComputerToolCoordinateSystem = * smaller set, such as `["click"]`. */ export function computerTools(options: ComputerToolsOptions = {}): Tool[] { - return createCuaActionToolDefinitions(resolveModeActions(options), options.mode ?? "os"); + return createCuaActionToolDefinitions(resolveModeActions(options), options.mode ?? "computer"); } /** Resolve the action list for a tools-options object: explicit list, or the mode's default set. */ export function resolveModeActions(options: ComputerToolsOptions = {}): readonly CuaActionType[] { - return options.actions ?? defaultActionsForMode(options.mode ?? "os", { javascriptExec: options.javascriptExec }); + return options.actions ?? defaultActionsForMode(options.mode ?? "computer", { javascriptExec: options.javascriptExec }); } -/** Guard for providers whose computer-use vocabulary only covers the OS plane. */ -export function assertOsModeOnly(provider: CuaProvider, options: ComputerToolsOptions = {}): void { - const mode = options.mode ?? "os"; - if (mode !== "os") throw new Error(`provider "${provider}" does not support mode "${mode}" (os only)`); +/** Guard for providers whose computer-use vocabulary only covers the computer plane. */ +export function assertComputerModeOnly(provider: CuaProvider, options: ComputerToolsOptions = {}): void { + const mode = options.mode ?? "computer"; + if (mode !== "computer") throw new Error(`provider "${provider}" does not support mode "${mode}" (computer only)`); } /** Build execution adapters for individual canonical CUA action tools. */ -export function createCuaActionToolExecutors(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "os"): CuaToolExecutorSpec[] { +export function createCuaActionToolExecutors(actions: readonly CuaActionType[] = CUA_ACTION_TYPES, mode: CuaMode = "computer"): CuaToolExecutorSpec[] { const definitions = createCuaActionToolDefinitions(actions, mode); return definitions.map((definition, index) => { const actionType = actions[index]!; @@ -194,7 +194,7 @@ export function createCuaBatchToolDefinition( return { name: options.name ?? CUA_BATCH_TOOL_NAME, description: options.description ?? CUA_BATCH_TOOL_DESCRIPTION, - parameters: createCuaBatchSchema(actions, options.mode ?? "os"), + parameters: createCuaBatchSchema(actions, options.mode ?? "computer"), }; } @@ -215,7 +215,7 @@ export function createCuaBatchToolExecutor( /** Build the provider's default CUA tool execution adapters. */ export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaToolExecutorSpec[] { - return createCuaActionToolExecutors(resolveModeActions(options), options.mode ?? "os"); + return createCuaActionToolExecutors(resolveModeActions(options), options.mode ?? "computer"); } function isBatchInput(value: unknown): value is CuaBatchInput { diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index e1eb49ae..87875f13 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -1,4 +1,4 @@ -import { assertOsModeOnly, computerToolExecutors, computerTools } from "../common"; +import { assertComputerModeOnly, computerToolExecutors, computerTools } from "../common"; import type { ComputerToolCoordinateSystem, ComputerToolsOptions, CuaProviderModule } from "../common"; export { @@ -31,14 +31,14 @@ export function buildGeminiSystemPrompt(opts: { suffix?: string } = {}): string export const providerModule = { // Gemini's computer-use coordinate convention is normalized 0-999, which - // only maps onto the OS plane today; DOM-plane viewport coordinates are + // only maps onto the computer plane today; browser-plane viewport coordinates are // unvalidated for it. toolDefinitions: (options?: ComputerToolsOptions) => { - assertOsModeOnly("google", options); + assertComputerModeOnly("google", options); return computerTools(options); }, toolExecutors: (options?: ComputerToolsOptions) => { - assertOsModeOnly("google", options); + assertComputerModeOnly("google", options); return computerToolExecutors(options); }, coordinateSystem, diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index a8c6df24..7b33f684 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -31,13 +31,13 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { export const OPENAI_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use the available tools for browser interaction and request explicit url, cursor_position, or screenshot reads when you need updated state.`; -export const OPENAI_DOM_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; +export const OPENAI_BROWSER_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; export const OPENAI_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and page_* tools read and act on the page itself by element reference. Prefer page_snapshot/page_find for reading and locating, and computer_* input for interaction; use page_* interaction for elements that are hard to hit by coordinate.`; export function buildOpenAISystemPrompt(opts: { suffix?: string; mode?: CuaMode } = {}): string { const base = - opts.mode === "dom" ? OPENAI_DOM_INSTRUCTIONS : opts.mode === "hybrid" ? OPENAI_HYBRID_INSTRUCTIONS : OPENAI_COMPUTER_INSTRUCTIONS; + opts.mode === "browser" ? OPENAI_BROWSER_INSTRUCTIONS : opts.mode === "hybrid" ? OPENAI_HYBRID_INSTRUCTIONS : OPENAI_COMPUTER_INSTRUCTIONS; return [base, opts.suffix].filter(Boolean).join("\n\n"); } diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 2dc5ae03..61065218 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -1,5 +1,5 @@ import { - assertOsModeOnly, + assertComputerModeOnly, computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, @@ -52,11 +52,11 @@ export function buildTzafonSystemPrompt(opts: { suffix?: string } = {}): string export const providerModule = { toolDefinitions: (options?: ComputerToolsOptions) => { - assertOsModeOnly("tzafon", options); + assertComputerModeOnly("tzafon", options); return computerTools(options); }, toolExecutors: (options?: ComputerToolsOptions) => { - assertOsModeOnly("tzafon", options); + assertComputerModeOnly("tzafon", options); return computerToolExecutors(options); }, coordinateSystem, diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index 5f2244b6..1fa91f5d 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,4 +1,4 @@ -import { assertOsModeOnly, type ComputerToolCoordinateSystem, type ComputerToolsOptions, type CuaProviderModule } from "../common"; +import { assertComputerModeOnly, type ComputerToolCoordinateSystem, type ComputerToolsOptions, type CuaProviderModule } from "../common"; import { computerToolExecutors } from "./actions"; import { yutoriCuaOnPayload } from "./provider"; @@ -64,11 +64,11 @@ export function buildYutoriSystemPrompt(opts: { suffix?: string } = {}): string export const providerModule = { toolDefinitions: (options?: ComputerToolsOptions) => { - assertOsModeOnly("yutori", options); + assertComputerModeOnly("yutori", options); return []; }, toolExecutors: (options?: ComputerToolsOptions) => { - assertOsModeOnly("yutori", options); + assertComputerModeOnly("yutori", options); return computerToolExecutors(options); }, coordinateSystem, diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 37367b4a..76261c99 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -29,8 +29,8 @@ export interface CuaRuntimeSpecOptions extends ComputerToolsOptions { /** * Drive the model through a provider-native tool declaration instead of * CUA's canonical function tools. The native tool determines (and is - * validated against) the mode: `computer_20260601` requires `"os"`, - * `browser_20260701` requires `"dom"`. When `mode` is omitted it is + * validated against) the mode: `computer_20260601` requires `"computer"`, + * `browser_20260701` requires `"browser"`. When `mode` is omitted it is * inferred from the native tool. */ nativeTool?: CuaNativeToolSpec; @@ -48,7 +48,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options: CuaRu const model = typeof input === "string" ? getCuaModel(input) : routeCuaApi(input); const provider = providerForModel(model); const mod: CuaProviderModule = PROVIDERS[provider]; - const mode = options.mode ?? (options.nativeTool ? modeForNativeTool(options.nativeTool) : "os"); + const mode = options.mode ?? (options.nativeTool ? modeForNativeTool(options.nativeTool) : "computer"); if (options.nativeTool) { const nativeTool = resolveNativeTool(options.nativeTool, model, mode); diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index 357ad161..0d3d8fb7 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from "vitest"; import { CUA_ACTION_TYPES, - CUA_DEFAULT_DOM_ACTION_TYPES, - CUA_HYBRID_DOM_ACTION_TYPES, - CUA_HYBRID_OS_ACTION_TYPES, + CUA_DEFAULT_BROWSER_ACTION_TYPES, + CUA_HYBRID_BROWSER_ACTION_TYPES, + CUA_HYBRID_COMPUTER_ACTION_TYPES, anthropic, computerTools, cuaToolNameForAction, @@ -13,20 +13,20 @@ import { } from "../src/index"; describe("mode action sets", () => { - it("os mode defaults to the legacy action set", () => { - expect(defaultActionsForMode("os")).toEqual(CUA_ACTION_TYPES); + it("computer mode defaults to the legacy action set", () => { + expect(defaultActionsForMode("computer")).toEqual(CUA_ACTION_TYPES); }); - it("dom mode defaults to DOM actions plus wait, without page_evaluate", () => { - const actions = defaultActionsForMode("dom"); + it("browser mode defaults to DOM actions plus wait, without page_evaluate", () => { + const actions = defaultActionsForMode("browser"); expect(actions).toContain("page_snapshot"); expect(actions).toContain("wait"); expect(actions).not.toContain("page_evaluate"); expect(actions).not.toContain("click"); }); - it("dom mode exposes page_evaluate only with javascriptExec", () => { - expect(defaultActionsForMode("dom", { javascriptExec: true })).toContain("page_evaluate"); + it("browser mode exposes page_evaluate only with javascriptExec", () => { + expect(defaultActionsForMode("browser", { javascriptExec: true })).toContain("page_evaluate"); }); it("hybrid mode dedupes to one tool per capability", () => { @@ -42,34 +42,34 @@ describe("mode action sets", () => { // Pointer/keyboard stays OS-level. expect(actions).not.toContain("page_type"); expect(actions).not.toContain("page_scroll"); - expect(actions).toEqual([...CUA_HYBRID_OS_ACTION_TYPES, ...CUA_HYBRID_DOM_ACTION_TYPES]); + expect(actions).toEqual([...CUA_HYBRID_COMPUTER_ACTION_TYPES, ...CUA_HYBRID_BROWSER_ACTION_TYPES]); }); }); describe("mode tool naming", () => { - it("os mode keeps canonical action ids", () => { - expect(cuaToolNameForAction("click", "os")).toBe("click"); + it("computer mode keeps canonical action ids", () => { + expect(cuaToolNameForAction("click", "computer")).toBe("click"); }); - it("dom mode strips the page_ prefix", () => { - expect(cuaToolNameForAction("page_snapshot", "dom")).toBe("snapshot"); - expect(cuaToolNameForAction("page_click", "dom")).toBe("click"); - expect(cuaToolNameForAction("wait", "dom")).toBe("wait"); + it("browser mode strips the page_ prefix", () => { + expect(cuaToolNameForAction("page_snapshot", "browser")).toBe("snapshot"); + expect(cuaToolNameForAction("page_click", "browser")).toBe("click"); + expect(cuaToolNameForAction("wait", "browser")).toBe("wait"); }); - it("hybrid mode prefixes OS actions and keeps page_ names", () => { + it("hybrid mode prefixes computer actions and keeps page_ names", () => { expect(cuaToolNameForAction("click", "hybrid")).toBe("computer_click"); expect(cuaToolNameForAction("page_click", "hybrid")).toBe("page_click"); }); - it("os mode rejects DOM actions", () => { - expect(() => cuaToolNameForAction("page_click", "os")).toThrow(/not available in os mode/); + it("computer mode rejects DOM actions", () => { + expect(() => cuaToolNameForAction("page_click", "computer")).toThrow(/not available in computer mode/); }); }); describe("mode tool schemas", () => { - it("dom mode click accepts refs or viewport coordinates", () => { - const tools = computerTools({ mode: "dom" }); + it("browser mode click accepts refs or viewport coordinates", () => { + const tools = computerTools({ mode: "browser" }); const click = tools.find((tool) => tool.name === "click")!; expect(click.parameters.properties.ref).toBeDefined(); expect(click.parameters.properties.x).toBeDefined(); @@ -83,21 +83,21 @@ describe("mode tool schemas", () => { expect(pageClick.parameters.required).toContain("ref"); }); - it("dom mode exposes every default DOM action under its unprefixed name", () => { - const tools = computerTools({ mode: "dom" }); + it("browser mode exposes every default DOM action under its unprefixed name", () => { + const tools = computerTools({ mode: "browser" }); const names = tools.map((tool) => tool.name); - for (const action of CUA_DEFAULT_DOM_ACTION_TYPES) { + for (const action of CUA_DEFAULT_BROWSER_ACTION_TYPES) { expect(names).toContain(action.slice("page_".length)); } }); }); describe("mode runtime specs", () => { - it("resolves dom mode for anthropic with mode-specific prompt and tools", () => { - const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "dom" }); - expect(spec.mode).toBe("dom"); + it("resolves browser mode for anthropic with mode-specific prompt and tools", () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "browser" }); + expect(spec.mode).toBe("browser"); expect(spec.toolDefinitions.map((tool) => tool.name)).toContain("snapshot"); - expect(spec.defaultSystemPrompt).toBe(anthropic.buildAnthropicSystemPrompt({ mode: "dom" })); + expect(spec.defaultSystemPrompt).toBe(anthropic.buildAnthropicSystemPrompt({ mode: "browser" })); }); it("resolves hybrid mode for openai", () => { @@ -109,14 +109,14 @@ describe("mode runtime specs", () => { expect(spec.defaultSystemPrompt).toBe(openai.buildOpenAISystemPrompt({ mode: "hybrid" })); }); - it("rejects non-os modes for os-only providers", () => { - expect(() => resolveCuaRuntimeSpec("yutori:n1.5-latest", { mode: "dom" })).toThrow(/os only/); - expect(() => resolveCuaRuntimeSpec("google:gemini-3-flash-preview", { mode: "hybrid" })).toThrow(/os only/); + it("rejects non-computer modes for computer-only providers", () => { + expect(() => resolveCuaRuntimeSpec("yutori:n1.5-latest", { mode: "browser" })).toThrow(/computer only/); + expect(() => resolveCuaRuntimeSpec("google:gemini-3-flash-preview", { mode: "hybrid" })).toThrow(/computer only/); }); - it("keeps os mode byte-compatible with the pre-modes default", () => { + it("keeps computer mode byte-compatible with the pre-modes default", () => { const before = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5"); - const after = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "os" }); + const after = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "computer" }); expect(after.toolDefinitions.map((tool) => tool.name)).toEqual(before.toolDefinitions.map((tool) => tool.name)); }); }); diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index f551ae83..326fbfdd 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -15,8 +15,8 @@ import { describe("native tool validation", () => { it("infers mode from the native tool", () => { - expect(modeForNativeTool({ type: "computer_20260601" })).toBe("os"); - expect(modeForNativeTool({ type: "browser_20260701" })).toBe("dom"); + expect(modeForNativeTool({ type: "computer_20260601" })).toBe("computer"); + expect(modeForNativeTool({ type: "browser_20260701" })).toBe("browser"); }); it("carries the beta header per tool", () => { @@ -25,12 +25,12 @@ describe("native tool validation", () => { }); it("rejects a native tool with a conflicting mode", () => { - expect(() => resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "dom", nativeTool: { type: "computer_20260601" } })).toThrow( - /requires mode "os"/, + expect(() => resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "browser", nativeTool: { type: "computer_20260601" } })).toThrow( + /requires mode "computer"/, ); expect(() => resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "hybrid", nativeTool: { type: "browser_20260701" } }), - ).toThrow(/requires mode "dom"/); + ).toThrow(/requires mode "browser"/); }); it("rejects native tools on non-anthropic models", () => { @@ -43,7 +43,7 @@ describe("native tool validation", () => { describe("native runtime specs", () => { it("routes computer_20260601 to the native api with a single placeholder tool", () => { const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260601", enable_zoom: true } }); - expect(spec.mode).toBe("os"); + expect(spec.mode).toBe("computer"); expect(spec.model.api).toBe(ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API); expect(spec.nativeTool?.betaHeader).toBe("computer-use-2026-06-01"); expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["computer"]); @@ -51,7 +51,7 @@ describe("native runtime specs", () => { it("routes browser_20260701 to the native api under the default name", () => { const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "browser_20260701" } }); - expect(spec.mode).toBe("dom"); + expect(spec.mode).toBe("browser"); expect(spec.model.api).toBe(ANTHROPIC_NATIVE_BROWSER_MESSAGES_API); expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["browser"]); }); diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index 97c45713..88e81af8 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_OS_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; +import { anthropic, CUA_COMPUTER_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; import type { CuaProviderModule } from "../src/providers/common"; const MODULES: Record = { @@ -62,7 +62,7 @@ describe("provider namespaces export a uniform surface", () => { expect(Array.isArray(actionTypes), `${prefix}_CUA_ACTION_TYPES must be exported`).toBe(true); expect((actionTypes as unknown[]).length).toBeGreaterThan(0); for (const action of actionTypes as string[]) { - expect(CUA_OS_ACTION_TYPES).toContain(action); + expect(CUA_COMPUTER_ACTION_TYPES).toContain(action); } expect(namespace[`${prefix}_COMPUTER_INSTRUCTIONS`], `${prefix}_COMPUTER_INSTRUCTIONS must be exported`).toBeTypeOf( diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 8310a94f..d1efdcdf 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -456,8 +456,8 @@ function providerBaseUrlOverride(provider: string): string | undefined { function parseMode(raw: string | undefined): CuaMode | undefined { if (raw === undefined) return undefined; const value = raw.trim().toLowerCase(); - if (value === "os" || value === "dom" || value === "hybrid") return value; - throw new Error(`invalid --mode value "${raw}"; expected one of: os | dom | hybrid`); + if (value === "computer" || value === "browser" || value === "hybrid") return value; + throw new Error(`invalid --mode value "${raw}"; expected one of: computer | browser | hybrid`); } function parseNativeTool(raw: string | undefined, jsExec: boolean | undefined): CuaNativeToolSpec | undefined { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index a1eed50c..2023ad45 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -47,15 +47,15 @@ Options: --max-steps Max turns for action subcommands (default 3) --playwright Add the playwright_execute tool so the model can run Playwright code against the browser session - --mode Action plane(s) to expose: os (default) | dom | hybrid - os: OS-level input only. dom: CDP page tools (snapshot, - find, click-by-ref, navigate, tabs). hybrid: both, - deduplicated (computer_* + page_* tools). + --mode Action plane(s) to expose: computer (default) | browser | hybrid + computer: OS-level input only. browser: CDP page tools + (snapshot, find, click-by-ref, navigate, tabs). + hybrid: both, deduplicated (computer_* + page_* tools). --native-tool Drive an Anthropic model through its native tool schema: - computer_20260601 (requires --mode os) or - browser_20260701 (requires --mode dom) + computer_20260601 (requires --mode computer) or + browser_20260701 (requires --mode browser) --js-exec Expose page_evaluate (arbitrary JS in the page) in - dom/hybrid modes + browser/hybrid modes --out Output file for screenshot subcommand -o, --output Output format for --print: text (default) | jsonl --jsonl-include-deltas Include assistant_text_delta events (default off) diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 83406a0e..1f6de3f0 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -33,11 +33,11 @@ export interface BuildCuaHarnessOptions { /** Context files (AGENTS.md, CLAUDE.md, …) appended to the system prompt. */ contextFiles?: ContextFile[]; thinkingLevel?: ThinkingLevel; - /** Which canonical action plane(s) to expose: "os" (default), "dom", or "hybrid". */ + /** Which canonical action plane(s) to expose: "computer" (default), "browser", or "hybrid". */ mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `page_evaluate` in dom/hybrid modes. */ + /** Expose `page_evaluate` in browser/hybrid modes. */ javascriptExec?: boolean; /** Expose the playwright_execute tool that runs Playwright code against the browser session. */ playwright?: boolean; diff --git a/packages/cli/src/tui/main.ts b/packages/cli/src/tui/main.ts index 6c6a9a7d..449b31a8 100644 --- a/packages/cli/src/tui/main.ts +++ b/packages/cli/src/tui/main.ts @@ -493,8 +493,8 @@ async function applyModelCommand( async function applyModeCommand(opts: InteractiveOptions, messages: MessageList, argument: string): Promise { const value = argument.trim().toLowerCase(); - if (value !== "os" && value !== "dom" && value !== "hybrid") { - messages.addError("usage: /mode "); + if (value !== "computer" && value !== "browser" && value !== "hybrid") { + messages.addError("usage: /mode "); return; } try { diff --git a/packages/cli/src/tui/slash-commands.ts b/packages/cli/src/tui/slash-commands.ts index 34169c88..e18a417a 100644 --- a/packages/cli/src/tui/slash-commands.ts +++ b/packages/cli/src/tui/slash-commands.ts @@ -29,8 +29,8 @@ export function buildAutocompleteProvider( commands.push({ name: "mode", - description: "Switch the action plane(s): os | dom | hybrid", - argumentHint: "", + description: "Switch the action plane(s): computer | browser | hybrid", + argumentHint: "", getArgumentCompletions: (prefix: string) => modeCompletions(prefix), }); @@ -66,8 +66,8 @@ function modelCompletions(prefix: string): AutocompleteItem[] { } const MODES: ReadonlyArray<{ value: string; description: string }> = [ - { value: "os", description: "OS-level input only (default)" }, - { value: "dom", description: "CDP page tools: snapshot, find, click-by-ref, navigate, tabs" }, + { value: "computer", description: "OS-level input only (default)" }, + { value: "browser", description: "CDP page tools: snapshot, find, click-by-ref, navigate, tabs" }, { value: "hybrid", description: "Both planes: computer_* input + ref-only page_* tools" }, ]; From e7d5fa1208e5cf57df2caa31d255ed1cf8df4e78 Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 18:46:46 +0000 Subject: [PATCH 04/34] Rename browser-plane action ids from page_* to browser_* Hybrid mode now exposes computer_* and browser_* tools, matching the mode names and Anthropic's tool vocabulary. Browser mode still strips the prefix. The CDP executor is BrowserExecutor (translator/browser.ts). Live-verified hybrid mode post-rename: model mixed browser_navigate, browser_snapshot, browser_click (by ref), and computer_wait. --- docs/architecture.md | 6 +- packages/agent/src/agent.ts | 4 +- packages/agent/src/index.ts | 2 +- packages/agent/src/tools.ts | 8 +- .../src/translator/{page.ts => browser.ts} | 98 ++++----- packages/agent/src/translator/cdp.ts | 2 +- packages/agent/src/translator/translator.ts | 22 +- packages/agent/src/translator/types.ts | 2 +- packages/agent/test/agent.test.ts | 2 +- ...age.test.ts => translator-browser.test.ts} | 18 +- packages/ai/src/actions/browser.ts | 202 +++++++++--------- packages/ai/src/modes.ts | 84 ++++---- packages/ai/src/providers/anthropic/index.ts | 2 +- packages/ai/src/providers/anthropic/native.ts | 40 ++-- packages/ai/src/providers/common.ts | 2 +- packages/ai/src/providers/openai/index.ts | 2 +- packages/ai/test/modes.test.ts | 38 ++-- packages/ai/test/native-tools.test.ts | 22 +- packages/cli/src/cli.ts | 4 +- packages/cli/src/harness.ts | 2 +- packages/cli/src/tui/slash-commands.ts | 2 +- 21 files changed, 282 insertions(+), 282 deletions(-) rename packages/agent/src/translator/{page.ts => browser.ts} (88%) rename packages/agent/test/{translator-page.test.ts => translator-browser.test.ts} (83%) diff --git a/docs/architecture.md b/docs/architecture.md index 8dfb0d72..27f29401 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,7 +101,7 @@ under `packages/ai/src/actions/`: VM: mouse, keyboard, display capture, executed through Kernel's `browsers.computer` REST API. Coordinates are pixels in the OS screenshot frame. -- **Browser plane** (`actions/browser.ts`, ids prefixed `page_`) — CDP-driven page +- **Browser plane** (`actions/browser.ts`, ids prefixed `browser_`) — CDP-driven page tools: accessibility snapshots with element refs, element-targeted interaction, navigation, tabs, viewport screenshots. Executed by `packages/agent/src/translator/page.ts` over a raw CDP websocket @@ -113,8 +113,8 @@ A `CuaMode` selects which plane(s) the model sees: | mode | tools | coordinate frame | | --- | --- | --- | | `computer` (default) | computer actions under their canonical ids (`click`, `screenshot`, …) | OS screenshot pixels | -| `browser` | browser actions with the `page_` prefix stripped (`snapshot`, `click`, …) plus `wait` | none for refs; viewport pixels where coordinates are allowed | -| `hybrid` | both planes, one tool per capability: computer actions as `computer_*`, browser reads/element-writes as `page_*` (ref-only) | OS screenshot pixels — the single live frame | +| `browser` | browser actions with the `browser_` prefix stripped (`snapshot`, `click`, …) plus `wait` | none for refs; viewport pixels where coordinates are allowed | +| `hybrid` | both planes, one tool per capability: computer actions as `computer_*`, browser reads/element-writes as `browser_*` (ref-only) | OS screenshot pixels — the single live frame | Hybrid deduplicates capabilities: navigation and tabs live on the browser plane, pointer/keyboard input and the (only) screenshot live on the OS plane, and diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index ba3c5607..69684872 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -76,7 +76,7 @@ export type CuaAgentOptions = Omit & { mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `page_evaluate` in browser/hybrid modes. Default false. */ + /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; @@ -116,7 +116,7 @@ export type CuaAgentHarnessOptions< mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `page_evaluate` in browser/hybrid modes. Default false. */ + /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index e8e8e7c4..ba05a2b1 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -3,7 +3,7 @@ export { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node"; export type { KernelBrowser } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; -export { PageExecutor } from "./translator/page"; +export { BrowserExecutor } from "./translator/browser"; export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index f47bc6e7..5a82c86a 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -37,7 +37,7 @@ export interface BatchDetails { | { type: "url"; url: string } | { type: "screenshot"; bytes: number } | { type: "cursor_position"; x: number; y: number } - | { type: "page_text"; label: string; bytes: number } + | { type: "browser_text"; label: string; bytes: number } >; } @@ -168,8 +168,8 @@ async function executeBatchTool( } else if (read.type === "cursor_position") { readResults.push({ type: "cursor_position", x: read.x, y: read.y }); content.push({ type: "text", text: `cursor_position(): ${read.x},${read.y}` }); - } else if (read.type === "page_text") { - readResults.push({ type: "page_text", label: read.label, bytes: read.text.length }); + } else if (read.type === "browser_text") { + readResults.push({ type: "browser_text", label: read.label, bytes: read.text.length }); content.push({ type: "text", text: read.text }); } else { readResults.push({ type: "screenshot", bytes: read.data.length }); @@ -179,7 +179,7 @@ async function executeBatchTool( if (content.length === 0) { // Post-action grounding capture: the OS display in os/hybrid mode, // the browser viewport in browser mode (the only frame the model sees). - const screenshot = mode === "browser" ? await translator.page().screenshot() : await translator.screenshot(); + const screenshot = mode === "browser" ? await translator.browser().screenshot() : await translator.screenshot(); readResults.push({ type: "screenshot", bytes: screenshot.data.length }); content.push({ type: "image", data: screenshot.data.toString("base64"), mimeType: screenshot.mimeType }); } diff --git a/packages/agent/src/translator/page.ts b/packages/agent/src/translator/browser.ts similarity index 88% rename from packages/agent/src/translator/page.ts rename to packages/agent/src/translator/browser.ts index b9086c3a..018262b5 100644 --- a/packages/agent/src/translator/page.ts +++ b/packages/agent/src/translator/browser.ts @@ -1,15 +1,15 @@ import { normalizeGotoUrl, - type CuaActionPageClick, - type CuaActionPageDrag, - type CuaActionPageFill, - type CuaActionPageFind, - type CuaActionPageHover, - type CuaActionPageKey, - type CuaActionPageNavigate, - type CuaActionPageScroll, - type CuaActionPageScrollTo, - type CuaActionPageSnapshot, + type CuaActionBrowserClick, + type CuaActionBrowserDrag, + type CuaActionBrowserFill, + type CuaActionBrowserFind, + type CuaActionBrowserHover, + type CuaActionBrowserKey, + type CuaActionBrowserNavigate, + type CuaActionBrowserScroll, + type CuaActionBrowserScrollTo, + type CuaActionBrowserSnapshot, type CuaBrowserAction, } from "@onkernel/cua-ai"; import { CdpConnection } from "./cdp"; @@ -46,7 +46,7 @@ interface RefEntry { * navigation bumps the generation, and refs from earlier generations resolve * to a stale-ref error whose message tells the model how to recover. */ -export class PageExecutor { +export class BrowserExecutor { private readonly refs = new Map(); private readonly generations = new Map(); private refCounter = 0; @@ -56,48 +56,48 @@ export class PageExecutor { async execute(action: CuaBrowserAction): Promise { switch (action.type) { - case "page_snapshot": - return [{ type: "page_text", label: "snapshot", text: await this.snapshot(action) }]; - case "page_text": - return [{ type: "page_text", label: "text", text: await this.pageText(tabOf(action)) }]; - case "page_find": - return [{ type: "page_text", label: "find", text: await this.find(action) }]; - case "page_click": + case "browser_snapshot": + return [{ type: "browser_text", label: "snapshot", text: await this.snapshot(action) }]; + case "browser_text": + return [{ type: "browser_text", label: "text", text: await this.pageText(tabOf(action)) }]; + case "browser_find": + return [{ type: "browser_text", label: "find", text: await this.find(action) }]; + case "browser_click": await this.click(action); return []; - case "page_hover": + case "browser_hover": await this.hover(action); return []; - case "page_drag": + case "browser_drag": await this.drag(action); return []; - case "page_fill": + case "browser_fill": await this.fill(action); return []; - case "page_scroll_to": + case "browser_scroll_to": await this.scrollTo(action); return []; - case "page_scroll": + case "browser_scroll": await this.scroll(action); return []; - case "page_type": { + case "browser_type": { const session = await this.session(tabOf(action)); await this.cdp.send("Input.insertText", { text: action.text }, session); return []; } - case "page_key": + case "browser_key": await this.key(action); return []; - case "page_navigate": - return [{ type: "page_text", label: "navigate", text: await this.navigate(action) }]; - case "page_list_tabs": - return [{ type: "page_text", label: "tabs", text: await this.listTabs() }]; - case "page_new_tab": - return [{ type: "page_text", label: "new_tab", text: await this.newTab() }]; - case "page_screenshot": + case "browser_navigate": + return [{ type: "browser_text", label: "navigate", text: await this.navigate(action) }]; + case "browser_list_tabs": + return [{ type: "browser_text", label: "tabs", text: await this.listTabs() }]; + case "browser_new_tab": + return [{ type: "browser_text", label: "new_tab", text: await this.newTab() }]; + case "browser_screenshot": return [{ type: "screenshot", ...(await this.screenshot(action.region, action.tab_id)) }]; - case "page_evaluate": - return [{ type: "page_text", label: "evaluate", text: await this.evaluate(action.code, tabOf(action)) }]; + case "browser_evaluate": + return [{ type: "browser_text", label: "evaluate", text: await this.evaluate(action.code, tabOf(action)) }]; } } @@ -110,7 +110,7 @@ export class PageExecutor { return { data: Buffer.from(data, "base64"), mimeType: "image/png" }; } - private async snapshot(action: CuaActionPageSnapshot): Promise { + private async snapshot(action: CuaActionBrowserSnapshot): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); @@ -161,7 +161,7 @@ export class PageExecutor { return line; } - private async find(action: CuaActionPageFind): Promise { + private async find(action: CuaActionBrowserFind): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); @@ -182,7 +182,7 @@ export class PageExecutor { .join("\n"); } - private async click(action: CuaActionPageClick): Promise { + private async click(action: CuaActionBrowserClick): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const point = await this.resolvePoint(action, targetId, session); @@ -202,21 +202,21 @@ export class PageExecutor { ); } - private async hover(action: CuaActionPageHover): Promise { + private async hover(action: CuaActionBrowserHover): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const point = await this.resolvePoint(action, targetId, session); await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y }, session); } - private async drag(action: CuaActionPageDrag): Promise { + private async drag(action: CuaActionBrowserDrag): Promise { const session = await this.session(tabOf(action)); await this.cdp.send("Input.dispatchMouseEvent", { type: "mousePressed", x: action.from.x, y: action.from.y, button: "left", clickCount: 1 }, session); await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: action.to.x, y: action.to.y, button: "left" }, session); await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseReleased", x: action.to.x, y: action.to.y, button: "left", clickCount: 1 }, session); } - private async fill(action: CuaActionPageFill): Promise { + private async fill(action: CuaActionBrowserFill): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const entry = this.resolveRef(action.ref, targetId); @@ -231,18 +231,18 @@ export class PageExecutor { session, ); if (exceptionDetails) { - throw new Error(`page_fill failed: ${exceptionDetails.exception?.description ?? "element rejected the value"}`); + throw new Error(`browser_fill failed: ${exceptionDetails.exception?.description ?? "element rejected the value"}`); } } - private async scrollTo(action: CuaActionPageScrollTo): Promise { + private async scrollTo(action: CuaActionBrowserScrollTo): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const entry = this.resolveRef(action.ref, targetId); await this.scrollIntoView(entry, action.ref, session); } - private async scroll(action: CuaActionPageScroll): Promise { + private async scroll(action: CuaActionBrowserScroll): Promise { const session = await this.session(tabOf(action)); const notches = action.amount ?? 3; const delta = Math.trunc(notches) * SCROLL_NOTCH_PX; @@ -251,7 +251,7 @@ export class PageExecutor { await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseWheel", x: action.x, y: action.y, deltaX, deltaY }, session); } - private async key(action: CuaActionPageKey): Promise { + private async key(action: CuaActionBrowserKey): Promise { const session = await this.session(tabOf(action)); const repeat = Math.min(Math.max(1, Math.trunc(action.repeat ?? 1)), 100); const chords = action.text.trim().split(/\s+/).filter(Boolean); @@ -273,7 +273,7 @@ export class PageExecutor { await this.cdp.send("Input.dispatchKeyEvent", { type: "keyUp", ...base }, session); } - private async navigate(action: CuaActionPageNavigate): Promise { + private async navigate(action: CuaActionBrowserNavigate): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const direction = action.url.trim().toLowerCase(); @@ -314,7 +314,7 @@ export class PageExecutor { result: { value?: unknown; description?: string; type?: string }; exceptionDetails?: { exception?: { description?: string } }; }>("Runtime.evaluate", { expression: code, returnByValue: true, awaitPromise: true }, session); - if (exceptionDetails) throw new Error(`page_evaluate failed: ${exceptionDetails.exception?.description ?? "evaluation threw"}`); + if (exceptionDetails) throw new Error(`browser_evaluate failed: ${exceptionDetails.exception?.description ?? "evaluation threw"}`); if (result.value === undefined) return result.description ?? String(result.type ?? "undefined"); return typeof result.value === "string" ? result.value : JSON.stringify(result.value); } @@ -330,7 +330,7 @@ export class PageExecutor { } private async resolvePoint( - action: CuaActionPageClick | CuaActionPageHover, + action: CuaActionBrowserClick | CuaActionBrowserHover, targetId: string, session: string, ): Promise<{ x: number; y: number }> { @@ -555,6 +555,6 @@ const INTERACTIVE_ROLES: ReadonlySet = new Set([ const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); -export function createPageExecutor(cdpWsUrl: string): PageExecutor { - return new PageExecutor(new CdpConnection(cdpWsUrl)); +export function createBrowserExecutor(cdpWsUrl: string): BrowserExecutor { + return new BrowserExecutor(new CdpConnection(cdpWsUrl)); } diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index b369444f..f06330d0 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -3,7 +3,7 @@ * * Connects to a Kernel browser's `cdp_ws_url` over a plain WebSocket and * speaks the DevTools JSON-RPC protocol directly — no Playwright and no - * driver dependency. Only what the page executor needs: command dispatch on + * driver dependency. Only what the browser executor needs: command dispatch on * the browser connection and on attached page sessions. */ diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 48b497cd..554130f4 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -21,7 +21,7 @@ import { type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import sharp from "sharp"; -import { createPageExecutor, type PageExecutor } from "./page"; +import { createBrowserExecutor, type BrowserExecutor } from "./browser"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult } from "./types"; @@ -32,8 +32,8 @@ export interface InternalComputerTranslatorOptions { client: Kernel; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; - /** Page executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ - createPageExecutor?: (cdpWsUrl: string) => PageExecutor; + /** Browser executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ + createBrowserExecutor?: (cdpWsUrl: string) => BrowserExecutor; } export class InternalComputerTranslator { @@ -43,8 +43,8 @@ export class InternalComputerTranslator { private readonly screenshotSpec?: CuaScreenshotSpec; private readonly viewport: { width: number; height: number }; private readonly cdpWsUrl?: string; - private readonly pageExecutorFactory: (cdpWsUrl: string) => PageExecutor; - private pageExecutor?: PageExecutor; + private readonly browserExecutorFactory: (cdpWsUrl: string) => BrowserExecutor; + private browserExecutor?: BrowserExecutor; constructor(opts: InternalComputerTranslatorOptions) { this.sessionId = opts.browser.session_id; @@ -53,16 +53,16 @@ export class InternalComputerTranslator { this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; this.cdpWsUrl = opts.browser.cdp_ws_url; - this.pageExecutorFactory = opts.createPageExecutor ?? createPageExecutor; + this.browserExecutorFactory = opts.createBrowserExecutor ?? createBrowserExecutor; } /** The browser-plane executor, connected lazily over the browser's CDP websocket. */ - page(): PageExecutor { - if (!this.pageExecutor) { + browser(): BrowserExecutor { + if (!this.browserExecutor) { if (!this.cdpWsUrl) throw new Error("browser has no cdp_ws_url; browser actions are unavailable"); - this.pageExecutor = this.pageExecutorFactory(this.cdpWsUrl); + this.browserExecutor = this.browserExecutorFactory(this.cdpWsUrl); } - return this.pageExecutor; + return this.browserExecutor; } async screenshotRaw(): Promise { @@ -128,7 +128,7 @@ export class InternalComputerTranslator { for (const action of actions) { if (isCuaBrowserAction(action)) { await flush(); - result.readResults.push(...(await this.page().execute(action))); + result.readResults.push(...(await this.browser().execute(action))); continue; } switch (action.type) { diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 4958fd53..55f793d7 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -2,7 +2,7 @@ export type BatchReadResult = | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number } - | { type: "page_text"; label: string; text: string }; + | { type: "browser_text"; label: string; text: string }; export interface BatchExecutionResult { readResults: BatchReadResult[]; diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index da022560..df5f2b3f 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -440,7 +440,7 @@ describe("CuaAgentHarness", () => { expect(harness.getMode()).toBe("hybrid"); const names = harness.getTools().map((tool) => tool.name); expect(names).toContain("computer_click"); - expect(names).toContain("page_snapshot"); + expect(names).toContain("browser_snapshot"); }); it("appends extraTools in harness construction", async () => { diff --git a/packages/agent/test/translator-page.test.ts b/packages/agent/test/translator-browser.test.ts similarity index 83% rename from packages/agent/test/translator-page.test.ts rename to packages/agent/test/translator-browser.test.ts index d2f165e7..a8c30090 100644 --- a/packages/agent/test/translator-page.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -2,7 +2,7 @@ import type Kernel from "@onkernel/sdk"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; -import type { PageExecutor } from "../src/translator/page"; +import type { BrowserExecutor } from "../src/translator/browser"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -29,11 +29,11 @@ function createFakeDom() { const dom = { execute: async (action: CuaBrowserAction): Promise => { executed.push(action); - if (action.type === "page_text") return [{ type: "page_text", label: "text", text: "hello" }]; + if (action.type === "browser_text") return [{ type: "browser_text", label: "text", text: "hello" }]; return []; }, screenshot: async () => ({ data: Buffer.from("png"), mimeType: "image/png" }), - } as unknown as PageExecutor; + } as unknown as BrowserExecutor; return { executed, dom }; } @@ -41,23 +41,23 @@ describe("InternalComputerTranslator DOM plane", () => { it("dispatches DOM actions to the DOM executor, flushing pending OS input first", async () => { const { batches, client } = createClient(); const { executed, dom } = createFakeDom(); - const translator = new InternalComputerTranslator({ browser, client, createPageExecutor: () => dom }); + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => dom }); const result = await translator.executeBatch([ { type: "click", x: 1, y: 2 }, - { type: "page_text" }, - { type: "page_click", ref: "e3" }, + { type: "browser_text" }, + { type: "browser_click", ref: "e3" }, ]); expect(batches).toHaveLength(1); - expect(executed.map((action) => action.type)).toEqual(["page_text", "page_click"]); - expect(result.readResults).toEqual([{ type: "page_text", label: "text", text: "hello" }]); + expect(executed.map((action) => action.type)).toEqual(["browser_text", "browser_click"]); + expect(result.readResults).toEqual([{ type: "browser_text", label: "text", text: "hello" }]); }); it("errors on DOM actions when the browser has no cdp_ws_url", async () => { const { client } = createClient(); const translator = new InternalComputerTranslator({ browser: { session_id: "b" } as KernelBrowser, client }); - await expect(translator.executeBatch([{ type: "page_text" }])).rejects.toThrow(/cdp_ws_url/); + await expect(translator.executeBatch([{ type: "browser_text" }])).rejects.toThrow(/cdp_ws_url/); }); }); diff --git a/packages/ai/src/actions/browser.ts b/packages/ai/src/actions/browser.ts index fab54f36..768fa2cd 100644 --- a/packages/ai/src/actions/browser.ts +++ b/packages/ai/src/actions/browser.ts @@ -6,66 +6,66 @@ import { Type, type TSchema } from "@earendil-works/pi-ai"; * These execute over CDP against the browser itself: accessibility-tree * reads with element references, element-targeted interaction, navigation, * tabs, and viewport screenshots. Where a browser action takes coordinates - * (`page_click`, `page_hover`, `page_drag`, `page_scroll`), they are pixels + * (`browser_click`, `browser_hover`, `browser_drag`, `browser_scroll`), they are pixels * in the browser viewport — a different frame from the computer-plane actions in * `./computer`. Modes that expose both planes (hybrid) therefore restrict browser * actions to element references so exactly one coordinate frame is live. * * Element references (`ref`) are snapshot-scoped opaque ids (`e12`) minted - * by `page_snapshot` / `page_find`; a stale ref is an error instructing the + * by `browser_snapshot` / `browser_find`; a stale ref is an error instructing the * model to re-snapshot. */ export const CUA_BROWSER_ACTION_TYPES = [ - "page_snapshot", - "page_text", - "page_find", - "page_click", - "page_hover", - "page_drag", - "page_fill", - "page_scroll_to", - "page_scroll", - "page_type", - "page_key", - "page_navigate", - "page_list_tabs", - "page_new_tab", - "page_screenshot", - "page_evaluate", + "browser_snapshot", + "browser_text", + "browser_find", + "browser_click", + "browser_hover", + "browser_drag", + "browser_fill", + "browser_scroll_to", + "browser_scroll", + "browser_type", + "browser_key", + "browser_navigate", + "browser_list_tabs", + "browser_new_tab", + "browser_screenshot", + "browser_evaluate", ] as const; export type CuaBrowserActionType = (typeof CUA_BROWSER_ACTION_TYPES)[number]; /** - * The default browser-mode toolset: everything except `page_evaluate`, which + * The default browser-mode toolset: everything except `browser_evaluate`, which * runs arbitrary JavaScript in the page and must be enabled explicitly * (`javascriptExec: true`). */ export const CUA_DEFAULT_BROWSER_ACTION_TYPES = CUA_BROWSER_ACTION_TYPES.filter( - (action): action is Exclude => action !== "page_evaluate", + (action): action is Exclude => action !== "browser_evaluate", ); -export interface CuaActionPageSnapshot { - type: "page_snapshot"; +export interface CuaActionBrowserSnapshot { + type: "browser_snapshot"; filter?: "all" | "interactive"; ref?: string; depth?: number; tab_id?: string; } -export interface CuaActionPageText { - type: "page_text"; +export interface CuaActionBrowserText { + type: "browser_text"; tab_id?: string; } -export interface CuaActionPageFind { - type: "page_find"; +export interface CuaActionBrowserFind { + type: "browser_find"; query: string; tab_id?: string; } -export interface CuaActionPageClick { - type: "page_click"; +export interface CuaActionBrowserClick { + type: "browser_click"; ref?: string; x?: number; y?: number; @@ -75,36 +75,36 @@ export interface CuaActionPageClick { tab_id?: string; } -export interface CuaActionPageHover { - type: "page_hover"; +export interface CuaActionBrowserHover { + type: "browser_hover"; ref?: string; x?: number; y?: number; tab_id?: string; } -export interface CuaActionPageDrag { - type: "page_drag"; +export interface CuaActionBrowserDrag { + type: "browser_drag"; from: { x: number; y: number }; to: { x: number; y: number }; tab_id?: string; } -export interface CuaActionPageFill { - type: "page_fill"; +export interface CuaActionBrowserFill { + type: "browser_fill"; ref: string; value: string | number | boolean; tab_id?: string; } -export interface CuaActionPageScrollTo { - type: "page_scroll_to"; +export interface CuaActionBrowserScrollTo { + type: "browser_scroll_to"; ref: string; tab_id?: string; } -export interface CuaActionPageScroll { - type: "page_scroll"; +export interface CuaActionBrowserScroll { + type: "browser_scroll"; x: number; y: number; direction: "up" | "down" | "left" | "right"; @@ -112,70 +112,70 @@ export interface CuaActionPageScroll { tab_id?: string; } -export interface CuaActionPageType { - type: "page_type"; +export interface CuaActionBrowserType { + type: "browser_type"; text: string; tab_id?: string; } -export interface CuaActionPageKey { - type: "page_key"; +export interface CuaActionBrowserKey { + type: "browser_key"; text: string; repeat?: number; tab_id?: string; } -export interface CuaActionPageNavigate { - type: "page_navigate"; +export interface CuaActionBrowserNavigate { + type: "browser_navigate"; /** A URL, or the sentinels "back" / "forward" for history navigation. */ url: string; tab_id?: string; } -export interface CuaActionPageListTabs { - type: "page_list_tabs"; +export interface CuaActionBrowserListTabs { + type: "browser_list_tabs"; } -export interface CuaActionPageNewTab { - type: "page_new_tab"; +export interface CuaActionBrowserNewTab { + type: "browser_new_tab"; } -export interface CuaActionPageScreenshot { - type: "page_screenshot"; +export interface CuaActionBrowserScreenshot { + type: "browser_screenshot"; /** Optional crop region, [x0, y0, x1, y1] in viewport pixels. */ region?: [number, number, number, number]; tab_id?: string; } -export interface CuaActionPageEvaluate { - type: "page_evaluate"; +export interface CuaActionBrowserEvaluate { + type: "browser_evaluate"; code: string; tab_id?: string; } export type CuaBrowserAction = - | CuaActionPageSnapshot - | CuaActionPageText - | CuaActionPageFind - | CuaActionPageClick - | CuaActionPageHover - | CuaActionPageDrag - | CuaActionPageFill - | CuaActionPageScrollTo - | CuaActionPageScroll - | CuaActionPageType - | CuaActionPageKey - | CuaActionPageNavigate - | CuaActionPageListTabs - | CuaActionPageNewTab - | CuaActionPageScreenshot - | CuaActionPageEvaluate; + | CuaActionBrowserSnapshot + | CuaActionBrowserText + | CuaActionBrowserFind + | CuaActionBrowserClick + | CuaActionBrowserHover + | CuaActionBrowserDrag + | CuaActionBrowserFill + | CuaActionBrowserScrollTo + | CuaActionBrowserScroll + | CuaActionBrowserType + | CuaActionBrowserKey + | CuaActionBrowserNavigate + | CuaActionBrowserListTabs + | CuaActionBrowserNewTab + | CuaActionBrowserScreenshot + | CuaActionBrowserEvaluate; /** Options for building browser action schemas. */ export interface CuaBrowserSchemaOptions { /** - * Whether coordinate targeting is allowed on `page_click` / `page_hover` - * and whether `page_drag` / `page_scroll` are expressible at all. Browser + * Whether coordinate targeting is allowed on `browser_click` / `browser_hover` + * and whether `browser_drag` / `browser_scroll` are expressible at all. Browser * mode allows viewport coordinates (they are the only frame); hybrid mode * must disallow them so the OS screenshot frame stays the single live * coordinate frame. @@ -185,7 +185,7 @@ export interface CuaBrowserSchemaOptions { const TabId = () => Type.Optional(Type.String({ description: "Tab to act on. Defaults to the active tab." })); -const RefProperty = () => Type.String({ description: "Element reference from page_snapshot or page_find, e.g. \"e12\"." }); +const RefProperty = () => Type.String({ description: "Element reference from browser_snapshot or browser_find, e.g. \"e12\"." }); export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOptions): Record { const clickTarget: Record = options.coordinates @@ -197,9 +197,9 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti : { ref: RefProperty() }; return { - page_snapshot: Type.Object( + browser_snapshot: Type.Object( { - type: Type.Literal("page_snapshot"), + type: Type.Literal("browser_snapshot"), filter: Type.Optional(Type.Union([Type.Literal("all"), Type.Literal("interactive")])), ref: Type.Optional(Type.String({ description: "Restrict the snapshot to the subtree rooted at this element reference." })), depth: Type.Optional(Type.Number({ description: "Maximum tree depth (default 15)." })), @@ -207,24 +207,24 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), - page_text: Type.Object( + browser_text: Type.Object( { - type: Type.Literal("page_text"), + type: Type.Literal("browser_text"), tab_id: TabId(), }, { additionalProperties: false }, ), - page_find: Type.Object( + browser_find: Type.Object( { - type: Type.Literal("page_find"), + type: Type.Literal("browser_find"), query: Type.String({ description: "Natural-language element description, e.g. \"the cookie consent accept button\"." }), tab_id: TabId(), }, { additionalProperties: false }, ), - page_click: Type.Object( + browser_click: Type.Object( { - type: Type.Literal("page_click"), + type: Type.Literal("browser_click"), ...clickTarget, button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")])), num_clicks: Type.Optional(Type.Number()), @@ -233,26 +233,26 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), - page_hover: Type.Object( + browser_hover: Type.Object( { - type: Type.Literal("page_hover"), + type: Type.Literal("browser_hover"), ...clickTarget, tab_id: TabId(), }, { additionalProperties: false }, ), - page_drag: Type.Object( + browser_drag: Type.Object( { - type: Type.Literal("page_drag"), + type: Type.Literal("browser_drag"), from: Type.Object({ x: Type.Number(), y: Type.Number() }, { additionalProperties: false }), to: Type.Object({ x: Type.Number(), y: Type.Number() }, { additionalProperties: false }), tab_id: TabId(), }, { additionalProperties: false }, ), - page_fill: Type.Object( + browser_fill: Type.Object( { - type: Type.Literal("page_fill"), + type: Type.Literal("browser_fill"), ref: RefProperty(), value: Type.Union([Type.String(), Type.Number(), Type.Boolean()], { description: "Value to set. Use a boolean for checkboxes, an option value or label for selects.", @@ -261,17 +261,17 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), - page_scroll_to: Type.Object( + browser_scroll_to: Type.Object( { - type: Type.Literal("page_scroll_to"), + type: Type.Literal("browser_scroll_to"), ref: RefProperty(), tab_id: TabId(), }, { additionalProperties: false }, ), - page_scroll: Type.Object( + browser_scroll: Type.Object( { - type: Type.Literal("page_scroll"), + type: Type.Literal("browser_scroll"), x: Type.Number({ description: "Viewport x anchor in pixels." }), y: Type.Number({ description: "Viewport y anchor in pixels." }), direction: Type.Union([Type.Literal("up"), Type.Literal("down"), Type.Literal("left"), Type.Literal("right")]), @@ -280,36 +280,36 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), - page_type: Type.Object( + browser_type: Type.Object( { - type: Type.Literal("page_type"), + type: Type.Literal("browser_type"), text: Type.String(), tab_id: TabId(), }, { additionalProperties: false }, ), - page_key: Type.Object( + browser_key: Type.Object( { - type: Type.Literal("page_key"), + type: Type.Literal("browser_key"), text: Type.String({ description: "Key or chord, e.g. \"Return\", \"ctrl+a\"." }), repeat: Type.Optional(Type.Number()), tab_id: TabId(), }, { additionalProperties: false }, ), - page_navigate: Type.Object( + browser_navigate: Type.Object( { - type: Type.Literal("page_navigate"), + type: Type.Literal("browser_navigate"), url: Type.String({ description: "URL to navigate to, or \"back\" / \"forward\" for history navigation." }), tab_id: TabId(), }, { additionalProperties: false }, ), - page_list_tabs: Type.Object({ type: Type.Literal("page_list_tabs") }, { additionalProperties: false }), - page_new_tab: Type.Object({ type: Type.Literal("page_new_tab") }, { additionalProperties: false }), - page_screenshot: Type.Object( + browser_list_tabs: Type.Object({ type: Type.Literal("browser_list_tabs") }, { additionalProperties: false }), + browser_new_tab: Type.Object({ type: Type.Literal("browser_new_tab") }, { additionalProperties: false }), + browser_screenshot: Type.Object( { - type: Type.Literal("page_screenshot"), + type: Type.Literal("browser_screenshot"), // Not Type.Tuple: tuples emit draft-07 `items: [...]`, which Anthropic's // draft 2020-12 schema validation rejects. region: Type.Optional( @@ -323,9 +323,9 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), - page_evaluate: Type.Object( + browser_evaluate: Type.Object( { - type: Type.Literal("page_evaluate"), + type: Type.Literal("browser_evaluate"), code: Type.String({ description: "JavaScript to evaluate in the page context. The value of the last expression is returned." }), tab_id: TabId(), }, diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index e4c7d0b5..983240a8 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -20,21 +20,21 @@ import { * Pairs with Anthropic's native `browser_20260701` tool. * - `hybrid` — both planes, deduplicated to one tool per capability. * Computer tools are prefixed `computer_`, browser tools keep their - * `page_` prefix and accept element refs only, and the OS screenshot + * `browser_` prefix and accept element refs only, and the OS screenshot * frame is the single live coordinate frame. */ export type CuaMode = "computer" | "browser" | "hybrid"; /** Options for resolving a mode's action set. */ export interface CuaModeOptions { - /** Expose `page_evaluate` (arbitrary JavaScript in the page). Default false. */ + /** Expose `browser_evaluate` (arbitrary JavaScript in the page). Default false. */ javascriptExec?: boolean; } /** * Computer actions exposed in hybrid mode: navigation reads/writes are - * excluded because they live on the browser plane (`page_navigate`, - * `page_list_tabs`), and `zoom` is included since the OS screenshot is + * excluded because they live on the browser plane (`browser_navigate`, + * `browser_list_tabs`), and `zoom` is included since the OS screenshot is * hybrid's only capture. */ export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] = [ @@ -55,21 +55,21 @@ export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] /** * Browser actions exposed in hybrid mode: reads and element-targeted writes - * only. Pointer/keyboard capabilities (`page_click` by coordinate, - * `page_type`, `page_key`, `page_scroll`, `page_hover`, `page_drag`) and - * `page_screenshot` are excluded — real OS input and the OS screenshot cover + * only. Pointer/keyboard capabilities (`browser_click` by coordinate, + * `browser_type`, `browser_key`, `browser_scroll`, `browser_hover`, `browser_drag`) and + * `browser_screenshot` are excluded — real OS input and the OS screenshot cover * those, keeping one tool per capability and one coordinate frame. */ export const CUA_HYBRID_BROWSER_ACTION_TYPES: readonly CuaBrowserActionType[] = [ - "page_snapshot", - "page_text", - "page_find", - "page_click", - "page_fill", - "page_scroll_to", - "page_navigate", - "page_list_tabs", - "page_new_tab", + "browser_snapshot", + "browser_text", + "browser_find", + "browser_click", + "browser_fill", + "browser_scroll_to", + "browser_navigate", + "browser_list_tabs", + "browser_new_tab", ]; /** Resolve the default canonical action set for a mode. */ @@ -78,12 +78,12 @@ export function defaultActionsForMode(mode: CuaMode, options: CuaModeOptions = { case "computer": return CUA_DEFAULT_COMPUTER_ACTION_TYPES; case "browser": - return [...CUA_DEFAULT_BROWSER_ACTION_TYPES, ...(options.javascriptExec ? (["page_evaluate"] as const) : []), "wait"]; + return [...CUA_DEFAULT_BROWSER_ACTION_TYPES, ...(options.javascriptExec ? (["browser_evaluate"] as const) : []), "wait"]; case "hybrid": return [ ...CUA_HYBRID_COMPUTER_ACTION_TYPES, ...CUA_HYBRID_BROWSER_ACTION_TYPES, - ...(options.javascriptExec ? (["page_evaluate"] as const) : []), + ...(options.javascriptExec ? (["browser_evaluate"] as const) : []), ]; } } @@ -100,10 +100,10 @@ export function schemaOptionsForMode(mode: CuaMode): CuaActionSchemaOptions { * The model-facing tool name for a canonical action in a mode. * * - `computer`: canonical action ids as-is (`click`, `screenshot`, …). - * - `browser`: browser ids with the `page_` prefix stripped (`snapshot`, + * - `browser`: browser ids with the `browser_` prefix stripped (`snapshot`, * `click`, …); the prefix only exists to disambiguate planes, and * browser mode has one. - * - `hybrid`: computer ids prefixed `computer_`, browser ids kept as `page_*`. + * - `hybrid`: computer ids prefixed `computer_`, browser ids kept as `browser_*`. */ export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): string { switch (mode) { @@ -111,31 +111,31 @@ export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): stri if (!isCuaComputerActionType(action)) throw new Error(`browser action "${action}" is not available in computer mode`); return action; case "browser": - return isCuaComputerActionType(action) ? action : action.slice("page_".length); + return isCuaComputerActionType(action) ? action : action.slice("browser_".length); case "hybrid": return isCuaComputerActionType(action) ? `computer_${action}` : action; } } const BROWSER_ACTION_DESCRIPTIONS: Record = { - page_snapshot: + browser_snapshot: "Return an accessibility-tree snapshot of the page with element references like [e12]. " + "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale.", - page_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", - page_find: "Find elements matching a natural-language description and return them with element references, like a filtered snapshot.", - page_click: "Click an element. Prefer targeting by element reference from a snapshot.", - page_hover: "Move the pointer over an element without clicking.", - page_drag: "Drag from one viewport coordinate to another.", - page_fill: "Set the value of a form element (input, textarea, select, checkbox) by element reference.", - page_scroll_to: "Scroll an element into view by element reference.", - page_scroll: "Scroll the page at a viewport position by wheel notches.", - page_type: "Type a literal string at the current focus.", - page_key: "Press a key or chord, e.g. \"Return\" or \"ctrl+a\".", - page_navigate: "Navigate the page to a URL, or \"back\" / \"forward\" in history.", - page_list_tabs: "List open tabs with each tab's id, title, and URL.", - page_new_tab: "Open a new empty tab and return its tab id.", - page_screenshot: "Capture the current browser viewport.", - page_evaluate: "Execute JavaScript in the page context and return the value of the last expression.", + browser_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", + browser_find: "Find elements matching a natural-language description and return them with element references, like a filtered snapshot.", + browser_click: "Click an element. Prefer targeting by element reference from a snapshot.", + browser_hover: "Move the pointer over an element without clicking.", + browser_drag: "Drag from one viewport coordinate to another.", + browser_fill: "Set the value of a form element (input, textarea, select, checkbox) by element reference.", + browser_scroll_to: "Scroll an element into view by element reference.", + browser_scroll: "Scroll the page at a viewport position by wheel notches.", + browser_type: "Type a literal string at the current focus.", + browser_key: "Press a key or chord, e.g. \"Return\" or \"ctrl+a\".", + browser_navigate: "Navigate the page to a URL, or \"back\" / \"forward\" in history.", + browser_list_tabs: "List open tabs with each tab's id, title, and URL.", + browser_new_tab: "Open a new empty tab and return its tab id.", + browser_screenshot: "Capture the current browser viewport.", + browser_evaluate: "Execute JavaScript in the page context and return the value of the last expression.", }; // Hybrid exposes both planes, so tool descriptions carry the arbitration @@ -144,7 +144,7 @@ const BROWSER_ACTION_DESCRIPTIONS: Record = { const HYBRID_COMPUTER_DESCRIPTION_OVERRIDES: Partial> = { click: "Click at a coordinate in OS screenshot pixels using real OS-level input. " + - "Preferred over page_click when the target is visible in the screenshot — OS input is indistinguishable from a human user.", + "Preferred over browser_click when the target is visible in the screenshot — OS input is indistinguishable from a human user.", screenshot: "Capture the display. This is the only screenshot tool; all coordinates refer to this image's pixels.", zoom: "Return a cropped view of the current display for closer inspection. Coordinates in later actions still refer to the full screenshot, not the crop.", scroll: "Scroll with the OS-level mouse wheel at a coordinate in OS screenshot pixels.", @@ -153,10 +153,10 @@ const HYBRID_COMPUTER_DESCRIPTION_OVERRIDES: Partial> = { - page_click: - "Click an element by reference from a page_snapshot. Dispatched via CDP, which protected sites may detect — " + - "prefer computer_click when the element is visible in the screenshot; use page_click for elements that are hard to hit by coordinate.", - page_snapshot: + browser_click: + "Click an element by reference from a browser_snapshot. Dispatched via CDP, which protected sites may detect — " + + "prefer computer_click when the element is visible in the screenshot; use browser_click for elements that are hard to hit by coordinate.", + browser_snapshot: "Return an accessibility-tree snapshot of the page with element references like [e12]. " + "This is the high-fidelity way to read page structure — prefer it over screenshots for reading and locating elements. " + "Refs are only valid until the page changes; re-snapshot when told a ref is stale.", diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index e16642a9..ff056198 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -37,7 +37,7 @@ export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud brows export const ANTHROPIC_BROWSER_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; -export const ANTHROPIC_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and page_* tools read and act on the page itself by element reference. Prefer page_snapshot/page_find for reading and locating, and computer_* input for interaction; use page_* interaction for elements that are hard to hit by coordinate.`; +export const ANTHROPIC_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and browser_* tools read and act on the page itself by element reference. Prefer browser_snapshot/browser_find for reading and locating, and computer_* input for interaction; use browser_* interaction for elements that are hard to hit by coordinate.`; export function buildAnthropicSystemPrompt(opts: { suffix?: string; mode?: CuaMode } = {}): string { const base = diff --git a/packages/ai/src/providers/anthropic/native.ts b/packages/ai/src/providers/anthropic/native.ts index 31b4ff77..bd3a4c1e 100644 --- a/packages/ai/src/providers/anthropic/native.ts +++ b/packages/ai/src/providers/anthropic/native.ts @@ -137,15 +137,15 @@ export function mapNativeBrowserInput(input: NativeInput): CuaAction[] { const tab = tabId(input); switch (input.action) { case "navigate": - return [{ type: "page_navigate", url: requireString(input.url, "url"), ...tab }]; + return [{ type: "browser_navigate", url: requireString(input.url, "url"), ...tab }]; case "list_tabs": - return [{ type: "page_list_tabs" }]; + return [{ type: "browser_list_tabs" }]; case "new_tab": - return [{ type: "page_new_tab" }]; + return [{ type: "browser_new_tab" }]; case "read_page": return [ { - type: "page_snapshot", + type: "browser_snapshot", ...(input.filter === "interactive" || input.filter === "all" ? { filter: input.filter } : {}), ...(typeof input.depth === "number" ? { depth: input.depth } : {}), ...(typeof input.ref === "string" ? { ref: input.ref } : {}), @@ -153,33 +153,33 @@ export function mapNativeBrowserInput(input: NativeInput): CuaAction[] { }, ]; case "get_page_text": - return [{ type: "page_text", ...tab }]; + return [{ type: "browser_text", ...tab }]; case "find": - return [{ type: "page_find", query: requireString(input.query, "query"), ...tab }]; + return [{ type: "browser_find", query: requireString(input.query, "query"), ...tab }]; case "form_input": - return [{ type: "page_fill", ref: refTarget(input.target), value: fillValue(input.value), ...tab }]; + return [{ type: "browser_fill", ref: refTarget(input.target), value: fillValue(input.value), ...tab }]; case "scroll_to": - return [{ type: "page_scroll_to", ref: refTarget(input.target), ...tab }]; + return [{ type: "browser_scroll_to", ref: refTarget(input.target), ...tab }]; case "screenshot": - return [{ type: "page_screenshot", ...tab }]; + return [{ type: "browser_screenshot", ...tab }]; case "zoom": - return [{ type: "page_screenshot", region: region(input.region), ...tab }]; + return [{ type: "browser_screenshot", region: region(input.region), ...tab }]; case "left_click": - return [{ type: "page_click", ...pageTarget(input.target), ...modifiers(input.modifiers), ...tab }]; + return [{ type: "browser_click", ...pageTarget(input.target), ...modifiers(input.modifiers), ...tab }]; case "right_click": - return [{ type: "page_click", ...pageTarget(input.target), button: "right", ...modifiers(input.modifiers), ...tab }]; + return [{ type: "browser_click", ...pageTarget(input.target), button: "right", ...modifiers(input.modifiers), ...tab }]; case "double_click": - return [{ type: "page_click", ...pageTarget(input.target), num_clicks: 2, ...modifiers(input.modifiers), ...tab }]; + return [{ type: "browser_click", ...pageTarget(input.target), num_clicks: 2, ...modifiers(input.modifiers), ...tab }]; case "triple_click": - return [{ type: "page_click", ...pageTarget(input.target), num_clicks: 3, ...modifiers(input.modifiers), ...tab }]; + return [{ type: "browser_click", ...pageTarget(input.target), num_clicks: 3, ...modifiers(input.modifiers), ...tab }]; case "hover": - return [{ type: "page_hover", ...pageTarget(input.target), ...tab }]; + return [{ type: "browser_hover", ...pageTarget(input.target), ...tab }]; case "left_click_drag": - return [{ type: "page_drag", from: coordinateTarget(input.from, "from"), to: coordinateTarget(input.target, "target"), ...tab }]; + return [{ type: "browser_drag", from: coordinateTarget(input.from, "from"), to: coordinateTarget(input.target, "target"), ...tab }]; case "scroll": return [ { - type: "page_scroll", + type: "browser_scroll", ...coordinateTarget(input.target, "target"), direction: scrollDirection(input.scroll_direction), ...(typeof input.scroll_amount === "number" ? { amount: input.scroll_amount } : {}), @@ -187,15 +187,15 @@ export function mapNativeBrowserInput(input: NativeInput): CuaAction[] { }, ]; case "type": - return [{ type: "page_type", text: text(input), ...tab }]; + return [{ type: "browser_type", text: text(input), ...tab }]; case "key": { const repeat = clampRepeat(input.repeat); - return Array.from({ length: repeat }, () => ({ type: "page_key" as const, text: text(input), ...tab })); + return Array.from({ length: repeat }, () => ({ type: "browser_key" as const, text: text(input), ...tab })); } case "wait": return [{ type: "wait", ms: durationSeconds(input) * 1000 }]; case "javascript_exec": - return [{ type: "page_evaluate", code: text(input), ...tab }]; + return [{ type: "browser_evaluate", code: text(input), ...tab }]; default: throw new Error(`unsupported browser_20260701 action "${input.action}"`); } diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 81eeb4c2..6edd8894 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -119,7 +119,7 @@ export interface ComputerToolsOptions { actions?: readonly CuaActionType[]; /** Which action plane(s) to expose. Default "computer". */ mode?: CuaMode; - /** Expose `page_evaluate` in browser/hybrid modes. Default false. */ + /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; } diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 7b33f684..324bbcde 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -33,7 +33,7 @@ export const OPENAI_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser export const OPENAI_BROWSER_INSTRUCTIONS = `You control a Kernel cloud browser through page tools. Prefer reading the page with snapshot or find and targeting elements by reference; use screenshots when you need to inspect visual state. Element references go stale when the page changes — re-snapshot when told so.`; -export const OPENAI_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and page_* tools read and act on the page itself by element reference. Prefer page_snapshot/page_find for reading and locating, and computer_* input for interaction; use page_* interaction for elements that are hard to hit by coordinate.`; +export const OPENAI_HYBRID_INSTRUCTIONS = `You control a Kernel cloud browser through two kinds of tools: computer_* tools perform real OS-level input (coordinates are pixels in the most recent computer_screenshot), and browser_* tools read and act on the page itself by element reference. Prefer browser_snapshot/browser_find for reading and locating, and computer_* input for interaction; use browser_* interaction for elements that are hard to hit by coordinate.`; export function buildOpenAISystemPrompt(opts: { suffix?: string; mode?: CuaMode } = {}): string { const base = diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index 0d3d8fb7..ff0a1fe6 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -17,16 +17,16 @@ describe("mode action sets", () => { expect(defaultActionsForMode("computer")).toEqual(CUA_ACTION_TYPES); }); - it("browser mode defaults to DOM actions plus wait, without page_evaluate", () => { + it("browser mode defaults to DOM actions plus wait, without browser_evaluate", () => { const actions = defaultActionsForMode("browser"); - expect(actions).toContain("page_snapshot"); + expect(actions).toContain("browser_snapshot"); expect(actions).toContain("wait"); - expect(actions).not.toContain("page_evaluate"); + expect(actions).not.toContain("browser_evaluate"); expect(actions).not.toContain("click"); }); - it("browser mode exposes page_evaluate only with javascriptExec", () => { - expect(defaultActionsForMode("browser", { javascriptExec: true })).toContain("page_evaluate"); + it("browser mode exposes browser_evaluate only with javascriptExec", () => { + expect(defaultActionsForMode("browser", { javascriptExec: true })).toContain("browser_evaluate"); }); it("hybrid mode dedupes to one tool per capability", () => { @@ -34,14 +34,14 @@ describe("mode action sets", () => { // Navigation lives on the DOM plane. expect(actions).not.toContain("goto"); expect(actions).not.toContain("url"); - expect(actions).toContain("page_navigate"); + expect(actions).toContain("browser_navigate"); // One screenshot: the OS display. expect(actions).toContain("screenshot"); expect(actions).toContain("zoom"); - expect(actions).not.toContain("page_screenshot"); + expect(actions).not.toContain("browser_screenshot"); // Pointer/keyboard stays OS-level. - expect(actions).not.toContain("page_type"); - expect(actions).not.toContain("page_scroll"); + expect(actions).not.toContain("browser_type"); + expect(actions).not.toContain("browser_scroll"); expect(actions).toEqual([...CUA_HYBRID_COMPUTER_ACTION_TYPES, ...CUA_HYBRID_BROWSER_ACTION_TYPES]); }); }); @@ -51,19 +51,19 @@ describe("mode tool naming", () => { expect(cuaToolNameForAction("click", "computer")).toBe("click"); }); - it("browser mode strips the page_ prefix", () => { - expect(cuaToolNameForAction("page_snapshot", "browser")).toBe("snapshot"); - expect(cuaToolNameForAction("page_click", "browser")).toBe("click"); + it("browser mode strips the browser_ prefix", () => { + expect(cuaToolNameForAction("browser_snapshot", "browser")).toBe("snapshot"); + expect(cuaToolNameForAction("browser_click", "browser")).toBe("click"); expect(cuaToolNameForAction("wait", "browser")).toBe("wait"); }); - it("hybrid mode prefixes computer actions and keeps page_ names", () => { + it("hybrid mode prefixes computer actions and keeps browser_ names", () => { expect(cuaToolNameForAction("click", "hybrid")).toBe("computer_click"); - expect(cuaToolNameForAction("page_click", "hybrid")).toBe("page_click"); + expect(cuaToolNameForAction("browser_click", "hybrid")).toBe("browser_click"); }); it("computer mode rejects DOM actions", () => { - expect(() => cuaToolNameForAction("page_click", "computer")).toThrow(/not available in computer mode/); + expect(() => cuaToolNameForAction("browser_click", "computer")).toThrow(/not available in computer mode/); }); }); @@ -75,9 +75,9 @@ describe("mode tool schemas", () => { expect(click.parameters.properties.x).toBeDefined(); }); - it("hybrid mode page_click is ref-only, keeping one coordinate frame", () => { + it("hybrid mode browser_click is ref-only, keeping one coordinate frame", () => { const tools = computerTools({ mode: "hybrid" }); - const pageClick = tools.find((tool) => tool.name === "page_click")!; + const pageClick = tools.find((tool) => tool.name === "browser_click")!; expect(pageClick.parameters.properties.ref).toBeDefined(); expect(pageClick.parameters.properties.x).toBeUndefined(); expect(pageClick.parameters.required).toContain("ref"); @@ -87,7 +87,7 @@ describe("mode tool schemas", () => { const tools = computerTools({ mode: "browser" }); const names = tools.map((tool) => tool.name); for (const action of CUA_DEFAULT_BROWSER_ACTION_TYPES) { - expect(names).toContain(action.slice("page_".length)); + expect(names).toContain(action.slice("browser_".length)); } }); }); @@ -105,7 +105,7 @@ describe("mode runtime specs", () => { expect(spec.mode).toBe("hybrid"); const names = spec.toolDefinitions.map((tool) => tool.name); expect(names).toContain("computer_click"); - expect(names).toContain("page_snapshot"); + expect(names).toContain("browser_snapshot"); expect(spec.defaultSystemPrompt).toBe(openai.buildOpenAISystemPrompt({ mode: "hybrid" })); }); diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index 326fbfdd..60c003ce 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -114,36 +114,36 @@ describe("computer_20260601 action mapping", () => { describe("browser_20260701 action mapping", () => { it("maps DOM reads", () => { expect(mapNativeBrowserInput({ action: "read_page", filter: "interactive", depth: 5 })).toEqual([ - { type: "page_snapshot", filter: "interactive", depth: 5 }, + { type: "browser_snapshot", filter: "interactive", depth: 5 }, ]); - expect(mapNativeBrowserInput({ action: "find", query: "search bar" })).toEqual([{ type: "page_find", query: "search bar" }]); - expect(mapNativeBrowserInput({ action: "get_page_text", tab_id: "T1" })).toEqual([{ type: "page_text", tab_id: "T1" }]); + expect(mapNativeBrowserInput({ action: "find", query: "search bar" })).toEqual([{ type: "browser_find", query: "search bar" }]); + expect(mapNativeBrowserInput({ action: "get_page_text", tab_id: "T1" })).toEqual([{ type: "browser_text", tab_id: "T1" }]); }); it("maps ref and coordinate click targets", () => { expect(mapNativeBrowserInput({ action: "left_click", target: { type: "ref", ref: "e7" } })).toEqual([ - { type: "page_click", ref: "e7" }, + { type: "browser_click", ref: "e7" }, ]); expect(mapNativeBrowserInput({ action: "left_click", target: { type: "coordinate", x: 4, y: 5 }, modifiers: "shift" })).toEqual([ - { type: "page_click", x: 4, y: 5, modifiers: ["shift"] }, + { type: "browser_click", x: 4, y: 5, modifiers: ["shift"] }, ]); }); it("requires ref targets on form_input and scroll_to", () => { expect(mapNativeBrowserInput({ action: "form_input", target: { type: "ref", ref: "e7" }, value: "hi" })).toEqual([ - { type: "page_fill", ref: "e7", value: "hi" }, + { type: "browser_fill", ref: "e7", value: "hi" }, ]); expect(() => mapNativeBrowserInput({ action: "scroll_to", target: { type: "coordinate", x: 1, y: 2 } })).toThrow(/requires a ref/); }); it("maps navigation, tabs, zoom, and javascript_exec", () => { - expect(mapNativeBrowserInput({ action: "navigate", url: "back" })).toEqual([{ type: "page_navigate", url: "back" }]); - expect(mapNativeBrowserInput({ action: "list_tabs" })).toEqual([{ type: "page_list_tabs" }]); + expect(mapNativeBrowserInput({ action: "navigate", url: "back" })).toEqual([{ type: "browser_navigate", url: "back" }]); + expect(mapNativeBrowserInput({ action: "list_tabs" })).toEqual([{ type: "browser_list_tabs" }]); expect(mapNativeBrowserInput({ action: "zoom", region: [1, 2, 3, 4] })).toEqual([ - { type: "page_screenshot", region: [1, 2, 3, 4] }, + { type: "browser_screenshot", region: [1, 2, 3, 4] }, ]); expect(mapNativeBrowserInput({ action: "javascript_exec", text: "document.title" })).toEqual([ - { type: "page_evaluate", code: "document.title" }, + { type: "browser_evaluate", code: "document.title" }, ]); }); }); @@ -153,7 +153,7 @@ describe("native tool executors", () => { const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "browser_20260701" } }); const executor = spec.toolExecutors[0]!; const actions: CuaAction[] = executor.toActions({ action: "left_click", target: { type: "ref", ref: "e3" } }); - expect(actions).toEqual([{ type: "page_click", ref: "e3" }]); + expect(actions).toEqual([{ type: "browser_click", ref: "e3" }]); }); it("exports the anthropic namespace surface", () => { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 2023ad45..fab1f844 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -50,11 +50,11 @@ Options: --mode Action plane(s) to expose: computer (default) | browser | hybrid computer: OS-level input only. browser: CDP page tools (snapshot, find, click-by-ref, navigate, tabs). - hybrid: both, deduplicated (computer_* + page_* tools). + hybrid: both, deduplicated (computer_* + browser_* tools). --native-tool Drive an Anthropic model through its native tool schema: computer_20260601 (requires --mode computer) or browser_20260701 (requires --mode browser) - --js-exec Expose page_evaluate (arbitrary JS in the page) in + --js-exec Expose browser_evaluate (arbitrary JS in the page) in browser/hybrid modes --out Output file for screenshot subcommand -o, --output Output format for --print: text (default) | jsonl diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 1f6de3f0..c0228c65 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -37,7 +37,7 @@ export interface BuildCuaHarnessOptions { mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `page_evaluate` in browser/hybrid modes. */ + /** Expose `browser_evaluate` in browser/hybrid modes. */ javascriptExec?: boolean; /** Expose the playwright_execute tool that runs Playwright code against the browser session. */ playwright?: boolean; diff --git a/packages/cli/src/tui/slash-commands.ts b/packages/cli/src/tui/slash-commands.ts index e18a417a..fe83941f 100644 --- a/packages/cli/src/tui/slash-commands.ts +++ b/packages/cli/src/tui/slash-commands.ts @@ -68,7 +68,7 @@ function modelCompletions(prefix: string): AutocompleteItem[] { const MODES: ReadonlyArray<{ value: string; description: string }> = [ { value: "computer", description: "OS-level input only (default)" }, { value: "browser", description: "CDP page tools: snapshot, find, click-by-ref, navigate, tabs" }, - { value: "hybrid", description: "Both planes: computer_* input + ref-only page_* tools" }, + { value: "hybrid", description: "Both planes: computer_* input + ref-only browser_* tools" }, ]; function modeCompletions(prefix: string): AutocompleteItem[] { From a43276921a46b2f3dba3de65de55e309ebcc6ccd Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 18:50:22 +0000 Subject: [PATCH 05/34] docs: sync architecture.md with computer/browser naming and runtime mode switching --- docs/architecture.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 27f29401..108e7329 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -104,7 +104,7 @@ under `packages/ai/src/actions/`: - **Browser plane** (`actions/browser.ts`, ids prefixed `browser_`) — CDP-driven page tools: accessibility snapshots with element refs, element-targeted interaction, navigation, tabs, viewport screenshots. Executed by - `packages/agent/src/translator/page.ts` over a raw CDP websocket + `packages/agent/src/translator/browser.ts` (`BrowserExecutor`) over a raw CDP websocket (`translator/cdp.ts`) to the browser's `cdp_ws_url` — no Playwright. Coordinates, where used, are viewport pixels. @@ -117,11 +117,16 @@ A `CuaMode` selects which plane(s) the model sees: | `hybrid` | both planes, one tool per capability: computer actions as `computer_*`, browser reads/element-writes as `browser_*` (ref-only) | OS screenshot pixels — the single live frame | Hybrid deduplicates capabilities: navigation and tabs live on the browser plane, -pointer/keyboard input and the (only) screenshot live on the OS plane, and -page tools take element refs only so exactly one coordinate frame exists. +pointer/keyboard input and the (only) screenshot live on the computer plane, and +hybrid browser tools take element refs only so exactly one coordinate frame exists. Element refs are snapshot-scoped (`e12`); a stale ref resolves to an error string that tells the model to re-snapshot. +The mode is set at construction (`mode` on `CuaAgent`/`CuaAgentHarness`, +`--mode` in the CLI) and can be switched at runtime with `setMode()` (the +TUI's `/mode` command), which refreshes CUA-owned tools and the default +system prompt. + **Native tools.** `resolveCuaRuntimeSpec(model, { nativeTool })` drives an Anthropic model through its provider-defined tool schema instead of the canonical function tools: `computer_20260601` pairs with `computer` mode and From e6219bb674927019828a17b70c7b25e89cb04a7c Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 19:14:31 +0000 Subject: [PATCH 06/34] Update native computer tool to shipped version computer_20260701 The early-access PDF documented computer_20260601 with beta header computer-use-2026-06-01, but the API ships computer_20260701 behind computer-use-2026-07-01 (the 0601 tag is not accepted under any header). Live-verified end-to-end against a Kernel browser. --- docs/architecture.md | 2 +- .../agent/examples/anthropic-native-smoke.ts | 4 ++-- packages/ai/src/modes.ts | 2 +- packages/ai/src/native-tools.ts | 6 ++--- packages/ai/src/providers/anthropic/native.ts | 10 ++++----- packages/ai/src/runtime-spec.ts | 2 +- packages/ai/test/native-tools.test.ts | 22 +++++++++---------- packages/cli/src/cli-harness.ts | 4 ++-- packages/cli/src/cli.ts | 2 +- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 108e7329..5840bd8a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -129,7 +129,7 @@ system prompt. **Native tools.** `resolveCuaRuntimeSpec(model, { nativeTool })` drives an Anthropic model through its provider-defined tool schema instead of the -canonical function tools: `computer_20260601` pairs with `computer` mode and +canonical function tools: `computer_20260701` pairs with `computer` mode and `browser_20260701` with `browser` mode (mismatches throw, mirroring the API's own rejection of mixed frames). The spec routes the model to a CUA-owned api id; the registered `anthropic` provider dispatches it to pi's builtin diff --git a/packages/agent/examples/anthropic-native-smoke.ts b/packages/agent/examples/anthropic-native-smoke.ts index 3a89eb61..3061d665 100644 --- a/packages/agent/examples/anthropic-native-smoke.ts +++ b/packages/agent/examples/anthropic-native-smoke.ts @@ -6,7 +6,7 @@ // computer (default) canonical computer-plane (OS input) tools // browser canonical browser-plane (CDP page) tools // hybrid both planes, deduplicated -// native-computer Anthropic computer_20260601 (requires the computer-use beta) +// native-computer Anthropic computer_20260701 (requires the computer-use beta) // native-browser Anthropic browser_20260701 (requires the browser-use beta) import Kernel from "@onkernel/sdk"; import { requireCuaEnvApiKeyForModel, type CuaModelRef, type CuaMode, type CuaNativeToolSpec } from "@onkernel/cua-ai"; @@ -20,7 +20,7 @@ const CONFIGS: Record = { - computer_20260601: { mode: "computer", provider: "anthropic", betaHeader: "computer-use-2026-06-01", defaultName: "computer" }, + computer_20260701: { mode: "computer", provider: "anthropic", betaHeader: "computer-use-2026-07-01", defaultName: "computer" }, browser_20260701: { mode: "browser", provider: "anthropic", betaHeader: "browser-use-2026-07-01", defaultName: "browser" }, }; diff --git a/packages/ai/src/providers/anthropic/native.ts b/packages/ai/src/providers/anthropic/native.ts index bd3a4c1e..418b2d05 100644 --- a/packages/ai/src/providers/anthropic/native.ts +++ b/packages/ai/src/providers/anthropic/native.ts @@ -13,12 +13,12 @@ export const ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API = "anthropic-cua-native-comp export const ANTHROPIC_NATIVE_BROWSER_MESSAGES_API = "anthropic-cua-native-browser-messages"; export const ANTHROPIC_NATIVE_API_BETA_HEADERS: Record = { - [ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API]: "computer-use-2026-06-01", + [ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API]: "computer-use-2026-07-01", [ANTHROPIC_NATIVE_BROWSER_MESSAGES_API]: "browser-use-2026-07-01", }; export function nativeApiForToolType(type: ResolvedCuaNativeTool["spec"]["type"]): string { - return type === "computer_20260601" ? ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API : ANTHROPIC_NATIVE_BROWSER_MESSAGES_API; + return type === "computer_20260701" ? ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API : ANTHROPIC_NATIVE_BROWSER_MESSAGES_API; } /** Merge a native tool's `anthropic-beta` header into stream options. */ @@ -45,7 +45,7 @@ export function nativeToolExecutors(resolved: ResolvedCuaNativeTool): CuaToolExe parameters: NativeActionSchema, }; const toActions = - resolved.spec.type === "computer_20260601" + resolved.spec.type === "computer_20260701" ? (args: unknown) => mapNativeComputerInput(asNativeInput(args)) : (args: unknown) => mapNativeBrowserInput(asNativeInput(args)); return [{ definition, toActions }]; @@ -83,7 +83,7 @@ function asNativeInput(args: unknown): NativeInput { const MAX_KEY_REPEAT = 100; -/** Map one `computer_20260601` tool input onto canonical computer-plane actions. */ +/** Map one `computer_20260701` tool input onto canonical computer-plane actions. */ export function mapNativeComputerInput(input: NativeInput): CuaAction[] { switch (input.action) { case "screenshot": @@ -128,7 +128,7 @@ export function mapNativeComputerInput(input: NativeInput): CuaAction[] { case "zoom": return [{ type: "zoom", region: region(input.region) }]; default: - throw new Error(`unsupported computer_20260601 action "${input.action}"`); + throw new Error(`unsupported computer_20260701 action "${input.action}"`); } } diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 76261c99..1bf43805 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -29,7 +29,7 @@ export interface CuaRuntimeSpecOptions extends ComputerToolsOptions { /** * Drive the model through a provider-native tool declaration instead of * CUA's canonical function tools. The native tool determines (and is - * validated against) the mode: `computer_20260601` requires `"computer"`, + * validated against) the mode: `computer_20260701` requires `"computer"`, * `browser_20260701` requires `"browser"`. When `mode` is omitted it is * inferred from the native tool. */ diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index 60c003ce..c275765e 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -15,17 +15,17 @@ import { describe("native tool validation", () => { it("infers mode from the native tool", () => { - expect(modeForNativeTool({ type: "computer_20260601" })).toBe("computer"); + expect(modeForNativeTool({ type: "computer_20260701" })).toBe("computer"); expect(modeForNativeTool({ type: "browser_20260701" })).toBe("browser"); }); it("carries the beta header per tool", () => { - expect(betaHeaderForNativeTool({ type: "computer_20260601" })).toBe("computer-use-2026-06-01"); + expect(betaHeaderForNativeTool({ type: "computer_20260701" })).toBe("computer-use-2026-07-01"); expect(betaHeaderForNativeTool({ type: "browser_20260701" })).toBe("browser-use-2026-07-01"); }); it("rejects a native tool with a conflicting mode", () => { - expect(() => resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "browser", nativeTool: { type: "computer_20260601" } })).toThrow( + expect(() => resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { mode: "browser", nativeTool: { type: "computer_20260701" } })).toThrow( /requires mode "computer"/, ); expect(() => @@ -34,18 +34,18 @@ describe("native tool validation", () => { }); it("rejects native tools on non-anthropic models", () => { - expect(() => resolveCuaRuntimeSpec("openai:gpt-5.5", { nativeTool: { type: "computer_20260601" } })).toThrow( + expect(() => resolveCuaRuntimeSpec("openai:gpt-5.5", { nativeTool: { type: "computer_20260701" } })).toThrow( /requires an anthropic model/, ); }); }); describe("native runtime specs", () => { - it("routes computer_20260601 to the native api with a single placeholder tool", () => { - const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260601", enable_zoom: true } }); + it("routes computer_20260701 to the native api with a single placeholder tool", () => { + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260701", enable_zoom: true } }); expect(spec.mode).toBe("computer"); expect(spec.model.api).toBe(ANTHROPIC_NATIVE_COMPUTER_MESSAGES_API); - expect(spec.nativeTool?.betaHeader).toBe("computer-use-2026-06-01"); + expect(spec.nativeTool?.betaHeader).toBe("computer-use-2026-07-01"); expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["computer"]); }); @@ -57,7 +57,7 @@ describe("native runtime specs", () => { }); it("swaps the placeholder tool for the native declaration in the payload", async () => { - const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260601", enable_zoom: true } }); + const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260701", enable_zoom: true } }); const payload = { tools: [ { name: "computer", description: "placeholder", input_schema: {} }, @@ -65,12 +65,12 @@ describe("native runtime specs", () => { ], }; const next = (await spec.onPayload?.(payload, spec.model)) as { tools: Array> }; - expect(next.tools[0]).toEqual({ type: "computer_20260601", name: "computer", enable_zoom: true }); + expect(next.tools[0]).toEqual({ type: "computer_20260701", name: "computer", enable_zoom: true }); expect(next.tools[1]!.name).toBe("playwright_execute"); }); }); -describe("computer_20260601 action mapping", () => { +describe("computer_20260701 action mapping", () => { it("maps clicks with coordinates, buttons, and modifier chords", () => { expect(mapNativeComputerInput({ action: "left_click", coordinate: [10, 20], text: "ctrl+shift" })).toEqual([ { type: "click", x: 10, y: 20, button: "left", hold_keys: ["ctrl", "shift"] }, @@ -107,7 +107,7 @@ describe("computer_20260601 action mapping", () => { }); it("rejects unknown actions", () => { - expect(() => mapNativeComputerInput({ action: "warp" })).toThrow(/unsupported computer_20260601 action/); + expect(() => mapNativeComputerInput({ action: "warp" })).toThrow(/unsupported computer_20260701 action/); }); }); diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index d1efdcdf..59a55d98 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -465,9 +465,9 @@ function parseNativeTool(raw: string | undefined, jsExec: boolean | undefined): const value = raw.trim().toLowerCase(); // enable_zoom follows Anthropic's own recommendation for fine-grained // visual targeting; the executor implements the zoom crop locally. - if (value === "computer_20260601") return { type: "computer_20260601", enable_zoom: true }; + if (value === "computer_20260701") return { type: "computer_20260701", enable_zoom: true }; if (value === "browser_20260701") return { type: "browser_20260701", ...(jsExec ? { enable_javascript_exec: true } : {}) }; - throw new Error(`invalid --native-tool value "${raw}"; expected one of: computer_20260601 | browser_20260701`); + throw new Error(`invalid --native-tool value "${raw}"; expected one of: computer_20260701 | browser_20260701`); } function mapThinkingLevel(raw: string | undefined): "off" | "minimal" | "low" | "medium" | "high" | "xhigh" { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index fab1f844..275130bf 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -52,7 +52,7 @@ Options: (snapshot, find, click-by-ref, navigate, tabs). hybrid: both, deduplicated (computer_* + browser_* tools). --native-tool Drive an Anthropic model through its native tool schema: - computer_20260601 (requires --mode computer) or + computer_20260701 (requires --mode computer) or browser_20260701 (requires --mode browser) --js-exec Expose browser_evaluate (arbitrary JS in the page) in browser/hybrid modes From a6f06a40d126086a253c267f6ffd5f9a8fc378af Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 19:36:10 +0000 Subject: [PATCH 07/34] Address bugbot review findings - CDP: reject pending commands when the socket closes unexpectedly - CuaRuntimeController disposes the previous translator (closing its CDP connection) on setModel/setMode - CuaAgentHarness.setMode keeps the requested activation state of tools that survive the mode switch instead of reactivating everything - resolveCuaRuntimeSpec folds javascriptExec into the browser_20260701 declaration (explicit enable_javascript_exec wins) - browser_new_tab makes the new tab active - browser_screenshot region clip normalizes reversed corners - CLI validates --mode/--native-tool at parse time (usage error, exit 2) and cleans up the provisioned browser when harness setup fails - browser mode skips the OS-display initial screenshot in the TUI and print paths so the viewport stays the only visual frame --- packages/agent/src/agent.ts | 16 ++++++++-- packages/agent/src/translator/browser.ts | 16 +++++++++- packages/agent/src/translator/cdp.ts | 6 +++- packages/agent/src/translator/translator.ts | 6 ++++ packages/agent/test/agent.test.ts | 21 +++++++++++++ packages/ai/src/runtime-spec.ts | 10 +++++- packages/ai/test/native-tools.test.ts | 14 +++++++++ packages/cli/src/cli-harness.ts | 35 +++++++++++++++++++-- packages/cli/src/cli.ts | 8 +++++ packages/cli/src/print.ts | 3 ++ packages/cli/src/tui/main.ts | 3 ++ 11 files changed, 130 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 69684872..daee9968 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -175,6 +175,11 @@ class CuaRuntimeController { setMode(mode: CuaMode): void { this.runtimeSpec = this.resolveSpec(this.runtimeSpec.model, mode); this.currentMode = mode; + this.replaceTranslator(); + } + + private replaceTranslator(): void { + this.translator.dispose(); this.translator = this.createTranslator(); } @@ -184,7 +189,7 @@ class CuaRuntimeController { setModel(model: CuaRuntimeInput): void { this.runtimeSpec = this.resolveSpec(model); - this.translator = this.createTranslator(); + this.replaceTranslator(); } tools(): AgentTool[] { @@ -474,9 +479,16 @@ export class CuaAgentHarness< * plane conflicts with the requested mode. */ async setMode(mode: CuaMode): Promise { + const previousNames = new Set(this.getTools().map((tool) => tool.name)); this.runtime.setMode(mode); const tools = this.runtime.tools(); - await super.setTools(tools, tools.map((tool) => tool.name)); + // Tools that survive the mode switch (extraTools, shared names) keep + // their requested activation state; names new in this mode activate. + const requested = this.requestedActiveToolNames; + const active = requested + ? tools.map((tool) => tool.name).filter((name) => !previousNames.has(name) || requested.includes(name)) + : tools.map((tool) => tool.name); + await super.setTools(tools, active); } /** The action plane(s) currently exposed to the model. */ diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 018262b5..54dc10ba 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -54,6 +54,11 @@ export class BrowserExecutor { constructor(private readonly cdp: CdpConnection) {} + /** Close the underlying CDP connection. Safe to call when never connected. */ + close(): void { + this.cdp.close(); + } + async execute(action: CuaBrowserAction): Promise { switch (action.type) { case "browser_snapshot": @@ -104,7 +109,15 @@ export class BrowserExecutor { async screenshot(region?: [number, number, number, number], tabId?: string): Promise<{ data: Buffer; mimeType: string }> { const session = await this.session(tabId); const clip = region - ? { clip: { x: region[0], y: region[1], width: Math.max(1, region[2] - region[0]), height: Math.max(1, region[3] - region[1]), scale: 1 } } + ? { + clip: { + x: Math.min(region[0], region[2]), + y: Math.min(region[1], region[3]), + width: Math.max(1, Math.abs(region[2] - region[0])), + height: Math.max(1, Math.abs(region[3] - region[1])), + scale: 1, + }, + } : {}; const { data } = await this.cdp.send<{ data: string }>("Page.captureScreenshot", { format: "png", ...clip }, session); return { data: Buffer.from(data, "base64"), mimeType: "image/png" }; @@ -305,6 +318,7 @@ export class BrowserExecutor { private async newTab(): Promise { const targetId = await this.cdp.createTarget("about:blank"); + this.activeTargetId = targetId; return `Opened tab_id ${shortTabId(targetId)}.\n${await this.tabContext(targetId)}`; } diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index f06330d0..a0fff625 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -77,7 +77,10 @@ export class CdpConnection { this.socket = undefined; this.opening = undefined; this.sessionsByTarget.clear(); - const error = new Error("CDP connection closed"); + this.rejectPending(new Error("CDP connection closed")); + } + + private rejectPending(error: Error): void { for (const pending of this.pending.values()) pending.reject(error); this.pending.clear(); } @@ -98,6 +101,7 @@ export class CdpConnection { this.socket = undefined; this.opening = undefined; this.sessionsByTarget.clear(); + this.rejectPending(new Error("CDP connection closed")); }); socket.addEventListener("message", (event) => { this.handleMessage(typeof event.data === "string" ? event.data : String(event.data)); diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 554130f4..79c852e2 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -56,6 +56,12 @@ export class InternalComputerTranslator { this.browserExecutorFactory = opts.createBrowserExecutor ?? createBrowserExecutor; } + /** Release held resources: closes the browser executor's CDP connection if one was opened. */ + dispose(): void { + this.browserExecutor?.close(); + this.browserExecutor = undefined; + } + /** The browser-plane executor, connected lazily over the browser's CDP websocket. */ browser(): BrowserExecutor { if (!this.browserExecutor) { diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index df5f2b3f..7c50ddf3 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -443,6 +443,27 @@ describe("CuaAgentHarness", () => { expect(names).toContain("browser_snapshot"); }); + it("setMode keeps the requested activation state of surviving tools", async () => { + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "anthropic:claude-opus-4-5", + extraTools: [createCustomTool()], + }); + const withoutCustom = harness + .getTools() + .map((tool) => tool.name) + .filter((name) => name !== "custom"); + await harness.setActiveTools(withoutCustom); + + await harness.setMode("browser"); + + const active = harness.getActiveTools().map((tool) => tool.name); + expect(active).toContain("snapshot"); + expect(active).not.toContain("custom"); + }); + it("appends extraTools in harness construction", async () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const tool = createCustomTool(); diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 1bf43805..2c8ef501 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -51,7 +51,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options: CuaRu const mode = options.mode ?? (options.nativeTool ? modeForNativeTool(options.nativeTool) : "computer"); if (options.nativeTool) { - const nativeTool = resolveNativeTool(options.nativeTool, model, mode); + const nativeTool = resolveNativeTool(withJavascriptExec(options.nativeTool, options.javascriptExec), model, mode); const nativeModel: Model = { ...model, api: nativeApiForToolType(nativeTool.spec.type) as Model["api"] }; const executors = nativeToolExecutors(nativeTool); return { @@ -82,6 +82,14 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options: CuaRu }; } +// Fold the mode-level javascriptExec option into the native browser tool +// declaration so it behaves like canonical browser mode. An explicit +// enable_javascript_exec on the spec wins. +function withJavascriptExec(spec: CuaNativeToolSpec, javascriptExec: boolean | undefined): CuaNativeToolSpec { + if (!javascriptExec || spec.type !== "browser_20260701" || spec.enable_javascript_exec !== undefined) return spec; + return { ...spec, enable_javascript_exec: true }; +} + function composePayloadHooks(first: CuaPayloadHook, second: CuaPayloadHook | undefined): CuaPayloadHook { if (!second) return first; return async (payload, model, context) => { diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index c275765e..775b0f7c 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -56,6 +56,20 @@ describe("native runtime specs", () => { expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["browser"]); }); + it("folds javascriptExec into the native browser declaration unless the spec is explicit", () => { + const folded = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { + nativeTool: { type: "browser_20260701" }, + javascriptExec: true, + }); + expect(folded.nativeTool?.declaration.enable_javascript_exec).toBe(true); + + const explicit = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { + nativeTool: { type: "browser_20260701", enable_javascript_exec: false }, + javascriptExec: true, + }); + expect(explicit.nativeTool?.declaration.enable_javascript_exec).toBe(false); + }); + it("swaps the placeholder tool for the native declaration in the payload", async () => { const spec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "computer_20260701", enable_zoom: true } }); const payload = { diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 59a55d98..3434f040 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -377,10 +377,39 @@ async function setupHarnessRuntime( disabled: flags.noSkills, }); + // Validate mode/native-tool flags before provisioning so a bad combination + // never leaves an orphaned browser behind. + const mode = parseMode(flags.mode); + const nativeTool = parseNativeTool(flags.nativeTool, flags.jsExec); + const provisioned = await provisionForFlags(flags, auth); + try { + return await finishHarnessRuntime(flags, auth, provisioned, { cwd, skills, contextFiles, mode, nativeTool, skipDisk: opts.skipDiskSession === true }); + } catch (err) { + await provisioned.handle.close().catch(() => {}); + throw err; + } +} + +interface FinishHarnessRuntimeContext { + cwd: string; + skills: Skill[]; + contextFiles: ContextFile[]; + mode: CuaMode | undefined; + nativeTool: CuaNativeToolSpec | undefined; + skipDisk: boolean; +} + +async function finishHarnessRuntime( + flags: HarnessCliFlags, + auth: ResolvedAuth, + provisioned: ProvisionedBrowser, + context: FinishHarnessRuntimeContext, +): Promise { + const { cwd, skills, contextFiles, mode, nativeTool } = context; const repo = createSessionRepo(flags.sessionDir); - const skipDisk = opts.skipDiskSession === true && !hasExplicitSessionFlag(flags); + const skipDisk = context.skipDisk && !hasExplicitSessionFlag(flags); const resolved = skipDisk ? undefined : await resolveSession(repo, cwd, flags, provisioned.named); let inMemorySession: Session | undefined; @@ -419,8 +448,8 @@ async function setupHarnessRuntime( skills, contextFiles, thinkingLevel, - mode: parseMode(flags.mode), - nativeTool: parseNativeTool(flags.nativeTool, flags.jsExec), + mode, + nativeTool, javascriptExec: flags.jsExec, playwright: flags.playwright, modelBaseUrl: baseUrlOverride, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 275130bf..d3cd16c1 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -186,6 +186,14 @@ function parseCliArgs(argv: string[]): CliFlags { ); } } + const modeRaw = parsed.values.mode as string | undefined; + if (modeRaw !== undefined && !["computer", "browser", "hybrid"].includes(modeRaw.trim().toLowerCase())) { + throw new Error(`invalid --mode value "${modeRaw}"; expected one of: computer | browser | hybrid`); + } + const nativeToolRaw = parsed.values["native-tool"] as string | undefined; + if (nativeToolRaw !== undefined && !["computer_20260701", "browser_20260701"].includes(nativeToolRaw.trim().toLowerCase())) { + throw new Error(`invalid --native-tool value "${nativeToolRaw}"; expected one of: computer_20260701 | browser_20260701`); + } return { help: !!parsed.values.help, diff --git a/packages/cli/src/print.ts b/packages/cli/src/print.ts index 8d244a9e..40275419 100644 --- a/packages/cli/src/print.ts +++ b/packages/cli/src/print.ts @@ -94,6 +94,9 @@ export async function runPrint(opts: RunPrintOptions): Promise { async function maybeInitialScreenshot(opts: RunPrintOptions): Promise { if (opts.skipInitialScreenshot) return undefined; + // Browser mode's only frame is the viewport; skip the OS-display capture + // rather than mix coordinate frames on the first turn. + if (opts.harness.getMode() === "browser") return undefined; const hasPriorTurn = await sessionHasPriorTurn(opts.session); if (hasPriorTurn) return undefined; const png = await captureScreenshot(opts.browserHandle.client, opts.browserHandle.browser.session_id); diff --git a/packages/cli/src/tui/main.ts b/packages/cli/src/tui/main.ts index 449b31a8..ef665b47 100644 --- a/packages/cli/src/tui/main.ts +++ b/packages/cli/src/tui/main.ts @@ -447,6 +447,9 @@ async function maybeInitialScreenshot( ): Promise { if (firstPromptSent) return undefined; if (opts.skipInitialScreenshot) return undefined; + // Browser mode's only frame is the viewport; skip the OS-display capture + // rather than mix coordinate frames on the first turn. + if (opts.harness.getMode() === "browser") return undefined; if (await sessionHasPriorTurn(opts.session)) return undefined; const png = await captureScreenshot(opts.browserHandle.client, opts.browserHandle.browser.session_id); if (!png) return undefined; From a6d40b7261d4d00bd11384f2fc1c4b95e993d0a0 Mon Sep 17 00:00:00 2001 From: hypeship Date: Wed, 8 Jul 2026 19:51:30 +0000 Subject: [PATCH 08/34] BrowserExecutor owns its CdpConnection Construct from the cdp_ws_url instead of an injected connection, so close() disposes a resource the executor actually owns. Drops the now-redundant createBrowserExecutor helper; the translator's factory option remains the test seam. --- packages/agent/src/translator/browser.ts | 12 +++++++----- packages/agent/src/translator/translator.ts | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 54dc10ba..537f89a1 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -52,9 +52,13 @@ export class BrowserExecutor { private refCounter = 0; private activeTargetId?: string; - constructor(private readonly cdp: CdpConnection) {} + private readonly cdp: CdpConnection; - /** Close the underlying CDP connection. Safe to call when never connected. */ + constructor(cdpWsUrl: string) { + this.cdp = new CdpConnection(cdpWsUrl); + } + + /** Close the CDP connection. Safe to call when never connected. */ close(): void { this.cdp.close(); } @@ -569,6 +573,4 @@ const INTERACTIVE_ROLES: ReadonlySet = new Set([ const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); -export function createBrowserExecutor(cdpWsUrl: string): BrowserExecutor { - return new BrowserExecutor(new CdpConnection(cdpWsUrl)); -} + diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 79c852e2..5610d31b 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -21,7 +21,7 @@ import { type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import sharp from "sharp"; -import { createBrowserExecutor, type BrowserExecutor } from "./browser"; +import { BrowserExecutor } from "./browser"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult } from "./types"; @@ -53,7 +53,7 @@ export class InternalComputerTranslator { this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; this.cdpWsUrl = opts.browser.cdp_ws_url; - this.browserExecutorFactory = opts.createBrowserExecutor ?? createBrowserExecutor; + this.browserExecutorFactory = opts.createBrowserExecutor ?? ((cdpWsUrl) => new BrowserExecutor(cdpWsUrl)); } /** Release held resources: closes the browser executor's CDP connection if one was opened. */ From 1eef797b1cda8bb150e3ed79c822f1b9b999f08e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:23:48 +0000 Subject: [PATCH 09/34] Harden browser executor ref lifecycle, depth rendering, and dialog handling - Prune stale ref entries when a target's generation bumps, and drop all refs/generations for a target when its session detaches - Enable Page events per attached session and invalidate refs on main-frame Page.frameNavigated, suppressing the bump for our own navigate() so it is counted once - Indent snapshot lines by rendered depth so skipped wrappers neither indent children nor consume the depth budget - Replace the dead textarea AX role with treeitem in interactive roles - Auto-dismiss native JavaScript dialogs and surface the dialog message as an extra read result on the next action - Accept an injected CdpConnection in BrowserExecutor for tests --- packages/agent/src/translator/browser.ts | 106 +++++++++-- packages/agent/src/translator/cdp.ts | 2 +- .../agent/test/translator-browser.test.ts | 166 +++++++++++++++++- 3 files changed, 258 insertions(+), 16 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 537f89a1..8efb7042 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -12,7 +12,7 @@ import { type CuaActionBrowserSnapshot, type CuaBrowserAction, } from "@onkernel/cua-ai"; -import { CdpConnection } from "./cdp"; +import { CdpConnection, type CdpEventMessage } from "./cdp"; import type { BatchReadResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; @@ -42,20 +42,58 @@ interface RefEntry { * Executes browser-plane canonical actions over CDP. * * Element refs are snapshot-scoped: each snapshot/find mints `e` ids - * mapped to CDP backend node ids for the target's current generation. A - * navigation bumps the generation, and refs from earlier generations resolve - * to a stale-ref error whose message tells the model how to recover. + * mapped to CDP backend node ids for the target's current generation. Any + * main-frame navigation — our own navigate() or a page-initiated one seen + * via Page.frameNavigated — bumps the generation and prunes that target's + * refs, so refs from earlier generations resolve to a stale-ref error whose + * message tells the model how to recover. + * + * Native JavaScript dialogs (alert/confirm/prompt) are auto-dismissed so + * they never wedge the CDP session; the dialog message is surfaced as an + * extra read result on the next executed action. */ export class BrowserExecutor { private readonly refs = new Map(); private readonly generations = new Map(); + private readonly targetsBySession = new Map(); + private readonly selfNavigations = new Set(); + private readonly dialogNotes: string[] = []; private refCounter = 0; private activeTargetId?: string; private readonly cdp: CdpConnection; - constructor(cdpWsUrl: string) { - this.cdp = new CdpConnection(cdpWsUrl); + constructor(cdp: string | CdpConnection) { + this.cdp = typeof cdp === "string" ? new CdpConnection(cdp) : cdp; + this.cdp.onEvent((event) => this.handleCdpEvent(event)); + } + + private handleCdpEvent(event: CdpEventMessage): void { + switch (event.method) { + case "Page.frameNavigated": { + const frame = event.params.frame as { parentId?: string } | undefined; + if (!event.sessionId || frame?.parentId) return; + const targetId = this.targetsBySession.get(event.sessionId); + if (!targetId) return; + if (!this.selfNavigations.delete(targetId)) this.invalidateRefs(targetId); + return; + } + case "Page.javascriptDialogOpening": { + if (!event.sessionId) return; + const { type, message } = event.params as { type?: string; message?: string }; + void this.cdp.send("Page.handleJavaScriptDialog", { accept: false }, event.sessionId).catch(() => {}); + this.dialogNotes.push(`Auto-dismissed a JavaScript ${type ?? "dialog"} dialog: ${JSON.stringify(message ?? "")}`); + return; + } + case "Target.detachedFromTarget": { + const sessionId = event.params.sessionId; + if (typeof sessionId !== "string") return; + const targetId = this.targetsBySession.get(sessionId); + this.targetsBySession.delete(sessionId); + if (targetId) this.dropTarget(targetId); + return; + } + } } /** Close the CDP connection. Safe to call when never connected. */ @@ -64,6 +102,13 @@ export class BrowserExecutor { } async execute(action: CuaBrowserAction): Promise { + const results = await this.dispatch(action); + const dialogs = this.drainDialogNotes(); + if (dialogs) results.push({ type: "browser_text", label: "dialog", text: dialogs }); + return results; + } + + private async dispatch(action: CuaBrowserAction): Promise { switch (action.type) { case "browser_snapshot": return [{ type: "browser_text", label: "snapshot", text: await this.snapshot(action) }]; @@ -148,12 +193,16 @@ export class BrowserExecutor { const walk = (nodeId: string, depth: number): void => { const node = byId.get(nodeId); if (!node) return; - if (depth <= maxDepth && !node.ignored) { + let childDepth = depth; + if (!node.ignored) { const line = this.renderNode(node, targetId, generation, depth, interactiveOnly); - if (line) lines.push(line); + if (line) { + lines.push(line); + childDepth = depth + 1; + } } - if (depth < maxDepth) { - for (const childId of node.childIds ?? []) walk(childId, depth + 1); + if (childDepth <= maxDepth) { + for (const childId of node.childIds ?? []) walk(childId, childDepth); } }; for (const rootId of rootIds) walk(rootId, 0); @@ -302,14 +351,19 @@ export class BrowserExecutor { ); const entry = history.entries[history.currentIndex + (direction === "back" ? -1 : 1)]; if (!entry) throw new Error(`cannot go ${direction}: no history entry`); + this.selfNavigations.add(targetId); await this.cdp.send("Page.navigateToHistoryEntry", { entryId: entry.id }, session); this.invalidateRefs(targetId); return `Navigated ${direction}.\n${await this.tabContext(targetId)}`; } const url = normalizeGotoUrl(action.url); if (!url) throw new Error("invalid url"); + this.selfNavigations.add(targetId); const { errorText } = await this.cdp.send<{ errorText?: string }>("Page.navigate", { url }, session); - if (errorText) throw new Error(`navigation to ${url} failed: ${errorText}`); + if (errorText) { + this.selfNavigations.delete(targetId); + throw new Error(`navigation to ${url} failed: ${errorText}`); + } this.invalidateRefs(targetId); return `Navigated to ${url}.\n${await this.tabContext(targetId)}`; } @@ -408,7 +462,26 @@ export class BrowserExecutor { } private invalidateRefs(targetId: string): void { - this.generations.set(targetId, this.generation(targetId) + 1); + const generation = this.generation(targetId) + 1; + this.generations.set(targetId, generation); + for (const [ref, entry] of this.refs) { + if (entry.targetId === targetId && entry.generation < generation) this.refs.delete(ref); + } + } + + private dropTarget(targetId: string): void { + this.generations.delete(targetId); + this.selfNavigations.delete(targetId); + for (const [ref, entry] of this.refs) { + if (entry.targetId === targetId) this.refs.delete(ref); + } + } + + private drainDialogNotes(): string | undefined { + if (this.dialogNotes.length === 0) return undefined; + const text = this.dialogNotes.join("\n"); + this.dialogNotes.length = 0; + return text; } private async session(tabId?: string): Promise { @@ -416,7 +489,12 @@ export class BrowserExecutor { } private async attach(targetId: string): Promise { - return this.cdp.attachToTarget(targetId); + const session = await this.cdp.attachToTarget(targetId); + if (!this.targetsBySession.has(session)) { + this.targetsBySession.set(session, targetId); + await this.cdp.send("Page.enable", {}, session); + } + return session; } private async resolveTarget(tabId?: string): Promise { @@ -568,7 +646,7 @@ const INTERACTIVE_ROLES: ReadonlySet = new Set([ "spinbutton", "switch", "tab", - "textarea", + "treeitem", ]); const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index a0fff625..546da8f7 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -12,7 +12,7 @@ interface PendingCommand { reject(error: Error): void; } -interface CdpEventMessage { +export interface CdpEventMessage { method: string; params: Record; sessionId?: string; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index a8c30090..b55b4f4b 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -2,7 +2,8 @@ import type Kernel from "@onkernel/sdk"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; -import type { BrowserExecutor } from "../src/translator/browser"; +import { BrowserExecutor } from "../src/translator/browser"; +import type { CdpConnection } from "../src/translator/cdp"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -86,3 +87,166 @@ describe("InternalComputerTranslator OS additions", () => { ]); }); }); + +interface FakeCdpEvent { + method: string; + params: Record; + sessionId?: string; +} + +interface SentCommand { + method: string; + params: Record; + sessionId?: string; +} + +function createFakeCdp(nodes: unknown[] = []) { + const sent: SentCommand[] = []; + const listeners: Array<(event: FakeCdpEvent) => void> = []; + const fake = { + onEvent: (listener: (event: FakeCdpEvent) => void) => { + listeners.push(listener); + }, + send: async (method: string, params: Record = {}, sessionId?: string) => { + sent.push({ method, params, sessionId }); + switch (method) { + case "Accessibility.getFullAXTree": + return { nodes }; + case "DOM.getBoxModel": + return { model: { content: [0, 0, 10, 0, 10, 10, 0, 10] } }; + case "Runtime.evaluate": + return { result: { value: "hello" } }; + default: + return {}; + } + }, + pageTargets: async () => [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], + attachToTarget: async () => "session-1", + createTarget: async () => "TARGET-2", + close: () => {}, + }; + const emit = (event: FakeCdpEvent) => { + for (const listener of listeners) listener(event); + }; + return { sent, emit, cdp: fake as unknown as CdpConnection }; +} + +interface AXNodeSpec { + nodeId: string; + role?: string; + name?: string; + backendDOMNodeId?: number; + parentId?: string; + childIds?: string[]; +} + +function ax(spec: AXNodeSpec) { + return { + nodeId: spec.nodeId, + parentId: spec.parentId, + childIds: spec.childIds, + backendDOMNodeId: spec.backendDOMNodeId, + role: spec.role !== undefined ? { value: spec.role } : undefined, + name: spec.name !== undefined ? { value: spec.name } : undefined, + }; +} + +function refsOf(executor: BrowserExecutor): Map { + return (executor as unknown as { refs: Map }).refs; +} + +async function snapshotText(executor: BrowserExecutor, action: Record = {}): Promise { + const results = await executor.execute({ type: "browser_snapshot", ...action } as CuaBrowserAction); + const read = results[0]!; + if (read.type !== "browser_text") throw new Error("expected browser_text read result"); + return read.text; +} + +const BUTTON_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), +]; + +describe("BrowserExecutor ref lifecycle", () => { + it("prunes stale refs when a navigation bumps the generation", async () => { + const { cdp } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + expect(refsOf(executor).size).toBe(1); + await executor.execute({ type: "browser_navigate", url: "https://b.test" } as CuaBrowserAction); + expect(refsOf(executor).size).toBe(0); + }); + + it("invalidates refs on main-frame frameNavigated but not on subframe navigation", async () => { + const { cdp, emit, sent } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "Save" [e1]'); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "F2", parentId: "F1" } }, sessionId: "session-1" }); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent")).toBe(true); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "F1" } }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + expect(refsOf(executor).size).toBe(0); + }); + + it("does not double-bump the generation for its own navigate", async () => { + const { cdp, emit } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await executor.execute({ type: "browser_navigate", url: "https://b.test" } as CuaBrowserAction); + const text = await snapshotText(executor); + expect(text).toContain('button "Save" [e1]'); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "F1" } }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).resolves.toEqual([]); + }); +}); + +describe("BrowserExecutor snapshot rendering", () => { + it("indents by rendered depth so skipped wrappers neither indent nor consume the depth budget", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "generic", parentId: "1", childIds: ["3"] }), + ax({ nodeId: "3", role: "generic", parentId: "2", childIds: ["4"] }), + ax({ nodeId: "4", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "3" }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + expect(await snapshotText(executor)).toBe('RootWebArea "Page"\n button "Save" [e1]'); + expect(await snapshotText(executor, { depth: 1 })).toBe('RootWebArea "Page"\n button "Save" [e2]'); + }); + + it("treats treeitem as interactive and the bogus textarea role as non-interactive", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "treeitem", name: "Reports", backendDOMNodeId: 10, parentId: "1" }), + ax({ nodeId: "3", role: "textarea", name: "Notes", backendDOMNodeId: 11, parentId: "1" }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toContain('treeitem "Reports" [e1]'); + expect(text).toContain('textarea "Notes"'); + expect(text).not.toContain('textarea "Notes" ['); + }); +}); + +describe("BrowserExecutor dialog guard", () => { + it("auto-dismisses JavaScript dialogs and surfaces the message on the next action", async () => { + const { cdp, emit, sent } = createFakeCdp(); + const executor = new BrowserExecutor(cdp); + await executor.execute({ type: "browser_text" } as CuaBrowserAction); + + emit({ method: "Page.javascriptDialogOpening", params: { type: "confirm", message: "Leave page?" }, sessionId: "session-1" }); + const handled = sent.find((cmd) => cmd.method === "Page.handleJavaScriptDialog"); + expect(handled).toEqual({ method: "Page.handleJavaScriptDialog", params: { accept: false }, sessionId: "session-1" }); + + const results = await executor.execute({ type: "browser_text" } as CuaBrowserAction); + expect(results).toEqual([ + { type: "browser_text", label: "text", text: "hello" }, + { type: "browser_text", label: "dialog", text: 'Auto-dismissed a JavaScript confirm dialog: "Leave page?"' }, + ]); + }); +}); From bc58ba0985af2a216e6fa514db1238348ccd3c63 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:35:52 +0000 Subject: [PATCH 10/34] Render AX states, self-heal stale refs, dedupe StaticText, add gated cursor hints --- packages/agent/src/agent.ts | 11 + packages/agent/src/tools.ts | 2 + packages/agent/src/translator/browser.ts | 198 ++++++++++++++++-- packages/agent/src/translator/translator.ts | 17 +- .../agent/test/translator-browser.test.ts | 180 +++++++++++++++- 5 files changed, 380 insertions(+), 28 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index daee9968..585c8f1c 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -78,6 +78,8 @@ export type CuaAgentOptions = Omit & { nativeTool?: CuaNativeToolSpec; /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only; default false. */ + cursorHints?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ @@ -118,6 +120,8 @@ export type CuaAgentHarnessOptions< nativeTool?: CuaNativeToolSpec; /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ javascriptExec?: boolean; + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only; default false. */ + cursorHints?: boolean; /** Expose a helper for browser navigation and URL reads. */ computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ @@ -146,6 +150,7 @@ class CuaRuntimeController { mode?: CuaMode; nativeTool?: CuaNativeToolSpec; javascriptExec?: boolean; + cursorHints?: boolean; computerUseExtra?: boolean; playwright?: boolean; onPayload?: SimpleStreamOptions["onPayload"]; @@ -233,6 +238,8 @@ class CuaRuntimeController { client: this.options.client, coordinateSystem: this.runtimeSpec.coordinateSystem, screenshot: this.runtimeSpec.screenshot, + mode: this.runtimeSpec.mode, + cursorHints: this.options.cursorHints, }); } } @@ -267,6 +274,7 @@ export class CuaAgent extends Agent { mode, nativeTool, javascriptExec, + cursorHints, computerUseExtra, playwright, ...agentOptions @@ -279,6 +287,7 @@ export class CuaAgent extends Agent { mode, nativeTool, javascriptExec, + cursorHints, computerUseExtra, playwright, onPayload, @@ -415,6 +424,7 @@ export class CuaAgentHarness< mode, nativeTool, javascriptExec, + cursorHints, computerUseExtra, playwright, systemPrompt, @@ -430,6 +440,7 @@ export class CuaAgentHarness< mode, nativeTool, javascriptExec, + cursorHints, computerUseExtra, playwright, onPayload, diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 5a82c86a..80b574cd 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -25,6 +25,8 @@ export interface ComputerToolOptions { screenshot?: CuaScreenshotSpec; /** Action plane(s) in play; controls whether the post-action fallback capture is the OS display or the viewport. Default "computer". */ mode?: CuaMode; + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode. Default false. */ + cursorHints?: boolean; computerUseExtra?: boolean; playwright?: boolean; } diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 8efb7042..c6c2b8a4 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -27,6 +27,8 @@ interface AXNode { ignored?: boolean; role?: { value?: string }; name?: { value?: string }; + value?: { value?: unknown }; + properties?: Array<{ name: string; value?: { value?: unknown } }>; backendDOMNodeId?: number; parentId?: string; childIds?: string[]; @@ -36,6 +38,22 @@ interface RefEntry { backendNodeId: number; targetId: string; generation: number; + role: string; + name: string; + nth: number; +} + +interface RenderContext { + targetId: string; + generation: number; + interactiveOnly: boolean; + nthIndex: Map; + cursorIds?: ReadonlySet; +} + +export interface BrowserExecutorOptions { + /** Mark elements whose computed cursor is "pointer" as clickable hints in snapshots. Default false. */ + cursorHints?: boolean; } /** @@ -48,6 +66,11 @@ interface RefEntry { * refs, so refs from earlier generations resolve to a stale-ref error whose * message tells the model how to recover. * + * A ref whose backend node vanished without a navigation (DOM churn) is + * self-healed: the AX tree is re-fetched and the ref re-resolved by the + * (role, name, nth) triple recorded at mint time, but only when the fresh + * match is unambiguous. + * * Native JavaScript dialogs (alert/confirm/prompt) are auto-dismissed so * they never wedge the CDP session; the dialog message is surfaced as an * extra read result on the next executed action. @@ -60,10 +83,12 @@ export class BrowserExecutor { private readonly dialogNotes: string[] = []; private refCounter = 0; private activeTargetId?: string; + private readonly cursorHints: boolean; private readonly cdp: CdpConnection; - constructor(cdp: string | CdpConnection) { + constructor(cdp: string | CdpConnection, options: BrowserExecutorOptions = {}) { + this.cursorHints = options.cursorHints ?? false; this.cdp = typeof cdp === "string" ? new CdpConnection(cdp) : cdp; this.cdp.onEvent((event) => this.handleCdpEvent(event)); } @@ -181,31 +206,37 @@ export class BrowserExecutor { let rootIds = roots.map((node) => node.nodeId); if (action.ref) { const entry = this.resolveRef(action.ref, targetId); - const rootNode = nodes.find((node) => node.backendDOMNodeId === entry.backendNodeId); - if (!rootNode) throw new Error(`ref ${action.ref} is stale or not on the current page. ${STALE_REF_HINT}`); + const rootNode = + nodes.find((node) => node.backendDOMNodeId === entry.backendNodeId) ?? this.healEntry(action.ref, entry, nodes); rootIds = [rootNode.nodeId]; } - const generation = this.generation(targetId); + const ctx: RenderContext = { + targetId, + generation: this.generation(targetId), + interactiveOnly: action.filter === "interactive", + nthIndex: buildNthIndex(nodes), + cursorIds: this.cursorHints ? await this.cursorPointerIds(session) : undefined, + }; const lines: string[] = []; const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; - const interactiveOnly = action.filter === "interactive"; - const walk = (nodeId: string, depth: number): void => { + const walk = (nodeId: string, depth: number, parentName: string): void => { const node = byId.get(nodeId); if (!node) return; let childDepth = depth; if (!node.ignored) { - const line = this.renderNode(node, targetId, generation, depth, interactiveOnly); + const line = this.renderNode(node, depth, parentName, ctx); if (line) { lines.push(line); childDepth = depth + 1; } } if (childDepth <= maxDepth) { - for (const childId of node.childIds ?? []) walk(childId, childDepth); + const name = node.name?.value ?? ""; + for (const childId of node.childIds ?? []) walk(childId, childDepth, name || parentName); } }; - for (const rootId of rootIds) walk(rootId, 0); + for (const rootId of rootIds) walk(rootId, 0, ""); let text = lines.join("\n"); if (text.length > SNAPSHOT_CHAR_LIMIT) { @@ -214,19 +245,48 @@ export class BrowserExecutor { return text || "(empty accessibility tree)"; } - private renderNode(node: AXNode, targetId: string, generation: number, depth: number, interactiveOnly: boolean): string | undefined { + private renderNode(node: AXNode, depth: number, parentName: string, ctx: RenderContext): string | undefined { const role = node.role?.value ?? ""; const name = node.name?.value ?? ""; const interactive = INTERACTIVE_ROLES.has(role); - if (interactiveOnly && !interactive) return undefined; - if (!interactiveOnly && !name && !interactive && SKIPPED_ROLES.has(role)) return undefined; + const pointer = node.backendDOMNodeId !== undefined && (ctx.cursorIds?.has(node.backendDOMNodeId) ?? false); + if (ctx.interactiveOnly && !interactive && !pointer) return undefined; + if (role === "StaticText" && name === parentName) return undefined; + if (!ctx.interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; - if (node.backendDOMNodeId !== undefined && interactive) { - line += ` [${this.mintRef(node.backendDOMNodeId, targetId, generation)}]`; + if (node.backendDOMNodeId !== undefined && (interactive || pointer)) { + line += ` [${this.mintRef(node, ctx.targetId, ctx.generation, ctx.nthIndex)}]`; } + const states = collectStates(node); + if (pointer && !interactive) states.push("cursor:pointer"); + if (states.length > 0) line += ` [${states.join(", ")}]`; return line; } + /** Resolve backend node ids for elements whose own computed cursor is "pointer", without touching the DOM. */ + private async cursorPointerIds(session: string): Promise> { + const ids = new Set(); + const { result } = await this.cdp.send<{ result: { objectId?: string } }>( + "Runtime.evaluate", + { expression: CURSOR_POINTER_SCAN, returnByValue: false }, + session, + ); + if (!result.objectId) return ids; + const { result: properties } = await this.cdp.send<{ result: Array<{ name: string; value?: { objectId?: string } }> }>( + "Runtime.getProperties", + { objectId: result.objectId, ownProperties: true }, + session, + ); + for (const property of properties) { + const objectId = property.value?.objectId; + if (!/^\d+$/.test(property.name) || !objectId) continue; + const { node } = await this.cdp.send<{ node: { backendNodeId?: number } }>("DOM.describeNode", { objectId }, session); + if (node.backendNodeId !== undefined) ids.add(node.backendNodeId); + } + await this.cdp.send("Runtime.releaseObject", { objectId: result.objectId }, session); + return ids; + } + private async find(action: CuaActionBrowserFind): Promise { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); @@ -239,11 +299,12 @@ export class BrowserExecutor { .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); if (scored.length === 0) return `No elements matched ${JSON.stringify(action.query)}. Try snapshot for the full tree.`; + const nthIndex = buildNthIndex(nodes); return scored .map(({ node }) => { const role = node.role?.value ?? "node"; const name = node.name?.value ? ` ${JSON.stringify(node.name.value)}` : ""; - return `${role}${name} [${this.mintRef(node.backendDOMNodeId!, targetId, this.generation(targetId))}]`; + return `${role}${name} [${this.mintRef(node, targetId, this.generation(targetId), nthIndex)}]`; }) .join("\n"); } @@ -425,7 +486,8 @@ export class BrowserExecutor { try { await this.cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: entry.backendNodeId }, session); } catch (err) { - throw new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, { cause: err }); + await this.healRef(ref, entry, session, err); + await this.cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: entry.backendNodeId }, session); } } @@ -438,21 +500,58 @@ export class BrowserExecutor { ); return object.objectId; } catch (err) { - throw new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, { cause: err }); + await this.healRef(ref, entry, session, err); + const { object } = await this.cdp.send<{ object: { objectId: string } }>( + "DOM.resolveNode", + { backendNodeId: entry.backendNodeId }, + session, + ); + return object.objectId; } } - private mintRef(backendNodeId: number, targetId: string, generation: number): string { + private async healRef(ref: string, entry: RefEntry, session: string, cause: unknown): Promise { + const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); + this.healEntry(ref, entry, nodes, cause); + } + + /** + * Re-resolve a stale entry by its (role, name, nth) triple against a fresh + * AX tree. Heals only the unambiguous case — the ref was minted as the + * first of its role+name cohort and exactly one fresh node matches. + */ + private healEntry(ref: string, entry: RefEntry, nodes: AXNode[], cause?: unknown): AXNode { + const candidates = nodes.filter( + (node) => + !node.ignored && + node.backendDOMNodeId !== undefined && + (node.role?.value ?? "") === entry.role && + (node.name?.value ?? "") === entry.name, + ); + const match = candidates.length === 1 && entry.nth === 0 && (entry.role || entry.name) ? candidates[0] : undefined; + if (!match) throw staleRefError(ref, cause); + entry.backendNodeId = match.backendDOMNodeId!; + return match; + } + + private mintRef(node: AXNode, targetId: string, generation: number, nthIndex: Map): string { this.refCounter += 1; const ref = `e${this.refCounter}`; - this.refs.set(ref, { backendNodeId, targetId, generation }); + this.refs.set(ref, { + backendNodeId: node.backendDOMNodeId!, + targetId, + generation, + role: node.role?.value ?? "", + name: node.name?.value ?? "", + nth: nthIndex.get(node.nodeId) ?? 0, + }); return ref; } private resolveRef(ref: string, targetId: string): RefEntry { const entry = this.refs.get(ref); if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(targetId)) { - throw new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`); + throw staleRefError(ref); } return entry; } @@ -518,6 +617,49 @@ function tabOf(action: { tab_id?: string }): string | undefined { return action.tab_id; } +function staleRefError(ref: string, cause?: unknown): Error { + return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); +} + +/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ +function buildNthIndex(nodes: AXNode[]): Map { + const counts = new Map(); + const index = new Map(); + for (const node of nodes) { + if (node.ignored || node.backendDOMNodeId === undefined) continue; + const key = `${node.role?.value ?? ""}\u0000${node.name?.value ?? ""}`; + const nth = counts.get(key) ?? 0; + counts.set(key, nth + 1); + index.set(node.nodeId, nth); + } + return index; +} + +function collectStates(node: AXNode): string[] { + const states: string[] = []; + for (const property of node.properties ?? []) { + const value = property.value?.value; + switch (property.name) { + case "checked": + case "pressed": + if (value === true || value === "true") states.push(property.name); + else if (value === "mixed") states.push(`${property.name}=mixed`); + break; + case "expanded": + case "disabled": + case "selected": + case "required": + if (value === true || value === "true") states.push(property.name); + break; + } + } + const value = node.value?.value; + if (value !== undefined && value !== "" && String(value) !== (node.name?.value ?? "")) { + states.push(`value=${JSON.stringify(String(value))}`); + } + return states; +} + function shortTabId(targetId: string): string { return targetId.slice(0, 10).toUpperCase(); } @@ -651,4 +793,20 @@ const INTERACTIVE_ROLES: ReadonlySet = new Set([ const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); +const CURSOR_POINTER_SCAN = `(() => { + const matches = []; + if (!document.body) return matches; + for (const el of document.body.querySelectorAll("*")) { + if (matches.length >= 100) break; + if (el.closest("a, button, input, select, textarea, summary")) continue; + if (getComputedStyle(el).cursor !== "pointer") continue; + const parent = el.parentElement; + if (parent && getComputedStyle(parent).cursor === "pointer") continue; + const rect = el.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) continue; + matches.push(el); + } + return matches; +})()`; + diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 5610d31b..f38f156c 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -17,11 +17,12 @@ import { type CuaActionZoom, type CuaBrowserAction, type CuaDragMouseButton, + type CuaMode, type CuaMouseButton, type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import sharp from "sharp"; -import { BrowserExecutor } from "./browser"; +import { BrowserExecutor, type BrowserExecutorOptions } from "./browser"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult } from "./types"; @@ -32,8 +33,12 @@ export interface InternalComputerTranslatorOptions { client: Kernel; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; + /** Action plane(s) in play; browser-executor extras like cursor hints are gated to "browser". */ + mode?: CuaMode; + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode. Default false. */ + cursorHints?: boolean; /** Browser executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ - createBrowserExecutor?: (cdpWsUrl: string) => BrowserExecutor; + createBrowserExecutor?: (cdpWsUrl: string, options: BrowserExecutorOptions) => BrowserExecutor; } export class InternalComputerTranslator { @@ -43,7 +48,8 @@ export class InternalComputerTranslator { private readonly screenshotSpec?: CuaScreenshotSpec; private readonly viewport: { width: number; height: number }; private readonly cdpWsUrl?: string; - private readonly browserExecutorFactory: (cdpWsUrl: string) => BrowserExecutor; + private readonly browserExecutorOptions: BrowserExecutorOptions; + private readonly browserExecutorFactory: (cdpWsUrl: string, options: BrowserExecutorOptions) => BrowserExecutor; private browserExecutor?: BrowserExecutor; constructor(opts: InternalComputerTranslatorOptions) { @@ -53,7 +59,8 @@ export class InternalComputerTranslator { this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; this.cdpWsUrl = opts.browser.cdp_ws_url; - this.browserExecutorFactory = opts.createBrowserExecutor ?? ((cdpWsUrl) => new BrowserExecutor(cdpWsUrl)); + this.browserExecutorOptions = { cursorHints: opts.cursorHints === true && opts.mode === "browser" }; + this.browserExecutorFactory = opts.createBrowserExecutor ?? ((cdpWsUrl, options) => new BrowserExecutor(cdpWsUrl, options)); } /** Release held resources: closes the browser executor's CDP connection if one was opened. */ @@ -66,7 +73,7 @@ export class InternalComputerTranslator { browser(): BrowserExecutor { if (!this.browserExecutor) { if (!this.cdpWsUrl) throw new Error("browser has no cdp_ws_url; browser actions are unavailable"); - this.browserExecutor = this.browserExecutorFactory(this.cdpWsUrl); + this.browserExecutor = this.browserExecutorFactory(this.cdpWsUrl, this.browserExecutorOptions); } return this.browserExecutor; } diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index b55b4f4b..fda1089b 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -2,7 +2,7 @@ import type Kernel from "@onkernel/sdk"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; -import { BrowserExecutor } from "../src/translator/browser"; +import { BrowserExecutor, type BrowserExecutorOptions } from "../src/translator/browser"; import type { CdpConnection } from "../src/translator/cdp"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -100,9 +100,14 @@ interface SentCommand { sessionId?: string; } -function createFakeCdp(nodes: unknown[] = []) { +function createFakeCdp(initialNodes: unknown[] = []) { const sent: SentCommand[] = []; const listeners: Array<(event: FakeCdpEvent) => void> = []; + let nodes = initialNodes as Array<{ backendDOMNodeId?: number }>; + let cursorBackendIds: number[] = []; + const requireBackendId = (id: unknown) => { + if (!nodes.some((node) => node.backendDOMNodeId === id)) throw new Error("No node with given id found"); + }; const fake = { onEvent: (listener: (event: FakeCdpEvent) => void) => { listeners.push(listener); @@ -112,10 +117,27 @@ function createFakeCdp(nodes: unknown[] = []) { switch (method) { case "Accessibility.getFullAXTree": return { nodes }; + case "DOM.scrollIntoViewIfNeeded": + requireBackendId(params.backendNodeId); + return {}; case "DOM.getBoxModel": + requireBackendId(params.backendNodeId); return { model: { content: [0, 0, 10, 0, 10, 10, 0, 10] } }; + case "DOM.resolveNode": + requireBackendId(params.backendNodeId); + return { object: { objectId: "node-obj" } }; case "Runtime.evaluate": + if (params.returnByValue === false) return { result: { objectId: "cursor-scan" } }; return { result: { value: "hello" } }; + case "Runtime.getProperties": + return { + result: [ + ...cursorBackendIds.map((id, index) => ({ name: String(index), value: { objectId: `el-${id}` } })), + { name: "length", value: {} }, + ], + }; + case "DOM.describeNode": + return { node: { backendNodeId: Number(String(params.objectId).slice(3)) } }; default: return {}; } @@ -128,13 +150,21 @@ function createFakeCdp(nodes: unknown[] = []) { const emit = (event: FakeCdpEvent) => { for (const listener of listeners) listener(event); }; - return { sent, emit, cdp: fake as unknown as CdpConnection }; + const setNodes = (next: unknown[]) => { + nodes = next as Array<{ backendDOMNodeId?: number }>; + }; + const setCursorBackendIds = (ids: number[]) => { + cursorBackendIds = ids; + }; + return { sent, emit, setNodes, setCursorBackendIds, cdp: fake as unknown as CdpConnection }; } interface AXNodeSpec { nodeId: string; role?: string; name?: string; + value?: unknown; + properties?: Array<{ name: string; value?: unknown }>; backendDOMNodeId?: number; parentId?: string; childIds?: string[]; @@ -148,6 +178,8 @@ function ax(spec: AXNodeSpec) { backendDOMNodeId: spec.backendDOMNodeId, role: spec.role !== undefined ? { value: spec.role } : undefined, name: spec.name !== undefined ? { value: spec.name } : undefined, + value: spec.value !== undefined ? { value: spec.value } : undefined, + properties: spec.properties?.map((property) => ({ name: property.name, value: { value: property.value } })), }; } @@ -231,6 +263,148 @@ describe("BrowserExecutor snapshot rendering", () => { expect(text).toContain('textarea "Notes"'); expect(text).not.toContain('textarea "Notes" ['); }); + + it("renders node states in a compact bracket after the ref", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3", "4", "5"] }), + ax({ + nodeId: "2", + role: "checkbox", + name: "Terms", + backendDOMNodeId: 10, + parentId: "1", + properties: [{ name: "checked", value: "true" }, { name: "required", value: true }], + }), + ax({ nodeId: "3", role: "checkbox", name: "Maybe", backendDOMNodeId: 11, parentId: "1", properties: [{ name: "checked", value: "mixed" }] }), + ax({ + nodeId: "4", + role: "button", + name: "Save", + backendDOMNodeId: 12, + parentId: "1", + properties: [{ name: "disabled", value: true }, { name: "expanded", value: false }], + }), + ax({ nodeId: "5", role: "textbox", name: "Email", backendDOMNodeId: 13, parentId: "1", value: "a@b.c" }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toContain('checkbox "Terms" [e1] [checked, required]'); + expect(text).toContain('checkbox "Maybe" [e2] [checked=mixed]'); + expect(text).toContain('button "Save" [e3] [disabled]'); + expect(text).not.toContain("expanded"); + expect(text).toContain('textbox "Email" [e4] [value="a@b.c"]'); + }); + + it("skips StaticText duplicating the parent name and collapses wrappers without losing text", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "4", "7"] }), + ax({ nodeId: "2", role: "heading", name: "Title", parentId: "1", childIds: ["3"] }), + ax({ nodeId: "3", role: "StaticText", name: "Title", parentId: "2" }), + ax({ nodeId: "4", role: "link", name: "Docs", backendDOMNodeId: 20, parentId: "1", childIds: ["5"] }), + ax({ nodeId: "5", role: "generic", parentId: "4", childIds: ["6"] }), + ax({ nodeId: "6", role: "StaticText", name: "Docs", parentId: "5" }), + ax({ nodeId: "7", role: "StaticText", name: "Standalone", parentId: "1" }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toBe(['RootWebArea "Page"', ' heading "Title"', ' link "Docs" [e1]', ' StaticText "Standalone"'].join("\n")); + expect(text.split("\n")).toHaveLength(4); + }); +}); + +describe("BrowserExecutor stale-ref self-healing", () => { + it("heals a ref whose backend node moved when exactly one node matches the role/name triple", async () => { + const { cdp, sent, setNodes } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 99, parentId: "1" }), + ]); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(sent.some((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 99)).toBe(true); + expect(sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent")).toBe(true); + }); + + it("refuses to heal when multiple nodes match the stored role and name", async () => { + const { cdp, setNodes } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 99, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Save", backendDOMNodeId: 100, parentId: "1" }), + ]); + await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + + it("refuses to heal a ref minted as a later duplicate", async () => { + const { cdp, setNodes } = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Save", backendDOMNodeId: 43, parentId: "1" }), + ]); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 99, parentId: "1" }), + ]); + await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); +}); + +describe("BrowserExecutor cursor-pointer hints", () => { + const POINTER_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "generic", name: "Buy now", backendDOMNodeId: 77, parentId: "1" }), + ]; + + it("does not run the cursor scan by default", async () => { + const { cdp, sent, setCursorBackendIds } = createFakeCdp(POINTER_TREE); + setCursorBackendIds([77]); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toContain('generic "Buy now"'); + expect(text).not.toContain("cursor:pointer"); + expect(sent.some((cmd) => cmd.method === "Runtime.evaluate")).toBe(false); + }); + + it("marks cursor:pointer elements as clickable hints when enabled", async () => { + const { cdp, sent, setCursorBackendIds } = createFakeCdp(POINTER_TREE); + setCursorBackendIds([77]); + const executor = new BrowserExecutor(cdp, { cursorHints: true }); + const text = await snapshotText(executor); + expect(text).toContain('generic "Buy now" [e1] [cursor:pointer]'); + expect(sent.some((cmd) => cmd.method === "DOM.describeNode")).toBe(true); + expect(sent.some((cmd) => cmd.method === "Runtime.releaseObject")).toBe(true); + }); + + it("only enables cursor hints on the executor in browser mode", () => { + const recordedFor = (mode?: "computer" | "browser" | "hybrid") => { + const recorded: BrowserExecutorOptions[] = []; + const { dom } = createFakeDom(); + const { client } = createClient(); + const translator = new InternalComputerTranslator({ + browser, + client, + mode, + cursorHints: true, + createBrowserExecutor: (_cdpWsUrl, options) => { + recorded.push(options); + return dom; + }, + }); + translator.browser(); + return recorded[0]!; + }; + expect(recordedFor("browser").cursorHints).toBe(true); + expect(recordedFor("hybrid").cursorHints).toBe(false); + expect(recordedFor("computer").cursorHints).toBe(false); + expect(recordedFor(undefined).cursorHints).toBe(false); + }); }); describe("BrowserExecutor dialog guard", () => { From 0fa08900aea2e36a04c742f080fcc76aa76b7e9b Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:46:02 +0000 Subject: [PATCH 11/34] Stitch iframe AX trees into snapshots and short-circuit unchanged re-snapshots --- packages/agent/src/translator/browser.ts | 241 ++++++++++++++---- .../agent/test/translator-browser.test.ts | 154 ++++++++++- packages/ai/src/modes.ts | 10 +- 3 files changed, 338 insertions(+), 67 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index c6c2b8a4..e9eb4762 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -21,6 +21,8 @@ const FIND_MATCH_LIMIT = 20; const SCROLL_NOTCH_PX = 120; const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; +const REF_PLACEHOLDER = "\u0000"; +const UNCHANGED_SNAPSHOT = "Page unchanged since the last snapshot; previous element refs are still valid."; interface AXNode { nodeId: string; @@ -37,6 +39,10 @@ interface AXNode { interface RefEntry { backendNodeId: number; targetId: string; + /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ + frameId: string; + /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ + sessionId: string; generation: number; role: string; name: string; @@ -45,12 +51,26 @@ interface RefEntry { interface RenderContext { targetId: string; + frameKey: string; + sessionId: string; generation: number; interactiveOnly: boolean; nthIndex: Map; cursorIds?: ReadonlySet; } +interface RenderedLine { + text: string; + refNode?: AXNode; + ctx: RenderContext; +} + +interface FrameStitch { + byId: Map; + roots: string[]; + ctx: RenderContext; +} + export interface BrowserExecutorOptions { /** Mark elements whose computed cursor is "pointer" as clickable hints in snapshots. Default false. */ cursorHints?: boolean; @@ -71,6 +91,14 @@ export interface BrowserExecutorOptions { * (role, name, nth) triple recorded at mint time, but only when the fresh * match is unambiguous. * + * Snapshots stitch iframe content under each iframe node: same-process + * frames via the page session's AX tree with a frameId, out-of-process + * frames via their auto-attached session. Refs record their frame and + * session so actions resolve through the right one, and each frame's refs + * are invalidated independently when that frame navigates. Re-snapshotting + * an unchanged page with the same params returns a short unchanged notice + * instead of the full tree. + * * Native JavaScript dialogs (alert/confirm/prompt) are auto-dismissed so * they never wedge the CDP session; the dialog message is surfaced as an * extra read result on the next executed action. @@ -79,6 +107,9 @@ export class BrowserExecutor { private readonly refs = new Map(); private readonly generations = new Map(); private readonly targetsBySession = new Map(); + private readonly frameSessions = new Map(); + private readonly frameTargets = new Set(); + private readonly lastSnapshots = new Map(); private readonly selfNavigations = new Set(); private readonly dialogNotes: string[] = []; private refCounter = 0; @@ -96,13 +127,30 @@ export class BrowserExecutor { private handleCdpEvent(event: CdpEventMessage): void { switch (event.method) { case "Page.frameNavigated": { - const frame = event.params.frame as { parentId?: string } | undefined; - if (!event.sessionId || frame?.parentId) return; + const frame = event.params.frame as { id?: string; parentId?: string } | undefined; + if (!event.sessionId || !frame) return; const targetId = this.targetsBySession.get(event.sessionId); if (!targetId) return; + if (this.frameTargets.has(targetId)) { + if (frame.id === targetId) this.invalidateFrame(targetId); + return; + } + if (frame.parentId) { + if (frame.id) this.invalidateFrame(frame.id); + return; + } if (!this.selfNavigations.delete(targetId)) this.invalidateRefs(targetId); return; } + case "Target.attachedToTarget": { + const { sessionId, targetInfo } = event.params as { sessionId?: string; targetInfo?: { targetId?: string; type?: string } }; + if (!sessionId || !targetInfo?.targetId || targetInfo.type !== "iframe") return; + this.frameSessions.set(targetInfo.targetId, sessionId); + this.frameTargets.add(targetInfo.targetId); + this.targetsBySession.set(sessionId, targetInfo.targetId); + void this.cdp.send("Page.enable", {}, sessionId).catch(() => {}); + return; + } case "Page.javascriptDialogOpening": { if (!event.sessionId) return; const { type, message } = event.params as { type?: string; message?: string }; @@ -199,53 +247,70 @@ export class BrowserExecutor { private async snapshot(action: CuaActionBrowserSnapshot): Promise { const targetId = await this.resolveTarget(action.tab_id); - const session = await this.attach(targetId); - const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); + const pageSession = await this.attach(targetId); + const refEntry = action.ref ? this.resolveRef(action.ref, targetId) : undefined; + const frameKey = refEntry?.frameId ?? targetId; + const { nodes, sessionId } = await this.frameAxTree(frameKey, targetId, pageSession); const byId = new Map(nodes.map((node) => [node.nodeId, node])); - const roots = nodes.filter((node) => !node.parentId); - let rootIds = roots.map((node) => node.nodeId); - if (action.ref) { - const entry = this.resolveRef(action.ref, targetId); + let rootIds = nodes.filter((node) => !node.parentId).map((node) => node.nodeId); + if (action.ref && refEntry) { const rootNode = - nodes.find((node) => node.backendDOMNodeId === entry.backendNodeId) ?? this.healEntry(action.ref, entry, nodes); + nodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, nodes); rootIds = [rootNode.nodeId]; } + const interactiveOnly = action.filter === "interactive"; const ctx: RenderContext = { targetId, - generation: this.generation(targetId), - interactiveOnly: action.filter === "interactive", + frameKey, + sessionId, + generation: this.generation(frameKey), + interactiveOnly, nthIndex: buildNthIndex(nodes), - cursorIds: this.cursorHints ? await this.cursorPointerIds(session) : undefined, + cursorIds: this.cursorHints && frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, }; - const lines: string[] = []; + const stitches = frameKey === targetId ? await this.stitchFrames(nodes, targetId, pageSession, interactiveOnly) : new Map(); + const lines: RenderedLine[] = []; const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; - const walk = (nodeId: string, depth: number, parentName: string): void => { - const node = byId.get(nodeId); + const walk = (tree: Map, treeCtx: RenderContext, nodeId: string, depth: number, parentName: string): void => { + const node = tree.get(nodeId); if (!node) return; let childDepth = depth; if (!node.ignored) { - const line = this.renderNode(node, depth, parentName, ctx); - if (line) { - lines.push(line); + const rendered = this.renderNode(node, depth, parentName, treeCtx); + if (rendered) { + lines.push({ ...rendered, ctx: treeCtx }); childDepth = depth + 1; } } - if (childDepth <= maxDepth) { - const name = node.name?.value ?? ""; - for (const childId of node.childIds ?? []) walk(childId, childDepth, name || parentName); + if (childDepth > maxDepth) return; + const stitch = treeCtx === ctx && node.backendDOMNodeId !== undefined ? stitches.get(node.backendDOMNodeId) : undefined; + if (stitch) { + for (const frameRootId of stitch.roots) walk(stitch.byId, stitch.ctx, frameRootId, childDepth, ""); + return; } + const name = node.name?.value ?? ""; + for (const childId of node.childIds ?? []) walk(tree, treeCtx, childId, childDepth, name || parentName); }; - for (const rootId of rootIds) walk(rootId, 0, ""); + for (const rootId of rootIds) walk(byId, ctx, rootId, 0, ""); - let text = lines.join("\n"); + const shape = lines.map((line) => line.text).join("\n"); + const frameGenerations = [...stitches.values()].map((stitch) => `${stitch.ctx.frameKey}:${stitch.ctx.generation}`); + const key = [action.ref ?? "", action.depth ?? "", action.filter ?? "", `${frameKey}:${ctx.generation}`, ...frameGenerations].join("|"); + const cached = this.lastSnapshots.get(targetId); + this.lastSnapshots.set(targetId, { key, shape }); + if (cached && cached.key === key && cached.shape === shape) return UNCHANGED_SNAPSHOT; + + let text = lines + .map((line) => (line.refNode ? line.text.replace(REF_PLACEHOLDER, this.mintRef(line.refNode, line.ctx)) : line.text)) + .join("\n"); if (text.length > SNAPSHOT_CHAR_LIMIT) { text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; } return text || "(empty accessibility tree)"; } - private renderNode(node: AXNode, depth: number, parentName: string, ctx: RenderContext): string | undefined { + private renderNode(node: AXNode, depth: number, parentName: string, ctx: RenderContext): { text: string; refNode?: AXNode } | undefined { const role = node.role?.value ?? ""; const name = node.name?.value ?? ""; const interactive = INTERACTIVE_ROLES.has(role); @@ -254,13 +319,65 @@ export class BrowserExecutor { if (role === "StaticText" && name === parentName) return undefined; if (!ctx.interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; + let refNode: AXNode | undefined; if (node.backendDOMNodeId !== undefined && (interactive || pointer)) { - line += ` [${this.mintRef(node, ctx.targetId, ctx.generation, ctx.nthIndex)}]`; + line += ` [${REF_PLACEHOLDER}]`; + refNode = node; } const states = collectStates(node); if (pointer && !interactive) states.push("cursor:pointer"); if (states.length > 0) line += ` [${states.join(", ")}]`; - return line; + return { text: line, refNode }; + } + + /** Fetch a frame's AX tree: OOPIFs through their own session, same-process frames through the page session with a frameId. */ + private async frameAxTree(frameKey: string, targetId: string, pageSession: string): Promise<{ nodes: AXNode[]; sessionId: string }> { + const frameSession = this.frameSessions.get(frameKey); + if (frameSession) { + const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, frameSession); + return { nodes, sessionId: frameSession }; + } + const params = frameKey === targetId ? {} : { frameId: frameKey }; + const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", params, pageSession); + return { nodes, sessionId: pageSession }; + } + + /** Resolve each iframe node's child frame and fetch its AX tree for stitching. One nesting level only. */ + private async stitchFrames( + nodes: AXNode[], + targetId: string, + pageSession: string, + interactiveOnly: boolean, + ): Promise> { + const stitches = new Map(); + for (const node of nodes) { + if (node.ignored || node.role?.value !== "Iframe" || node.backendDOMNodeId === undefined) continue; + try { + const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( + "DOM.describeNode", + { backendNodeId: node.backendDOMNodeId, depth: 1 }, + pageSession, + ); + const frameId = dom.contentDocument?.frameId ?? dom.frameId; + if (!frameId || frameId === targetId) continue; + const { nodes: frameNodes, sessionId } = await this.frameAxTree(frameId, targetId, pageSession); + stitches.set(node.backendDOMNodeId, { + byId: new Map(frameNodes.map((frameNode) => [frameNode.nodeId, frameNode])), + roots: frameNodes.filter((frameNode) => !frameNode.parentId).map((frameNode) => frameNode.nodeId), + ctx: { + targetId, + frameKey: frameId, + sessionId, + generation: this.generation(frameId), + interactiveOnly, + nthIndex: buildNthIndex(frameNodes), + }, + }); + } catch { + // Cross-origin or already-detached frames can refuse the fetch; the iframe renders without children. + } + } + return stitches; } /** Resolve backend node ids for elements whose own computed cursor is "pointer", without touching the DOM. */ @@ -299,12 +416,19 @@ export class BrowserExecutor { .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); if (scored.length === 0) return `No elements matched ${JSON.stringify(action.query)}. Try snapshot for the full tree.`; - const nthIndex = buildNthIndex(nodes); + const ctx: RenderContext = { + targetId, + frameKey: targetId, + sessionId: session, + generation: this.generation(targetId), + interactiveOnly: false, + nthIndex: buildNthIndex(nodes), + }; return scored .map(({ node }) => { const role = node.role?.value ?? "node"; const name = node.name?.value ? ` ${JSON.stringify(node.name.value)}` : ""; - return `${role}${name} [${this.mintRef(node, targetId, this.generation(targetId), nthIndex)}]`; + return `${role}${name} [${this.mintRef(node, ctx)}]`; }) .join("\n"); } @@ -316,16 +440,16 @@ export class BrowserExecutor { const modifiers = modifierBits(action.modifiers); const button = action.button ?? "left"; const clickCount = action.num_clicks ?? 1; - await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y, modifiers }, session); + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y, modifiers }, point.session); await this.cdp.send( "Input.dispatchMouseEvent", { type: "mousePressed", x: point.x, y: point.y, button, clickCount, modifiers }, - session, + point.session, ); await this.cdp.send( "Input.dispatchMouseEvent", { type: "mouseReleased", x: point.x, y: point.y, button, clickCount, modifiers }, - session, + point.session, ); } @@ -333,7 +457,7 @@ export class BrowserExecutor { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const point = await this.resolvePoint(action, targetId, session); - await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y }, session); + await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y }, point.session); } private async drag(action: CuaActionBrowserDrag): Promise { @@ -345,8 +469,8 @@ export class BrowserExecutor { private async fill(action: CuaActionBrowserFill): Promise { const targetId = await this.resolveTarget(action.tab_id); - const session = await this.attach(targetId); const entry = this.resolveRef(action.ref, targetId); + const session = entry.sessionId; const objectId = await this.resolveObject(entry, action.ref, session); const { exceptionDetails } = await this.cdp.send<{ exceptionDetails?: { exception?: { description?: string } } }>( "Runtime.callFunctionOn", @@ -364,9 +488,8 @@ export class BrowserExecutor { private async scrollTo(action: CuaActionBrowserScrollTo): Promise { const targetId = await this.resolveTarget(action.tab_id); - const session = await this.attach(targetId); const entry = this.resolveRef(action.ref, targetId); - await this.scrollIntoView(entry, action.ref, session); + await this.scrollIntoView(entry, action.ref, entry.sessionId); } private async scroll(action: CuaActionBrowserScroll): Promise { @@ -466,19 +589,19 @@ export class BrowserExecutor { action: CuaActionBrowserClick | CuaActionBrowserHover, targetId: string, session: string, - ): Promise<{ x: number; y: number }> { + ): Promise<{ x: number; y: number; session: string }> { if (action.ref !== undefined) { const entry = this.resolveRef(action.ref, targetId); - await this.scrollIntoView(entry, action.ref, session); + await this.scrollIntoView(entry, action.ref, entry.sessionId); const { model } = await this.cdp.send<{ model: { content: number[] } }>( "DOM.getBoxModel", { backendNodeId: entry.backendNodeId }, - session, + entry.sessionId, ); const quad = model.content; - return { x: (quad[0]! + quad[4]!) / 2, y: (quad[1]! + quad[5]!) / 2 }; + return { x: (quad[0]! + quad[4]!) / 2, y: (quad[1]! + quad[5]!) / 2, session: entry.sessionId }; } - if (typeof action.x === "number" && typeof action.y === "number") return { x: action.x, y: action.y }; + if (typeof action.x === "number" && typeof action.y === "number") return { x: action.x, y: action.y, session }; throw new Error("page target required: pass a ref or viewport coordinates"); } @@ -486,7 +609,7 @@ export class BrowserExecutor { try { await this.cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: entry.backendNodeId }, session); } catch (err) { - await this.healRef(ref, entry, session, err); + await this.healRef(ref, entry, err); await this.cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: entry.backendNodeId }, session); } } @@ -500,7 +623,7 @@ export class BrowserExecutor { ); return object.objectId; } catch (err) { - await this.healRef(ref, entry, session, err); + await this.healRef(ref, entry, err); const { object } = await this.cdp.send<{ object: { objectId: string } }>( "DOM.resolveNode", { backendNodeId: entry.backendNodeId }, @@ -510,8 +633,8 @@ export class BrowserExecutor { } } - private async healRef(ref: string, entry: RefEntry, session: string, cause: unknown): Promise { - const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); + private async healRef(ref: string, entry: RefEntry, cause: unknown): Promise { + const { nodes } = await this.frameAxTree(entry.frameId, entry.targetId, entry.sessionId); this.healEntry(ref, entry, nodes, cause); } @@ -534,23 +657,25 @@ export class BrowserExecutor { return match; } - private mintRef(node: AXNode, targetId: string, generation: number, nthIndex: Map): string { + private mintRef(node: AXNode, ctx: RenderContext): string { this.refCounter += 1; const ref = `e${this.refCounter}`; this.refs.set(ref, { backendNodeId: node.backendDOMNodeId!, - targetId, - generation, + targetId: ctx.targetId, + frameId: ctx.frameKey, + sessionId: ctx.sessionId, + generation: ctx.generation, role: node.role?.value ?? "", name: node.name?.value ?? "", - nth: nthIndex.get(node.nodeId) ?? 0, + nth: ctx.nthIndex.get(node.nodeId) ?? 0, }); return ref; } private resolveRef(ref: string, targetId: string): RefEntry { const entry = this.refs.get(ref); - if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(targetId)) { + if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(entry.frameId)) { throw staleRefError(ref); } return entry; @@ -561,18 +686,27 @@ export class BrowserExecutor { } private invalidateRefs(targetId: string): void { - const generation = this.generation(targetId) + 1; - this.generations.set(targetId, generation); + this.generations.set(targetId, this.generation(targetId) + 1); + for (const [ref, entry] of this.refs) { + if (entry.targetId === targetId) this.refs.delete(ref); + } + } + + private invalidateFrame(frameKey: string): void { + this.generations.set(frameKey, this.generation(frameKey) + 1); for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId && entry.generation < generation) this.refs.delete(ref); + if (entry.frameId === frameKey) this.refs.delete(ref); } } private dropTarget(targetId: string): void { this.generations.delete(targetId); this.selfNavigations.delete(targetId); + this.lastSnapshots.delete(targetId); + this.frameSessions.delete(targetId); + this.frameTargets.delete(targetId); for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId) this.refs.delete(ref); + if (entry.targetId === targetId || entry.frameId === targetId) this.refs.delete(ref); } } @@ -592,6 +726,7 @@ export class BrowserExecutor { if (!this.targetsBySession.has(session)) { this.targetsBySession.set(session, targetId); await this.cdp.send("Page.enable", {}, session); + await this.cdp.send("Target.setAutoAttach", { autoAttach: true, flatten: true, waitForDebuggerOnStart: false }, session); } return session; } diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index fda1089b..31fbc32d 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -105,8 +105,21 @@ function createFakeCdp(initialNodes: unknown[] = []) { const listeners: Array<(event: FakeCdpEvent) => void> = []; let nodes = initialNodes as Array<{ backendDOMNodeId?: number }>; let cursorBackendIds: number[] = []; - const requireBackendId = (id: unknown) => { - if (!nodes.some((node) => node.backendDOMNodeId === id)) throw new Error("No node with given id found"); + const sessionTrees = new Map>(); + const frameTrees = new Map>(); + const iframeFrameIds = new Map(); + const autoAttachFrames: Array<{ targetId: string; sessionId: string }> = []; + const emit = (event: FakeCdpEvent) => { + for (const listener of listeners) listener(event); + }; + const treeFor = (sessionId?: string, frameId?: unknown) => { + if (typeof frameId === "string" && frameTrees.has(frameId)) return frameTrees.get(frameId)!; + if (sessionId && sessionTrees.has(sessionId)) return sessionTrees.get(sessionId)!; + return nodes; + }; + const requireBackendId = (id: unknown, sessionId?: string) => { + const candidates = sessionId && sessionTrees.has(sessionId) ? [sessionTrees.get(sessionId)!] : [nodes, ...frameTrees.values()]; + if (!candidates.some((tree) => tree.some((node) => node.backendDOMNodeId === id))) throw new Error("No node with given id found"); }; const fake = { onEvent: (listener: (event: FakeCdpEvent) => void) => { @@ -116,15 +129,24 @@ function createFakeCdp(initialNodes: unknown[] = []) { sent.push({ method, params, sessionId }); switch (method) { case "Accessibility.getFullAXTree": - return { nodes }; + return { nodes: treeFor(sessionId, params.frameId) }; + case "Target.setAutoAttach": + for (const frame of autoAttachFrames.splice(0)) { + emit({ + method: "Target.attachedToTarget", + params: { sessionId: frame.sessionId, targetInfo: { targetId: frame.targetId, type: "iframe" } }, + sessionId, + }); + } + return {}; case "DOM.scrollIntoViewIfNeeded": - requireBackendId(params.backendNodeId); + requireBackendId(params.backendNodeId, sessionId); return {}; case "DOM.getBoxModel": - requireBackendId(params.backendNodeId); + requireBackendId(params.backendNodeId, sessionId); return { model: { content: [0, 0, 10, 0, 10, 10, 0, 10] } }; case "DOM.resolveNode": - requireBackendId(params.backendNodeId); + requireBackendId(params.backendNodeId, sessionId); return { object: { objectId: "node-obj" } }; case "Runtime.evaluate": if (params.returnByValue === false) return { result: { objectId: "cursor-scan" } }; @@ -137,6 +159,9 @@ function createFakeCdp(initialNodes: unknown[] = []) { ], }; case "DOM.describeNode": + if (typeof params.backendNodeId === "number") { + return { node: { backendNodeId: params.backendNodeId, frameId: iframeFrameIds.get(params.backendNodeId) } }; + } return { node: { backendNodeId: Number(String(params.objectId).slice(3)) } }; default: return {}; @@ -147,16 +172,35 @@ function createFakeCdp(initialNodes: unknown[] = []) { createTarget: async () => "TARGET-2", close: () => {}, }; - const emit = (event: FakeCdpEvent) => { - for (const listener of listeners) listener(event); - }; const setNodes = (next: unknown[]) => { nodes = next as Array<{ backendDOMNodeId?: number }>; }; const setCursorBackendIds = (ids: number[]) => { cursorBackendIds = ids; }; - return { sent, emit, setNodes, setCursorBackendIds, cdp: fake as unknown as CdpConnection }; + const setSessionTree = (sessionId: string, tree: unknown[]) => { + sessionTrees.set(sessionId, tree as Array<{ backendDOMNodeId?: number }>); + }; + const setFrameTree = (frameId: string, tree: unknown[]) => { + frameTrees.set(frameId, tree as Array<{ backendDOMNodeId?: number }>); + }; + const setIframeFrame = (backendNodeId: number, frameId: string) => { + iframeFrameIds.set(backendNodeId, frameId); + }; + const addAutoAttachFrame = (frame: { targetId: string; sessionId: string }) => { + autoAttachFrames.push(frame); + }; + return { + sent, + emit, + setNodes, + setCursorBackendIds, + setSessionTree, + setFrameTree, + setIframeFrame, + addAutoAttachFrame, + cdp: fake as unknown as CdpConnection, + }; } interface AXNodeSpec { @@ -424,3 +468,93 @@ describe("BrowserExecutor dialog guard", () => { ]); }); }); + +describe("BrowserExecutor snapshot diffing", () => { + const UNCHANGED = "Page unchanged since the last snapshot; previous element refs are still valid."; + + it("returns a short unchanged notice for an identical re-snapshot and the full tree after a change", async () => { + const { cdp, setNodes } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + expect(await snapshotText(executor)).toContain('button "Save" [e1]'); + expect(await snapshotText(executor)).toBe(UNCHANGED); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Delete", backendDOMNodeId: 43, parentId: "1" }), + ]); + expect(await snapshotText(executor)).toContain('button "Delete" [e2]'); + }); + + it("returns the full tree when the params differ from the previous snapshot", async () => { + const { cdp } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + expect(await snapshotText(executor, { filter: "interactive" })).toContain('button "Save" [e2]'); + }); +}); + +describe("BrowserExecutor iframe stitching", () => { + it("stitches a same-process iframe subtree indented under its iframe node", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const { cdp, emit, setFrameTree, setIframeFrame } = createFakeCdp(tree); + setIframeFrame(50, "FRAME-SP"); + setFrameTree("FRAME-SP", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Embed", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Inside", backendDOMNodeId: 60, parentId: "f1" }), + ]); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toBe(['RootWebArea "Page"', " Iframe", ' RootWebArea "Embed"', ' button "Inside" [e1]'].join("\n")); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "F0" } }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + + const OOPIF_PAGE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Top", backendDOMNodeId: 40, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const OOPIF_CHILD = [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Widget", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Pay", backendDOMNodeId: 70, parentId: "f1" }), + ]; + const setupOopif = () => { + const fake = createFakeCdp(OOPIF_PAGE); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", OOPIF_CHILD); + return fake; + }; + + it("resolves a ref inside an OOPIF through the child frame's session", async () => { + const { cdp, sent } = setupOopif(); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "Top" [e1]'); + expect(text).toContain(' button "Pay" [e2]'); + + await executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + const scrolled = sent.find((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 70); + expect(scrolled?.sessionId).toBe("session-oop"); + const pressed = sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); + expect(pressed?.sessionId).toBe("session-oop"); + }); + + it("invalidates only the child frame's refs when the child frame navigates", async () => { + const { cdp, emit } = setupOopif(); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-OOP" } }, sessionId: "session-oop" }); + await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + + const text = await snapshotText(executor); + expect(text).toContain('button "Pay" [e'); + }); +}); diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index ef6d4961..1543007a 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -119,8 +119,9 @@ export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): stri const BROWSER_ACTION_DESCRIPTIONS: Record = { browser_snapshot: - "Return an accessibility-tree snapshot of the page with element references like [e12]. " + - "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale.", + "Return an accessibility-tree snapshot of the page, including iframe content, with element references like [e12]. " + + "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale. " + + "If the page has not changed since your previous snapshot, a short unchanged notice is returned instead and earlier refs remain valid.", browser_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", browser_find: "Find elements matching a natural-language description and return them with element references, like a filtered snapshot.", browser_click: "Click an element. Prefer targeting by element reference from a snapshot.", @@ -157,9 +158,10 @@ const HYBRID_BROWSER_DESCRIPTION_OVERRIDES: Partial Date: Wed, 8 Jul 2026 23:01:32 +0000 Subject: [PATCH 12/34] Address review findings in browser executor - Accept alert/beforeunload dialogs instead of dismissing everything, so pages with unload guards can be left; word dialog notes per type - Consume the self-navigation flag on Page.navigatedWithinDocument so a same-document navigation can't suppress the next real invalidation - Render checked/pressed/expanded=false and heading level states - Mint refs for named content roles (heading, listitem, navigation, ...) and iframe nodes so they can be targeted by scroll_to and ref-scoped snapshots; stitch IframePresentational frames too - Merge consecutive StaticText siblings into one snapshot line - Heal duplicate refs by nth when the role+name cohort size is unchanged, instead of only healing singletons - Dispatch mouse input for OOPIF refs on the page session (box-model quads are main-viewport coordinates) - Bound ref growth: stop minting past the snapshot char limit, evict the oldest refs per target, and skip generation entries for frames that were never referenced - Run the cursor-hint DOM.describeNode calls in parallel and release scan objects via an object group - Note in browser_find's description that it searches the main frame --- packages/agent/src/translator/browser.ts | 214 ++++++++++++++---- .../agent/test/translator-browser.test.ts | 144 ++++++++++-- packages/ai/src/modes.ts | 3 +- 3 files changed, 302 insertions(+), 59 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index e9eb4762..dfcdf488 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -18,6 +18,7 @@ import type { BatchReadResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; const FIND_MATCH_LIMIT = 20; +const REF_LIMIT_PER_TARGET = 1000; const SCROLL_NOTCH_PX = 120; const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; @@ -47,6 +48,13 @@ interface RefEntry { role: string; name: string; nth: number; + /** Size of the (role, name) cohort in the tree the ref was minted from. */ + cohort: number; +} + +interface NthIndex { + index: Map; + cohorts: Map; } interface RenderContext { @@ -55,7 +63,7 @@ interface RenderContext { sessionId: string; generation: number; interactiveOnly: boolean; - nthIndex: Map; + nthIndex: NthIndex; cursorIds?: ReadonlySet; } @@ -89,7 +97,7 @@ export interface BrowserExecutorOptions { * A ref whose backend node vanished without a navigation (DOM churn) is * self-healed: the AX tree is re-fetched and the ref re-resolved by the * (role, name, nth) triple recorded at mint time, but only when the fresh - * match is unambiguous. + * cohort has the same size as at mint time. * * Snapshots stitch iframe content under each iframe node: same-process * frames via the page session's AX tree with a frameId, out-of-process @@ -99,9 +107,10 @@ export interface BrowserExecutorOptions { * an unchanged page with the same params returns a short unchanged notice * instead of the full tree. * - * Native JavaScript dialogs (alert/confirm/prompt) are auto-dismissed so - * they never wedge the CDP session; the dialog message is surfaced as an - * extra read result on the next executed action. + * Native JavaScript dialogs are auto-handled so they never wedge the CDP + * session: alert and beforeunload dialogs are accepted (so navigation can + * proceed), confirm and prompt dialogs are dismissed. The dialog message + * is surfaced as an extra read result on the next executed action. */ export class BrowserExecutor { private readonly refs = new Map(); @@ -142,6 +151,15 @@ export class BrowserExecutor { if (!this.selfNavigations.delete(targetId)) this.invalidateRefs(targetId); return; } + case "Page.navigatedWithinDocument": { + // A navigate() that turns out same-document never fires frameNavigated; + // consume the pending flag here so it can't swallow the next real navigation. + if (!event.sessionId) return; + const targetId = this.targetsBySession.get(event.sessionId); + const { frameId } = event.params as { frameId?: string }; + if (targetId && frameId === targetId) this.selfNavigations.delete(targetId); + return; + } case "Target.attachedToTarget": { const { sessionId, targetInfo } = event.params as { sessionId?: string; targetInfo?: { targetId?: string; type?: string } }; if (!sessionId || !targetInfo?.targetId || targetInfo.type !== "iframe") return; @@ -154,8 +172,15 @@ export class BrowserExecutor { case "Page.javascriptDialogOpening": { if (!event.sessionId) return; const { type, message } = event.params as { type?: string; message?: string }; - void this.cdp.send("Page.handleJavaScriptDialog", { accept: false }, event.sessionId).catch(() => {}); - this.dialogNotes.push(`Auto-dismissed a JavaScript ${type ?? "dialog"} dialog: ${JSON.stringify(message ?? "")}`); + const accept = type === "alert" || type === "beforeunload"; + void this.cdp.send("Page.handleJavaScriptDialog", { accept }, event.sessionId).catch(() => {}); + const summary = + type === "beforeunload" + ? "Accepted a beforeunload dialog so navigation could proceed" + : accept + ? "Acknowledged a JavaScript alert dialog" + : `Dismissed a JavaScript ${type ?? "dialog"} dialog (answered No/cancel)`; + this.dialogNotes.push(`${summary}: ${JSON.stringify(message ?? "")}`); return; } case "Target.detachedFromTarget": { @@ -290,7 +315,18 @@ export class BrowserExecutor { return; } const name = node.name?.value ?? ""; - for (const childId of node.childIds ?? []) walk(tree, treeCtx, childId, childDepth, name || parentName); + const childName = name || parentName; + const childIds = node.childIds ?? []; + for (let i = 0; i < childIds.length; i += 1) { + const run = staticTextRun(tree, childIds, i); + if (run) { + const rendered = this.renderNode(run.node, childDepth, childName, treeCtx); + if (rendered) lines.push({ ...rendered, ctx: treeCtx }); + i = run.end; + continue; + } + walk(tree, treeCtx, childIds[i]!, childDepth, childName); + } }; for (const rootId of rootIds) walk(byId, ctx, rootId, 0, ""); @@ -301,12 +337,16 @@ export class BrowserExecutor { this.lastSnapshots.set(targetId, { key, shape }); if (cached && cached.key === key && cached.shape === shape) return UNCHANGED_SNAPSHOT; - let text = lines - .map((line) => (line.refNode ? line.text.replace(REF_PLACEHOLDER, this.mintRef(line.refNode, line.ctx)) : line.text)) - .join("\n"); + let text = ""; + for (const line of lines) { + if (text.length > SNAPSHOT_CHAR_LIMIT) break; + const rendered = line.refNode ? line.text.replace(REF_PLACEHOLDER, this.mintRef(line.refNode, line.ctx)) : line.text; + text = text ? `${text}\n${rendered}` : rendered; + } if (text.length > SNAPSHOT_CHAR_LIMIT) { text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; } + this.pruneRefs(targetId); return text || "(empty accessibility tree)"; } @@ -320,7 +360,8 @@ export class BrowserExecutor { if (!ctx.interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; let refNode: AXNode | undefined; - if (node.backendDOMNodeId !== undefined && (interactive || pointer)) { + const refWorthy = interactive || pointer || FRAME_ROLES.has(role) || (name !== "" && CONTENT_ROLES.has(role)); + if (node.backendDOMNodeId !== undefined && refWorthy) { line += ` [${REF_PLACEHOLDER}]`; refNode = node; } @@ -351,7 +392,7 @@ export class BrowserExecutor { ): Promise> { const stitches = new Map(); for (const node of nodes) { - if (node.ignored || node.role?.value !== "Iframe" || node.backendDOMNodeId === undefined) continue; + if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; try { const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( "DOM.describeNode", @@ -385,22 +426,28 @@ export class BrowserExecutor { const ids = new Set(); const { result } = await this.cdp.send<{ result: { objectId?: string } }>( "Runtime.evaluate", - { expression: CURSOR_POINTER_SCAN, returnByValue: false }, + { expression: CURSOR_POINTER_SCAN, returnByValue: false, objectGroup: CURSOR_SCAN_GROUP }, session, ); if (!result.objectId) return ids; - const { result: properties } = await this.cdp.send<{ result: Array<{ name: string; value?: { objectId?: string } }> }>( - "Runtime.getProperties", - { objectId: result.objectId, ownProperties: true }, - session, - ); - for (const property of properties) { - const objectId = property.value?.objectId; - if (!/^\d+$/.test(property.name) || !objectId) continue; - const { node } = await this.cdp.send<{ node: { backendNodeId?: number } }>("DOM.describeNode", { objectId }, session); - if (node.backendNodeId !== undefined) ids.add(node.backendNodeId); + try { + const { result: properties } = await this.cdp.send<{ result: Array<{ name: string; value?: { objectId?: string } }> }>( + "Runtime.getProperties", + { objectId: result.objectId, ownProperties: true }, + session, + ); + const objectIds = properties + .filter((property) => /^\d+$/.test(property.name) && property.value?.objectId) + .map((property) => property.value!.objectId!); + const described = await Promise.all( + objectIds.map((objectId) => this.cdp.send<{ node: { backendNodeId?: number } }>("DOM.describeNode", { objectId }, session)), + ); + for (const { node } of described) { + if (node.backendNodeId !== undefined) ids.add(node.backendNodeId); + } + } finally { + await this.cdp.send("Runtime.releaseObjectGroup", { objectGroup: CURSOR_SCAN_GROUP }, session).catch(() => {}); } - await this.cdp.send("Runtime.releaseObject", { objectId: result.objectId }, session); return ids; } @@ -424,13 +471,15 @@ export class BrowserExecutor { interactiveOnly: false, nthIndex: buildNthIndex(nodes), }; - return scored + const text = scored .map(({ node }) => { const role = node.role?.value ?? "node"; const name = node.name?.value ? ` ${JSON.stringify(node.name.value)}` : ""; return `${role}${name} [${this.mintRef(node, ctx)}]`; }) .join("\n"); + this.pruneRefs(targetId); + return text; } private async click(action: CuaActionBrowserClick): Promise { @@ -599,7 +648,9 @@ export class BrowserExecutor { entry.sessionId, ); const quad = model.content; - return { x: (quad[0]! + quad[4]!) / 2, y: (quad[1]! + quad[5]!) / 2, session: entry.sessionId }; + // Box-model quads are main-viewport coordinates even through an OOPIF's + // session, so input always dispatches on the page target's session. + return { x: (quad[0]! + quad[4]!) / 2, y: (quad[1]! + quad[5]!) / 2, session }; } if (typeof action.x === "number" && typeof action.y === "number") return { x: action.x, y: action.y, session }; throw new Error("page target required: pass a ref or viewport coordinates"); @@ -640,8 +691,8 @@ export class BrowserExecutor { /** * Re-resolve a stale entry by its (role, name, nth) triple against a fresh - * AX tree. Heals only the unambiguous case — the ref was minted as the - * first of its role+name cohort and exactly one fresh node matches. + * AX tree. Heals only when the fresh role+name cohort has the same size as + * at mint time, so the nth position still identifies the same element. */ private healEntry(ref: string, entry: RefEntry, nodes: AXNode[], cause?: unknown): AXNode { const candidates = nodes.filter( @@ -651,13 +702,18 @@ export class BrowserExecutor { (node.role?.value ?? "") === entry.role && (node.name?.value ?? "") === entry.name, ); - const match = candidates.length === 1 && entry.nth === 0 && (entry.role || entry.name) ? candidates[0] : undefined; + const match = + (entry.role || entry.name) && candidates.length === entry.cohort && entry.nth < candidates.length + ? candidates[entry.nth] + : undefined; if (!match) throw staleRefError(ref, cause); entry.backendNodeId = match.backendDOMNodeId!; return match; } private mintRef(node: AXNode, ctx: RenderContext): string { + const role = node.role?.value ?? ""; + const name = node.name?.value ?? ""; this.refCounter += 1; const ref = `e${this.refCounter}`; this.refs.set(ref, { @@ -666,15 +722,18 @@ export class BrowserExecutor { frameId: ctx.frameKey, sessionId: ctx.sessionId, generation: ctx.generation, - role: node.role?.value ?? "", - name: node.name?.value ?? "", - nth: ctx.nthIndex.get(node.nodeId) ?? 0, + role, + name, + nth: ctx.nthIndex.index.get(node.nodeId) ?? 0, + cohort: ctx.nthIndex.cohorts.get(cohortKey(role, name)) ?? 1, }); return ref; } private resolveRef(ref: string, targetId: string): RefEntry { const entry = this.refs.get(ref); + // Entries are deleted eagerly on invalidation; the generation check only + // guards refs resolved while a navigation event is still in flight. if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(entry.frameId)) { throw staleRefError(ref); } @@ -693,10 +752,16 @@ export class BrowserExecutor { } private invalidateFrame(frameKey: string): void { - this.generations.set(frameKey, this.generation(frameKey) + 1); + let tracked = this.generations.has(frameKey) || this.frameSessions.has(frameKey); for (const [ref, entry] of this.refs) { - if (entry.frameId === frameKey) this.refs.delete(ref); + if (entry.frameId === frameKey) { + this.refs.delete(ref); + tracked = true; + } } + // Frames we never referenced don't get a generation entry, or pages with + // rotating ad iframes would grow the map without bound. + if (tracked) this.generations.set(frameKey, this.generation(frameKey) + 1); } private dropTarget(targetId: string): void { @@ -706,10 +771,22 @@ export class BrowserExecutor { this.frameSessions.delete(targetId); this.frameTargets.delete(targetId); for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId || entry.frameId === targetId) this.refs.delete(ref); + if (entry.targetId === targetId || entry.frameId === targetId) { + if (entry.frameId !== targetId) this.generations.delete(entry.frameId); + this.refs.delete(ref); + } } } + /** SPAs can mint refs indefinitely without ever navigating; bound per-target growth by evicting the oldest. */ + private pruneRefs(targetId: string): void { + const owned: string[] = []; + for (const [ref, entry] of this.refs) { + if (entry.targetId === targetId) owned.push(ref); + } + for (const ref of owned.slice(0, Math.max(0, owned.length - REF_LIMIT_PER_TARGET))) this.refs.delete(ref); + } + private drainDialogNotes(): string | undefined { if (this.dialogNotes.length === 0) return undefined; const text = this.dialogNotes.join("\n"); @@ -757,17 +834,37 @@ function staleRefError(ref: string, cause?: unknown): Error { } /** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ -function buildNthIndex(nodes: AXNode[]): Map { - const counts = new Map(); +function buildNthIndex(nodes: AXNode[]): NthIndex { + const cohorts = new Map(); const index = new Map(); for (const node of nodes) { if (node.ignored || node.backendDOMNodeId === undefined) continue; - const key = `${node.role?.value ?? ""}\u0000${node.name?.value ?? ""}`; - const nth = counts.get(key) ?? 0; - counts.set(key, nth + 1); + const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); + const nth = cohorts.get(key) ?? 0; + cohorts.set(key, nth + 1); index.set(node.nodeId, nth); } - return index; + return { index, cohorts }; +} + +function cohortKey(role: string, name: string): string { + return `${role}\u0000${name}`; +} + +/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ +function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { + let end = start; + const parts: string[] = []; + while (end < childIds.length) { + const node = tree.get(childIds[end]!); + if (!node || node.ignored || node.role?.value !== "StaticText") break; + const text = node.name?.value ?? ""; + if (text) parts.push(text); + end += 1; + } + if (end - start < 2) return undefined; + const first = tree.get(childIds[start]!)!; + return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; } function collectStates(node: AXNode): string[] { @@ -777,15 +874,21 @@ function collectStates(node: AXNode): string[] { switch (property.name) { case "checked": case "pressed": + case "expanded": + // False is meaningful here: it distinguishes an unchecked checkbox or + // collapsed disclosure from an element without the state at all. if (value === true || value === "true") states.push(property.name); + else if (value === false || value === "false") states.push(`${property.name}=false`); else if (value === "mixed") states.push(`${property.name}=mixed`); break; - case "expanded": case "disabled": case "selected": case "required": if (value === true || value === "true") states.push(property.name); break; + case "level": + if (typeof value === "number") states.push(`level=${value}`); + break; } } const value = node.value?.value; @@ -928,6 +1031,31 @@ const INTERACTIVE_ROLES: ReadonlySet = new Set([ const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); +const FRAME_ROLES: ReadonlySet = new Set(["Iframe", "IframePresentational"]); + +/** Non-interactive roles that get refs when named, so scroll_to / ref-scoped snapshots can target them. */ +const CONTENT_ROLES: ReadonlySet = new Set([ + "heading", + "cell", + "gridcell", + "columnheader", + "rowheader", + "row", + "listitem", + "article", + "region", + "main", + "navigation", + "banner", + "contentinfo", + "complementary", + "tabpanel", + "figure", + "image", +]); + +const CURSOR_SCAN_GROUP = "cua-cursor-scan"; + const CURSOR_POINTER_SCAN = `(() => { const matches = []; if (!document.body) return matches; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 31fbc32d..0cf3f9b9 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -268,6 +268,17 @@ describe("BrowserExecutor ref lifecycle", () => { expect(refsOf(executor).size).toBe(0); }); + it("does not let a same-document navigation suppress the next real navigation's invalidation", async () => { + const { cdp, emit } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await executor.execute({ type: "browser_navigate", url: "https://a.test/#section" } as CuaBrowserAction); + emit({ method: "Page.navigatedWithinDocument", params: { frameId: "TARGET-1", url: "https://a.test/#section" }, sessionId: "session-1" }); + await snapshotText(executor); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + it("does not double-bump the generation for its own navigate", async () => { const { cdp, emit } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); @@ -335,11 +346,63 @@ describe("BrowserExecutor snapshot rendering", () => { const text = await snapshotText(executor); expect(text).toContain('checkbox "Terms" [e1] [checked, required]'); expect(text).toContain('checkbox "Maybe" [e2] [checked=mixed]'); - expect(text).toContain('button "Save" [e3] [disabled]'); - expect(text).not.toContain("expanded"); + expect(text).toContain('button "Save" [e3] [disabled, expanded=false]'); expect(text).toContain('textbox "Email" [e4] [value="a@b.c"]'); }); + it("renders false checked state, expanded, pressed, selected, and heading level", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3", "4", "5"] }), + ax({ nodeId: "2", role: "radio", name: "Solo", backendDOMNodeId: 10, parentId: "1", properties: [{ name: "checked", value: "false" }] }), + ax({ nodeId: "3", role: "button", name: "Bold", backendDOMNodeId: 11, parentId: "1", properties: [{ name: "pressed", value: "true" }] }), + ax({ + nodeId: "4", + role: "tab", + name: "Overview", + backendDOMNodeId: 12, + parentId: "1", + properties: [{ name: "selected", value: true }, { name: "expanded", value: true }], + }), + ax({ nodeId: "5", role: "heading", name: "Pricing", backendDOMNodeId: 13, parentId: "1", properties: [{ name: "level", value: 2 }] }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + const text = await snapshotText(executor); + expect(text).toContain('radio "Solo" [e1] [checked=false]'); + expect(text).toContain('button "Bold" [e2] [pressed]'); + expect(text).toContain('tab "Overview" [e3] [selected, expanded]'); + expect(text).toContain('heading "Pricing" [e4] [level=2]'); + }); + + it("merges consecutive StaticText siblings into one line", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3", "4"] }), + ax({ nodeId: "2", role: "StaticText", name: "Fast", parentId: "1" }), + ax({ nodeId: "3", role: "StaticText", name: "browsers", parentId: "1" }), + ax({ nodeId: "4", role: "button", name: "Go", backendDOMNodeId: 42, parentId: "1" }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + expect(await snapshotText(executor)).toBe('RootWebArea "Page"\n StaticText "Fast browsers"\n button "Go" [e1]'); + }); + + it("scopes a snapshot to a named content role's ref", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "4"] }), + ax({ nodeId: "2", role: "navigation", name: "Menu", backendDOMNodeId: 30, parentId: "1", childIds: ["3"] }), + ax({ nodeId: "3", role: "link", name: "Home", backendDOMNodeId: 31, parentId: "2" }), + ax({ nodeId: "4", role: "button", name: "Save", backendDOMNodeId: 32, parentId: "1" }), + ]; + const { cdp } = createFakeCdp(tree); + const executor = new BrowserExecutor(cdp); + const full = await snapshotText(executor); + expect(full).toContain('navigation "Menu" [e1]'); + const scoped = await snapshotText(executor, { ref: "e1" }); + expect(scoped).toContain('navigation "Menu"'); + expect(scoped).toContain('link "Home"'); + expect(scoped).not.toContain('button "Save"'); + }); + it("skips StaticText duplicating the parent name and collapses wrappers without losing text", async () => { const tree = [ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "4", "7"] }), @@ -372,6 +435,39 @@ describe("BrowserExecutor stale-ref self-healing", () => { expect(sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent")).toBe(true); }); + it("heals a duplicate ref by position when the cohort size is unchanged", async () => { + const { cdp, sent, setNodes } = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Save", backendDOMNodeId: 43, parentId: "1" }), + ]); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 99, parentId: "1" }), + ax({ nodeId: "3", role: "button", name: "Save", backendDOMNodeId: 100, parentId: "1" }), + ]); + await executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + expect(sent.some((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 100)).toBe(true); + }); + + it("heals a stale ref on browser_fill and retries the resolve", async () => { + const { cdp, sent, setNodes } = createFakeCdp([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "textbox", name: "Email", backendDOMNodeId: 42, parentId: "1" }), + ]); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + setNodes([ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "textbox", name: "Email", backendDOMNodeId: 99, parentId: "1" }), + ]); + await executor.execute({ type: "browser_fill", ref: "e1", value: "a@b.c" } as CuaBrowserAction); + expect(sent.some((cmd) => cmd.method === "DOM.resolveNode" && cmd.params.backendNodeId === 99)).toBe(true); + expect(sent.some((cmd) => cmd.method === "Runtime.callFunctionOn")).toBe(true); + }); + it("refuses to heal when multiple nodes match the stored role and name", async () => { const { cdp, setNodes } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); @@ -384,7 +480,7 @@ describe("BrowserExecutor stale-ref self-healing", () => { await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); - it("refuses to heal a ref minted as a later duplicate", async () => { + it("refuses to heal a duplicate ref when the cohort shrank", async () => { const { cdp, setNodes } = createFakeCdp([ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), ax({ nodeId: "2", role: "button", name: "Save", backendDOMNodeId: 42, parentId: "1" }), @@ -423,7 +519,7 @@ describe("BrowserExecutor cursor-pointer hints", () => { const text = await snapshotText(executor); expect(text).toContain('generic "Buy now" [e1] [cursor:pointer]'); expect(sent.some((cmd) => cmd.method === "DOM.describeNode")).toBe(true); - expect(sent.some((cmd) => cmd.method === "Runtime.releaseObject")).toBe(true); + expect(sent.some((cmd) => cmd.method === "Runtime.releaseObjectGroup")).toBe(true); }); it("only enables cursor hints on the executor in browser mode", () => { @@ -452,21 +548,39 @@ describe("BrowserExecutor cursor-pointer hints", () => { }); describe("BrowserExecutor dialog guard", () => { - it("auto-dismisses JavaScript dialogs and surfaces the message on the next action", async () => { + it("dismisses confirm/prompt dialogs and surfaces the message on the next action", async () => { const { cdp, emit, sent } = createFakeCdp(); const executor = new BrowserExecutor(cdp); await executor.execute({ type: "browser_text" } as CuaBrowserAction); - emit({ method: "Page.javascriptDialogOpening", params: { type: "confirm", message: "Leave page?" }, sessionId: "session-1" }); + emit({ method: "Page.javascriptDialogOpening", params: { type: "confirm", message: "Delete item?" }, sessionId: "session-1" }); const handled = sent.find((cmd) => cmd.method === "Page.handleJavaScriptDialog"); expect(handled).toEqual({ method: "Page.handleJavaScriptDialog", params: { accept: false }, sessionId: "session-1" }); const results = await executor.execute({ type: "browser_text" } as CuaBrowserAction); expect(results).toEqual([ { type: "browser_text", label: "text", text: "hello" }, - { type: "browser_text", label: "dialog", text: 'Auto-dismissed a JavaScript confirm dialog: "Leave page?"' }, + { type: "browser_text", label: "dialog", text: 'Dismissed a JavaScript confirm dialog (answered No/cancel): "Delete item?"' }, ]); }); + + it("accepts alert and beforeunload dialogs so navigation can proceed", async () => { + const { cdp, emit, sent } = createFakeCdp(); + const executor = new BrowserExecutor(cdp); + await executor.execute({ type: "browser_text" } as CuaBrowserAction); + + emit({ method: "Page.javascriptDialogOpening", params: { type: "beforeunload", message: "" }, sessionId: "session-1" }); + emit({ method: "Page.javascriptDialogOpening", params: { type: "alert", message: "Saved!" }, sessionId: "session-1" }); + const handled = sent.filter((cmd) => cmd.method === "Page.handleJavaScriptDialog"); + expect(handled.map((cmd) => cmd.params)).toEqual([{ accept: true }, { accept: true }]); + + const results = await executor.execute({ type: "browser_text" } as CuaBrowserAction); + expect(results[1]).toEqual({ + type: "browser_text", + label: "dialog", + text: 'Accepted a beforeunload dialog so navigation could proceed: ""\nAcknowledged a JavaScript alert dialog: "Saved!"', + }); + }); }); describe("BrowserExecutor snapshot diffing", () => { @@ -507,11 +621,11 @@ describe("BrowserExecutor iframe stitching", () => { ]); const executor = new BrowserExecutor(cdp); const text = await snapshotText(executor); - expect(text).toBe(['RootWebArea "Page"', " Iframe", ' RootWebArea "Embed"', ' button "Inside" [e1]'].join("\n")); - await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(text).toBe(['RootWebArea "Page"', " Iframe [e1]", ' RootWebArea "Embed"', ' button "Inside" [e2]'].join("\n")); + await executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "F0" } }, sessionId: "session-1" }); - await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); const OOPIF_PAGE = [ @@ -531,18 +645,18 @@ describe("BrowserExecutor iframe stitching", () => { return fake; }; - it("resolves a ref inside an OOPIF through the child frame's session", async () => { + it("resolves an OOPIF ref's node through the child session but dispatches input on the page session", async () => { const { cdp, sent } = setupOopif(); const executor = new BrowserExecutor(cdp); const text = await snapshotText(executor); expect(text).toContain('button "Top" [e1]'); - expect(text).toContain(' button "Pay" [e2]'); + expect(text).toContain(' button "Pay" [e3]'); - await executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + await executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); const scrolled = sent.find((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 70); expect(scrolled?.sessionId).toBe("session-oop"); const pressed = sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); - expect(pressed?.sessionId).toBe("session-oop"); + expect(pressed?.sessionId).toBe("session-1"); }); it("invalidates only the child frame's refs when the child frame navigates", async () => { @@ -551,7 +665,7 @@ describe("BrowserExecutor iframe stitching", () => { await snapshotText(executor); emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-OOP" } }, sessionId: "session-oop" }); - await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); const text = await snapshotText(executor); diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index 1543007a..5e31a115 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -123,7 +123,8 @@ const BROWSER_ACTION_DESCRIPTIONS: Record = { "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale. " + "If the page has not changed since your previous snapshot, a short unchanged notice is returned instead and earlier refs remain valid.", browser_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", - browser_find: "Find elements matching a natural-language description and return them with element references, like a filtered snapshot.", + browser_find: + "Find elements in the main frame matching a natural-language description and return them with element references, like a filtered snapshot.", browser_click: "Click an element. Prefer targeting by element reference from a snapshot.", browser_hover: "Move the pointer over an element without clicking.", browser_drag: "Drag from one viewport coordinate to another.", From 5082f9085e4bb03a1e222fe189c402f73d3e7c04 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 23:25:21 +0000 Subject: [PATCH 13/34] Enable JS exec and navigation helper by default; finish naming sweep - Remove the javascriptExec option: browser_evaluate is part of the default browser/hybrid action sets, and the native browser tool declaration gets enable_javascript_exec unless the spec sets it explicitly. Drop the CLI --js-exec flag. Opt out by passing an explicit actions list or native spec. - Remove the computerUseExtra option: the computer_use_extra navigation helper is always registered (deduped by name against caller executors). - Delete CUA_DEFAULT_BROWSER_ACTION_TYPES; the default browser set is now just CUA_BROWSER_ACTION_TYPES. - Rename the remaining OS/DOM-era identifiers, comments, and test titles to the computer/browser plane vocabulary (COMPUTER_ACTION_TYPE_SET, BROWSER_ACTION_TYPE_SET, createFakeBrowserExecutor, ...). --- packages/agent/README.md | 3 +-- packages/agent/src/agent.ts | 22 +----------------- packages/agent/src/tools.ts | 9 ++++---- packages/agent/test/agent.test.ts | 22 +++++++++--------- .../agent/test/tool-exhaustiveness.test.ts | 2 +- .../agent/test/translator-browser.test.ts | 22 +++++++++--------- packages/ai/src/actions/browser.ts | 9 -------- packages/ai/src/actions/computer.ts | 4 ++-- packages/ai/src/actions/index.ts | 14 +++++------ packages/ai/src/modes.ts | 23 ++++++------------- .../ai/src/providers/anthropic/actions.ts | 2 +- packages/ai/src/providers/common.ts | 6 ++--- packages/ai/src/runtime-spec.ts | 10 ++++---- packages/ai/test/modes.test.ts | 18 ++++++--------- packages/ai/test/native-tools.test.ts | 10 ++++---- packages/cli/src/cli-harness.ts | 8 +++---- packages/cli/src/cli.ts | 6 ----- packages/cli/src/harness.ts | 4 ---- 18 files changed, 67 insertions(+), 127 deletions(-) diff --git a/packages/agent/README.md b/packages/agent/README.md index cb60d4f3..5371d013 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -97,7 +97,6 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions: - `client` (Kernel SDK client) - CUA model refs (`"provider:model"`) accepted where pi expects a concrete model - `extraTools` to add your own pi tools alongside the built-in browser tools -- `computerUseExtra: true` to let the model use a small navigation helper - `playwright: true` to let the model run Playwright/TypeScript against the live browser session @@ -122,7 +121,7 @@ or handing off to another service while it also controls the browser. Not every provider's native computer-use vocabulary includes browser navigation — some models can click and type but have no direct way to open a -URL or go back. `computerUseExtra: true` adds `computer_use_extra`, a +URL or go back. The classes therefore always add `computer_use_extra`, a provider-neutral escape hatch exposing `goto`, `back`, `forward`, and `url` so navigation works uniformly regardless of which model is driving. diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 585c8f1c..5a8b1dd2 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -76,12 +76,8 @@ export type CuaAgentOptions = Omit & { mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ - javascriptExec?: boolean; /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only; default false. */ cursorHints?: boolean; - /** Expose a helper for browser navigation and URL reads. */ - computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; }; @@ -118,12 +114,8 @@ export type CuaAgentHarnessOptions< mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ - javascriptExec?: boolean; /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only; default false. */ cursorHints?: boolean; - /** Expose a helper for browser navigation and URL reads. */ - computerUseExtra?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */ @@ -149,9 +141,7 @@ class CuaRuntimeController { extraTools?: AgentTool[]; mode?: CuaMode; nativeTool?: CuaNativeToolSpec; - javascriptExec?: boolean; cursorHints?: boolean; - computerUseExtra?: boolean; playwright?: boolean; onPayload?: SimpleStreamOptions["onPayload"]; }, @@ -165,7 +155,6 @@ class CuaRuntimeController { return resolveCuaRuntimeSpec(model, { mode, nativeTool: this.options.nativeTool, - javascriptExec: this.options.javascriptExec, }); } @@ -203,7 +192,6 @@ class CuaRuntimeController { { toolExecutors: this.runtimeSpec.toolExecutors, mode: this.runtimeSpec.mode, - computerUseExtra: this.options.computerUseExtra, playwright: this.options.playwright, }, this.translator, @@ -227,7 +215,7 @@ class CuaRuntimeController { keepToolNames(): string[] { return [ ...(this.options.extraTools ?? []).map((tool) => tool.name), - ...(this.options.computerUseExtra ? [CUA_NAVIGATION_TOOL_NAME] : []), + CUA_NAVIGATION_TOOL_NAME, ...(this.options.playwright ? [CUA_PLAYWRIGHT_TOOL_NAME] : []), ]; } @@ -273,9 +261,7 @@ export class CuaAgent extends Agent { extraTools, mode, nativeTool, - javascriptExec, cursorHints, - computerUseExtra, playwright, ...agentOptions } = options; @@ -286,9 +272,7 @@ export class CuaAgent extends Agent { extraTools, mode, nativeTool, - javascriptExec, cursorHints, - computerUseExtra, playwright, onPayload, }); @@ -423,9 +407,7 @@ export class CuaAgentHarness< extraTools, mode, nativeTool, - javascriptExec, cursorHints, - computerUseExtra, playwright, systemPrompt, onPayload, @@ -439,9 +421,7 @@ export class CuaAgentHarness< extraTools, mode, nativeTool, - javascriptExec, cursorHints, - computerUseExtra, playwright, onPayload, }); diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 80b574cd..0a73796a 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -27,7 +27,6 @@ export interface ComputerToolOptions { mode?: CuaMode; /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode. Default false. */ cursorHints?: boolean; - computerUseExtra?: boolean; playwright?: boolean; } @@ -88,16 +87,16 @@ export function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTo /** Build executor tools against an existing translator (internal; not part of the package surface). */ export function buildCuaComputerTools( - args: Pick, + args: Pick, translator: InternalComputerTranslator, ): CuaExecutorTool[] { return withExtraTools(args).map((executor) => createExecutorTool(executor, translator, args.mode ?? "computer")); } -function withExtraTools(args: Pick): ComputerExecutorSpec[] { +function withExtraTools(args: Pick): ComputerExecutorSpec[] { const executors: ComputerExecutorSpec[] = [...args.toolExecutors]; const existing = new Set(executors.map((executor) => executor.definition.name)); - if (args.computerUseExtra && !existing.has(CUA_NAVIGATION_TOOL_NAME)) { + if (!existing.has(CUA_NAVIGATION_TOOL_NAME)) { executors.push({ kind: "navigation", definition: createCuaNavigationToolDefinition() }); } if (args.playwright && !existing.has(CUA_PLAYWRIGHT_TOOL_NAME)) { @@ -179,7 +178,7 @@ async function executeBatchTool( } } if (content.length === 0) { - // Post-action grounding capture: the OS display in os/hybrid mode, + // Post-action grounding capture: the OS display in computer/hybrid mode, // the browser viewport in browser mode (the only frame the model sees). const screenshot = mode === "browser" ? await translator.browser().screenshot() : await translator.screenshot(); readResults.push({ type: "screenshot", bytes: screenshot.data.length }); diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 7c50ddf3..7790404d 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -92,7 +92,7 @@ describe("CuaAgent", () => { }, }); - expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "custom"]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "computer_use_extra", "custom"]); }); it("always keeps provider CUA tools when adding extra tools", () => { @@ -109,7 +109,7 @@ describe("CuaAgent", () => { }, }); - expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "custom"]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "computer_use_extra", "custom"]); expect(agent.state.systemPrompt).toBe("Use the browser carefully."); }); @@ -124,15 +124,14 @@ describe("CuaAgent", () => { }); expect(runtime.toolDefinitions.map((tool) => tool.name)).toContain(ANTHROPIC_BATCH_TOOL_NAME); - expect(agent.state.tools.map((tool) => tool.name)).toEqual(runtime.toolExecutors.map((tool) => tool.definition.name)); + expect(agent.state.tools.map((tool) => tool.name)).toEqual([...runtime.toolExecutors.map((tool) => tool.definition.name), "computer_use_extra"]); }); - it("synthesizes navigation tools when requested", () => { + it("synthesizes navigation tools by default", () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const agent = new CuaAgent({ browser, client, - computerUseExtra: true, initialState: { model: "openai:gpt-5.5", }, @@ -157,6 +156,7 @@ describe("CuaAgent", () => { expect(agent.state.tools.map((tool) => tool.name)).toEqual([ ...runtime.toolExecutors.map((tool) => tool.definition.name), + "computer_use_extra", "playwright_execute", ]); }); @@ -175,7 +175,7 @@ describe("CuaAgent", () => { expect(agent.state.model.id).toBe(runtime.model.id); expect(agent.state.systemPrompt).toBe(runtime.defaultSystemPrompt); - expect(agent.state.tools).toHaveLength(runtime.toolExecutors.length); + expect(agent.state.tools).toHaveLength(runtime.toolExecutors.length + 1); }); it("switches action planes through setMode", () => { @@ -227,7 +227,7 @@ describe("CuaAgent", () => { agent.state.model = "google:gemini-3-flash-preview"; const runtime = resolveCuaRuntimeSpec("google:gemini-3-flash-preview"); - expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "custom"]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolExecutors.map((item) => item.definition.name), "computer_use_extra", "custom"]); expect(agent.state.systemPrompt).toBe("custom prompt"); }); @@ -299,7 +299,6 @@ describe("CuaAgent", () => { client: screenshotClient, streamFn, extraTools: [createCustomTool("custom_tool")], - computerUseExtra: true, initialState: { model: "yutori:n1.5-latest", }, @@ -342,7 +341,7 @@ describe("CuaAgent", () => { const update = await agent.prepareNextTurn?.(undefined); expect(update?.model?.id).toBe(runtime.model.id); - expect(update?.context?.tools).toHaveLength(runtime.toolExecutors.length); + expect(update?.context?.tools).toHaveLength(runtime.toolExecutors.length + 1); await expect(agent.prepareNextTurn?.(undefined)).resolves.toBeUndefined(); }); @@ -423,7 +422,7 @@ describe("CuaAgentHarness", () => { await harness.setModel("google:gemini-3-flash-preview"); expect(harness.getModel().id).toBe(runtime.model.id); - expect(harness.getTools()).toHaveLength(runtime.toolExecutors.length); + expect(harness.getTools()).toHaveLength(runtime.toolExecutors.length + 1); }); it("switches action planes through setMode", async () => { @@ -477,6 +476,7 @@ describe("CuaAgentHarness", () => { expect(harness.getTools().map((item) => item.name)).toEqual([ ...runtime.toolExecutors.map((item) => item.definition.name), + "computer_use_extra", "custom", ]); }); @@ -509,7 +509,7 @@ describe("CuaAgentHarness", () => { await harness.setModel("google:gemini-3-flash-preview"); expect(harness.getTools()).toHaveLength( - resolveCuaRuntimeSpec("google:gemini-3-flash-preview").toolExecutors.length, + resolveCuaRuntimeSpec("google:gemini-3-flash-preview").toolExecutors.length + 1, ); expect(harness.getActiveTools().map((tool) => tool.name)).toEqual(["click", "screenshot"]); diff --git a/packages/agent/test/tool-exhaustiveness.test.ts b/packages/agent/test/tool-exhaustiveness.test.ts index af8c1ba9..a0a67ed8 100644 --- a/packages/agent/test/tool-exhaustiveness.test.ts +++ b/packages/agent/test/tool-exhaustiveness.test.ts @@ -24,7 +24,7 @@ describe("Cua tool executor coverage", () => { it("instantiates one executor per provider execution adapter", () => { const toolExecutors = resolveCuaRuntimeSpec("openai:gpt-5.5").toolExecutors; const tools = createCuaComputerTools({ browser, client, toolExecutors }); - expect(tools.map((tool) => tool.name).sort()).toEqual(toolExecutors.map((tool) => tool.definition.name).sort()); + expect(tools.map((tool) => tool.name).sort()).toEqual([...toolExecutors.map((tool) => tool.definition.name), "computer_use_extra"].sort()); }); it("executes Yutori local canonical action tools", async () => { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 0cf3f9b9..ab5820fe 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -25,9 +25,9 @@ function createClient() { return { batches, client }; } -function createFakeDom() { +function createFakeBrowserExecutor() { const executed: CuaBrowserAction[] = []; - const dom = { + const executor = { execute: async (action: CuaBrowserAction): Promise => { executed.push(action); if (action.type === "browser_text") return [{ type: "browser_text", label: "text", text: "hello" }]; @@ -35,14 +35,14 @@ function createFakeDom() { }, screenshot: async () => ({ data: Buffer.from("png"), mimeType: "image/png" }), } as unknown as BrowserExecutor; - return { executed, dom }; + return { executed, executor }; } -describe("InternalComputerTranslator DOM plane", () => { - it("dispatches DOM actions to the DOM executor, flushing pending OS input first", async () => { +describe("InternalComputerTranslator browser plane", () => { + it("dispatches browser actions to the browser executor, flushing pending OS input first", async () => { const { batches, client } = createClient(); - const { executed, dom } = createFakeDom(); - const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => dom }); + const { executed, executor } = createFakeBrowserExecutor(); + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => executor }); const result = await translator.executeBatch([ { type: "click", x: 1, y: 2 }, @@ -55,14 +55,14 @@ describe("InternalComputerTranslator DOM plane", () => { expect(result.readResults).toEqual([{ type: "browser_text", label: "text", text: "hello" }]); }); - it("errors on DOM actions when the browser has no cdp_ws_url", async () => { + it("errors on browser actions when the browser has no cdp_ws_url", async () => { const { client } = createClient(); const translator = new InternalComputerTranslator({ browser: { session_id: "b" } as KernelBrowser, client }); await expect(translator.executeBatch([{ type: "browser_text" }])).rejects.toThrow(/cdp_ws_url/); }); }); -describe("InternalComputerTranslator OS additions", () => { +describe("InternalComputerTranslator computer additions", () => { it("crops the OS screenshot for zoom, staying in the screenshot frame", async () => { const { client } = createClient(); const translator = new InternalComputerTranslator({ browser, client }); @@ -525,7 +525,7 @@ describe("BrowserExecutor cursor-pointer hints", () => { it("only enables cursor hints on the executor in browser mode", () => { const recordedFor = (mode?: "computer" | "browser" | "hybrid") => { const recorded: BrowserExecutorOptions[] = []; - const { dom } = createFakeDom(); + const { executor } = createFakeBrowserExecutor(); const { client } = createClient(); const translator = new InternalComputerTranslator({ browser, @@ -534,7 +534,7 @@ describe("BrowserExecutor cursor-pointer hints", () => { cursorHints: true, createBrowserExecutor: (_cdpWsUrl, options) => { recorded.push(options); - return dom; + return executor; }, }); translator.browser(); diff --git a/packages/ai/src/actions/browser.ts b/packages/ai/src/actions/browser.ts index 768fa2cd..55589ac6 100644 --- a/packages/ai/src/actions/browser.ts +++ b/packages/ai/src/actions/browser.ts @@ -36,15 +36,6 @@ export const CUA_BROWSER_ACTION_TYPES = [ export type CuaBrowserActionType = (typeof CUA_BROWSER_ACTION_TYPES)[number]; -/** - * The default browser-mode toolset: everything except `browser_evaluate`, which - * runs arbitrary JavaScript in the page and must be enabled explicitly - * (`javascriptExec: true`). - */ -export const CUA_DEFAULT_BROWSER_ACTION_TYPES = CUA_BROWSER_ACTION_TYPES.filter( - (action): action is Exclude => action !== "browser_evaluate", -); - export interface CuaActionBrowserSnapshot { type: "browser_snapshot"; filter?: "all" | "interactive"; diff --git a/packages/ai/src/actions/computer.ts b/packages/ai/src/actions/computer.ts index c6c55b8d..181269b0 100644 --- a/packages/ai/src/actions/computer.ts +++ b/packages/ai/src/actions/computer.ts @@ -31,8 +31,8 @@ export const CUA_COMPUTER_ACTION_TYPES = [ export type CuaComputerActionType = (typeof CUA_COMPUTER_ACTION_TYPES)[number]; /** - * The default OS-mode toolset. This is the pre-modes canonical action list: - * every OS action except `zoom`, which is only exposed by default in hybrid + * The default computer-mode toolset. This is the pre-modes canonical action list: + * every computer action except `zoom`, which is only exposed by default in hybrid * mode and by Anthropic's native computer tool (`enable_zoom`). */ export const CUA_DEFAULT_COMPUTER_ACTION_TYPES = CUA_COMPUTER_ACTION_TYPES.filter( diff --git a/packages/ai/src/actions/index.ts b/packages/ai/src/actions/index.ts index 6fec9fab..a2d164e5 100644 --- a/packages/ai/src/actions/index.ts +++ b/packages/ai/src/actions/index.ts @@ -5,31 +5,31 @@ import { CUA_COMPUTER_ACTION_SCHEMA_BY_TYPE, CUA_COMPUTER_ACTION_TYPES, type Cua export * from "./browser"; export * from "./computer"; -/** Any canonical CUA action type, across the OS and browser planes. */ +/** Any canonical CUA action type, across the computer and browser planes. */ export type CuaActionType = CuaComputerActionType | CuaBrowserActionType; -/** Any canonical CUA action, across the OS and browser planes. */ +/** Any canonical CUA action, across the computer and browser planes. */ export type CuaAction = CuaComputerAction | CuaBrowserAction; /** Every canonical action type: the computer plane followed by the browser plane. */ export const CUA_ALL_ACTION_TYPES: readonly CuaActionType[] = [...CUA_COMPUTER_ACTION_TYPES, ...CUA_BROWSER_ACTION_TYPES]; -const OS_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_COMPUTER_ACTION_TYPES); -const DOM_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_BROWSER_ACTION_TYPES); +const COMPUTER_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_COMPUTER_ACTION_TYPES); +const BROWSER_ACTION_TYPE_SET: ReadonlySet = new Set(CUA_BROWSER_ACTION_TYPES); /** Whether a canonical action type belongs to the computer plane. */ export function isCuaComputerActionType(action: CuaActionType): action is CuaComputerActionType { - return OS_ACTION_TYPE_SET.has(action); + return COMPUTER_ACTION_TYPE_SET.has(action); } /** Whether a canonical action type belongs to the browser plane. */ export function isCuaBrowserActionType(action: CuaActionType): action is CuaBrowserActionType { - return DOM_ACTION_TYPE_SET.has(action); + return BROWSER_ACTION_TYPE_SET.has(action); } /** Whether a canonical action belongs to the browser plane. */ export function isCuaBrowserAction(action: CuaAction): action is CuaBrowserAction { - return DOM_ACTION_TYPE_SET.has(action.type); + return BROWSER_ACTION_TYPE_SET.has(action.type); } /** Options for building canonical action schemas. */ diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index 5e31a115..a3fa4a00 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -1,5 +1,5 @@ import { - CUA_DEFAULT_BROWSER_ACTION_TYPES, + CUA_BROWSER_ACTION_TYPES, CUA_DEFAULT_COMPUTER_ACTION_TYPES, isCuaComputerActionType, type CuaActionSchemaOptions, @@ -25,12 +25,6 @@ import { */ export type CuaMode = "computer" | "browser" | "hybrid"; -/** Options for resolving a mode's action set. */ -export interface CuaModeOptions { - /** Expose `browser_evaluate` (arbitrary JavaScript in the page). Default false. */ - javascriptExec?: boolean; -} - /** * Computer actions exposed in hybrid mode: navigation reads/writes are * excluded because they live on the browser plane (`browser_navigate`, @@ -54,8 +48,8 @@ export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] ]; /** - * Browser actions exposed in hybrid mode: reads and element-targeted writes - * only. Pointer/keyboard capabilities (`browser_click` by coordinate, + * Browser actions exposed in hybrid mode: reads, element-targeted writes, + * and JavaScript evaluation. Pointer/keyboard capabilities (`browser_click` by coordinate, * `browser_type`, `browser_key`, `browser_scroll`, `browser_hover`, `browser_drag`) and * `browser_screenshot` are excluded — real OS input and the OS screenshot cover * those, keeping one tool per capability and one coordinate frame. @@ -70,21 +64,18 @@ export const CUA_HYBRID_BROWSER_ACTION_TYPES: readonly CuaBrowserActionType[] = "browser_navigate", "browser_list_tabs", "browser_new_tab", + "browser_evaluate", ]; /** Resolve the default canonical action set for a mode. */ -export function defaultActionsForMode(mode: CuaMode, options: CuaModeOptions = {}): readonly CuaActionType[] { +export function defaultActionsForMode(mode: CuaMode): readonly CuaActionType[] { switch (mode) { case "computer": return CUA_DEFAULT_COMPUTER_ACTION_TYPES; case "browser": - return [...CUA_DEFAULT_BROWSER_ACTION_TYPES, ...(options.javascriptExec ? (["browser_evaluate"] as const) : []), "wait"]; + return [...CUA_BROWSER_ACTION_TYPES, "wait"]; case "hybrid": - return [ - ...CUA_HYBRID_COMPUTER_ACTION_TYPES, - ...CUA_HYBRID_BROWSER_ACTION_TYPES, - ...(options.javascriptExec ? (["browser_evaluate"] as const) : []), - ]; + return [...CUA_HYBRID_COMPUTER_ACTION_TYPES, ...CUA_HYBRID_BROWSER_ACTION_TYPES]; } } diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts index 2058fa06..af51bda8 100644 --- a/packages/ai/src/providers/anthropic/actions.ts +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -70,7 +70,7 @@ function resolveAnthropicActions(options: AnthropicComputerToolsOptions): readon options.actions ?? (mode === "computer" ? ANTHROPIC_CUA_ACTION_TYPES.filter((action) => action !== "zoom") - : defaultActionsForMode(mode, { javascriptExec: options.javascriptExec }).filter( + : defaultActionsForMode(mode).filter( (action) => isCuaBrowserActionType(action) || isAnthropicCanonicalAction(action), )); const supported: AnthropicCanonicalActionType[] = []; diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 6edd8894..6430fae2 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -19,7 +19,7 @@ export * from "../modes"; export * from "../native-tools"; /** - * The default os-mode action set: every computer-plane action except `zoom`. + * The default computer-mode action set: every computer-plane action except `zoom`. * The full canonical vocabulary is split by plane into * {@link CUA_COMPUTER_ACTION_TYPES} and {@link CUA_BROWSER_ACTION_TYPES}. */ @@ -119,8 +119,6 @@ export interface ComputerToolsOptions { actions?: readonly CuaActionType[]; /** Which action plane(s) to expose. Default "computer". */ mode?: CuaMode; - /** Expose `browser_evaluate` in browser/hybrid modes. Default false. */ - javascriptExec?: boolean; } export type ComputerToolCoordinateSystem = @@ -145,7 +143,7 @@ export function computerTools(options: ComputerToolsOptions = {}): Tool[] { /** Resolve the action list for a tools-options object: explicit list, or the mode's default set. */ export function resolveModeActions(options: ComputerToolsOptions = {}): readonly CuaActionType[] { - return options.actions ?? defaultActionsForMode(options.mode ?? "computer", { javascriptExec: options.javascriptExec }); + return options.actions ?? defaultActionsForMode(options.mode ?? "computer"); } /** Guard for providers whose computer-use vocabulary only covers the computer plane. */ diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 2c8ef501..e8fbee66 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -51,7 +51,7 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options: CuaRu const mode = options.mode ?? (options.nativeTool ? modeForNativeTool(options.nativeTool) : "computer"); if (options.nativeTool) { - const nativeTool = resolveNativeTool(withJavascriptExec(options.nativeTool, options.javascriptExec), model, mode); + const nativeTool = resolveNativeTool(withDefaultJavascriptExec(options.nativeTool), model, mode); const nativeModel: Model = { ...model, api: nativeApiForToolType(nativeTool.spec.type) as Model["api"] }; const executors = nativeToolExecutors(nativeTool); return { @@ -82,11 +82,11 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options: CuaRu }; } -// Fold the mode-level javascriptExec option into the native browser tool -// declaration so it behaves like canonical browser mode. An explicit +// JavaScript execution is on by default, matching canonical browser mode +// where `browser_evaluate` is part of the default toolset. An explicit // enable_javascript_exec on the spec wins. -function withJavascriptExec(spec: CuaNativeToolSpec, javascriptExec: boolean | undefined): CuaNativeToolSpec { - if (!javascriptExec || spec.type !== "browser_20260701" || spec.enable_javascript_exec !== undefined) return spec; +function withDefaultJavascriptExec(spec: CuaNativeToolSpec): CuaNativeToolSpec { + if (spec.type !== "browser_20260701" || spec.enable_javascript_exec !== undefined) return spec; return { ...spec, enable_javascript_exec: true }; } diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index ff0a1fe6..d9b2f030 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { CUA_ACTION_TYPES, - CUA_DEFAULT_BROWSER_ACTION_TYPES, + CUA_BROWSER_ACTION_TYPES, CUA_HYBRID_BROWSER_ACTION_TYPES, CUA_HYBRID_COMPUTER_ACTION_TYPES, anthropic, @@ -17,21 +17,17 @@ describe("mode action sets", () => { expect(defaultActionsForMode("computer")).toEqual(CUA_ACTION_TYPES); }); - it("browser mode defaults to DOM actions plus wait, without browser_evaluate", () => { + it("browser mode defaults to every browser action plus wait", () => { const actions = defaultActionsForMode("browser"); expect(actions).toContain("browser_snapshot"); expect(actions).toContain("wait"); - expect(actions).not.toContain("browser_evaluate"); + expect(actions).toContain("browser_evaluate"); expect(actions).not.toContain("click"); }); - it("browser mode exposes browser_evaluate only with javascriptExec", () => { - expect(defaultActionsForMode("browser", { javascriptExec: true })).toContain("browser_evaluate"); - }); - it("hybrid mode dedupes to one tool per capability", () => { const actions = defaultActionsForMode("hybrid"); - // Navigation lives on the DOM plane. + // Navigation lives on the browser plane. expect(actions).not.toContain("goto"); expect(actions).not.toContain("url"); expect(actions).toContain("browser_navigate"); @@ -62,7 +58,7 @@ describe("mode tool naming", () => { expect(cuaToolNameForAction("browser_click", "hybrid")).toBe("browser_click"); }); - it("computer mode rejects DOM actions", () => { + it("computer mode rejects browser actions", () => { expect(() => cuaToolNameForAction("browser_click", "computer")).toThrow(/not available in computer mode/); }); }); @@ -83,10 +79,10 @@ describe("mode tool schemas", () => { expect(pageClick.parameters.required).toContain("ref"); }); - it("browser mode exposes every default DOM action under its unprefixed name", () => { + it("browser mode exposes every default browser action under its unprefixed name", () => { const tools = computerTools({ mode: "browser" }); const names = tools.map((tool) => tool.name); - for (const action of CUA_DEFAULT_BROWSER_ACTION_TYPES) { + for (const action of CUA_BROWSER_ACTION_TYPES) { expect(names).toContain(action.slice("browser_".length)); } }); diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index 775b0f7c..fa815fc7 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -56,16 +56,14 @@ describe("native runtime specs", () => { expect(spec.toolDefinitions.map((tool) => tool.name)).toEqual(["browser"]); }); - it("folds javascriptExec into the native browser declaration unless the spec is explicit", () => { - const folded = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { + it("enables javascript exec on the native browser declaration unless the spec is explicit", () => { + const defaulted = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "browser_20260701" }, - javascriptExec: true, }); - expect(folded.nativeTool?.declaration.enable_javascript_exec).toBe(true); + expect(defaulted.nativeTool?.declaration.enable_javascript_exec).toBe(true); const explicit = resolveCuaRuntimeSpec("anthropic:claude-opus-4-5", { nativeTool: { type: "browser_20260701", enable_javascript_exec: false }, - javascriptExec: true, }); expect(explicit.nativeTool?.declaration.enable_javascript_exec).toBe(false); }); @@ -126,7 +124,7 @@ describe("computer_20260701 action mapping", () => { }); describe("browser_20260701 action mapping", () => { - it("maps DOM reads", () => { + it("maps browser reads", () => { expect(mapNativeBrowserInput({ action: "read_page", filter: "interactive", depth: 5 })).toEqual([ { type: "browser_snapshot", filter: "interactive", depth: 5 }, ]); diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 3434f040..9c028047 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -181,7 +181,6 @@ export interface HarnessCliFlags { playwright: boolean; mode?: string; nativeTool?: string; - jsExec?: boolean; model?: string; thinking?: string; browserProfile?: string; @@ -380,7 +379,7 @@ async function setupHarnessRuntime( // Validate mode/native-tool flags before provisioning so a bad combination // never leaves an orphaned browser behind. const mode = parseMode(flags.mode); - const nativeTool = parseNativeTool(flags.nativeTool, flags.jsExec); + const nativeTool = parseNativeTool(flags.nativeTool); const provisioned = await provisionForFlags(flags, auth); try { @@ -450,7 +449,6 @@ async function finishHarnessRuntime( thinkingLevel, mode, nativeTool, - javascriptExec: flags.jsExec, playwright: flags.playwright, modelBaseUrl: baseUrlOverride, }); @@ -489,13 +487,13 @@ function parseMode(raw: string | undefined): CuaMode | undefined { throw new Error(`invalid --mode value "${raw}"; expected one of: computer | browser | hybrid`); } -function parseNativeTool(raw: string | undefined, jsExec: boolean | undefined): CuaNativeToolSpec | undefined { +function parseNativeTool(raw: string | undefined): CuaNativeToolSpec | undefined { if (raw === undefined) return undefined; const value = raw.trim().toLowerCase(); // enable_zoom follows Anthropic's own recommendation for fine-grained // visual targeting; the executor implements the zoom crop locally. if (value === "computer_20260701") return { type: "computer_20260701", enable_zoom: true }; - if (value === "browser_20260701") return { type: "browser_20260701", ...(jsExec ? { enable_javascript_exec: true } : {}) }; + if (value === "browser_20260701") return { type: "browser_20260701" }; throw new Error(`invalid --native-tool value "${raw}"; expected one of: computer_20260701 | browser_20260701`); } diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index d3cd16c1..00a1aa22 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -54,8 +54,6 @@ Options: --native-tool Drive an Anthropic model through its native tool schema: computer_20260701 (requires --mode computer) or browser_20260701 (requires --mode browser) - --js-exec Expose browser_evaluate (arbitrary JS in the page) in - browser/hybrid modes --out Output file for screenshot subcommand -o, --output Output format for --print: text (default) | jsonl --jsonl-include-deltas Include assistant_text_delta events (default off) @@ -112,7 +110,6 @@ interface CliFlags { playwright: boolean; mode?: string; nativeTool?: string; - jsExec: boolean; model?: string; thinking?: string; browserProfile?: string; @@ -164,7 +161,6 @@ function parseCliArgs(argv: string[]): CliFlags { playwright: { type: "boolean", default: false }, mode: { type: "string" }, "native-tool": { type: "string" }, - "js-exec": { type: "boolean", default: false }, }, allowPositionals: true, strict: true, @@ -222,7 +218,6 @@ function parseCliArgs(argv: string[]): CliFlags { playwright: !!parsed.values.playwright, mode: parsed.values.mode as string | undefined, nativeTool: parsed.values["native-tool"] as string | undefined, - jsExec: !!parsed.values["js-exec"], positionals: parsed.positionals, }; } @@ -241,7 +236,6 @@ function toHarnessFlags(flags: CliFlags): HarnessCliFlags { playwright: flags.playwright, mode: flags.mode, nativeTool: flags.nativeTool, - jsExec: flags.jsExec, model: flags.model, thinking: flags.thinking, browserProfile: flags.browserProfile, diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index c0228c65..5dfaa970 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -37,8 +37,6 @@ export interface BuildCuaHarnessOptions { mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Expose `browser_evaluate` in browser/hybrid modes. */ - javascriptExec?: boolean; /** Expose the playwright_execute tool that runs Playwright code against the browser session. */ playwright?: boolean; /** Override the default coding-tools extraTools (bash/read/edit/write/grep/find/ls). */ @@ -75,7 +73,6 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { extraTools, mode: opts.mode, nativeTool: opts.nativeTool, - javascriptExec: opts.javascriptExec, playwright: opts.playwright, resources: { skills }, thinkingLevel: opts.thinkingLevel, @@ -83,7 +80,6 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { const runtime = resolveCuaRuntimeSpec(activeModel, { mode: harness?.getMode() ?? opts.mode, nativeTool: opts.nativeTool, - javascriptExec: opts.javascriptExec, }); return composeSystemPrompt(runtime.defaultSystemPrompt, resources.skills ?? [], contextFiles); }, From 72aad8766ebd802e12f913300804b453d1c4b3ee Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 23:29:04 +0000 Subject: [PATCH 14/34] Address bugbot findings on navigation frame and mode-switch tool state - computer_use_extra captures the browser viewport in browser mode instead of the OS display, matching post-action grounding captures - setMode updates requestedActiveToolNames so a later setModel re-applies the current mode's active subset rather than pre-switch names --- packages/agent/src/agent.ts | 3 +++ packages/agent/src/tools.ts | 12 +++++++--- packages/agent/test/agent.test.ts | 22 ++++++++++++++++++ .../agent/test/translator-browser.test.ts | 23 +++++++++++++++++++ 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 5a8b1dd2..908774cc 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -480,6 +480,9 @@ export class CuaAgentHarness< ? tools.map((tool) => tool.name).filter((name) => !previousNames.has(name) || requested.includes(name)) : tools.map((tool) => tool.name); await super.setTools(tools, active); + // The requested subset now reflects this mode's toolset; without this a + // later setModel would restore the pre-switch names. + if (requested) this.requestedActiveToolNames = active; } /** The action plane(s) currently exposed to the model. */ diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 0a73796a..79647204 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -114,7 +114,7 @@ function createExecutorTool(executor: ComputerExecutorSpec, translator: Internal description: definition.description, parameters: definition.parameters, async execute(_toolCallId: string, params: unknown): Promise> { - return executeNavigationTool(translator, asNavigationInput(params)); + return executeNavigationTool(translator, asNavigationInput(params), mode); }, }; return tool; @@ -190,7 +190,11 @@ async function executeBatchTool( return { content, details: { statusText: "Actions executed successfully.", readResults } }; } -async function executeNavigationTool(translator: InternalComputerTranslator, params: CuaNavigationInput): Promise> { +async function executeNavigationTool( + translator: InternalComputerTranslator, + params: CuaNavigationInput, + mode: CuaMode, +): Promise> { const action = params.action; try { let statusText = `${action} executed successfully.`; @@ -203,7 +207,9 @@ async function executeNavigationTool(translator: InternalComputerTranslator, par } else { await translator.executeBatch([{ type: action }]); } - const screenshot = await translator.screenshot(); + // Same grounding frame as post-action captures: the browser viewport in + // browser mode, the OS display otherwise. + const screenshot = mode === "browser" ? await translator.browser().screenshot() : await translator.screenshot(); return { content: [ { type: "text", text: statusText }, diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 7790404d..dc088b59 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -463,6 +463,28 @@ describe("CuaAgentHarness", () => { expect(active).not.toContain("custom"); }); + it("setModel after setMode keeps the mode's active tool subset", async () => { + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "anthropic:claude-opus-4-5", + extraTools: [createCustomTool()], + }); + const withoutCustom = harness + .getTools() + .map((tool) => tool.name) + .filter((name) => name !== "custom"); + await harness.setActiveTools(withoutCustom); + await harness.setMode("browser"); + + await harness.setModel("anthropic:claude-opus-4-7"); + + const active = harness.getActiveTools().map((tool) => tool.name); + expect(active).toContain("snapshot"); + expect(active).not.toContain("custom"); + }); + it("appends extraTools in harness construction", async () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const tool = createCustomTool(); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index ab5820fe..86506bbc 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; import { BrowserExecutor, type BrowserExecutorOptions } from "../src/translator/browser"; import type { CdpConnection } from "../src/translator/cdp"; +import { buildCuaComputerTools } from "../src/tools"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -672,3 +673,25 @@ describe("BrowserExecutor iframe stitching", () => { expect(text).toContain('button "Pay" [e'); }); }); + +describe("navigation tool grounding frame", () => { + const navTool = (mode: "computer" | "browser") => { + const { client } = createClient(); + const { executor } = createFakeBrowserExecutor(); + const translator = new InternalComputerTranslator({ browser, client, mode, createBrowserExecutor: () => executor }); + return buildCuaComputerTools({ toolExecutors: [], mode }, translator).find((tool) => tool.name === "computer_use_extra")!; + }; + + it("captures the viewport in browser mode and the OS display otherwise", async () => { + const viewportData = Buffer.from("png").toString("base64"); + + const browserResult = await navTool("browser").execute("call_1", { action: "back" }); + const viewportImage = browserResult.content.find((block) => block.type === "image"); + expect(viewportImage).toMatchObject({ type: "image", data: viewportData }); + + const computerResult = await navTool("computer").execute("call_2", { action: "back" }); + const osImage = computerResult.content.find((block) => block.type === "image"); + expect(osImage?.type).toBe("image"); + expect((osImage as { data: string }).data).not.toBe(viewportData); + }); +}); From 9fa56c481af4bcc1ae5cf48286e546b27769bdc7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 23:41:06 +0000 Subject: [PATCH 15/34] Address bugbot findings on frame invalidation, find scope, and setMode rollback - A navigation observed in an OOPIF target's session invalidates that target's refs even when it's a same-process subframe (their AX nodes are inlined in the frame target's tree, so its refs share the target's key) - browser_find searches stitched iframe trees too, matching what browser_snapshot renders; found refs resolve through their frame session - CuaAgentHarness.setMode restores the previous runtime mode when super.setTools fails, keeping the runtime in step with the exposed tools --- packages/agent/src/agent.ts | 9 +++- packages/agent/src/translator/browser.ts | 43 +++++++++++++------ .../agent/test/translator-browser.test.ts | 23 ++++++++++ packages/ai/src/modes.ts | 2 +- 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 908774cc..121a764a 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -470,6 +470,7 @@ export class CuaAgentHarness< * plane conflicts with the requested mode. */ async setMode(mode: CuaMode): Promise { + const previousMode = this.runtime.mode; const previousNames = new Set(this.getTools().map((tool) => tool.name)); this.runtime.setMode(mode); const tools = this.runtime.tools(); @@ -479,7 +480,13 @@ export class CuaAgentHarness< const active = requested ? tools.map((tool) => tool.name).filter((name) => !previousNames.has(name) || requested.includes(name)) : tools.map((tool) => tool.name); - await super.setTools(tools, active); + try { + await super.setTools(tools, active); + } catch (err) { + // Keep the runtime in step with the exposed tools when the switch fails. + this.runtime.setMode(previousMode); + throw err; + } // The requested subset now reflects this mode's toolset; without this a // later setModel would restore the pre-switch names. if (requested) this.requestedActiveToolNames = active; diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index dfcdf488..a8259f03 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -141,7 +141,10 @@ export class BrowserExecutor { const targetId = this.targetsBySession.get(event.sessionId); if (!targetId) return; if (this.frameTargets.has(targetId)) { - if (frame.id === targetId) this.invalidateFrame(targetId); + // Refs from a frame target's tree (its root and any same-process + // subframes inlined in it) are all minted against the target's key, + // so any navigation observed in its session stales them. + this.invalidateFrame(targetId); return; } if (frame.parentId) { @@ -455,24 +458,38 @@ export class BrowserExecutor { const targetId = await this.resolveTarget(action.tab_id); const session = await this.attach(targetId); const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); + const pools: Array<{ nodes: AXNode[]; ctx: RenderContext }> = [ + { + nodes, + ctx: { + targetId, + frameKey: targetId, + sessionId: session, + generation: this.generation(targetId), + interactiveOnly: false, + nthIndex: buildNthIndex(nodes), + }, + }, + ]; + // Search stitched frames too so find sees everything snapshot renders. + for (const stitch of (await this.stitchFrames(nodes, targetId, session, false)).values()) { + pools.push({ nodes: [...stitch.byId.values()], ctx: stitch.ctx }); + } const queryTokens = tokenize(action.query); - const scored = nodes - .filter((node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? ""))) - .map((node) => ({ node, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })) + const scored = pools + .flatMap(({ nodes: poolNodes, ctx }) => + poolNodes + .filter( + (node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), + ) + .map((node) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })), + ) .filter((entry) => entry.score > 0) .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); if (scored.length === 0) return `No elements matched ${JSON.stringify(action.query)}. Try snapshot for the full tree.`; - const ctx: RenderContext = { - targetId, - frameKey: targetId, - sessionId: session, - generation: this.generation(targetId), - interactiveOnly: false, - nthIndex: buildNthIndex(nodes), - }; const text = scored - .map(({ node }) => { + .map(({ node, ctx }) => { const role = node.role?.value ?? "node"; const name = node.name?.value ? ` ${JSON.stringify(node.name.value)}` : ""; return `${role}${name} [${this.mintRef(node, ctx)}]`; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 86506bbc..ffbcaea2 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -660,6 +660,29 @@ describe("BrowserExecutor iframe stitching", () => { expect(pressed?.sessionId).toBe("session-1"); }); + it("invalidates a frame target's refs when a subframe inside it navigates", async () => { + const { cdp, emit } = setupOopif(); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-INNER", parentId: "FRAME-OOP" } }, sessionId: "session-oop" }); + await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + + it("finds elements inside stitched iframes", async () => { + const { cdp, sent } = setupOopif(); + const executor = new BrowserExecutor(cdp); + + const results = await executor.execute({ type: "browser_find", query: "pay button" } as CuaBrowserAction); + const text = (results[0] as { text: string }).text; + expect(text).toContain('button "Pay" [e'); + + const ref = /\[(e\d+)\]/.exec(text)![1]!; + await executor.execute({ type: "browser_click", ref } as CuaBrowserAction); + const resolved = sent.find((cmd) => cmd.method === "DOM.getBoxModel" && cmd.params.backendNodeId === 70); + expect(resolved?.sessionId).toBe("session-oop"); + }); + it("invalidates only the child frame's refs when the child frame navigates", async () => { const { cdp, emit } = setupOopif(); const executor = new BrowserExecutor(cdp); diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index a3fa4a00..cbc83f7d 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -115,7 +115,7 @@ const BROWSER_ACTION_DESCRIPTIONS: Record = { "If the page has not changed since your previous snapshot, a short unchanged notice is returned instead and earlier refs remain valid.", browser_text: "Return the page's visible text content as plain text. Best for articles and text-heavy pages.", browser_find: - "Find elements in the main frame matching a natural-language description and return them with element references, like a filtered snapshot.", + "Find elements on the page (including iframe content) matching a natural-language description and return them with element references, like a filtered snapshot.", browser_click: "Click an element. Prefer targeting by element reference from a snapshot.", browser_hover: "Move the pointer over an element without clicking.", browser_drag: "Drag from one viewport coordinate to another.", From b6c8f0a70f1bb5689aa22ac12a5f53f712c0aa0e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 23:52:38 +0000 Subject: [PATCH 16/34] Route browser-mode navigation helper through CDP; no-op repeated setMode - computer_use_extra's goto/back/forward/url use the browser plane in browser mode (Page.navigate / history / target url) instead of OS keyboard shortcuts, so navigation invalidates refs and matches the viewport grounding frame; adds BrowserExecutor.currentUrl() - setMode with the current mode is a no-op at the agent, harness, and runtime-controller level, so a repeated /mode selection no longer disposes the translator and its CDP-backed refs and tab state --- packages/agent/src/agent.ts | 5 ++++ packages/agent/src/tools.ts | 7 ++++- packages/agent/src/translator/browser.ts | 7 +++++ packages/agent/test/agent.test.ts | 15 +++++++++++ .../agent/test/translator-browser.test.ts | 27 +++++++++++++++---- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 121a764a..c4f02128 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -167,6 +167,9 @@ class CuaRuntimeController { } setMode(mode: CuaMode): void { + // A repeated selection must not replace the translator: disposing it + // would drop snapshot refs, tab context, and the CDP connection. + if (mode === this.runtimeSpec.mode) return; this.runtimeSpec = this.resolveSpec(this.runtimeSpec.model, mode); this.currentMode = mode; this.replaceTranslator(); @@ -357,6 +360,7 @@ export class CuaAgent extends Agent { /** Switch the action plane(s) exposed to the model; takes effect next turn. */ setMode(mode: CuaMode): void { + if (mode === this.runtime.mode) return; this.runtime.setMode(mode); this.runtimeDirty = true; const state = super.state; @@ -470,6 +474,7 @@ export class CuaAgentHarness< * plane conflicts with the requested mode. */ async setMode(mode: CuaMode): Promise { + if (mode === this.runtime.mode) return; const previousMode = this.runtime.mode; const previousNames = new Set(this.getTools().map((tool) => tool.name)); this.runtime.setMode(mode); diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 79647204..eacb710d 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -199,9 +199,14 @@ async function executeNavigationTool( try { let statusText = `${action} executed successfully.`; let url: string | undefined; + // In browser mode navigation stays on the browser plane (CDP) so it + // invalidates refs and matches the viewport grounding frame; the OS + // keyboard-shortcut path would navigate outside the plane the model sees. if (action === "url") { - url = await translator.currentUrl(); + url = mode === "browser" ? await translator.browser().currentUrl() : await translator.currentUrl(); statusText = `Current URL: ${url}`; + } else if (mode === "browser") { + await translator.executeBatch([{ type: "browser_navigate", url: action === "goto" ? (params.url ?? "") : action }]); } else if (action === "goto") { await translator.executeBatch([{ type: "goto", url: params.url ?? "" }]); } else { diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index a8259f03..21069083 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -618,6 +618,13 @@ export class BrowserExecutor { return `Navigated to ${url}.\n${await this.tabContext(targetId)}`; } + /** URL of the active tab. */ + async currentUrl(): Promise { + const targetId = await this.resolveTarget(); + const targets = await this.cdp.pageTargets(); + return targets.find((target) => target.targetId === targetId)?.url ?? ""; + } + private async listTabs(): Promise { const targets = await this.cdp.pageTargets(); if (targets.length === 0) return "No open tabs."; diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index dc088b59..cd10aac1 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -485,6 +485,21 @@ describe("CuaAgentHarness", () => { expect(active).not.toContain("custom"); }); + it("treats a repeated setMode as a no-op", async () => { + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "anthropic:claude-opus-4-5", + }); + const before = harness.getTools(); + + await harness.setMode("computer"); + + // Same tool instances: the translator and its CDP state were not replaced. + expect(harness.getTools()[0]).toBe(before[0]); + }); + it("appends extraTools in harness construction", async () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const tool = createCustomTool(); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index ffbcaea2..1507b247 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -699,22 +699,39 @@ describe("BrowserExecutor iframe stitching", () => { describe("navigation tool grounding frame", () => { const navTool = (mode: "computer" | "browser") => { - const { client } = createClient(); - const { executor } = createFakeBrowserExecutor(); + const { client, batches } = createClient(); + const { executor, executed } = createFakeBrowserExecutor(); const translator = new InternalComputerTranslator({ browser, client, mode, createBrowserExecutor: () => executor }); - return buildCuaComputerTools({ toolExecutors: [], mode }, translator).find((tool) => tool.name === "computer_use_extra")!; + const tool = buildCuaComputerTools({ toolExecutors: [], mode }, translator).find((tool) => tool.name === "computer_use_extra")!; + return { tool, batches, executed }; }; it("captures the viewport in browser mode and the OS display otherwise", async () => { const viewportData = Buffer.from("png").toString("base64"); - const browserResult = await navTool("browser").execute("call_1", { action: "back" }); + const browserResult = await navTool("browser").tool.execute("call_1", { action: "back" }); const viewportImage = browserResult.content.find((block) => block.type === "image"); expect(viewportImage).toMatchObject({ type: "image", data: viewportData }); - const computerResult = await navTool("computer").execute("call_2", { action: "back" }); + const computerResult = await navTool("computer").tool.execute("call_2", { action: "back" }); const osImage = computerResult.content.find((block) => block.type === "image"); expect(osImage?.type).toBe("image"); expect((osImage as { data: string }).data).not.toBe(viewportData); }); + + it("navigates on the browser plane in browser mode and the OS plane otherwise", async () => { + const inBrowser = navTool("browser"); + await inBrowser.tool.execute("call_1", { action: "goto", url: "https://example.com" }); + await inBrowser.tool.execute("call_2", { action: "back" }); + expect(inBrowser.executed).toEqual([ + { type: "browser_navigate", url: "https://example.com" }, + { type: "browser_navigate", url: "back" }, + ]); + expect(inBrowser.batches).toEqual([]); + + const inComputer = navTool("computer"); + await inComputer.tool.execute("call_3", { action: "back" }); + expect(inComputer.executed).toEqual([]); + expect(inComputer.batches).toHaveLength(1); + }); }); From 05b5c2fb1b8e9a7d9154dc3c12100fdbefc0db32 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 8 Jul 2026 23:59:55 +0000 Subject: [PATCH 17/34] Extend CDP navigation to hybrid mode and add setModel rollback - computer_use_extra routes navigation through the browser plane whenever it is exposed (browser and hybrid modes), so refs invalidate and URL reads are tab-aware; only computer mode keeps the OS shortcut path - setModel restores the previous runtime model when super.setTools fails, mirroring setMode's rollback --- packages/agent/src/agent.ts | 9 ++++++++- packages/agent/src/tools.ts | 10 +++++----- packages/agent/test/translator-browser.test.ts | 11 ++++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index c4f02128..1f9b30d0 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -457,9 +457,16 @@ export class CuaAgentHarness< * concrete model selected by `@onkernel/cua-ai`. */ override async setModel(model: CuaRuntimeInput): Promise { + const previousModel = this.runtime.model; this.runtime.setModel(model); const tools = this.runtime.tools(); - await super.setTools(tools, this.requestedActiveToolNames ?? tools.map((tool) => tool.name)); + try { + await super.setTools(tools, this.requestedActiveToolNames ?? tools.map((tool) => tool.name)); + } catch (err) { + // Keep the runtime in step with the exposed tools when the switch fails. + this.runtime.setModel(previousModel); + throw err; + } await super.setModel(this.runtime.model); } diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index eacb710d..1a10ad2f 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -199,13 +199,13 @@ async function executeNavigationTool( try { let statusText = `${action} executed successfully.`; let url: string | undefined; - // In browser mode navigation stays on the browser plane (CDP) so it - // invalidates refs and matches the viewport grounding frame; the OS - // keyboard-shortcut path would navigate outside the plane the model sees. + // When the browser plane is exposed (browser/hybrid), navigation stays + // on it (CDP) so it invalidates element refs and reads the tab-aware URL; + // the OS keyboard-shortcut path would navigate outside that plane. if (action === "url") { - url = mode === "browser" ? await translator.browser().currentUrl() : await translator.currentUrl(); + url = mode === "computer" ? await translator.currentUrl() : await translator.browser().currentUrl(); statusText = `Current URL: ${url}`; - } else if (mode === "browser") { + } else if (mode !== "computer") { await translator.executeBatch([{ type: "browser_navigate", url: action === "goto" ? (params.url ?? "") : action }]); } else if (action === "goto") { await translator.executeBatch([{ type: "goto", url: params.url ?? "" }]); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 1507b247..6176caa2 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -698,7 +698,7 @@ describe("BrowserExecutor iframe stitching", () => { }); describe("navigation tool grounding frame", () => { - const navTool = (mode: "computer" | "browser") => { + const navTool = (mode: "computer" | "browser" | "hybrid") => { const { client, batches } = createClient(); const { executor, executed } = createFakeBrowserExecutor(); const translator = new InternalComputerTranslator({ browser, client, mode, createBrowserExecutor: () => executor }); @@ -719,7 +719,7 @@ describe("navigation tool grounding frame", () => { expect((osImage as { data: string }).data).not.toBe(viewportData); }); - it("navigates on the browser plane in browser mode and the OS plane otherwise", async () => { + it("navigates on the browser plane in browser and hybrid modes and the OS plane in computer mode", async () => { const inBrowser = navTool("browser"); await inBrowser.tool.execute("call_1", { action: "goto", url: "https://example.com" }); await inBrowser.tool.execute("call_2", { action: "back" }); @@ -729,8 +729,13 @@ describe("navigation tool grounding frame", () => { ]); expect(inBrowser.batches).toEqual([]); + const inHybrid = navTool("hybrid"); + await inHybrid.tool.execute("call_3", { action: "forward" }); + expect(inHybrid.executed).toEqual([{ type: "browser_navigate", url: "forward" }]); + expect(inHybrid.batches).toEqual([]); + const inComputer = navTool("computer"); - await inComputer.tool.execute("call_3", { action: "back" }); + await inComputer.tool.execute("call_4", { action: "back" }); expect(inComputer.executed).toEqual([]); expect(inComputer.batches).toHaveLength(1); }); From 9680d7d400f6a09c4cb596f234a1a5a2ee702e91 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:49:54 +0000 Subject: [PATCH 18/34] Mention required mode in native tool provider mismatch error --- packages/ai/src/native-tools.ts | 2 +- packages/ai/test/native-tools.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ai/src/native-tools.ts b/packages/ai/src/native-tools.ts index fa8832af..99c2dbeb 100644 --- a/packages/ai/src/native-tools.ts +++ b/packages/ai/src/native-tools.ts @@ -99,7 +99,7 @@ export function resolveNativeTool(spec: CuaNativeToolSpec, model: Model, mo const info = NATIVE_TOOL_INFO[spec.type]; if (!info) throw new Error(`unknown native tool type "${(spec as { type: string }).type}"`); if (model.provider !== info.provider) { - throw new Error(`native tool "${spec.type}" requires an ${info.provider} model; got provider "${model.provider}"`); + throw new Error(`native tool "${spec.type}" requires an ${info.provider} model paired with mode "${info.mode}"; got provider "${model.provider}"`); } if (mode !== info.mode) { throw new Error(`native tool "${spec.type}" requires mode "${info.mode}"; got "${mode}"`); diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index fa815fc7..399bc498 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -35,7 +35,7 @@ describe("native tool validation", () => { it("rejects native tools on non-anthropic models", () => { expect(() => resolveCuaRuntimeSpec("openai:gpt-5.5", { nativeTool: { type: "computer_20260701" } })).toThrow( - /requires an anthropic model/, + /requires an anthropic model paired with mode "computer"/, ); }); }); From 0a8a202267ccb422ffe06a6ce221bfb40d94c6e7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:50:00 +0000 Subject: [PATCH 19/34] Strip markdown artifacts from url action output --- packages/cli/src/action/prompts.ts | 2 +- packages/cli/src/action/result.ts | 2 +- packages/cli/test/action-result.test.ts | 25 +++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 packages/cli/test/action-result.test.ts diff --git a/packages/cli/src/action/prompts.ts b/packages/cli/src/action/prompts.ts index 6772c0b8..18b552a5 100644 --- a/packages/cli/src/action/prompts.ts +++ b/packages/cli/src/action/prompts.ts @@ -90,5 +90,5 @@ Be concise and factual. Do NOT perform any actions. Only observe and respond.`; } function urlPrompt(): string { - return `Report the current page URL. Use the url action to read it. Do not perform any other actions.`; + return `Report the current page URL. Use the url action to read it. Do not perform any other actions. Respond with only the bare URL, no markdown or other formatting.`; } diff --git a/packages/cli/src/action/result.ts b/packages/cli/src/action/result.ts index 61b81e36..e656b5cd 100644 --- a/packages/cli/src/action/result.ts +++ b/packages/cli/src/action/result.ts @@ -94,7 +94,7 @@ function extractFirstUrl(text: string): string | undefined { /(?:https?:\/\/\S+|about:blank|file:\/\/\S+|chrome:\/\/\S+|chrome-extension:\/\/\S+|edge:\/\/\S+|brave:\/\/\S+)/gi, ); if (!matches || matches.length === 0) return undefined; - return matches[matches.length - 1]!.replace(/[),.;!?]+$/, ""); + return matches[matches.length - 1]!.replace(/[)*_`,.;!?]+$/, ""); } export function formatCompact(r: ActionResult): string { diff --git a/packages/cli/test/action-result.test.ts b/packages/cli/test/action-result.test.ts new file mode 100644 index 00000000..f1b25d11 --- /dev/null +++ b/packages/cli/test/action-result.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { parseResult } from "../src/action/result"; + +describe("parseResult url extraction", () => { + it("passes through a plain url", () => { + const res = parseResult("url", "https://quotes.toscrape.com/page/2/", [], 10); + expect(res.status).toBe("ok"); + expect(res.url).toBe("https://quotes.toscrape.com/page/2/"); + }); + + it("strips markdown bold markers", () => { + const res = parseResult("url", "**https://quotes.toscrape.com/page/2/**", [], 10); + expect(res.url).toBe("https://quotes.toscrape.com/page/2/"); + }); + + it("strips wrapping backticks", () => { + const res = parseResult("url", "`https://example.com/path`", [], 10); + expect(res.url).toBe("https://example.com/path"); + }); + + it("strips trailing punctuation", () => { + const res = parseResult("url", "The current URL is https://example.com/page.", [], 10); + expect(res.url).toBe("https://example.com/page"); + }); +}); From beed0fc313667b4e5dd3078be545fe2faed65928 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:52:16 +0000 Subject: [PATCH 20/34] Validate mode/native-tool combos before provisioning a browser --- packages/cli/src/cli-harness.ts | 2 + .../cli/test/cli-harness-validation.test.ts | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 packages/cli/test/cli-harness-validation.test.ts diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 9c028047..9655955a 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -12,6 +12,7 @@ import { type CuaNativeToolSpec, parseCuaModelRef, requireCuaEnvApiKey, + resolveCuaRuntimeSpec, } from "@onkernel/cua-ai"; import { parseArgs } from "node:util"; import { stderr, stdout } from "node:process"; @@ -380,6 +381,7 @@ async function setupHarnessRuntime( // never leaves an orphaned browser behind. const mode = parseMode(flags.mode); const nativeTool = parseNativeTool(flags.nativeTool); + resolveCuaRuntimeSpec(auth.modelRef, { mode, nativeTool }); const provisioned = await provisionForFlags(flags, auth); try { diff --git a/packages/cli/test/cli-harness-validation.test.ts b/packages/cli/test/cli-harness-validation.test.ts new file mode 100644 index 00000000..e76d639e --- /dev/null +++ b/packages/cli/test/cli-harness-validation.test.ts @@ -0,0 +1,67 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { type HarnessCliFlags, runActionCommand } from "../src/cli-harness"; +import { provisionBrowser } from "../src/harness-browser"; + +vi.mock("../src/harness-browser", () => ({ provisionBrowser: vi.fn() })); + +function flagsWith(overrides: Partial): HarnessCliFlags { + return { + verbose: false, + profileSaveChanges: false, + continueLatest: false, + resumePicker: false, + noSession: true, + noSkills: true, + debugTui: false, + jsonlIncludeDeltas: false, + jsonlIncludeImages: false, + playwright: false, + skillPaths: [], + ...overrides, + }; +} + +describe("mode/native-tool validation before provisioning", () => { + beforeEach(() => { + vi.stubEnv("KERNEL_API_KEY", "test-kernel-key"); + vi.stubEnv("ANTHROPIC_API_KEY", "test-anthropic-key"); + vi.stubEnv("OPENAI_API_KEY", "test-openai-key"); + vi.stubEnv("GOOGLE_API_KEY", "test-google-key"); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.mocked(provisionBrowser).mockClear(); + }); + + it("rejects a native tool whose mode conflicts without provisioning a browser", async () => { + await expect( + runActionCommand("url", [], flagsWith({ + model: "anthropic:claude-opus-4-8", + mode: "hybrid", + nativeTool: "computer_20260701", + })), + ).rejects.toThrow('native tool "computer_20260701" requires mode "computer"; got "hybrid"'); + expect(provisionBrowser).not.toHaveBeenCalled(); + }); + + it("rejects a native tool on a non-anthropic model without provisioning a browser", async () => { + await expect( + runActionCommand("url", [], flagsWith({ + model: "openai:gpt-5.5", + nativeTool: "computer_20260701", + })), + ).rejects.toThrow('native tool "computer_20260701" requires an anthropic model; got provider "openai"'); + expect(provisionBrowser).not.toHaveBeenCalled(); + }); + + it("rejects an unsupported provider/mode pair without provisioning a browser", async () => { + await expect( + runActionCommand("url", [], flagsWith({ + model: "google:gemini-3-flash-preview", + mode: "browser", + })), + ).rejects.toThrow('provider "google" does not support mode "browser" (computer only)'); + expect(provisionBrowser).not.toHaveBeenCalled(); + }); +}); From a0ccedc4218fc8c543a5147468a960814b37efbe Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:55:13 +0000 Subject: [PATCH 21/34] Persist named-session model and guard cross-provider response threading --- packages/ai/src/providers/common.ts | 12 +-- packages/ai/src/providers/openai/provider.ts | 2 +- packages/ai/src/providers/tzafon/provider.ts | 2 +- packages/ai/test/openai-threading.test.ts | 19 +++++ packages/ai/test/tzafon-threading.test.ts | 19 +++++ packages/cli/src/cli-harness.ts | 22 +++++ packages/cli/src/harness-named-sessions.ts | 21 +++++ .../cli/test/harness-named-sessions.test.ts | 83 +++++++++++++++++++ 8 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 packages/cli/test/harness-named-sessions.test.ts diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 6430fae2..3b667703 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -303,17 +303,19 @@ export interface ResponseThreadingDelta { * Anchors on the most recent assistant turn: returns its `responseId` and the * messages after it (the delta). An errored or aborted turn may carry a * `responseId` captured from an incomplete response the server never stored, so - * its id is ignored. When the anchor has no usable `responseId`, or there is no - * assistant turn yet, returns every message and no id so the caller replays the - * full history, rather than chaining to a phantom id and pruning past it. + * its id is ignored. A turn produced by a different `api` (e.g. after a + * mid-session `-m` provider switch) is ignored too — its id would be foreign to + * the current provider. When the anchor has no usable `responseId`, or there is + * no assistant turn yet, returns every message and no id so the caller replays + * the full history, rather than chaining to a phantom id and pruning past it. */ -export function responseThreadingDelta(messages: readonly Message[]): ResponseThreadingDelta { +export function responseThreadingDelta(messages: readonly Message[], api: Api): ResponseThreadingDelta { for (let index = messages.length - 1; index >= 0; index -= 1) { const message = messages[index]!; if (message.role !== "assistant") continue; const assistant = message as AssistantMessage; const failed = assistant.stopReason === "error" || assistant.stopReason === "aborted"; - const responseId = failed ? undefined : assistant.responseId; + const responseId = failed || assistant.api !== api ? undefined : assistant.responseId; return responseId ? { previousResponseId: responseId, deltaMessages: messages.slice(index + 1) } : { deltaMessages: [...messages] }; } return { deltaMessages: [...messages] }; diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index cb8bc75d..bca616a3 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -29,7 +29,7 @@ export function threadRequest( context: Context, options: (ResponseThreadingOptions & { onPayload?: OnPayload }) | undefined, ): { context: Context; onPayload: OnPayload } { - const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages) : undefined; + const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages, OPENAI_CUA_RESPONSES_API) : undefined; const previousResponseId = delta?.previousResponseId; const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; const onPayload: OnPayload = async (payload, model) => { diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index db64cc90..95f31de9 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -76,7 +76,7 @@ export function buildTzafonRequestInput(model: Model, context: Context, opt max_output_tokens: options?.maxTokens ?? model.maxTokens, }; if (!responseThreadingEnabled(options)) return body; - const { previousResponseId, deltaMessages } = responseThreadingDelta(context.messages); + const { previousResponseId, deltaMessages } = responseThreadingDelta(context.messages, TZAFON_RESPONSES_API); if (!previousResponseId) return body; return { ...body, input: convertMessages(deltaMessages), previous_response_id: previousResponseId, store: true }; } diff --git a/packages/ai/test/openai-threading.test.ts b/packages/ai/test/openai-threading.test.ts index 7ba25602..d4821d06 100644 --- a/packages/ai/test/openai-threading.test.ts +++ b/packages/ai/test/openai-threading.test.ts @@ -96,6 +96,25 @@ describe("openai threadRequest", () => { expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); }); + it("never anchors previous_response_id on an assistant turn from a different api", async () => { + const ctx = multiTurnContext(); + // A mid-session -m provider switch leaves the prior provider's turn (and its foreign id) as the anchor. + ctx.messages.push({ + role: "assistant", + content: [{ type: "text", text: "done" }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-opus-4-8", + responseId: "msg_anthropic", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "stop", + timestamp: 0, + }); + const { context, onPayload } = threadRequest(ctx, undefined); + expect(context).toBe(ctx); + expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); + }); + it("composes a caller onPayload on top of the threaded payload", async () => { const { onPayload } = threadRequest(multiTurnContext(), { onPayload: (payload) => ({ wrapped: payload }), diff --git a/packages/ai/test/tzafon-threading.test.ts b/packages/ai/test/tzafon-threading.test.ts index e7ba1f15..959b55c5 100644 --- a/packages/ai/test/tzafon-threading.test.ts +++ b/packages/ai/test/tzafon-threading.test.ts @@ -115,6 +115,25 @@ describe("buildTzafonRequestInput response threading", () => { expect(screenshotImageUrls(body.input)).toHaveLength(TURNS); }); + it("replays full history when the latest assistant turn is from a different api", () => { + const context = multiTurnContext(); + context.messages.push({ + role: "assistant", + content: [{ type: "text", text: "done" }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-opus-4-8", + responseId: "msg_anthropic", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "stop", + timestamp: 0, + }); + + const body = tzafon.buildTzafonRequestInput(model, context); + expect(body.previous_response_id).toBeUndefined(); + expect(screenshotImageUrls(body.input)).toHaveLength(TURNS); + }); + // Off-path screenshot count scales with turn count; on-path stays constant at one. it("grows the payload per turn when off but stays flat when on", () => { const counts = (turns: number, disable: boolean) => { diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 9c028047..3f67349b 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -29,6 +29,8 @@ import { formatRelativeAge, listNamedSessions, type NamedSessionMetadata, + readNamedSession, + recordSessionModel, recordTranscriptPath, shortKernelId, startNamedSession, @@ -362,10 +364,24 @@ export interface SetupHarnessRuntimeOptions { skipDiskSession?: boolean; } +/** Default -m/--mode/--native-tool from a named session's stored values when not passed explicitly. */ +export function applyNamedSessionDefaults(flags: HarnessCliFlags, meta: NamedSessionMetadata): HarnessCliFlags { + return { + ...flags, + model: flags.model ?? meta.model, + mode: flags.mode ?? meta.mode, + nativeTool: flags.nativeTool ?? meta.native_tool, + }; +} + async function setupHarnessRuntime( flags: HarnessCliFlags, opts: SetupHarnessRuntimeOptions = {}, ): Promise { + if (flags.namedSession) { + const named = await readNamedSession(flags.namedSession); + if (named) flags = applyNamedSessionDefaults(flags, named); + } const auth = resolveAuth(flags); const cwd = process.cwd(); const env = new NodeExecutionEnv({ cwd }); @@ -429,6 +445,11 @@ async function finishHarnessRuntime( }); if (provisioned.named) { await recordTranscriptPath(provisioned.named.name, resolved.transcriptPath); + await recordSessionModel(provisioned.named.name, { + model: auth.modelRef, + mode: flags.mode, + native_tool: flags.nativeTool, + }); } if (flags.verbose) { stderr.write(`[cua] session=${resolved.transcriptPath}\n`); @@ -653,6 +674,7 @@ export async function runSessionSubcommand(args: string[], flags: HarnessCliFlag browserTimeoutSeconds: flags.browserTimeout, profileSelector: flags.browserProfile, saveProfileChanges: flags.profileSaveChanges, + model: flags.model ? resolveCuaModelRef(flags.model) : undefined, }); stdout.write(`name=${meta.name}\n`); stdout.write(`kernel_session_id=${browser.session_id}\n`); diff --git a/packages/cli/src/harness-named-sessions.ts b/packages/cli/src/harness-named-sessions.ts index 9b703d3d..663b91b9 100644 --- a/packages/cli/src/harness-named-sessions.ts +++ b/packages/cli/src/harness-named-sessions.ts @@ -19,6 +19,10 @@ export interface NamedSessionMetadata { live_url?: string; profile_id?: string; transcript_path?: string; + /** Model ref last used with this session; chained invocations without -m default to it. */ + model?: string; + mode?: string; + native_tool?: string; created_at: number; } @@ -99,6 +103,8 @@ export interface StartNamedSessionOptions { /** Profile id or name (created if missing). Same semantics as `--profile`. */ profileSelector?: string; saveProfileChanges?: boolean; + /** Canonical model ref to seed the session with (same semantics as `-m`). */ + model?: string; } export interface StartNamedSessionResult { @@ -138,6 +144,7 @@ export async function startNamedSession(opts: StartNamedSessionOptions): Promise kernel_session_id: browser.session_id, live_url: browser.browser_live_view_url, profile_id: profileId, + model: opts.model, created_at: Date.now(), }; const metadataPath = await writeNamedSession(meta); @@ -232,6 +239,20 @@ export async function recordTranscriptPath(name: string, transcriptPath: string) await writeNamedSession(meta); } +/** Persist the model/mode/native-tool used with a named session so chained invocations reuse them. */ +export async function recordSessionModel( + name: string, + runtime: { model: string; mode?: string; native_tool?: string }, +): Promise { + const meta = await readNamedSession(name); + if (!meta) return; + if (meta.model === runtime.model && meta.mode === runtime.mode && meta.native_tool === runtime.native_tool) return; + meta.model = runtime.model; + meta.mode = runtime.mode; + meta.native_tool = runtime.native_tool; + await writeNamedSession(meta); +} + export function shortKernelId(id: string): string { return id.length > 10 ? `${id.slice(0, 8)}…` : id; } diff --git a/packages/cli/test/harness-named-sessions.test.ts b/packages/cli/test/harness-named-sessions.test.ts new file mode 100644 index 00000000..f8db3597 --- /dev/null +++ b/packages/cli/test/harness-named-sessions.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { applyNamedSessionDefaults, type HarnessCliFlags } from "../src/cli-harness"; +import { + type NamedSessionMetadata, + readNamedSession, + recordSessionModel, + writeNamedSession, +} from "../src/harness-named-sessions"; + +const originalXdg = process.env.XDG_DATA_HOME; + +function baseMeta(overrides: Partial = {}): NamedSessionMetadata { + return { name: "foo", kernel_session_id: "ks_123", created_at: Date.now(), ...overrides }; +} + +function baseFlags(overrides: Partial = {}): HarnessCliFlags { + return { + verbose: false, + profileSaveChanges: false, + continueLatest: false, + resumePicker: false, + noSession: false, + noSkills: false, + debugTui: false, + jsonlIncludeDeltas: false, + jsonlIncludeImages: false, + playwright: false, + namedSession: "foo", + skillPaths: [], + ...overrides, + }; +} + +describe("named session model persistence", () => { + beforeEach(() => { + process.env.XDG_DATA_HOME = mkdtempSync(join(tmpdir(), "cua-cli-named-")); + }); + + afterEach(() => { + if (originalXdg === undefined) delete process.env.XDG_DATA_HOME; + else process.env.XDG_DATA_HOME = originalXdg; + }); + + it("records the model/mode/native-tool onto the metadata file", async () => { + await writeNamedSession(baseMeta()); + await recordSessionModel("foo", { model: "anthropic:claude-opus-4-8", mode: "hybrid", native_tool: "computer_20260701" }); + const meta = await readNamedSession("foo"); + expect(meta?.model).toBe("anthropic:claude-opus-4-8"); + expect(meta?.mode).toBe("hybrid"); + expect(meta?.native_tool).toBe("computer_20260701"); + }); + + it("overwrites a previously recorded model on an explicit switch", async () => { + await writeNamedSession(baseMeta({ model: "openai:gpt-5.5" })); + await recordSessionModel("foo", { model: "anthropic:claude-opus-4-8", mode: "hybrid" }); + const meta = await readNamedSession("foo"); + expect(meta?.model).toBe("anthropic:claude-opus-4-8"); + expect(meta?.mode).toBe("hybrid"); + }); + + it("is a no-op for an unknown session", async () => { + await recordSessionModel("missing", { model: "openai:gpt-5.5" }); + expect(await readNamedSession("missing")).toBeUndefined(); + }); + + it("defaults flags from the stored session model when -m is omitted", () => { + const meta = baseMeta({ model: "anthropic:claude-opus-4-8", mode: "hybrid", native_tool: "computer_20260701" }); + const flags = applyNamedSessionDefaults(baseFlags(), meta); + expect(flags.model).toBe("anthropic:claude-opus-4-8"); + expect(flags.mode).toBe("hybrid"); + expect(flags.nativeTool).toBe("computer_20260701"); + }); + + it("keeps explicit flags over stored session values", () => { + const meta = baseMeta({ model: "anthropic:claude-opus-4-8", mode: "hybrid" }); + const flags = applyNamedSessionDefaults(baseFlags({ model: "openai:gpt-5.5", mode: "browser" }), meta); + expect(flags.model).toBe("openai:gpt-5.5"); + expect(flags.mode).toBe("browser"); + }); +}); From cce860191a9d08aeee4e505de2949ef2d2e5d570 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:57:16 +0000 Subject: [PATCH 22/34] Update validation test for enriched native tool error message --- packages/cli/test/cli-harness-validation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/cli-harness-validation.test.ts b/packages/cli/test/cli-harness-validation.test.ts index e76d639e..c8f20ce3 100644 --- a/packages/cli/test/cli-harness-validation.test.ts +++ b/packages/cli/test/cli-harness-validation.test.ts @@ -51,7 +51,7 @@ describe("mode/native-tool validation before provisioning", () => { model: "openai:gpt-5.5", nativeTool: "computer_20260701", })), - ).rejects.toThrow('native tool "computer_20260701" requires an anthropic model; got provider "openai"'); + ).rejects.toThrow('native tool "computer_20260701" requires an anthropic model paired with mode "computer"; got provider "openai"'); expect(provisionBrowser).not.toHaveBeenCalled(); }); From 8cffec59044262b2ec45b74e3ba8e7289de8abcd Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 01:05:16 +0000 Subject: [PATCH 23/34] Route deterministic subcommands directly through the executor, no model open, url, press, and screenshot no longer build the LLM harness or require a model API key; new snapshot, find, text, fill, tabs, and click subcommands run the same way. find/fill share a new structured BrowserExecutor.findCandidates; fill resolves a lexical query to a unique fillable element in-process (refs never cross invocations). click/type-by-description, observe, and do stay model-mediated. Dead open/press/url prompts removed; help text and the cua-cli skill doc updated to the shipped surface. --- packages/agent/src/index.ts | 2 + packages/agent/src/translator/browser.ts | 39 ++- packages/cli/src/action/harness-runner.ts | 38 +-- packages/cli/src/action/prompts.ts | 39 +-- packages/cli/src/action/result.ts | 53 ++-- packages/cli/src/cli-executor.ts | 236 ++++++++++++++++ packages/cli/src/cli-harness.ts | 35 +-- packages/cli/src/cli.ts | 51 +++- packages/cli/test/action-runner.test.ts | 18 -- packages/cli/test/cli-executor.test.ts | 326 ++++++++++++++++++++++ skills/cua-cli/SKILL.md | 68 +++-- 11 files changed, 728 insertions(+), 177 deletions(-) create mode 100644 packages/cli/src/cli-executor.ts create mode 100644 packages/cli/test/cli-executor.test.ts diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index ba05a2b1..b8dc5be1 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -2,8 +2,10 @@ export * from "@earendil-works/pi-agent-core"; export { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node"; export type { KernelBrowser } from "./translator/translator"; +export { InternalComputerTranslator } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; export { BrowserExecutor } from "./translator/browser"; +export type { BrowserFindCandidate } from "./translator/browser"; export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 21069083..637a0755 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -79,6 +79,13 @@ interface FrameStitch { ctx: RenderContext; } +export interface BrowserFindCandidate { + ref: string; + role: string; + name: string; + score: number; +} + export interface BrowserExecutorOptions { /** Mark elements whose computed cursor is "pointer" as clickable hints in snapshots. Default false. */ cursorHints?: boolean; @@ -455,7 +462,19 @@ export class BrowserExecutor { } private async find(action: CuaActionBrowserFind): Promise { - const targetId = await this.resolveTarget(action.tab_id); + const candidates = await this.findCandidates(action.query, action.tab_id); + if (candidates.length === 0) return `No elements matched ${JSON.stringify(action.query)}. Try snapshot for the full tree.`; + return candidates + .map(({ ref, role, name }) => `${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""} [${ref}]`) + .join("\n"); + } + + /** + * Score elements against a natural-language query and mint refs for the + * matches, best first. Structured counterpart of the `browser_find` action. + */ + async findCandidates(query: string, tabId?: string): Promise { + const targetId = await this.resolveTarget(tabId); const session = await this.attach(targetId); const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); const pools: Array<{ nodes: AXNode[]; ctx: RenderContext }> = [ @@ -475,7 +494,7 @@ export class BrowserExecutor { for (const stitch of (await this.stitchFrames(nodes, targetId, session, false)).values()) { pools.push({ nodes: [...stitch.byId.values()], ctx: stitch.ctx }); } - const queryTokens = tokenize(action.query); + const queryTokens = tokenize(query); const scored = pools .flatMap(({ nodes: poolNodes, ctx }) => poolNodes @@ -487,16 +506,14 @@ export class BrowserExecutor { .filter((entry) => entry.score > 0) .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); - if (scored.length === 0) return `No elements matched ${JSON.stringify(action.query)}. Try snapshot for the full tree.`; - const text = scored - .map(({ node, ctx }) => { - const role = node.role?.value ?? "node"; - const name = node.name?.value ? ` ${JSON.stringify(node.name.value)}` : ""; - return `${role}${name} [${this.mintRef(node, ctx)}]`; - }) - .join("\n"); + const candidates = scored.map(({ node, ctx, score }) => ({ + ref: this.mintRef(node, ctx), + role: node.role?.value ?? "", + name: node.name?.value ?? "", + score, + })); this.pruneRefs(targetId); - return text; + return candidates; } private async click(action: CuaActionBrowserClick): Promise { diff --git a/packages/cli/src/action/harness-runner.ts b/packages/cli/src/action/harness-runner.ts index da1f68da..bd883a8a 100644 --- a/packages/cli/src/action/harness-runner.ts +++ b/packages/cli/src/action/harness-runner.ts @@ -1,6 +1,5 @@ import type { AgentHarnessEvent, CuaAgentHarness, Session } from "@onkernel/cua-agent"; import type { AssistantMessage, ImageContent } from "@onkernel/cua-ai"; -import { writeFile } from "node:fs/promises"; import { stderr, stdout } from "node:process"; import { captureScreenshot, type CuaBrowserHandle } from "../harness-browser"; import { type ActionRequest, buildPrompt, DEFAULT_MAX_TURNS } from "./prompts"; @@ -15,53 +14,22 @@ export interface HarnessRunOptions { maxTurns?: number; } -export interface ScreenshotOutput { - out: string; // path or "-" for stdout -} - export interface RunActionResult { result: ActionResult; exitCode: number; } /** - * Run a single action subcommand against an existing harness + browser and - * return the parsed result plus exit code. The `screenshot` action is - * model-free — it captures directly through the SDK. All other actions - * drive the harness for at most `maxTurns` turns. + * Run a single model-mediated action subcommand against an existing + * harness + browser and return the parsed result plus exit code. Drives + * the harness for at most `maxTurns` turns. */ export async function runAction( req: ActionRequest, opts: HarnessRunOptions, - screenshot?: ScreenshotOutput, ): Promise { const startedAt = Date.now(); - if (req.action === "screenshot") { - const out = screenshot ?? { out: "screenshot.png" }; - const png = await captureScreenshot(opts.browserHandle.client, opts.browserHandle.browser.session_id); - if (!png) { - const elapsed = Date.now() - startedAt; - const result: ActionResult = { - action: "screenshot", - status: "error", - text: "failed to capture screenshot", - elapsedMs: elapsed, - timestamp: Date.now(), - }; - return { result, exitCode: exitCodeFor(result) }; - } - if (out.out === "-") { - stdout.write(png); - } else { - await writeFile(out.out, png); - } - const elapsed = Date.now() - startedAt; - const result = parseResult("screenshot", "", [], elapsed); - result.text = out.out === "-" ? "(stdout)" : out.out; - return { result, exitCode: 0 }; - } - const prompt = buildPrompt(req); const maxTurns = req.maxTurns ?? opts.maxTurns ?? DEFAULT_MAX_TURNS; diff --git a/packages/cli/src/action/prompts.ts b/packages/cli/src/action/prompts.ts index 6772c0b8..4776f306 100644 --- a/packages/cli/src/action/prompts.ts +++ b/packages/cli/src/action/prompts.ts @@ -2,21 +2,12 @@ * Constrained one-shot prompts for the agent-friendly CLI subcommands. */ -export type ActionType = - | "click" - | "type" - | "open" - | "press" - | "screenshot" - | "url" - | "observe" - | "do"; +export type ModelActionType = "click" | "type" | "observe" | "do"; export interface ActionRequest { - action: ActionType; + action: ModelActionType; target?: string; text?: string; - keys?: string[]; maxTurns?: number; } @@ -31,26 +22,14 @@ export function buildPrompt(req: ActionRequest): string { if (!req.target) throw new Error("type action requires a target description"); if (!req.text) throw new Error("type action requires text to type"); return typePrompt(req.target, req.text); - case "open": { - const url = req.text || req.target; - if (!url) throw new Error("open action requires a URL"); - return openPrompt(url); - } - case "press": - if (!req.keys || req.keys.length === 0) throw new Error("press action requires at least one key"); - return pressPrompt(req.keys); case "observe": if (req.text) return observeWithQuestionPrompt(req.text); return observePrompt(); - case "url": - return urlPrompt(); case "do": { const instruction = req.text || req.target; if (!instruction) throw new Error("do action requires an instruction"); return instruction; } - case "screenshot": - throw new Error("screenshot action does not use a prompt"); } } @@ -69,16 +48,6 @@ If no matching element is visible on screen, respond with the text: NOT_FOUND: f Do not perform any other actions.`; } -function openPrompt(url: string): string { - return `Navigate the browser to this URL: ${url} -Use the goto action. Perform only this navigation, then stop.`; -} - -function pressPrompt(keys: string[]): string { - return `Press the following key(s): ${keys.join("+")} -Perform exactly this key press, then stop. Do not perform any other actions.`; -} - function observePrompt(): string { return `Look at the current screen and describe what you see. Be concise and factual. Do NOT perform any actions. Only observe and describe.`; @@ -88,7 +57,3 @@ function observeWithQuestionPrompt(question: string): string { return `Look at the current screen and answer this question: ${JSON.stringify(question)} Be concise and factual. Do NOT perform any actions. Only observe and respond.`; } - -function urlPrompt(): string { - return `Report the current page URL. Use the url action to read it. Do not perform any other actions.`; -} diff --git a/packages/cli/src/action/result.ts b/packages/cli/src/action/result.ts index 61b81e36..0f759ffd 100644 --- a/packages/cli/src/action/result.ts +++ b/packages/cli/src/action/result.ts @@ -1,4 +1,19 @@ -import type { ActionType } from "./prompts"; +import type { ModelActionType } from "./prompts"; + +/** Subcommands that execute directly against the browser/OS planes, no model involved. */ +export type DeterministicActionType = + | "open" + | "url" + | "snapshot" + | "text" + | "find" + | "fill" + | "press" + | "click" + | "tabs" + | "screenshot"; + +export type ActionType = ModelActionType | DeterministicActionType; export type Status = "ok" | "not_found" | "error" | "timeout"; @@ -23,7 +38,7 @@ export interface ActionResult { * and any action events captured during the run. */ export function parseResult( - action: ActionType, + action: ModelActionType, textOutput: string, actionEvents: ActionEventInfo[], elapsedMs: number, @@ -68,35 +83,11 @@ export function parseResult( } } - switch (action) { - case "observe": - result.text = trimmed; - break; - case "url": { - const url = extractFirstUrl(trimmed); - if (url) { - result.url = url; - } else if (trimmed) { - result.status = "error"; - result.text = trimmed; - } - break; - } - default: - break; - } + if (action === "observe") result.text = trimmed; return result; } -function extractFirstUrl(text: string): string | undefined { - const matches = text.match( - /(?:https?:\/\/\S+|about:blank|file:\/\/\S+|chrome:\/\/\S+|chrome-extension:\/\/\S+|edge:\/\/\S+|brave:\/\/\S+)/gi, - ); - if (!matches || matches.length === 0) return undefined; - return matches[matches.length - 1]!.replace(/[),.;!?]+$/, ""); -} - export function formatCompact(r: ActionResult): string { switch (r.status) { case "not_found": @@ -117,12 +108,18 @@ export function formatCompact(r: ActionResult): string { return "ok"; case "press": return "ok pressed"; + case "fill": + return r.text ? `ok filled ${r.text}` : "ok filled"; case "observe": + case "snapshot": + case "text": + case "find": + case "tabs": return r.text ?? ""; case "url": return r.url ?? r.text ?? ""; case "screenshot": - return "ok"; + return r.text ?? "ok"; case "do": return r.text ?? "ok"; default: diff --git a/packages/cli/src/cli-executor.ts b/packages/cli/src/cli-executor.ts new file mode 100644 index 00000000..7bed15ed --- /dev/null +++ b/packages/cli/src/cli-executor.ts @@ -0,0 +1,236 @@ +import { InternalComputerTranslator, type BatchReadResult, type BrowserFindCandidate } from "@onkernel/cua-agent"; +import { writeFile } from "node:fs/promises"; +import { stderr, stdout } from "node:process"; +import { emitCompact, type RunActionResult } from "./action/harness-runner"; +import { exitCodeFor, type ActionResult, type DeterministicActionType } from "./action/result"; +import { provisionForFlags, requireKernelApiKey, type HarnessCliFlags } from "./cli-harness"; +import { captureScreenshot, type CuaBrowserHandle } from "./harness-browser"; + +/** + * Model-free subcommand plane. These commands validate argv, attach to (or + * provision) a Kernel browser, and call the executor directly over CDP or + * the computer batch API — no LLM harness, no model API key. + */ + +export type DeterministicRequest = + | { action: "open"; url: string } + | { action: "url" } + | { action: "snapshot"; filter?: "interactive" } + | { action: "text" } + | { action: "find"; query: string } + | { action: "fill"; query: string; value: string } + | { action: "press"; keys: string[] } + | { action: "click"; x: number; y: number } + | { action: "tabs" } + | { action: "screenshot"; out: string }; + +export const DETERMINISTIC_SUBCOMMANDS: ReadonlySet = new Set([ + "open", + "url", + "snapshot", + "text", + "find", + "fill", + "press", + "tabs", + "screenshot", +]); + +/** `cua click ` is deterministic; any other click argv is a model-mediated description. */ +export function isCoordinatePair(rest: string[]): boolean { + return rest.length === 2 && rest.every((token) => /^\d+$/.test(token)); +} + +/** Roles `cua fill` will target. Everything else is left to `click`/`type`. */ +const FILLABLE_ROLES: ReadonlySet = new Set([ + "textbox", + "searchbox", + "combobox", + "checkbox", + "radio", + "listbox", + "spinbutton", +]); + +/** Parse and validate a deterministic subcommand's argv. Throws before any Kernel API call. */ +export function parseDeterministicArgs( + action: DeterministicActionType, + rest: string[], + flags: HarnessCliFlags, +): DeterministicRequest { + switch (action) { + case "open": { + const url = (rest[0] ?? "").trim(); + if (!url || rest.length > 1) throw new Error("usage: cua open "); + return { action, url }; + } + case "url": + return { action }; + case "snapshot": { + if (rest.length > 0) throw new Error("usage: cua snapshot [--filter interactive]"); + const filter = flags.filter?.trim().toLowerCase(); + if (filter !== undefined && filter !== "interactive") { + throw new Error(`invalid --filter value "${flags.filter}"; expected: interactive`); + } + return { action, ...(filter === "interactive" ? { filter } : {}) }; + } + case "text": + return { action }; + case "find": { + const query = rest.join(" ").trim(); + if (!query) throw new Error('usage: cua find ""'); + return { action, query }; + } + case "fill": { + const query = (rest[0] ?? "").trim(); + const value = rest[1]; + if (!query || value === undefined || rest.length > 2) { + throw new Error('usage: cua fill "" ""'); + } + return { action, query, value }; + } + case "press": { + const keys = rest.map((key) => key.trim()).filter((key) => key.length > 0); + if (keys.length === 0) throw new Error("usage: cua press [key...]"); + return { action, keys }; + } + case "click": { + if (!isCoordinatePair(rest)) throw new Error("usage: cua click "); + return { action, x: Number(rest[0]), y: Number(rest[1]) }; + } + case "tabs": + return { action }; + case "screenshot": { + if (rest.length > 0) throw new Error("usage: cua screenshot [--out file|-]"); + return { action, out: flags.out ?? "screenshot.png" }; + } + } +} + +/** Run a deterministic subcommand end to end: parse, provision/attach, execute, print, tear down. */ +export async function runDeterministicCommand( + action: DeterministicActionType, + rest: string[], + flags: HarnessCliFlags, +): Promise { + const req = parseDeterministicArgs(action, rest, flags); + const { apiKey, baseUrl } = requireKernelApiKey(); + const provisioned = await provisionForFlags(flags, { kernelApiKey: apiKey, kernelBaseUrl: baseUrl }); + return runDeterministicOnHandle(req, provisioned.handle); +} + +/** Execute a parsed request against a browser handle. Split from provisioning for tests. */ +export async function runDeterministicOnHandle( + req: DeterministicRequest, + handle: CuaBrowserHandle, + createTranslator: (handle: CuaBrowserHandle) => InternalComputerTranslator = defaultTranslator, +): Promise { + const translator = createTranslator(handle); + try { + const res = await executeDeterministic(req, translator, handle); + return emitCompact(res); + } finally { + translator.dispose(); + try { + await handle.close(); + } catch (err) { + stderr.write(`[cua] cleanup warning: ${(err as Error).message}\n`); + } + } +} + +function defaultTranslator(handle: CuaBrowserHandle): InternalComputerTranslator { + return new InternalComputerTranslator({ browser: handle.browser, client: handle.client }); +} + +async function executeDeterministic( + req: DeterministicRequest, + translator: InternalComputerTranslator, + handle: CuaBrowserHandle, +): Promise { + const startedAt = Date.now(); + const finish = (partial: Omit): RunActionResult => { + const result: ActionResult = { ...partial, elapsedMs: Date.now() - startedAt, timestamp: Date.now() }; + return { result, exitCode: exitCodeFor(result) }; + }; + try { + switch (req.action) { + case "open": + await translator.browser().execute({ type: "browser_navigate", url: req.url }); + return finish({ action: req.action, status: "ok" }); + case "url": + return finish({ action: req.action, status: "ok", url: await translator.browser().currentUrl() }); + case "snapshot": { + const reads = await translator.browser().execute({ type: "browser_snapshot", ...(req.filter ? { filter: req.filter } : {}) }); + return finish({ action: req.action, status: "ok", text: readText(reads) }); + } + case "text": { + const reads = await translator.browser().execute({ type: "browser_text" }); + return finish({ action: req.action, status: "ok", text: readText(reads) }); + } + case "find": { + const candidates = await translator.browser().findCandidates(req.query); + if (candidates.length === 0) { + return finish({ action: req.action, status: "not_found", text: `no elements matched ${JSON.stringify(req.query)}` }); + } + return finish({ action: req.action, status: "ok", text: candidates.map(formatCandidate).join("\n") }); + } + case "fill": { + const executor = translator.browser(); + const candidates = (await executor.findCandidates(req.query)).filter((c) => FILLABLE_ROLES.has(c.role)); + if (candidates.length === 0) { + return finish({ action: req.action, status: "not_found", text: `no fillable element matched ${JSON.stringify(req.query)}` }); + } + const tied = candidates.filter((c) => c.score === candidates[0]!.score); + if (tied.length > 1) { + const listing = tied.map((c) => `${c.role} ${JSON.stringify(c.name)}`).join(", "); + return finish({ + action: req.action, + status: "not_found", + text: `ambiguous query ${JSON.stringify(req.query)} (${tied.length} matches): ${listing}`, + }); + } + const match = candidates[0]!; + await executor.execute({ type: "browser_fill", ref: match.ref, value: req.value }); + return finish({ action: req.action, status: "ok", text: `${match.role} ${JSON.stringify(match.name)}` }); + } + case "press": + await translator.executeBatch([{ type: "keypress", keys: req.keys }]); + return finish({ action: req.action, status: "ok" }); + case "click": + await translator.executeBatch([{ type: "click", x: req.x, y: req.y }]); + return finish({ action: req.action, status: "ok", coordinates: [req.x, req.y] }); + case "tabs": { + const reads = await translator.browser().execute({ type: "browser_list_tabs" }); + return finish({ action: req.action, status: "ok", text: readText(reads) }); + } + case "screenshot": { + const png = await captureScreenshot(handle.client, handle.browser.session_id); + if (!png) { + return finish({ action: req.action, status: "error", text: "failed to capture screenshot" }); + } + if (req.out === "-") { + stdout.write(png); + } else { + await writeFile(req.out, png); + } + return finish({ action: req.action, status: "ok", text: req.out === "-" ? "(stdout)" : req.out }); + } + } + } catch (err) { + return finish({ action: req.action, status: "error", text: (err as Error).message }); + } +} + +function formatCandidate(candidate: BrowserFindCandidate): string { + const name = candidate.name ? ` ${JSON.stringify(candidate.name)}` : ""; + return `${candidate.role || "node"}${name} [${candidate.ref}]`; +} + +function readText(reads: BatchReadResult[]): string { + const parts: string[] = []; + for (const read of reads) { + if (read.type === "browser_text") parts.push(read.text); + } + return parts.join("\n"); +} diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 9c028047..75dae6ce 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -18,7 +18,7 @@ import { stderr, stdout } from "node:process"; import type { CuaBrowserHandle } from "./harness-browser"; import { type ActionRequest, - type ActionType, + type ModelActionType, } from "./action/prompts"; import { runAction, emitCompact } from "./action/harness-runner"; import { buildCuaHarness } from "./harness"; @@ -188,6 +188,7 @@ export interface HarnessCliFlags { maxSteps?: number; out?: string; output?: string; + filter?: string; imageProtocol?: string; namedSession?: string; sessionRef?: string; @@ -195,13 +196,16 @@ export interface HarnessCliFlags { skillPaths: string[]; } -interface ResolvedAuth { +export interface KernelAuth { kernelApiKey: string; kernelBaseUrl?: string; +} + +interface ResolvedAuth extends KernelAuth { modelRef: CuaModelRef; } -function requireKernelApiKey(): { apiKey: string; baseUrl?: string } { +export function requireKernelApiKey(): { apiKey: string; baseUrl?: string } { const apiKey = process.env.KERNEL_API_KEY?.trim(); if (!apiKey) throw new Error("missing Kernel API key (set KERNEL_API_KEY)"); const baseUrl = process.env.KERNEL_BASE_URL?.trim() || undefined; @@ -217,12 +221,12 @@ function resolveAuth(flags: HarnessCliFlags): ResolvedAuth { return { kernelApiKey: apiKey, kernelBaseUrl: baseUrl, modelRef }; } -interface ProvisionedBrowser { +export interface ProvisionedBrowser { handle: CuaBrowserHandle; named?: NamedSessionMetadata; } -async function provisionForFlags(flags: HarnessCliFlags, auth: ResolvedAuth): Promise { +export async function provisionForFlags(flags: HarnessCliFlags, auth: KernelAuth): Promise { if (flags.namedSession) { const { client, browser, meta } = await attachNamedSession({ name: flags.namedSession, @@ -582,27 +586,22 @@ export async function runInteractiveCommand( } } -/** Run a one-shot action subcommand through the new harness wiring. */ +/** Run a one-shot model-mediated action subcommand through the harness wiring. */ export async function runActionCommand( - action: ActionType, + action: ModelActionType, rest: string[], flags: HarnessCliFlags, ): Promise { const runtime = await setupHarnessRuntime(flags, { skipDiskSession: true }); const req: ActionRequest = buildActionRequest(action, rest); if (flags.maxSteps !== undefined) req.maxTurns = flags.maxSteps; - const screenshotOut = flags.out - ? { out: flags.out } - : action === "screenshot" - ? { out: "screenshot.png" } - : undefined; try { const res = await runAction(req, { harness: runtime.harness, browserHandle: runtime.handle, session: runtime.session, skipInitialScreenshot: runtime.resolved?.resumed === true, - }, screenshotOut); + }); return emitCompact(res); } finally { try { @@ -613,22 +612,14 @@ export async function runActionCommand( } } -function buildActionRequest(action: ActionType, rest: string[]): ActionRequest { +function buildActionRequest(action: ModelActionType, rest: string[]): ActionRequest { switch (action) { - case "open": - return { action, text: rest[0] }; case "click": return { action, target: rest.join(" ") }; case "type": return { action, target: rest[0], text: rest[1] }; - case "press": - return { action, keys: rest }; case "observe": return { action, text: rest.join(" ") }; - case "url": - return { action }; - case "screenshot": - return { action }; case "do": return { action, text: rest.join(" ") }; } diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 00a1aa22..eb807e9d 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -1,7 +1,13 @@ #!/usr/bin/env node import { stderr, stdout } from "node:process"; import { parseArgs } from "node:util"; -import { type ActionType } from "./action/prompts"; +import { type ModelActionType } from "./action/prompts"; +import type { DeterministicActionType } from "./action/result"; +import { + DETERMINISTIC_SUBCOMMANDS, + isCoordinatePair, + runDeterministicCommand, +} from "./cli-executor"; import { runActionCommand, runInteractiveCommand, @@ -17,17 +23,30 @@ const HELP = `cua — Kernel-cloud-browser computer-use agent Usage: cua [options] [prompt...] cua --print "go to news.ycombinator.com and summarize" - cua open + cua open + cua url + cua snapshot [--filter interactive] + cua find "" + cua text + cua fill "" "" + cua press [key...] + cua click + cua tabs + cua screenshot [--out file|-] + cua click "" cua type "" "" - cua press [key...] cua observe [""] - cua url - cua screenshot [--out file|-] cua do "" cua models [-p provider] cua session start [name] | stop | list | show +Subcommands above the blank line are model-free: they run directly against +the browser (no LLM, no model API key; only KERNEL_API_KEY). \`click \` +with exactly two integer arguments clicks those viewport coordinates without +a model; any other \`click\` argument is a natural-language description +resolved by the model. Exit codes: 0 ok, 1 not_found, 2 error. + Options: -p, --print Run a single prompt and exit -m, --model Model ref (default: ${DEFAULT_CUA_MODEL_REF}) @@ -55,6 +74,7 @@ Options: computer_20260701 (requires --mode computer) or browser_20260701 (requires --mode browser) --out Output file for screenshot subcommand + --filter Restrict \`cua snapshot\` to interactive elements -o, --output Output format for --print: text (default) | jsonl --jsonl-include-deltas Include assistant_text_delta events (default off) --jsonl-include-images Include base64 screenshots (default off, only sizes) @@ -117,6 +137,7 @@ interface CliFlags { maxSteps?: number; out?: string; output?: string; + filter?: string; imageProtocol?: string; namedSession?: string; sessionRef?: string; @@ -145,6 +166,7 @@ function parseCliArgs(argv: string[]): CliFlags { "browser-timeout": { type: "string" }, "max-steps": { type: "string" }, out: { type: "string" }, + filter: { type: "string" }, "image-protocol": { type: "string" }, "session-name": { type: "string", short: "s" }, continue: { type: "boolean", short: "c", default: false }, @@ -207,6 +229,7 @@ function parseCliArgs(argv: string[]): CliFlags { browserTimeout: Number.isFinite(browserTimeout) ? browserTimeout : undefined, maxSteps: Number.isFinite(maxSteps) ? maxSteps : undefined, out: parsed.values.out as string | undefined, + filter: parsed.values.filter as string | undefined, imageProtocol: parsed.values["image-protocol"] as string | undefined, namedSession: parsed.values["session-name"] as string | undefined, sessionRef: parsed.values.session as string | undefined, @@ -243,6 +266,7 @@ function toHarnessFlags(flags: CliFlags): HarnessCliFlags { maxSteps: flags.maxSteps, out: flags.out, output: flags.output, + filter: flags.filter, imageProtocol: flags.imageProtocol, namedSession: flags.namedSession, sessionRef: flags.sessionRef, @@ -251,7 +275,7 @@ function toHarnessFlags(flags: CliFlags): HarnessCliFlags { }; } -const SUBCOMMANDS = new Set(["open", "click", "type", "press", "observe", "url", "screenshot", "do"]); +const MODEL_SUBCOMMANDS = new Set(["click", "type", "observe", "do"]); export async function main(argv: string[]): Promise { if (argv[0] === "models") { @@ -283,9 +307,20 @@ export async function main(argv: string[]): Promise { } } - if (first && SUBCOMMANDS.has(first)) { + const rest = positionals.slice(1); + + if (first && (DETERMINISTIC_SUBCOMMANDS.has(first) || (first === "click" && isCoordinatePair(rest)))) { + try { + return await runDeterministicCommand(first as DeterministicActionType, rest, toHarnessFlags(flags)); + } catch (err) { + stderr.write(`error: ${(err as Error).message}\n`); + return 2; + } + } + + if (first && MODEL_SUBCOMMANDS.has(first)) { try { - return await runActionCommand(first as ActionType, positionals.slice(1), toHarnessFlags(flags)); + return await runActionCommand(first as ModelActionType, rest, toHarnessFlags(flags)); } catch (err) { stderr.write(`error: ${(err as Error).message}\n`); return 2; diff --git a/packages/cli/test/action-runner.test.ts b/packages/cli/test/action-runner.test.ts index 4ab513d8..6833e79b 100644 --- a/packages/cli/test/action-runner.test.ts +++ b/packages/cli/test/action-runner.test.ts @@ -48,24 +48,6 @@ describe("action harness-runner", () => { expect(res.result.text).toBe("no match"); }); - it("captures a screenshot via the SDK without invoking the harness", async () => { - fixture = await buildTestHarness({ turns: [] }); - const originalWrite = process.stdout.write.bind(process.stdout); - process.stdout.write = ((_chunk: string | Uint8Array): boolean => true) as typeof process.stdout.write; - try { - const res = await runAction( - { action: "screenshot" }, - { harness: fixture.harness, browserHandle: handleFor(fixture), session: fixture.session }, - { out: "-" }, - ); - expect(res.exitCode).toBe(0); - expect(fixture.provider.callCount()).toBe(0); - expect(fixture.kernel.screenshots).toBe(1); - } finally { - process.stdout.write = originalWrite; - } - }); - it("exits 2 when the provider returns an error", async () => { fixture = await buildTestHarness({ turns: [{ steps: [{ type: "error", message: "boom" }] }], diff --git a/packages/cli/test/cli-executor.test.ts b/packages/cli/test/cli-executor.test.ts new file mode 100644 index 00000000..14428a1c --- /dev/null +++ b/packages/cli/test/cli-executor.test.ts @@ -0,0 +1,326 @@ +import type { BrowserExecutor, BrowserFindCandidate, InternalComputerTranslator as Translator } from "@onkernel/cua-agent"; +import { InternalComputerTranslator } from "@onkernel/cua-agent"; +import type { CuaBrowserAction } from "@onkernel/cua-ai"; +import { mkdtempSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + isCoordinatePair, + parseDeterministicArgs, + runDeterministicCommand, + runDeterministicOnHandle, +} from "../src/cli-executor"; +import type { HarnessCliFlags } from "../src/cli-harness"; +import type { CuaBrowserHandle } from "../src/harness-browser"; +import { createFakeKernelEnvironment, type FakeKernelEnvironment } from "./fixtures/fake-kernel"; + +const PROVIDER_ENV_KEYS = [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GOOGLE_API_KEY", + "GEMINI_API_KEY", + "TZAFON_API_KEY", + "YUTORI_API_KEY", +]; + +function baseFlags(overrides: Partial = {}): HarnessCliFlags { + return { + verbose: false, + profileSaveChanges: true, + continueLatest: false, + resumePicker: false, + noSession: false, + noSkills: false, + debugTui: false, + jsonlIncludeDeltas: false, + jsonlIncludeImages: false, + playwright: false, + skillPaths: [], + ...overrides, + }; +} + +interface FakeExecutorState { + actions: CuaBrowserAction[]; + closed: number; +} + +interface FakeExecutorScript { + candidates?: BrowserFindCandidate[]; + url?: string; + texts?: Partial>; + failWith?: Error; +} + +function fakeExecutor(script: FakeExecutorScript = {}): { executor: BrowserExecutor; state: FakeExecutorState } { + const state: FakeExecutorState = { actions: [], closed: 0 }; + const executor = { + async execute(action: CuaBrowserAction) { + if (script.failWith) throw script.failWith; + state.actions.push(action); + const text = script.texts?.[action.type]; + return text !== undefined ? [{ type: "browser_text", label: action.type, text }] : []; + }, + async findCandidates() { + if (script.failWith) throw script.failWith; + return script.candidates ?? []; + }, + async currentUrl() { + return script.url ?? ""; + }, + close() { + state.closed += 1; + }, + }; + return { executor: executor as unknown as BrowserExecutor, state }; +} + +interface TestSetup { + kernel: FakeKernelEnvironment; + handle: CuaBrowserHandle; + handleCloses: () => number; + createTranslator: (handle: CuaBrowserHandle) => Translator; + state: FakeExecutorState; +} + +function setup(script: FakeExecutorScript = {}): TestSetup { + const kernel = createFakeKernelEnvironment(); + let closes = 0; + const handle: CuaBrowserHandle = { + client: kernel.client, + browser: kernel.browser, + async close() { + closes += 1; + }, + }; + const { executor, state } = fakeExecutor(script); + const createTranslator = (h: CuaBrowserHandle) => + new InternalComputerTranslator({ browser: h.browser, client: h.client, createBrowserExecutor: () => executor }); + return { kernel, handle, handleCloses: () => closes, createTranslator, state }; +} + +let stdoutLines: string[] = []; +let originalWrite: typeof process.stdout.write; +let savedEnv: Record = {}; + +beforeEach(() => { + stdoutLines = []; + originalWrite = process.stdout.write.bind(process.stdout); + process.stdout.write = ((chunk: string | Uint8Array): boolean => { + stdoutLines.push(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("latin1")); + return true; + }) as typeof process.stdout.write; + savedEnv = {}; + for (const key of [...PROVIDER_ENV_KEYS, "KERNEL_API_KEY"]) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } +}); + +afterEach(() => { + process.stdout.write = originalWrite; + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + +describe("isCoordinatePair", () => { + it("accepts exactly two integer tokens", () => { + expect(isCoordinatePair(["10", "20"])).toBe(true); + }); + + it("rejects descriptions, partial pairs, and non-integers", () => { + expect(isCoordinatePair(["3", "dots", "menu"])).toBe(false); + expect(isCoordinatePair(["10"])).toBe(false); + expect(isCoordinatePair(["10", "20px"])).toBe(false); + expect(isCoordinatePair([])).toBe(false); + }); +}); + +describe("parseDeterministicArgs", () => { + it("rejects invalid argv before any provisioning", () => { + expect(() => parseDeterministicArgs("open", [], baseFlags())).toThrow("usage: cua open"); + expect(() => parseDeterministicArgs("find", [], baseFlags())).toThrow("usage: cua find"); + expect(() => parseDeterministicArgs("fill", ["query"], baseFlags())).toThrow("usage: cua fill"); + expect(() => parseDeterministicArgs("press", [], baseFlags())).toThrow("usage: cua press"); + expect(() => parseDeterministicArgs("click", ["a", "b"], baseFlags())).toThrow("usage: cua click"); + expect(() => parseDeterministicArgs("snapshot", [], baseFlags({ filter: "everything" }))).toThrow( + "invalid --filter", + ); + }); + + it("accepts the documented forms", () => { + expect(parseDeterministicArgs("open", ["back"], baseFlags())).toEqual({ action: "open", url: "back" }); + expect(parseDeterministicArgs("snapshot", [], baseFlags({ filter: "interactive" }))).toEqual({ + action: "snapshot", + filter: "interactive", + }); + expect(parseDeterministicArgs("fill", ["email", "a@b.c"], baseFlags())).toEqual({ + action: "fill", + query: "email", + value: "a@b.c", + }); + expect(parseDeterministicArgs("click", ["10", "20"], baseFlags())).toEqual({ action: "click", x: 10, y: 20 }); + }); + + it("runDeterministicCommand surfaces argv errors before touching the Kernel API", async () => { + // KERNEL_API_KEY is unset in this suite: reaching provisioning would + // throw "missing Kernel API key" instead of the usage error. + await expect(runDeterministicCommand("open", [], baseFlags())).rejects.toThrow("usage: cua open"); + }); +}); + +describe("runDeterministicOnHandle", () => { + it("open navigates via CDP and prints ok (no provider keys in env)", async () => { + const t = setup({ texts: { browser_navigate: "Navigated to https://example.test/." } }); + const code = await runDeterministicOnHandle({ action: "open", url: "example.test" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe("ok\n"); + expect(t.state.actions).toEqual([{ type: "browser_navigate", url: "example.test" }]); + expect(t.state.closed).toBe(1); + expect(t.handleCloses()).toBe(1); + }); + + it("url prints the current URL", async () => { + const t = setup({ url: "https://example.test/page" }); + const code = await runDeterministicOnHandle({ action: "url" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe("https://example.test/page\n"); + }); + + it("snapshot passes --filter through and prints the tree", async () => { + const t = setup({ texts: { browser_snapshot: 'button "Go" [e1]' } }); + const code = await runDeterministicOnHandle( + { action: "snapshot", filter: "interactive" }, + t.handle, + t.createTranslator, + ); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe('button "Go" [e1]\n'); + expect(t.state.actions).toEqual([{ type: "browser_snapshot", filter: "interactive" }]); + }); + + it("text prints the page text", async () => { + const t = setup({ texts: { browser_text: "hello world" } }); + const code = await runDeterministicOnHandle({ action: "text" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe("hello world\n"); + }); + + it("tabs prints one line per tab", async () => { + const t = setup({ texts: { browser_list_tabs: 'tab_id AAAA: "One" (https://a)\ntab_id BBBB: "Two" (https://b)' } }); + const code = await runDeterministicOnHandle({ action: "tabs" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toContain("tab_id AAAA"); + }); + + it("find exits 1 when no candidates match", async () => { + const t = setup({ candidates: [] }); + const code = await runDeterministicOnHandle({ action: "find", query: "missing thing" }, t.handle, t.createTranslator); + expect(code).toBe(1); + expect(stdoutLines.join("")).toBe('not_found no elements matched "missing thing"\n'); + }); + + it("find prints one candidate per line", async () => { + const t = setup({ + candidates: [ + { ref: "e1", role: "button", name: "Search", score: 2 }, + { ref: "e2", role: "link", name: "Search help", score: 1 }, + ], + }); + const code = await runDeterministicOnHandle({ action: "find", query: "search" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe('button "Search" [e1]\nlink "Search help" [e2]\n'); + }); + + it("fill exits 1 when nothing fillable matches", async () => { + const t = setup({ candidates: [{ ref: "e1", role: "button", name: "Email us", score: 1 }] }); + const code = await runDeterministicOnHandle( + { action: "fill", query: "email", value: "a@b.c" }, + t.handle, + t.createTranslator, + ); + expect(code).toBe(1); + expect(stdoutLines.join("")).toBe('not_found no fillable element matched "email"\n'); + expect(t.state.actions).toEqual([]); + }); + + it("fill exits 1 on a tied top score and lists the matches", async () => { + const t = setup({ + candidates: [ + { ref: "e1", role: "textbox", name: "Email", score: 1 }, + { ref: "e2", role: "textbox", name: "Email confirmation", score: 1 }, + ], + }); + const code = await runDeterministicOnHandle( + { action: "fill", query: "email", value: "a@b.c" }, + t.handle, + t.createTranslator, + ); + expect(code).toBe(1); + expect(stdoutLines.join("")).toBe( + 'not_found ambiguous query "email" (2 matches): textbox "Email", textbox "Email confirmation"\n', + ); + expect(t.state.actions).toEqual([]); + }); + + it("fill fills the unique best fillable match by ref", async () => { + const t = setup({ + candidates: [ + { ref: "e1", role: "button", name: "Email us", score: 3 }, + { ref: "e2", role: "textbox", name: "Email", score: 2 }, + { ref: "e3", role: "textbox", name: "Name", score: 1 }, + ], + }); + const code = await runDeterministicOnHandle( + { action: "fill", query: "email", value: "a@b.c" }, + t.handle, + t.createTranslator, + ); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe('ok filled textbox "Email"\n'); + expect(t.state.actions).toEqual([{ type: "browser_fill", ref: "e2", value: "a@b.c" }]); + }); + + it("press dispatches one key chord through the computer batch API", async () => { + const t = setup(); + const code = await runDeterministicOnHandle({ action: "press", keys: ["ctrl", "l"] }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe("ok pressed\n"); + expect(t.kernel.batchCalls).toHaveLength(1); + const body = t.kernel.batchCalls[0]!.body as { actions: Array<{ type: string; press_key?: { keys: string[]; hold_keys?: string[] } }> }; + expect(body.actions).toEqual([{ type: "press_key", press_key: { keys: ["l"], hold_keys: ["Control_L"] } }]); + }); + + it("click dispatches an OS-level click at the coordinates", async () => { + const t = setup(); + const code = await runDeterministicOnHandle({ action: "click", x: 10, y: 20 }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe("ok clicked (10, 20)\n"); + const body = t.kernel.batchCalls[0]!.body as { actions: Array<{ type: string; click_mouse?: { x: number; y: number } }> }; + expect(body.actions[0]!.type).toBe("click_mouse"); + expect(body.actions[0]!.click_mouse).toMatchObject({ x: 10, y: 20 }); + }); + + it("screenshot captures via the SDK and writes the file", async () => { + const t = setup(); + const out = join(mkdtempSync(join(tmpdir(), "cua-shot-")), "shot.png"); + const code = await runDeterministicOnHandle({ action: "screenshot", out }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe(`${out}\n`); + expect(t.kernel.screenshots).toBe(1); + expect((await readFile(out)).length).toBeGreaterThan(0); + }); + + it("exits 2 and still closes executor and handle when the executor throws", async () => { + const t = setup({ failWith: new Error("cdp exploded") }); + const code = await runDeterministicOnHandle({ action: "open", url: "example.test" }, t.handle, t.createTranslator); + expect(code).toBe(2); + expect(stdoutLines.join("")).toBe("error cdp exploded\n"); + expect(t.state.closed).toBe(1); + expect(t.handleCloses()).toBe(1); + }); +}); diff --git a/skills/cua-cli/SKILL.md b/skills/cua-cli/SKILL.md index 43e5d2ed..295e3a25 100644 --- a/skills/cua-cli/SKILL.md +++ b/skills/cua-cli/SKILL.md @@ -1,34 +1,54 @@ --- name: cua-cli -description: Drive a Kernel cloud browser from the shell using the `cua` CLI. Use this skill when you need to open URLs, click elements, type into fields, take screenshots, or chain multi-step browser tasks across shell calls. Supports named sessions for stateful workflows. +description: Drive a Kernel cloud browser from the shell using the `cua` CLI. Use this skill when you need to open URLs, click elements, type into fields, inspect pages, fill forms, take screenshots, or chain multi-step browser tasks across shell calls. Supports named sessions for stateful workflows. --- # cua-cli -`cua` is a single-binary CLI that drives a real Chrome session running in Kernel. It's designed for agentic use: each subcommand returns a one-line result on stdout and a deterministic exit code, so you can chain calls together and parse the output. +`cua` is a single-binary CLI that drives a real Chrome session running in Kernel. It's designed for agentic use: each subcommand returns a stable result on stdout and a deterministic exit code (0 ok, 1 not_found, 2 error), so you can chain calls together and parse the output. ## One-shot subcommands Each call below provisions a fresh Kernel browser by default, runs the action, and tears the browser down. Use `-s ` (see "Named sessions" below) to keep state across calls. +### Model-free subcommands + +These run directly against the browser (CDP or OS input) — no LLM involved, no model API key needed, only `KERNEL_API_KEY`. + +| Subcommand | What it does | Stdout | Exit code | +| --- | --- | --- | --- | +| `cua open ` | Navigate via CDP; `back`/`forward` walk history. | `ok` | 0 ok, 2 error | +| `cua url` | Print the active tab's URL. | the URL | 0 ok, 2 error | +| `cua snapshot [--filter interactive]` | Print the page's accessibility tree with element refs like `[e12]`. `--filter interactive` keeps only interactive elements. | the tree (multi-line) | 0 ok, 2 error | +| `cua find ""` | Lexically score elements against the query, best first. | one match per line: `role "name" [eN]` | 0 ok, 1 not_found, 2 error | +| `cua text` | Print the page's visible text (`innerText`). | the text (multi-line) | 0 ok, 2 error | +| `cua fill "" ""` | Find the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton) and set its value. Exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. | `ok filled ""` | 0 ok, 1 not_found, 2 error | +| `cua press [...]` | Send one key chord (e.g. `cua press ctrl l`, `cua press Return`). | `ok pressed` | 0 ok, 2 error | +| `cua click ` | OS-level click at viewport coordinates. Exactly two integer arguments — anything else routes to the model-mediated `click` below. | `ok clicked (x, y)` | 0 ok, 2 error | +| `cua tabs` | List open tabs. | one line per tab: `tab_id XXXX: "title" (url)` | 0 ok, 2 error | +| `cua screenshot [--out ]` | Save a PNG (default `screenshot.png`). `--out -` writes the bytes to stdout. | the path or `(stdout)` | 0 ok, 2 error | + +**Element refs are not valid across `cua` invocations.** Refs printed by `snapshot`/`find` (`[e12]`) live only for the process that minted them — there is no `fill ` form. Use `fill ""`, `click ""`, or `click ` instead; the refs are still useful as unique line handles when reading output. + +### Model-mediated subcommands + +These resolve a natural-language description with an LLM, so they need the model provider's API key (e.g. `OPENAI_API_KEY` for the default model). + | Subcommand | What it does | Stdout | Exit code | | --- | --- | --- | --- | -| `cua open ` | Navigate to a URL via the address bar. | `ok` | 0 ok, 2 error | | `cua click ""` | Find the element matching the visible, natural-language description and click it. | `ok clicked (x, y)` or `not_found ` | 0 ok, 1 not_found, 2 error | | `cua type "" ""` | Focus the field matching the visible, natural-language description and type text. | `ok typed` or `not_found ` | 0 ok, 1 not_found, 2 error | -| `cua press [...]` | Send a key combo (e.g. `cua press ctrl l`, `cua press Return`). | `ok pressed` | 0 ok, 2 error | -| `cua url` | Read and print the current URL. | the URL | 0 ok, 2 error | | `cua observe ["question"]` | Describe the page; optionally answer a question. | the description | 0 ok, 2 error | -| `cua screenshot --out ` | Save a PNG. `--out -` writes the bytes to stdout. | the path or `(stdout)` | 0 ok, 2 error | | `cua do ""` | Open-ended; let the agent plan and act. Bound by `--max-steps` (default 3). | the assistant's final text | 0 ok, 2 error | Useful flags: -- `-m ` — pick the LLM (default `gpt-5.5`). Other good picks: - `claude-opus-4-7`, `gemini-3-flash-preview`, `n1.5-latest`. +- `-m ` — pick the LLM for model-mediated subcommands (default `gpt-5.5`). + Other good picks: `claude-opus-4-7`, `gemini-3-flash-preview`, `n1.5-latest`. - `cua models` — list supported `-m` values and their providers; filter with `cua models -p openai|anthropic|gemini|yutori`. - `--max-steps ` — bound the agent loop on `cua do` (default 3). +- `--filter interactive` — restrict `cua snapshot` to interactive elements. - `--profile ` — load a Kernel browser profile for cookies / storage. Existing ids or names are reused; a non-id name is created if it does not exist. Use this whenever logged-in state or other persisted browser @@ -45,14 +65,23 @@ named session first: ```bash cua --profile github session start login # creates a Kernel browser, prints `name=login` cua -s login open https://github.com/login -cua -s login type "email field" "$EMAIL" -cua -s login type "password field" "$PASSWORD" -cua -s login click "Sign in" +cua -s login fill "email field" "$EMAIL" # model-free +cua -s login fill "password field" "$PASSWORD" # model-free +cua -s login click "Sign in" # model-mediated cua -s login url # prints the post-login URL cua session stop login # tears down the Kernel browser ``` -Inspect: +Inspecting a page mid-flow, entirely model-free: + +```bash +cua -s login snapshot --filter interactive # what can I interact with? +cua -s login find "sign in button" # score elements against a query +cua -s login text # read the page's visible text +cua -s login tabs # list open tabs +``` + +Inspect sessions: ```bash cua session list # tab-formatted: NAME, KERNEL_ID, AGE, LIVE_URL @@ -66,9 +95,11 @@ Liveness: Kernel browsers can time out from inactivity even between your calls. ## Session transcripts -Every `cua --print`, interactive TUI, and `cua -s ` invocation appends to -a JSONL transcript. Treat the on-disk directory name as internal; find the -exact path instead of trying to reconstruct it: +Every `cua --print`, interactive TUI, and model-mediated `cua -s ` +invocation appends to a JSONL transcript. Model-free subcommands do not touch +transcripts — there is no model conversation to record. Treat the on-disk +directory name as internal; find the exact path instead of trying to +reconstruct it: ```bash cua -v --print "..." # stderr includes: [cua] session= @@ -77,8 +108,8 @@ cua session show login | jq -r .transcript_path The default root is `$XDG_DATA_HOME/cua/sessions` or `~/.local/share/cua/sessions`. For named sessions, `transcript_path` appears -after the first `cua -s ...`, `cua -s --print ...`, or TUI attach -records a transcript. +after the first model-mediated `-s` call (`click ""`, `type`, `observe`, +`do`, `--print`, or a TUI attach) records a transcript. Each line is a JSON object with one of these `role` values: `user`, `assistant`, `toolResult`. There's also a custom `cua-browser` entry written once per session with `kernel_session_id` / `live_url` / `profile_id`. @@ -102,7 +133,8 @@ running until you Ctrl+C. ## Don't forget -- Subcommands that need an element or field description (`click`, `type`) match SEMANTICALLY, not by selector. Use natural-language descriptions of what the user would see on screen. +- Prefer the model-free subcommands when they can do the job — they're faster, cheaper, and deterministic. Reach for `click ""` / `type` / `do` only when you need semantic matching or planning. +- Subcommands that take an element or field description (`click ""`, `type`) match SEMANTICALLY, not by selector. Use natural-language descriptions of what the user would see on screen. `fill` matches lexically against accessible role/name — use the words from `snapshot`/`find` output. - Browser viewport defaults to 1920x1080. - Keyboard navigation (`Page_Down`, `Home`, arrow keys via `cua press`) is more reliable than mouse-wheel scrolling. - For multi-step state, you almost always want `-s `. Without it, the second subcommand can't see anything the first one did. From 929f11126caf5c92bf1047777fb645df9f0da09e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 01:18:03 +0000 Subject: [PATCH 24/34] Harden deterministic subcommand contracts - Apply the fillable-role filter inside findCandidates before the match limit so fill cannot miss a form field crowded out by non-fillable matches; findCandidates takes an optional role set - Map checkbox/radio fill values through an explicit true/false vocabulary instead of Boolean(string), rejecting anything else - Reject extra positionals on url/text/tabs and --filter outside snapshot before provisioning - Keep screenshot --out - stdout as pure PNG bytes; no status line - Guard translator.dispose() in teardown so handle.close() always runs - Route subcommand dispatch through deterministicActionFor and test it - Cover screenshot --out - and capture failure in the CLI suite - Align SKILL.md with the shipped surface (find/fill/screenshot rows, liveness message, transcript sentence) --- packages/agent/src/translator/browser.ts | 4 +- packages/cli/src/cli-executor.ts | 42 +++++++++++-- packages/cli/src/cli.ts | 12 ++-- packages/cli/test/cli-executor.test.ts | 80 +++++++++++++++++++++++- skills/cua-cli/SKILL.md | 10 +-- 5 files changed, 125 insertions(+), 23 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 637a0755..833e7adb 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -473,7 +473,7 @@ export class BrowserExecutor { * Score elements against a natural-language query and mint refs for the * matches, best first. Structured counterpart of the `browser_find` action. */ - async findCandidates(query: string, tabId?: string): Promise { + async findCandidates(query: string, tabId?: string, roles?: ReadonlySet): Promise { const targetId = await this.resolveTarget(tabId); const session = await this.attach(targetId); const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); @@ -503,7 +503,7 @@ export class BrowserExecutor { ) .map((node) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })), ) - .filter((entry) => entry.score > 0) + .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); const candidates = scored.map(({ node, ctx, score }) => ({ diff --git a/packages/cli/src/cli-executor.ts b/packages/cli/src/cli-executor.ts index 7bed15ed..63f50a0b 100644 --- a/packages/cli/src/cli-executor.ts +++ b/packages/cli/src/cli-executor.ts @@ -52,12 +52,33 @@ const FILLABLE_ROLES: ReadonlySet = new Set([ "spinbutton", ]); +/** Roles whose fill value is a checked state, not text. */ +const TOGGLE_ROLES: ReadonlySet = new Set(["checkbox", "radio"]); + +function parseToggleValue(raw: string): boolean { + const value = raw.trim().toLowerCase(); + if (["true", "1", "checked", "on"].includes(value)) return true; + if (["false", "0", "unchecked", "off"].includes(value)) return false; + throw new Error(`checkbox/radio value must be true|false|1|0|checked|unchecked|on|off, got ${JSON.stringify(raw)}`); +} + +/** Resolve argv to a deterministic subcommand, or undefined when the model plane should handle it. */ +export function deterministicActionFor(first: string | undefined, rest: string[]): DeterministicActionType | undefined { + if (!first) return undefined; + if (DETERMINISTIC_SUBCOMMANDS.has(first)) return first as DeterministicActionType; + if (first === "click" && isCoordinatePair(rest)) return "click"; + return undefined; +} + /** Parse and validate a deterministic subcommand's argv. Throws before any Kernel API call. */ export function parseDeterministicArgs( action: DeterministicActionType, rest: string[], flags: HarnessCliFlags, ): DeterministicRequest { + if (flags.filter !== undefined && action !== "snapshot") { + throw new Error("--filter only applies to cua snapshot"); + } switch (action) { case "open": { const url = (rest[0] ?? "").trim(); @@ -65,6 +86,7 @@ export function parseDeterministicArgs( return { action, url }; } case "url": + if (rest.length > 0) throw new Error("usage: cua url"); return { action }; case "snapshot": { if (rest.length > 0) throw new Error("usage: cua snapshot [--filter interactive]"); @@ -75,6 +97,7 @@ export function parseDeterministicArgs( return { action, ...(filter === "interactive" ? { filter } : {}) }; } case "text": + if (rest.length > 0) throw new Error("usage: cua text"); return { action }; case "find": { const query = rest.join(" ").trim(); @@ -99,6 +122,7 @@ export function parseDeterministicArgs( return { action, x: Number(rest[0]), y: Number(rest[1]) }; } case "tabs": + if (rest.length > 0) throw new Error("usage: cua tabs"); return { action }; case "screenshot": { if (rest.length > 0) throw new Error("usage: cua screenshot [--out file|-]"); @@ -130,7 +154,11 @@ export async function runDeterministicOnHandle( const res = await executeDeterministic(req, translator, handle); return emitCompact(res); } finally { - translator.dispose(); + try { + translator.dispose(); + } catch (err) { + stderr.write(`[cua] cleanup warning: ${(err as Error).message}\n`); + } try { await handle.close(); } catch (err) { @@ -177,7 +205,7 @@ async function executeDeterministic( } case "fill": { const executor = translator.browser(); - const candidates = (await executor.findCandidates(req.query)).filter((c) => FILLABLE_ROLES.has(c.role)); + const candidates = await executor.findCandidates(req.query, undefined, FILLABLE_ROLES); if (candidates.length === 0) { return finish({ action: req.action, status: "not_found", text: `no fillable element matched ${JSON.stringify(req.query)}` }); } @@ -191,7 +219,8 @@ async function executeDeterministic( }); } const match = candidates[0]!; - await executor.execute({ type: "browser_fill", ref: match.ref, value: req.value }); + const value = TOGGLE_ROLES.has(match.role) ? parseToggleValue(req.value) : req.value; + await executor.execute({ type: "browser_fill", ref: match.ref, value }); return finish({ action: req.action, status: "ok", text: `${match.role} ${JSON.stringify(match.name)}` }); } case "press": @@ -210,11 +239,12 @@ async function executeDeterministic( return finish({ action: req.action, status: "error", text: "failed to capture screenshot" }); } if (req.out === "-") { + // stdout is the PNG bytes; the compact status line would corrupt a pipe. stdout.write(png); - } else { - await writeFile(req.out, png); + return finish({ action: req.action, status: "ok", text: "" }); } - return finish({ action: req.action, status: "ok", text: req.out === "-" ? "(stdout)" : req.out }); + await writeFile(req.out, png); + return finish({ action: req.action, status: "ok", text: req.out }); } } } catch (err) { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index eb807e9d..0986dd74 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -2,12 +2,7 @@ import { stderr, stdout } from "node:process"; import { parseArgs } from "node:util"; import { type ModelActionType } from "./action/prompts"; -import type { DeterministicActionType } from "./action/result"; -import { - DETERMINISTIC_SUBCOMMANDS, - isCoordinatePair, - runDeterministicCommand, -} from "./cli-executor"; +import { deterministicActionFor, runDeterministicCommand } from "./cli-executor"; import { runActionCommand, runInteractiveCommand, @@ -309,9 +304,10 @@ export async function main(argv: string[]): Promise { const rest = positionals.slice(1); - if (first && (DETERMINISTIC_SUBCOMMANDS.has(first) || (first === "click" && isCoordinatePair(rest)))) { + const deterministic = deterministicActionFor(first, rest); + if (deterministic) { try { - return await runDeterministicCommand(first as DeterministicActionType, rest, toHarnessFlags(flags)); + return await runDeterministicCommand(deterministic, rest, toHarnessFlags(flags)); } catch (err) { stderr.write(`error: ${(err as Error).message}\n`); return 2; diff --git a/packages/cli/test/cli-executor.test.ts b/packages/cli/test/cli-executor.test.ts index 14428a1c..8241e958 100644 --- a/packages/cli/test/cli-executor.test.ts +++ b/packages/cli/test/cli-executor.test.ts @@ -7,6 +7,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { + deterministicActionFor, isCoordinatePair, parseDeterministicArgs, runDeterministicCommand, @@ -63,9 +64,10 @@ function fakeExecutor(script: FakeExecutorScript = {}): { executor: BrowserExecu const text = script.texts?.[action.type]; return text !== undefined ? [{ type: "browser_text", label: action.type, text }] : []; }, - async findCandidates() { + async findCandidates(_query: string, _tabId?: string, roles?: ReadonlySet) { if (script.failWith) throw script.failWith; - return script.candidates ?? []; + const candidates = script.candidates ?? []; + return roles ? candidates.filter((c) => roles.has(c.role)) : candidates; }, async currentUrl() { return script.url ?? ""; @@ -127,6 +129,24 @@ afterEach(() => { } }); +describe("deterministicActionFor", () => { + it("routes deterministic subcommands to the executor plane", () => { + expect(deterministicActionFor("url", [])).toBe("url"); + expect(deterministicActionFor("open", ["https://a"])).toBe("open"); + expect(deterministicActionFor("screenshot", [])).toBe("screenshot"); + expect(deterministicActionFor("click", ["10", "20"])).toBe("click"); + }); + + it("leaves model-mediated and free-form argv alone", () => { + expect(deterministicActionFor("click", ["3", "dots", "menu"])).toBeUndefined(); + expect(deterministicActionFor("click", ["sign in button"])).toBeUndefined(); + expect(deterministicActionFor("do", ["open hn"])).toBeUndefined(); + expect(deterministicActionFor("observe", [])).toBeUndefined(); + expect(deterministicActionFor("session", ["list"])).toBeUndefined(); + expect(deterministicActionFor(undefined, [])).toBeUndefined(); + }); +}); + describe("isCoordinatePair", () => { it("accepts exactly two integer tokens", () => { expect(isCoordinatePair(["10", "20"])).toBe(true); @@ -152,6 +172,18 @@ describe("parseDeterministicArgs", () => { ); }); + it("rejects extra positionals on url, text, and tabs", () => { + expect(() => parseDeterministicArgs("url", ["extra"], baseFlags())).toThrow("usage: cua url"); + expect(() => parseDeterministicArgs("text", ["extra"], baseFlags())).toThrow("usage: cua text"); + expect(() => parseDeterministicArgs("tabs", ["extra"], baseFlags())).toThrow("usage: cua tabs"); + }); + + it("rejects --filter on subcommands other than snapshot", () => { + expect(() => parseDeterministicArgs("text", [], baseFlags({ filter: "interactive" }))).toThrow( + "--filter only applies to cua snapshot", + ); + }); + it("accepts the documented forms", () => { expect(parseDeterministicArgs("open", ["back"], baseFlags())).toEqual({ action: "open", url: "back" }); expect(parseDeterministicArgs("snapshot", [], baseFlags({ filter: "interactive" }))).toEqual({ @@ -285,6 +317,30 @@ describe("runDeterministicOnHandle", () => { expect(t.state.actions).toEqual([{ type: "browser_fill", ref: "e2", value: "a@b.c" }]); }); + it("fill maps checkbox values to a checked state", async () => { + const t = setup({ candidates: [{ ref: "e1", role: "checkbox", name: "Subscribe", score: 2 }] }); + const code = await runDeterministicOnHandle( + { action: "fill", query: "subscribe", value: "false" }, + t.handle, + t.createTranslator, + ); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe('ok filled checkbox "Subscribe"\n'); + expect(t.state.actions).toEqual([{ type: "browser_fill", ref: "e1", value: false }]); + }); + + it("fill exits 2 on an unrecognized checkbox value", async () => { + const t = setup({ candidates: [{ ref: "e1", role: "checkbox", name: "Subscribe", score: 2 }] }); + const code = await runDeterministicOnHandle( + { action: "fill", query: "subscribe", value: "maybe" }, + t.handle, + t.createTranslator, + ); + expect(code).toBe(2); + expect(stdoutLines.join("")).toContain("error checkbox/radio value must be"); + expect(t.state.actions).toEqual([]); + }); + it("press dispatches one key chord through the computer batch API", async () => { const t = setup(); const code = await runDeterministicOnHandle({ action: "press", keys: ["ctrl", "l"] }, t.handle, t.createTranslator); @@ -315,6 +371,26 @@ describe("runDeterministicOnHandle", () => { expect((await readFile(out)).length).toBeGreaterThan(0); }); + it("screenshot --out - writes only the PNG bytes to stdout", async () => { + const t = setup(); + const code = await runDeterministicOnHandle({ action: "screenshot", out: "-" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines).toHaveLength(1); + expect(stdoutLines[0]!.startsWith("\x89PNG\r\n\x1a\n")).toBe(true); + expect(t.kernel.screenshots).toBe(1); + }); + + it("screenshot exits 2 when capture fails", async () => { + const t = setup(); + const computer = t.kernel.client.browsers.computer as unknown as { captureScreenshot: () => Promise }; + computer.captureScreenshot = async () => { + throw new Error("capture unavailable"); + }; + const code = await runDeterministicOnHandle({ action: "screenshot", out: "-" }, t.handle, t.createTranslator); + expect(code).toBe(2); + expect(stdoutLines.join("")).toBe("error failed to capture screenshot\n"); + }); + it("exits 2 and still closes executor and handle when the executor throws", async () => { const t = setup({ failWith: new Error("cdp exploded") }); const code = await runDeterministicOnHandle({ action: "open", url: "example.test" }, t.handle, t.createTranslator); diff --git a/skills/cua-cli/SKILL.md b/skills/cua-cli/SKILL.md index 295e3a25..1daa9681 100644 --- a/skills/cua-cli/SKILL.md +++ b/skills/cua-cli/SKILL.md @@ -20,13 +20,13 @@ These run directly against the browser (CDP or OS input) — no LLM involved, no | `cua open ` | Navigate via CDP; `back`/`forward` walk history. | `ok` | 0 ok, 2 error | | `cua url` | Print the active tab's URL. | the URL | 0 ok, 2 error | | `cua snapshot [--filter interactive]` | Print the page's accessibility tree with element refs like `[e12]`. `--filter interactive` keeps only interactive elements. | the tree (multi-line) | 0 ok, 2 error | -| `cua find ""` | Lexically score elements against the query, best first. | one match per line: `role "name" [eN]` | 0 ok, 1 not_found, 2 error | +| `cua find ""` | Lexically score elements against the query, best first. | one match per line: `role "name" [eN]` (the quoted name is omitted when the element has none; role falls back to `node`) | 0 ok, 1 not_found, 2 error | | `cua text` | Print the page's visible text (`innerText`). | the text (multi-line) | 0 ok, 2 error | -| `cua fill "" ""` | Find the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton) and set its value. Exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. | `ok filled ""` | 0 ok, 1 not_found, 2 error | +| `cua fill "" ""` | Find the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton) and set its value. Exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. For checkbox/radio the value must be `true\|false\|1\|0\|checked\|unchecked\|on\|off` (anything else exits 2). | `ok filled ""` | 0 ok, 1 not_found, 2 error | | `cua press [...]` | Send one key chord (e.g. `cua press ctrl l`, `cua press Return`). | `ok pressed` | 0 ok, 2 error | | `cua click ` | OS-level click at viewport coordinates. Exactly two integer arguments — anything else routes to the model-mediated `click` below. | `ok clicked (x, y)` | 0 ok, 2 error | | `cua tabs` | List open tabs. | one line per tab: `tab_id XXXX: "title" (url)` | 0 ok, 2 error | -| `cua screenshot [--out ]` | Save a PNG (default `screenshot.png`). `--out -` writes the bytes to stdout. | the path or `(stdout)` | 0 ok, 2 error | +| `cua screenshot [--out ]` | Save a PNG (default `screenshot.png`). `--out -` writes the bytes to stdout. | the saved path; with `--out -`, stdout is exactly the PNG bytes (safe to pipe) | 0 ok, 2 error | **Element refs are not valid across `cua` invocations.** Refs printed by `snapshot`/`find` (`[e12]`) live only for the process that minted them — there is no `fill ` form. Use `fill ""`, `click ""`, or `click ` instead; the refs are still useful as unique line handles when reading output. @@ -91,7 +91,7 @@ cua session show login # full JSON metadata Pass `--profile` when starting the named session; later `cua -s login ...` calls attach to that same browser, so they do not need the profile flag. -Liveness: Kernel browsers can time out from inactivity even between your calls. If `cua -s ...` returns `error session "" is no longer alive on Kernel ...`, run `cua session stop && cua --profile github session start ` to provision a fresh one with the same persisted profile. +Liveness: Kernel browsers can time out from inactivity even between your calls. If `cua -s ...` fails with `error: named session "" is no longer alive on Kernel ...` (printed to stderr, exit 2), run `cua session stop && cua --profile github session start ` to provision a fresh one with the same persisted profile. ## Session transcripts @@ -109,7 +109,7 @@ cua session show login | jq -r .transcript_path The default root is `$XDG_DATA_HOME/cua/sessions` or `~/.local/share/cua/sessions`. For named sessions, `transcript_path` appears after the first model-mediated `-s` call (`click ""`, `type`, `observe`, -`do`, `--print`, or a TUI attach) records a transcript. +`do`, `--print`, or a TUI attach). Each line is a JSON object with one of these `role` values: `user`, `assistant`, `toolResult`. There's also a custom `cua-browser` entry written once per session with `kernel_session_id` / `live_url` / `profile_id`. From dcce110cd27d00f224cab51e817327e124c38b3d Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:11:28 +0000 Subject: [PATCH 25/34] Persist element refs across invocations of a named session snapshot/find refs now survive to later cua invocations: the executor's ref table (entries, generations, ref counter, active tab) serializes to .refs.json alongside the named-session metadata and rehydrates on the next deterministic command, so cua -s x snapshot then cua -s x click e12 works across processes. New click and fill forms target refs directly over CDP; a stale ref exits 1 with the re-snapshot hint. Imported refs rebind their CDP session lazily (session ids are process-local), backend node ids stay valid for the life of the document, and the existing generation/self-heal machinery covers pages that changed between invocations. Refs files are removed on session stop. --- packages/agent/src/index.ts | 2 +- packages/agent/src/translator/browser.ts | 57 +++++++++++-- .../agent/test/translator-browser.test.ts | 29 +++++++ packages/cli/src/action/result.ts | 1 + packages/cli/src/cli-executor.ts | 82 +++++++++++++++++-- packages/cli/src/cli.ts | 10 ++- packages/cli/src/harness-named-sessions.ts | 26 +++++- packages/cli/test/cli-executor.test.ts | 79 +++++++++++++++++- skills/cua-cli/SKILL.md | 9 +- 9 files changed, 270 insertions(+), 25 deletions(-) diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index b8dc5be1..2f79b50f 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -5,7 +5,7 @@ export type { KernelBrowser } from "./translator/translator"; export { InternalComputerTranslator } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; export { BrowserExecutor } from "./translator/browser"; -export type { BrowserFindCandidate } from "./translator/browser"; +export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser"; export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 833e7adb..0bd377ae 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -91,6 +91,21 @@ export interface BrowserExecutorOptions { cursorHints?: boolean; } +/** + * Serializable ref state, so refs minted in one process (e.g. a `cua + * snapshot` invocation) can be resolved in a later one against the same + * browser. Session ids are process-local and deliberately not exported; + * imported refs rebind lazily. Backend node ids stay valid for the life of + * the document, and the usual generation/self-heal machinery covers pages + * that changed in between. + */ +export interface BrowserRefState { + refCounter: number; + activeTargetId?: string; + generations: Array<[string, number]>; + refs: Array<[string, Omit]>; +} + /** * Executes browser-plane canonical actions over CDP. * @@ -209,6 +224,24 @@ export class BrowserExecutor { this.cdp.close(); } + /** Snapshot the ref table for persistence across invocations; see {@link BrowserRefState}. */ + exportRefState(): BrowserRefState { + return { + refCounter: this.refCounter, + ...(this.activeTargetId ? { activeTargetId: this.activeTargetId } : {}), + generations: [...this.generations], + refs: [...this.refs].map(([ref, { sessionId: _sessionId, ...entry }]) => [ref, entry]), + }; + } + + /** Restore a ref table exported by a previous invocation against the same browser. */ + importRefState(state: BrowserRefState): void { + this.refCounter = Math.max(this.refCounter, state.refCounter); + this.activeTargetId = state.activeTargetId ?? this.activeTargetId; + for (const [frameId, generation] of state.generations) this.generations.set(frameId, generation); + for (const [ref, entry] of state.refs) this.refs.set(ref, { ...entry, sessionId: "" }); + } + async execute(action: CuaBrowserAction): Promise { const results = await this.dispatch(action); const dialogs = this.drainDialogNotes(); @@ -553,7 +586,7 @@ export class BrowserExecutor { private async fill(action: CuaActionBrowserFill): Promise { const targetId = await this.resolveTarget(action.tab_id); const entry = this.resolveRef(action.ref, targetId); - const session = entry.sessionId; + const session = await this.refSession(entry); const objectId = await this.resolveObject(entry, action.ref, session); const { exceptionDetails } = await this.cdp.send<{ exceptionDetails?: { exception?: { description?: string } } }>( "Runtime.callFunctionOn", @@ -572,7 +605,7 @@ export class BrowserExecutor { private async scrollTo(action: CuaActionBrowserScrollTo): Promise { const targetId = await this.resolveTarget(action.tab_id); const entry = this.resolveRef(action.ref, targetId); - await this.scrollIntoView(entry, action.ref, entry.sessionId); + await this.scrollIntoView(entry, action.ref, await this.refSession(entry)); } private async scroll(action: CuaActionBrowserScroll): Promise { @@ -682,11 +715,12 @@ export class BrowserExecutor { ): Promise<{ x: number; y: number; session: string }> { if (action.ref !== undefined) { const entry = this.resolveRef(action.ref, targetId); - await this.scrollIntoView(entry, action.ref, entry.sessionId); + const refSession = await this.refSession(entry); + await this.scrollIntoView(entry, action.ref, refSession); const { model } = await this.cdp.send<{ model: { content: number[] } }>( "DOM.getBoxModel", { backendNodeId: entry.backendNodeId }, - entry.sessionId, + refSession, ); const quad = model.content; // Box-model quads are main-viewport coordinates even through an OOPIF's @@ -726,7 +760,7 @@ export class BrowserExecutor { } private async healRef(ref: string, entry: RefEntry, cause: unknown): Promise { - const { nodes } = await this.frameAxTree(entry.frameId, entry.targetId, entry.sessionId); + const { nodes } = await this.frameAxTree(entry.frameId, entry.targetId, await this.refSession(entry)); this.healEntry(ref, entry, nodes, cause); } @@ -771,6 +805,19 @@ export class BrowserExecutor { return ref; } + /** + * Session for a ref's DOM/Input calls. Imported refs (see + * {@link importRefState}) carry no live session and rebind here: the + * frame's own session for OOPIFs when auto-attach has surfaced it, the + * page session otherwise. + */ + private async refSession(entry: RefEntry): Promise { + if (!entry.sessionId) { + entry.sessionId = this.frameSessions.get(entry.frameId) ?? (await this.attach(entry.targetId)); + } + return entry.sessionId; + } + private resolveRef(ref: string, targetId: string): RefEntry { const entry = this.refs.get(ref); // Entries are deleted eagerly on invalidation; the generation check only diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 6176caa2..4c9c84f5 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -740,3 +740,32 @@ describe("navigation tool grounding frame", () => { expect(inComputer.batches).toHaveLength(1); }); }); + +describe("BrowserExecutor ref state export/import", () => { + it("resolves refs imported from a previous executor against the same browser", async () => { + const first = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const { cdp, sent } = createFakeCdp(BUTTON_TREE); + const second = new BrowserExecutor(cdp); + second.importRefState(state); + await second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + const pressed = sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); + expect(pressed).toBeDefined(); + }); + + it("keeps minting unique refs after import and invalidates imported refs on navigation", async () => { + const first = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const { cdp, emit } = createFakeCdp(BUTTON_TREE); + const second = new BrowserExecutor(cdp); + second.importRefState(state); + expect(await snapshotText(second)).toContain('button "Save" [e2]'); + + emit({ method: "Page.frameNavigated", params: { frame: { id: "F0" } }, sessionId: "session-1" }); + await expect(second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); +}); diff --git a/packages/cli/src/action/result.ts b/packages/cli/src/action/result.ts index 0f759ffd..5b1983d4 100644 --- a/packages/cli/src/action/result.ts +++ b/packages/cli/src/action/result.ts @@ -101,6 +101,7 @@ export function formatCompact(r: ActionResult): string { switch (r.action as ActionType) { case "click": if (r.coordinates) return `ok clicked (${r.coordinates[0]}, ${r.coordinates[1]})`; + if (r.text) return `ok clicked ${r.text}`; return "ok clicked"; case "type": return "ok typed"; diff --git a/packages/cli/src/cli-executor.ts b/packages/cli/src/cli-executor.ts index 63f50a0b..ac8eccd0 100644 --- a/packages/cli/src/cli-executor.ts +++ b/packages/cli/src/cli-executor.ts @@ -1,9 +1,10 @@ -import { InternalComputerTranslator, type BatchReadResult, type BrowserFindCandidate } from "@onkernel/cua-agent"; +import { InternalComputerTranslator, type BatchReadResult, type BrowserFindCandidate, type BrowserRefState } from "@onkernel/cua-agent"; import { writeFile } from "node:fs/promises"; import { stderr, stdout } from "node:process"; import { emitCompact, type RunActionResult } from "./action/harness-runner"; import { exitCodeFor, type ActionResult, type DeterministicActionType } from "./action/result"; import { provisionForFlags, requireKernelApiKey, type HarnessCliFlags } from "./cli-harness"; +import { readNamedSessionRefs, writeNamedSessionRefs } from "./harness-named-sessions"; import { captureScreenshot, type CuaBrowserHandle } from "./harness-browser"; /** @@ -19,8 +20,10 @@ export type DeterministicRequest = | { action: "text" } | { action: "find"; query: string } | { action: "fill"; query: string; value: string } + | { action: "fill"; ref: string; value: string } | { action: "press"; keys: string[] } | { action: "click"; x: number; y: number } + | { action: "click"; ref: string } | { action: "tabs" } | { action: "screenshot"; out: string }; @@ -41,6 +44,11 @@ export function isCoordinatePair(rest: string[]): boolean { return rest.length === 2 && rest.every((token) => /^\d+$/.test(token)); } +/** An element ref minted by `cua snapshot` / `cua find`, e.g. `e12`. */ +export function isElementRef(token: string | undefined): token is string { + return token !== undefined && /^e\d+$/.test(token); +} + /** Roles `cua fill` will target. Everything else is left to `click`/`type`. */ const FILLABLE_ROLES: ReadonlySet = new Set([ "textbox", @@ -62,11 +70,24 @@ function parseToggleValue(raw: string): boolean { throw new Error(`checkbox/radio value must be true|false|1|0|checked|unchecked|on|off, got ${JSON.stringify(raw)}`); } +/** + * Value for a ref-addressed fill, where the element's role is unknown until + * the browser resolves it. Toggle words become booleans — lossless for text + * controls (the page-side fill stringifies) and correct for checkboxes. + * "1"/"0" stay strings so select options and numeric inputs keep their value. + */ +function refFillValue(raw: string): string | boolean { + const value = raw.trim().toLowerCase(); + if (["true", "checked", "on"].includes(value)) return true; + if (["false", "unchecked", "off"].includes(value)) return false; + return raw; +} + /** Resolve argv to a deterministic subcommand, or undefined when the model plane should handle it. */ export function deterministicActionFor(first: string | undefined, rest: string[]): DeterministicActionType | undefined { if (!first) return undefined; if (DETERMINISTIC_SUBCOMMANDS.has(first)) return first as DeterministicActionType; - if (first === "click" && isCoordinatePair(rest)) return "click"; + if (first === "click" && (isCoordinatePair(rest) || (rest.length === 1 && isElementRef(rest[0])))) return "click"; return undefined; } @@ -105,12 +126,13 @@ export function parseDeterministicArgs( return { action, query }; } case "fill": { - const query = (rest[0] ?? "").trim(); + const target = (rest[0] ?? "").trim(); const value = rest[1]; - if (!query || value === undefined || rest.length > 2) { - throw new Error('usage: cua fill "" ""'); + if (!target || value === undefined || rest.length > 2) { + throw new Error('usage: cua fill ""'); } - return { action, query, value }; + if (isElementRef(target)) return { action, ref: target, value }; + return { action, query: target, value }; } case "press": { const keys = rest.map((key) => key.trim()).filter((key) => key.length > 0); @@ -118,7 +140,8 @@ export function parseDeterministicArgs( return { action, keys }; } case "click": { - if (!isCoordinatePair(rest)) throw new Error("usage: cua click "); + if (rest.length === 1 && isElementRef(rest[0])) return { action, ref: rest[0] }; + if (!isCoordinatePair(rest)) throw new Error("usage: cua click | cua click "); return { action, x: Number(rest[0]), y: Number(rest[1]) }; } case "tabs": @@ -131,6 +154,16 @@ export function parseDeterministicArgs( } } +/** + * Persistence seam for element refs so they survive across invocations of + * the same named session. Absent for fresh (non `-s`) browsers, whose refs + * cannot outlive the browser anyway. + */ +export interface RefStateStore { + load(): Promise; + save(state: BrowserRefState): Promise; +} + /** Run a deterministic subcommand end to end: parse, provision/attach, execute, print, tear down. */ export async function runDeterministicCommand( action: DeterministicActionType, @@ -140,7 +173,14 @@ export async function runDeterministicCommand( const req = parseDeterministicArgs(action, rest, flags); const { apiKey, baseUrl } = requireKernelApiKey(); const provisioned = await provisionForFlags(flags, { kernelApiKey: apiKey, kernelBaseUrl: baseUrl }); - return runDeterministicOnHandle(req, provisioned.handle); + const name = flags.namedSession; + const refStore: RefStateStore | undefined = name + ? { + load: () => readNamedSessionRefs(name), + save: (state) => writeNamedSessionRefs(name, state), + } + : undefined; + return runDeterministicOnHandle(req, provisioned.handle, defaultTranslator, refStore); } /** Execute a parsed request against a browser handle. Split from provisioning for tests. */ @@ -148,12 +188,24 @@ export async function runDeterministicOnHandle( req: DeterministicRequest, handle: CuaBrowserHandle, createTranslator: (handle: CuaBrowserHandle) => InternalComputerTranslator = defaultTranslator, + refStore?: RefStateStore, ): Promise { const translator = createTranslator(handle); try { + if (refStore) { + const state = await refStore.load(); + if (state) translator.browser().importRefState(state); + } const res = await executeDeterministic(req, translator, handle); return emitCompact(res); } finally { + if (refStore) { + try { + await refStore.save(translator.browser().exportRefState()); + } catch (err) { + stderr.write(`[cua] cleanup warning: ${(err as Error).message}\n`); + } + } try { translator.dispose(); } catch (err) { @@ -205,6 +257,10 @@ async function executeDeterministic( } case "fill": { const executor = translator.browser(); + if ("ref" in req) { + await executor.execute({ type: "browser_fill", ref: req.ref, value: refFillValue(req.value) }); + return finish({ action: req.action, status: "ok", text: req.ref }); + } const candidates = await executor.findCandidates(req.query, undefined, FILLABLE_ROLES); if (candidates.length === 0) { return finish({ action: req.action, status: "not_found", text: `no fillable element matched ${JSON.stringify(req.query)}` }); @@ -227,6 +283,10 @@ async function executeDeterministic( await translator.executeBatch([{ type: "keypress", keys: req.keys }]); return finish({ action: req.action, status: "ok" }); case "click": + if ("ref" in req) { + await translator.browser().execute({ type: "browser_click", ref: req.ref }); + return finish({ action: req.action, status: "ok", text: req.ref }); + } await translator.executeBatch([{ type: "click", x: req.x, y: req.y }]); return finish({ action: req.action, status: "ok", coordinates: [req.x, req.y] }); case "tabs": { @@ -248,7 +308,11 @@ async function executeDeterministic( } } } catch (err) { - return finish({ action: req.action, status: "error", text: (err as Error).message }); + const message = (err as Error).message; + // A stale ref is "not found" (exit 1): the caller should re-snapshot, + // same as a failed description match — not an infrastructure error. + const status = /stale|not on the current page/i.test(message) ? "not_found" : "error"; + return finish({ action: req.action, status, text: message }); } } diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 0986dd74..f678ead2 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -23,9 +23,9 @@ Usage: cua snapshot [--filter interactive] cua find "" cua text - cua fill "" "" + cua fill "" cua press [key...] - cua click + cua click | cua click cua tabs cua screenshot [--out file|-] @@ -39,8 +39,10 @@ Usage: Subcommands above the blank line are model-free: they run directly against the browser (no LLM, no model API key; only KERNEL_API_KEY). \`click \` with exactly two integer arguments clicks those viewport coordinates without -a model; any other \`click\` argument is a natural-language description -resolved by the model. Exit codes: 0 ok, 1 not_found, 2 error. +a model, and \`click e12\` / \`fill e12 ...\` target an element ref minted by +\`snapshot\` or \`find\`; any other \`click\` argument is a natural-language +description resolved by the model. With \`-s \`, refs span invocations +(re-snapshot on a stale-ref error). Exit codes: 0 ok, 1 not_found, 2 error. Options: -p, --print Run a single prompt and exit diff --git a/packages/cli/src/harness-named-sessions.ts b/packages/cli/src/harness-named-sessions.ts index 663b91b9..c4a88902 100644 --- a/packages/cli/src/harness-named-sessions.ts +++ b/packages/cli/src/harness-named-sessions.ts @@ -1,4 +1,4 @@ -import type { KernelBrowser } from "@onkernel/cua-agent"; +import type { BrowserRefState, KernelBrowser } from "@onkernel/cua-agent"; import Kernel from "@onkernel/sdk"; import { mkdir, readdir, readFile, stat, unlink, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; @@ -72,11 +72,35 @@ export async function writeNamedSession(meta: NamedSessionMetadata): Promise { const path = sessionFilePath(name); + await unlink(refsFilePath(name)).catch(() => {}); if (!(await fileExists(path))) return false; await unlink(path); return true; } +function refsFilePath(name: string): string { + return join(namedSessionsDir(), `${name}.refs.json`); +} + +/** + * Element refs minted by one invocation (snapshot/find) survive to the next + * via this per-session file, so `cua -s x snapshot` then `cua -s x click e12` + * works across processes. Scoped to the named session's browser; stale state + * is caught by the executor's generation/self-heal machinery. + */ +export async function readNamedSessionRefs(name: string): Promise { + try { + return JSON.parse(await readFile(refsFilePath(name), "utf8")) as BrowserRefState; + } catch { + return undefined; + } +} + +export async function writeNamedSessionRefs(name: string, state: BrowserRefState): Promise { + await mkdir(namedSessionsDir(), { recursive: true }); + await writeFile(refsFilePath(name), JSON.stringify(state) + "\n", { mode: 0o600 }); +} + export async function listNamedSessions(): Promise { const dir = namedSessionsDir(); if (!(await fileExists(dir))) return []; diff --git a/packages/cli/test/cli-executor.test.ts b/packages/cli/test/cli-executor.test.ts index 8241e958..1c242627 100644 --- a/packages/cli/test/cli-executor.test.ts +++ b/packages/cli/test/cli-executor.test.ts @@ -1,4 +1,4 @@ -import type { BrowserExecutor, BrowserFindCandidate, InternalComputerTranslator as Translator } from "@onkernel/cua-agent"; +import type { BrowserExecutor, BrowserFindCandidate, BrowserRefState, InternalComputerTranslator as Translator } from "@onkernel/cua-agent"; import { InternalComputerTranslator } from "@onkernel/cua-agent"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; import { mkdtempSync } from "node:fs"; @@ -46,6 +46,8 @@ function baseFlags(overrides: Partial = {}): HarnessCliFlags { interface FakeExecutorState { actions: CuaBrowserAction[]; closed: number; + imported: BrowserRefState[]; + exported: number; } interface FakeExecutorScript { @@ -55,8 +57,14 @@ interface FakeExecutorScript { failWith?: Error; } +const FAKE_REF_STATE: BrowserRefState = { + refCounter: 7, + generations: [["F0", 0]], + refs: [["e7", { backendNodeId: 42, targetId: "F0", frameId: "F0", generation: 0, role: "button", name: "Save", nth: 0, cohort: 1 }]], +}; + function fakeExecutor(script: FakeExecutorScript = {}): { executor: BrowserExecutor; state: FakeExecutorState } { - const state: FakeExecutorState = { actions: [], closed: 0 }; + const state: FakeExecutorState = { actions: [], closed: 0, imported: [], exported: 0 }; const executor = { async execute(action: CuaBrowserAction) { if (script.failWith) throw script.failWith; @@ -72,6 +80,13 @@ function fakeExecutor(script: FakeExecutorScript = {}): { executor: BrowserExecu async currentUrl() { return script.url ?? ""; }, + importRefState(refState: BrowserRefState) { + state.imported.push(refState); + }, + exportRefState(): BrowserRefState { + state.exported += 1; + return FAKE_REF_STATE; + }, close() { state.closed += 1; }, @@ -135,11 +150,14 @@ describe("deterministicActionFor", () => { expect(deterministicActionFor("open", ["https://a"])).toBe("open"); expect(deterministicActionFor("screenshot", [])).toBe("screenshot"); expect(deterministicActionFor("click", ["10", "20"])).toBe("click"); + expect(deterministicActionFor("click", ["e12"])).toBe("click"); }); it("leaves model-mediated and free-form argv alone", () => { expect(deterministicActionFor("click", ["3", "dots", "menu"])).toBeUndefined(); expect(deterministicActionFor("click", ["sign in button"])).toBeUndefined(); + expect(deterministicActionFor("click", ["e12x"])).toBeUndefined(); + expect(deterministicActionFor("click", ["e12", "e13"])).toBeUndefined(); expect(deterministicActionFor("do", ["open hn"])).toBeUndefined(); expect(deterministicActionFor("observe", [])).toBeUndefined(); expect(deterministicActionFor("session", ["list"])).toBeUndefined(); @@ -196,6 +214,12 @@ describe("parseDeterministicArgs", () => { value: "a@b.c", }); expect(parseDeterministicArgs("click", ["10", "20"], baseFlags())).toEqual({ action: "click", x: 10, y: 20 }); + expect(parseDeterministicArgs("click", ["e12"], baseFlags())).toEqual({ action: "click", ref: "e12" }); + expect(parseDeterministicArgs("fill", ["e12", "a@b.c"], baseFlags())).toEqual({ + action: "fill", + ref: "e12", + value: "a@b.c", + }); }); it("runDeterministicCommand surfaces argv errors before touching the Kernel API", async () => { @@ -361,6 +385,57 @@ describe("runDeterministicOnHandle", () => { expect(body.actions[0]!.click_mouse).toMatchObject({ x: 10, y: 20 }); }); + it("click dispatches a CDP click on the ref", async () => { + const t = setup(); + const code = await runDeterministicOnHandle({ action: "click", ref: "e12" }, t.handle, t.createTranslator); + expect(code).toBe(0); + expect(stdoutLines.join("")).toBe("ok clicked e12\n"); + expect(t.state.actions).toEqual([{ type: "browser_click", ref: "e12" }]); + expect(t.kernel.batchCalls).toHaveLength(0); + }); + + it("click exits 1 when the ref is stale", async () => { + const t = setup({ failWith: new Error("ref e12 is stale or not on the current page. Call snapshot to get fresh refs.") }); + const code = await runDeterministicOnHandle({ action: "click", ref: "e12" }, t.handle, t.createTranslator); + expect(code).toBe(1); + expect(stdoutLines.join("")).toContain("not_found"); + }); + + it("fill fills that element, mapping toggle words to booleans", async () => { + const t = setup(); + expect(await runDeterministicOnHandle({ action: "fill", ref: "e7", value: "a@b.c" }, t.handle, t.createTranslator)).toBe(0); + expect(await runDeterministicOnHandle({ action: "fill", ref: "e8", value: "on" }, t.handle, t.createTranslator)).toBe(0); + expect(t.state.actions).toEqual([ + { type: "browser_fill", ref: "e7", value: "a@b.c" }, + { type: "browser_fill", ref: "e8", value: true }, + ]); + expect(stdoutLines.join("")).toBe("ok filled e7\nok filled e8\n"); + }); + + it("loads persisted ref state before executing and saves it after", async () => { + const t = setup(); + const saved: BrowserRefState[] = []; + const store = { + async load() { + return FAKE_REF_STATE; + }, + async save(state: BrowserRefState) { + saved.push(state); + }, + }; + const code = await runDeterministicOnHandle({ action: "click", ref: "e7" }, t.handle, t.createTranslator, store); + expect(code).toBe(0); + expect(t.state.imported).toEqual([FAKE_REF_STATE]); + expect(saved).toEqual([FAKE_REF_STATE]); + }); + + it("does not touch ref state without a store", async () => { + const t = setup(); + await runDeterministicOnHandle({ action: "click", ref: "e7" }, t.handle, t.createTranslator); + expect(t.state.imported).toEqual([]); + expect(t.state.exported).toBe(0); + }); + it("screenshot captures via the SDK and writes the file", async () => { const t = setup(); const out = join(mkdtempSync(join(tmpdir(), "cua-shot-")), "shot.png"); diff --git a/skills/cua-cli/SKILL.md b/skills/cua-cli/SKILL.md index 1daa9681..fd755328 100644 --- a/skills/cua-cli/SKILL.md +++ b/skills/cua-cli/SKILL.md @@ -22,13 +22,14 @@ These run directly against the browser (CDP or OS input) — no LLM involved, no | `cua snapshot [--filter interactive]` | Print the page's accessibility tree with element refs like `[e12]`. `--filter interactive` keeps only interactive elements. | the tree (multi-line) | 0 ok, 2 error | | `cua find ""` | Lexically score elements against the query, best first. | one match per line: `role "name" [eN]` (the quoted name is omitted when the element has none; role falls back to `node`) | 0 ok, 1 not_found, 2 error | | `cua text` | Print the page's visible text (`innerText`). | the text (multi-line) | 0 ok, 2 error | -| `cua fill "" ""` | Find the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton) and set its value. Exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. For checkbox/radio the value must be `true\|false\|1\|0\|checked\|unchecked\|on\|off` (anything else exits 2). | `ok filled ""` | 0 ok, 1 not_found, 2 error | +| `cua fill ""` | Set a form field's value. With a ref (`e12` from `snapshot`/`find`) it targets that exact element. With a query it finds the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton); exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. For checkbox/radio pass `true\|false\|checked\|unchecked\|on\|off` (query form also accepts `1\|0`). | `ok filled ""` (query) or `ok filled e12` (ref) | 0 ok, 1 not_found, 2 error | | `cua press [...]` | Send one key chord (e.g. `cua press ctrl l`, `cua press Return`). | `ok pressed` | 0 ok, 2 error | -| `cua click ` | OS-level click at viewport coordinates. Exactly two integer arguments — anything else routes to the model-mediated `click` below. | `ok clicked (x, y)` | 0 ok, 2 error | +| `cua click ` | OS-level click at viewport coordinates. Exactly two integer arguments. | `ok clicked (x, y)` | 0 ok, 2 error | +| `cua click ` | CDP click on an element ref from `snapshot`/`find`, e.g. `cua click e12`. Any other single `click` argument routes to the model-mediated `click` below. | `ok clicked e12` | 0 ok, 1 not_found (stale ref — re-snapshot), 2 error | | `cua tabs` | List open tabs. | one line per tab: `tab_id XXXX: "title" (url)` | 0 ok, 2 error | | `cua screenshot [--out ]` | Save a PNG (default `screenshot.png`). `--out -` writes the bytes to stdout. | the saved path; with `--out -`, stdout is exactly the PNG bytes (safe to pipe) | 0 ok, 2 error | -**Element refs are not valid across `cua` invocations.** Refs printed by `snapshot`/`find` (`[e12]`) live only for the process that minted them — there is no `fill ` form. Use `fill ""`, `click ""`, or `click ` instead; the refs are still useful as unique line handles when reading output. +**Element refs span invocations within a named session.** Refs printed by `snapshot`/`find` (`[e12]`) are persisted per `-s` session, so `cua -s x snapshot` then `cua -s x click e12` works. If the page changed in between, the ref self-heals when the element is still unambiguous; otherwise the command exits 1 with a stale-ref message — re-run `snapshot` and use a fresh ref. Without `-s` there is no shared browser, so refs from a previous invocation are meaningless. ### Model-mediated subcommands @@ -77,6 +78,8 @@ Inspecting a page mid-flow, entirely model-free: ```bash cua -s login snapshot --filter interactive # what can I interact with? cua -s login find "sign in button" # score elements against a query +cua -s login click e12 # click a ref from the snapshot/find output +cua -s login fill e7 "$EMAIL" # fill a ref directly cua -s login text # read the page's visible text cua -s login tabs # list open tabs ``` From 9caf2fc58d4e738a80e0528ee975a15f08d9390b Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:24:30 +0000 Subject: [PATCH 26/34] Validate runtime flags on deterministic commands; persist TUI mode/model switches - Deterministic subcommands reject invalid --mode/--native-tool values with the same usage error as harness-backed entry points, instead of silently ignoring them - TUI /mode and /model switches patch the named session's metadata so a later cua -s restores the selected plane and model instead of the startup values --- packages/cli/src/cli-executor.ts | 6 +++++- packages/cli/src/cli-harness.ts | 5 +++-- packages/cli/src/harness-named-sessions.ts | 12 ++++++++++++ packages/cli/src/tui/main.ts | 5 +++++ packages/cli/test/cli-executor.test.ts | 5 +++++ packages/cli/test/harness-named-sessions.test.ts | 16 ++++++++++++++++ 6 files changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/cli-executor.ts b/packages/cli/src/cli-executor.ts index ac8eccd0..b11dab39 100644 --- a/packages/cli/src/cli-executor.ts +++ b/packages/cli/src/cli-executor.ts @@ -3,7 +3,7 @@ import { writeFile } from "node:fs/promises"; import { stderr, stdout } from "node:process"; import { emitCompact, type RunActionResult } from "./action/harness-runner"; import { exitCodeFor, type ActionResult, type DeterministicActionType } from "./action/result"; -import { provisionForFlags, requireKernelApiKey, type HarnessCliFlags } from "./cli-harness"; +import { parseMode, parseNativeTool, provisionForFlags, requireKernelApiKey, type HarnessCliFlags } from "./cli-harness"; import { readNamedSessionRefs, writeNamedSessionRefs } from "./harness-named-sessions"; import { captureScreenshot, type CuaBrowserHandle } from "./harness-browser"; @@ -171,6 +171,10 @@ export async function runDeterministicCommand( flags: HarnessCliFlags, ): Promise { const req = parseDeterministicArgs(action, rest, flags); + // Deterministic commands ignore the runtime mode, but an invalid value is + // still a usage error, same as the harness-backed entry points. + parseMode(flags.mode); + parseNativeTool(flags.nativeTool); const { apiKey, baseUrl } = requireKernelApiKey(); const provisioned = await provisionForFlags(flags, { kernelApiKey: apiKey, kernelBaseUrl: baseUrl }); const name = flags.namedSession; diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 4cba3935..66e351d2 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -507,14 +507,14 @@ function providerBaseUrlOverride(provider: string): string | undefined { return value && value.length > 0 ? value : undefined; } -function parseMode(raw: string | undefined): CuaMode | undefined { +export function parseMode(raw: string | undefined): CuaMode | undefined { if (raw === undefined) return undefined; const value = raw.trim().toLowerCase(); if (value === "computer" || value === "browser" || value === "hybrid") return value; throw new Error(`invalid --mode value "${raw}"; expected one of: computer | browser | hybrid`); } -function parseNativeTool(raw: string | undefined): CuaNativeToolSpec | undefined { +export function parseNativeTool(raw: string | undefined): CuaNativeToolSpec | undefined { if (raw === undefined) return undefined; const value = raw.trim().toLowerCase(); // enable_zoom follows Anthropic's own recommendation for fine-grained @@ -599,6 +599,7 @@ export async function runInteractiveCommand( resumed: runtime.resolved?.resumed === true, transcriptPath: runtime.resolved?.transcriptPath, skipInitialScreenshot: runtime.resolved?.resumed === true, + namedSession: flags.namedSession, }); } finally { try { diff --git a/packages/cli/src/harness-named-sessions.ts b/packages/cli/src/harness-named-sessions.ts index c4a88902..80d30234 100644 --- a/packages/cli/src/harness-named-sessions.ts +++ b/packages/cli/src/harness-named-sessions.ts @@ -277,6 +277,18 @@ export async function recordSessionModel( await writeNamedSession(meta); } +/** Patch individual runtime fields (e.g. after a TUI /mode or /model switch) without clobbering the rest. */ +export async function updateNamedSessionRuntime(name: string, patch: { model?: string; mode?: string }): Promise { + const meta = await readNamedSession(name); + if (!meta) return; + const model = patch.model ?? meta.model; + const mode = patch.mode ?? meta.mode; + if (meta.model === model && meta.mode === mode) return; + meta.model = model; + meta.mode = mode; + await writeNamedSession(meta); +} + export function shortKernelId(id: string): string { return id.length > 10 ? `${id.slice(0, 8)}…` : id; } diff --git a/packages/cli/src/tui/main.ts b/packages/cli/src/tui/main.ts index ef665b47..25df0a11 100644 --- a/packages/cli/src/tui/main.ts +++ b/packages/cli/src/tui/main.ts @@ -25,6 +25,7 @@ import { homedir } from "node:os"; import type { ImageContent, Model } from "@onkernel/cua-ai"; import { captureScreenshot, type CuaBrowserHandle } from "../harness-browser"; import { resolveCuaModelRef } from "../harness-models"; +import { updateNamedSessionRuntime } from "../harness-named-sessions"; import type { ContextFile } from "../harness-skills"; import { openTuiDebugLog } from "./debug-log"; import { applyAndSummarizeImageProtocol } from "./diagnostics"; @@ -56,6 +57,8 @@ export interface InteractiveOptions { resumed?: boolean; /** Display path of the on-disk transcript, when one exists. */ transcriptPath?: string; + /** Named session (-s) backing this TUI; /mode and /model switches persist to it. */ + namedSession?: string; /** Enable extra TUI render diagnostics for manual repros. */ debugTui?: boolean; } @@ -481,6 +484,7 @@ async function applyModelCommand( try { const resolved = resolveCuaModelRef(ref); await opts.harness.setModel(resolved); + if (opts.namedSession) await updateNamedSessionRuntime(opts.namedSession, { model: resolved }); const model = opts.harness.getModel(); footer.update({ provider: model.provider, @@ -502,6 +506,7 @@ async function applyModeCommand(opts: InteractiveOptions, messages: MessageList, } try { await opts.harness.setMode(value); + if (opts.namedSession) await updateNamedSessionRuntime(opts.namedSession, { mode: value }); messages.addNotice(`mode → ${value}`); } catch (err) { messages.addError((err as Error).message); diff --git a/packages/cli/test/cli-executor.test.ts b/packages/cli/test/cli-executor.test.ts index 1c242627..6cfb791c 100644 --- a/packages/cli/test/cli-executor.test.ts +++ b/packages/cli/test/cli-executor.test.ts @@ -222,6 +222,11 @@ describe("parseDeterministicArgs", () => { }); }); + it("runDeterministicCommand rejects invalid --mode and --native-tool values", async () => { + await expect(runDeterministicCommand("url", [], baseFlags({ mode: "bogus" }))).rejects.toThrow(/invalid --mode/); + await expect(runDeterministicCommand("url", [], baseFlags({ nativeTool: "bogus" }))).rejects.toThrow(/invalid --native-tool/); + }); + it("runDeterministicCommand surfaces argv errors before touching the Kernel API", async () => { // KERNEL_API_KEY is unset in this suite: reaching provisioning would // throw "missing Kernel API key" instead of the usage error. diff --git a/packages/cli/test/harness-named-sessions.test.ts b/packages/cli/test/harness-named-sessions.test.ts index f8db3597..ce3ce444 100644 --- a/packages/cli/test/harness-named-sessions.test.ts +++ b/packages/cli/test/harness-named-sessions.test.ts @@ -7,6 +7,7 @@ import { type NamedSessionMetadata, readNamedSession, recordSessionModel, + updateNamedSessionRuntime, writeNamedSession, } from "../src/harness-named-sessions"; @@ -66,6 +67,21 @@ describe("named session model persistence", () => { expect(await readNamedSession("missing")).toBeUndefined(); }); + it("patches individual runtime fields without clobbering the rest", async () => { + await writeNamedSession(baseMeta({ model: "openai:gpt-5.5", mode: "computer", native_tool: "computer_20260701" })); + + await updateNamedSessionRuntime("foo", { mode: "browser" }); + let meta = await readNamedSession("foo"); + expect(meta?.mode).toBe("browser"); + expect(meta?.model).toBe("openai:gpt-5.5"); + expect(meta?.native_tool).toBe("computer_20260701"); + + await updateNamedSessionRuntime("foo", { model: "anthropic:claude-opus-4-8" }); + meta = await readNamedSession("foo"); + expect(meta?.model).toBe("anthropic:claude-opus-4-8"); + expect(meta?.mode).toBe("browser"); + }); + it("defaults flags from the stored session model when -m is omitted", () => { const meta = baseMeta({ model: "anthropic:claude-opus-4-8", mode: "hybrid", native_tool: "computer_20260701" }); const flags = applyNamedSessionDefaults(baseFlags(), meta); From 4a41199a2c9991e0d41676a9d855f0d2760ee45c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:34:03 +0000 Subject: [PATCH 27/34] Clear the self-navigation flag when a navigate command rejects A thrown Page.navigate / Page.navigateToHistoryEntry left the flag set, so the next main-frame frameNavigated consumed it and skipped ref invalidation. Both commands now run through a selfNavigate helper that disarms on rejection. --- packages/agent/src/translator/browser.ts | 24 +++++++++++++++---- .../agent/test/translator-browser.test.ts | 20 ++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 0bd377ae..8d3a2367 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -651,15 +651,15 @@ export class BrowserExecutor { ); const entry = history.entries[history.currentIndex + (direction === "back" ? -1 : 1)]; if (!entry) throw new Error(`cannot go ${direction}: no history entry`); - this.selfNavigations.add(targetId); - await this.cdp.send("Page.navigateToHistoryEntry", { entryId: entry.id }, session); + await this.selfNavigate(targetId, () => this.cdp.send("Page.navigateToHistoryEntry", { entryId: entry.id }, session)); this.invalidateRefs(targetId); return `Navigated ${direction}.\n${await this.tabContext(targetId)}`; } const url = normalizeGotoUrl(action.url); if (!url) throw new Error("invalid url"); - this.selfNavigations.add(targetId); - const { errorText } = await this.cdp.send<{ errorText?: string }>("Page.navigate", { url }, session); + const { errorText } = await this.selfNavigate(targetId, () => + this.cdp.send<{ errorText?: string }>("Page.navigate", { url }, session), + ); if (errorText) { this.selfNavigations.delete(targetId); throw new Error(`navigation to ${url} failed: ${errorText}`); @@ -668,6 +668,22 @@ export class BrowserExecutor { return `Navigated to ${url}.\n${await this.tabContext(targetId)}`; } + /** + * Run a navigation command with the self-navigation flag armed. The flag + * must be set before the command (frameNavigated can arrive first), and + * must not survive a rejected command — it would swallow the next real + * navigation's invalidation. + */ + private async selfNavigate(targetId: string, command: () => Promise): Promise { + this.selfNavigations.add(targetId); + try { + return await command(); + } catch (err) { + this.selfNavigations.delete(targetId); + throw err; + } + } + /** URL of the active tab. */ async currentUrl(): Promise { const targetId = await this.resolveTarget(); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 4c9c84f5..535f9ec8 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -110,6 +110,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { const frameTrees = new Map>(); const iframeFrameIds = new Map(); const autoAttachFrames: Array<{ targetId: string; sessionId: string }> = []; + const failMethods = new Set(); const emit = (event: FakeCdpEvent) => { for (const listener of listeners) listener(event); }; @@ -128,6 +129,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { }, send: async (method: string, params: Record = {}, sessionId?: string) => { sent.push({ method, params, sessionId }); + if (failMethods.has(method)) throw new Error(`${method} rejected`); switch (method) { case "Accessibility.getFullAXTree": return { nodes: treeFor(sessionId, params.frameId) }; @@ -191,6 +193,9 @@ function createFakeCdp(initialNodes: unknown[] = []) { const addAutoAttachFrame = (frame: { targetId: string; sessionId: string }) => { autoAttachFrames.push(frame); }; + const failOn = (method: string) => { + failMethods.add(method); + }; return { sent, emit, @@ -200,6 +205,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { setFrameTree, setIframeFrame, addAutoAttachFrame, + failOn, cdp: fake as unknown as CdpConnection, }; } @@ -254,6 +260,20 @@ describe("BrowserExecutor ref lifecycle", () => { expect(refsOf(executor).size).toBe(0); }); + it("does not let a rejected navigate suppress the next real navigation's invalidation", async () => { + const { cdp, emit, failOn } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + expect(refsOf(executor).size).toBe(1); + + failOn("Page.navigate"); + await expect(executor.execute({ type: "browser_navigate", url: "https://b.test" } as CuaBrowserAction)).rejects.toThrow(/rejected/); + + // A page-initiated navigation right after the failed command must still invalidate. + emit({ method: "Page.frameNavigated", params: { frame: { id: "F0" } }, sessionId: "session-1" }); + expect(refsOf(executor).size).toBe(0); + }); + it("invalidates refs on main-frame frameNavigated but not on subframe navigation", async () => { const { cdp, emit, sent } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); From 9314cc0db66de200167f98ded1b79b3f42d1ff3b Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 03:01:05 +0000 Subject: [PATCH 28/34] Fix session list crash on refs sidecars, focus on fill, and skill doc drift - Skip .refs.json sidecars and entries missing required fields in listNamedSessions so `cua session list` no longer crashes once a named session has persisted element refs - Focus the target element in browser_fill so a following `press Return` lands in the filled field and submits the form - Trim injected-script stacks from browser_fill errors to a single line - Correct SKILL.md: ref self-heal scope (navigation always invalidates), transcript JSONL record shape, session show/stop exit codes, and the models provider list (google/tzafon) --- packages/agent/src/translator/browser.ts | 5 +- .../agent/test/translator-browser.test.ts | 55 +++++++++++++++++++ packages/cli/src/harness-named-sessions.ts | 6 +- .../cli/test/harness-named-sessions.test.ts | 22 +++++++- skills/cua-cli/SKILL.md | 15 +++-- 5 files changed, 95 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 8d3a2367..4abcfd1c 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -598,7 +598,9 @@ export class BrowserExecutor { session, ); if (exceptionDetails) { - throw new Error(`browser_fill failed: ${exceptionDetails.exception?.description ?? "element rejected the value"}`); + const description = exceptionDetails.exception?.description ?? "element rejected the value"; + const message = description.split("\n", 1)[0]?.replace(/^[A-Za-z]*(?:Error|Exception): /, "") || "element rejected the value"; + throw new Error(`browser_fill failed: ${message}`); } } @@ -1109,6 +1111,7 @@ const FILL_FUNCTION = `function(value) { } else { throw new Error("element is not a form control"); } + el.focus(); el.dispatchEvent(new Event("input", { bubbles: true })); el.dispatchEvent(new Event("change", { bubbles: true })); }`; diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 535f9ec8..4f437ec2 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -517,6 +517,61 @@ describe("BrowserExecutor stale-ref self-healing", () => { }); }); +describe("BrowserExecutor fill", () => { + const FILL_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "textbox", name: "Email", backendDOMNodeId: 42, parentId: "1" }), + ]; + + it("focuses the element it fills before dispatching input events", async () => { + const { cdp, sent } = createFakeCdp(FILL_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + await executor.execute({ type: "browser_fill", ref: "e1", value: "a@b.c" } as CuaBrowserAction); + const call = sent.find((cmd) => cmd.method === "Runtime.callFunctionOn"); + const declaration = call?.params.functionDeclaration as string; + const fillFn = new Function(`return (${declaration})`)() as (value: unknown) => void; + const events: string[] = []; + const el = { + tagName: "INPUT", + type: "text", + value: "", + isContentEditable: false, + focus: () => events.push("focus"), + dispatchEvent: (event: Event) => events.push(event.type), + }; + fillFn.call(el, "hello"); + expect(el.value).toBe("hello"); + expect(events).toEqual(["focus", "input", "change"]); + }); + + it.each([ + ["Error: element is not a form control"], + ["TypeError: element is not a form control"], + ])("trims fill exception %j to a single line without the prefix or stack", async (firstLine) => { + const { cdp } = createFakeCdp(FILL_TREE); + const inner = cdp as unknown as { send: (method: string, params?: Record, sessionId?: string) => Promise }; + const wrapped = { + ...cdp, + send: async (method: string, params?: Record, sessionId?: string) => { + if (method === "Runtime.callFunctionOn") { + return { + exceptionDetails: { + exception: { description: `${firstLine}\n at HTMLAnchorElement. (:20:9)` }, + }, + }; + } + return inner.send(method, params, sessionId); + }, + } as unknown as CdpConnection; + const executor = new BrowserExecutor(wrapped); + await snapshotText(executor); + await expect(executor.execute({ type: "browser_fill", ref: "e1", value: "x" } as CuaBrowserAction)).rejects.toThrow( + /^browser_fill failed: element is not a form control$/, + ); + }); +}); + describe("BrowserExecutor cursor-pointer hints", () => { const POINTER_TREE = [ ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), diff --git a/packages/cli/src/harness-named-sessions.ts b/packages/cli/src/harness-named-sessions.ts index 80d30234..7182c0f2 100644 --- a/packages/cli/src/harness-named-sessions.ts +++ b/packages/cli/src/harness-named-sessions.ts @@ -107,10 +107,12 @@ export async function listNamedSessions(): Promise { const entries = await readdir(dir); const out: NamedSessionMetadata[] = []; for (const entry of entries) { - if (!entry.endsWith(".json")) continue; + if (!entry.endsWith(".json") || entry.endsWith(".refs.json")) continue; try { const raw = await readFile(join(dir, entry), "utf8"); - out.push(JSON.parse(raw) as NamedSessionMetadata); + const meta = JSON.parse(raw) as NamedSessionMetadata; + if (typeof meta.name !== "string" || typeof meta.kernel_session_id !== "string" || typeof meta.created_at !== "number") continue; + out.push(meta); } catch { // skip unreadable / malformed entries } diff --git a/packages/cli/test/harness-named-sessions.test.ts b/packages/cli/test/harness-named-sessions.test.ts index ce3ce444..0446b92b 100644 --- a/packages/cli/test/harness-named-sessions.test.ts +++ b/packages/cli/test/harness-named-sessions.test.ts @@ -1,14 +1,16 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { applyNamedSessionDefaults, type HarnessCliFlags } from "../src/cli-harness"; import { + listNamedSessions, type NamedSessionMetadata, readNamedSession, recordSessionModel, updateNamedSessionRuntime, writeNamedSession, + writeNamedSessionRefs, } from "../src/harness-named-sessions"; const originalXdg = process.env.XDG_DATA_HOME; @@ -96,4 +98,22 @@ describe("named session model persistence", () => { expect(flags.model).toBe("openai:gpt-5.5"); expect(flags.mode).toBe("browser"); }); + + it("excludes refs sidecar files from listNamedSessions", async () => { + await writeNamedSession(baseMeta()); + await writeNamedSessionRefs("foo", { refCounter: 3, generations: [], refs: [] }); + const sessions = await listNamedSessions(); + expect(sessions).toHaveLength(1); + expect(sessions[0]?.name).toBe("foo"); + }); + + it("skips metadata entries missing required fields", async () => { + await writeNamedSession(baseMeta()); + const dir = join(process.env.XDG_DATA_HOME!, "cua", "named-sessions"); + writeFileSync(join(dir, "bogus.json"), JSON.stringify({ unrelated: true })); + writeFileSync(join(dir, "no-age.json"), JSON.stringify({ name: "no-age", kernel_session_id: "k1" })); + const sessions = await listNamedSessions(); + expect(sessions).toHaveLength(1); + expect(sessions[0]?.name).toBe("foo"); + }); }); diff --git a/skills/cua-cli/SKILL.md b/skills/cua-cli/SKILL.md index fd755328..c45dea7a 100644 --- a/skills/cua-cli/SKILL.md +++ b/skills/cua-cli/SKILL.md @@ -22,14 +22,14 @@ These run directly against the browser (CDP or OS input) — no LLM involved, no | `cua snapshot [--filter interactive]` | Print the page's accessibility tree with element refs like `[e12]`. `--filter interactive` keeps only interactive elements. | the tree (multi-line) | 0 ok, 2 error | | `cua find ""` | Lexically score elements against the query, best first. | one match per line: `role "name" [eN]` (the quoted name is omitted when the element has none; role falls back to `node`) | 0 ok, 1 not_found, 2 error | | `cua text` | Print the page's visible text (`innerText`). | the text (multi-line) | 0 ok, 2 error | -| `cua fill ""` | Set a form field's value. With a ref (`e12` from `snapshot`/`find`) it targets that exact element. With a query it finds the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton); exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. For checkbox/radio pass `true\|false\|checked\|unchecked\|on\|off` (query form also accepts `1\|0`). | `ok filled ""` (query) or `ok filled e12` (ref) | 0 ok, 1 not_found, 2 error | +| `cua fill ""` | Set a form field's value. With a ref (`e12` from `snapshot`/`find`) it targets that exact element. With a query it finds the unique best-matching form field (textbox, searchbox, combobox, checkbox, radio, listbox, spinbutton); exit 1 with the tied matches listed if the query is ambiguous — tighten it and retry. For checkbox/radio pass `true\|false\|checked\|unchecked\|on\|off` (query form also accepts `1\|0`). `fill` leaves the field focused, so a following `cua press Return` submits the form. | `ok filled ""` (query) or `ok filled e12` (ref) | 0 ok, 1 not_found, 2 error | | `cua press [...]` | Send one key chord (e.g. `cua press ctrl l`, `cua press Return`). | `ok pressed` | 0 ok, 2 error | | `cua click ` | OS-level click at viewport coordinates. Exactly two integer arguments. | `ok clicked (x, y)` | 0 ok, 2 error | | `cua click ` | CDP click on an element ref from `snapshot`/`find`, e.g. `cua click e12`. Any other single `click` argument routes to the model-mediated `click` below. | `ok clicked e12` | 0 ok, 1 not_found (stale ref — re-snapshot), 2 error | | `cua tabs` | List open tabs. | one line per tab: `tab_id XXXX: "title" (url)` | 0 ok, 2 error | | `cua screenshot [--out ]` | Save a PNG (default `screenshot.png`). `--out -` writes the bytes to stdout. | the saved path; with `--out -`, stdout is exactly the PNG bytes (safe to pipe) | 0 ok, 2 error | -**Element refs span invocations within a named session.** Refs printed by `snapshot`/`find` (`[e12]`) are persisted per `-s` session, so `cua -s x snapshot` then `cua -s x click e12` works. If the page changed in between, the ref self-heals when the element is still unambiguous; otherwise the command exits 1 with a stale-ref message — re-run `snapshot` and use a fresh ref. Without `-s` there is no shared browser, so refs from a previous invocation are meaningless. +**Element refs span invocations within a named session.** Refs printed by `snapshot`/`find` (`[e12]`) are persisted per `-s` session, so `cua -s x snapshot` then `cua -s x click e12` works. Refs self-heal across in-page DOM changes when the element is still unambiguous, but any navigation — including reloading the same URL — invalidates them; the command then exits 1 with a stale-ref message — re-run `snapshot` and use a fresh ref. Without `-s` there is no shared browser, so refs from a previous invocation are meaningless. ### Model-mediated subcommands @@ -47,7 +47,10 @@ Useful flags: - `-m ` — pick the LLM for model-mediated subcommands (default `gpt-5.5`). Other good picks: `claude-opus-4-7`, `gemini-3-flash-preview`, `n1.5-latest`. - `cua models` — list supported `-m` values and their providers; filter - with `cua models -p openai|anthropic|gemini|yutori`. + with `cua models -p openai|anthropic|google|yutori|tzafon` (`gemini` is + accepted as an alias for `google`). Model refs print as `provider:model` + (e.g. `google:gemini-3-flash-preview`); `-m` accepts either the full ref or + a bare model id that matches exactly one entry. - `--max-steps ` — bound the agent loop on `cua do` (default 3). - `--filter interactive` — restrict `cua snapshot` to interactive elements. - `--profile ` — load a Kernel browser profile for cookies / @@ -91,6 +94,10 @@ cua session list # tab-formatted: NAME, KERNEL_ID, AGE, cua session show login # full JSON metadata ``` +`cua session show ` and `cua session stop ` exit 1 when the named +session does not exist (`no named session ""`); other session failures +exit 2. + Pass `--profile` when starting the named session; later `cua -s login ...` calls attach to that same browser, so they do not need the profile flag. @@ -114,7 +121,7 @@ The default root is `$XDG_DATA_HOME/cua/sessions` or after the first model-mediated `-s` call (`click ""`, `type`, `observe`, `do`, `--print`, or a TUI attach). -Each line is a JSON object with one of these `role` values: `user`, `assistant`, `toolResult`. There's also a custom `cua-browser` entry written once per session with `kernel_session_id` / `live_url` / `profile_id`. +Each line is a pi `SessionManager` record with a top-level `type` of `session`, `message`, or `custom`. Conversation entries have `type: "message"` with the role nested at `.message.role` (`user`, `assistant`, or `toolResult`). There's also a `type: "custom"` entry with `customType: "cua-browser"` written once per session whose `data` carries `sessionId` / `liveUrl` (and `profileId` when a profile is loaded). Use `cua --print -o jsonl "..."` only when you need live stdout events while a run is happening. That stream is a compact event feed (`tool_call`, From d52e723b522265a46304b992bca23dfac47bd6d7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:22:25 +0000 Subject: [PATCH 29/34] Bump cua-ai 0.5.0, cua-agent 0.5.0, cua-cli 0.3.0 for browser modes release --- package-lock.json | 12 ++++++------ packages/agent/CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ packages/agent/package.json | 4 ++-- packages/ai/CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ packages/ai/package.json | 2 +- packages/cli/package.json | 6 +++--- 6 files changed, 77 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 75af476a..ef407244 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6111,12 +6111,12 @@ }, "packages/agent": { "name": "@onkernel/cua-agent", - "version": "0.4.0", + "version": "0.5.0", "license": "MIT", "dependencies": { "@earendil-works/pi-agent-core": "0.80.3", "@earendil-works/pi-ai": "0.80.3", - "@onkernel/cua-ai": "0.4.0", + "@onkernel/cua-ai": "0.5.0", "@onkernel/sdk": "0.49.0", "sharp": "^0.34.5" }, @@ -6127,7 +6127,7 @@ }, "packages/ai": { "name": "@onkernel/cua-ai", - "version": "0.4.0", + "version": "0.5.0", "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "0.80.3", @@ -6141,13 +6141,13 @@ }, "packages/cli": { "name": "@onkernel/cua-cli", - "version": "0.2.0", + "version": "0.3.0", "license": "MIT", "dependencies": { "@earendil-works/pi-coding-agent": "0.80.3", "@earendil-works/pi-tui": "0.80.3", - "@onkernel/cua-agent": "0.4.0", - "@onkernel/cua-ai": "0.4.0", + "@onkernel/cua-agent": "0.5.0", + "@onkernel/cua-ai": "0.5.0", "@onkernel/sdk": "0.49.0" }, "bin": { diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index e8c88f6a..7d131565 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## 0.5.0 - 2026-07-09 + +Breaking: adds the browser action plane and runtime mode switching; removes +the `javascriptExec` and `computerUseExtra` options. + +- New `BrowserExecutor`: drives the browser plane over CDP. Accessibility + snapshots with element refs (`[e12]`) and node states + (checked/expanded/disabled/value/…), iframe and OOPIF stitching with + per-frame session-aware refs, StaticText dedupe and wrapper collapsing, an + unchanged-snapshot short-circuit, lexical `find`, `fill`, CDP navigation + and tab management, and a JavaScript dialog guard. Refs invalidate on real + navigations (`Page.frameNavigated`), self-heal via (role, name, nth) when + the page changes but the element is still unambiguous, and the ref table is + bounded (per-target cap, generation sweeps). `exportRefState()` / + `importRefState()` persist refs across processes against the same browser. +- `CuaAgent` and `CuaAgentHarness` accept `mode` (`"computer"` | `"browser"` + | `"hybrid"`) and `nativeTool`, and support runtime plane switching via + `setMode()` / `getMode()`; mode switches preserve the requested activation + state of surviving tools, dispose the previous translator's CDP + connection, and roll back cleanly on failure (as does `setModel()`). +- Post-action grounding captures and the navigation helper are mode-aware: + browser mode grounds on the viewport and routes navigation through CDP + (browser and hybrid modes both route `computer_use_extra` navigation over + the browser plane so refs invalidate correctly). +- New `cursorHints` option marks cursor:pointer elements as clickable hints + in browser-mode snapshots (default off). +- Breaking: `computerUseExtra` is removed — the `computer_use_extra` + navigation helper is always registered (deduped by name against caller + executors). `javascriptExec` is removed — see `@onkernel/cua-ai` 0.5.0. +- Breaking: `BrowserExecutor` is constructed from a CDP WebSocket URL and + owns its connection; `createBrowserExecutor` remains the injection seam on + the translator options. +- Updated `@onkernel/cua-ai` to 0.5.0. + ## 0.4.0 - 2026-07-07 Breaking: follows pi-agent-core 0.80's `Models`-based harness. diff --git a/packages/agent/package.json b/packages/agent/package.json index 8c0b0b78..94a5b25d 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-agent", - "version": "0.4.0", + "version": "0.5.0", "description": "Kernel browser computer-use Agent and AgentHarness classes built on pi-agent-core", "license": "MIT", "type": "module", @@ -42,7 +42,7 @@ "dependencies": { "@earendil-works/pi-agent-core": "0.80.3", "@earendil-works/pi-ai": "0.80.3", - "@onkernel/cua-ai": "0.4.0", + "@onkernel/cua-ai": "0.5.0", "@onkernel/sdk": "0.49.0", "sharp": "^0.34.5" }, diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 7f0dd654..73a353bd 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## 0.5.0 - 2026-07-09 + +Breaking: introduces action planes (modes) and Anthropic native tools; +removes the `javascriptExec` option and `CUA_DEFAULT_BROWSER_ACTION_TYPES`. + +- New `mode` option (`"computer"` | `"browser"` | `"hybrid"`, exported as + `CuaMode`) on `resolveCuaRuntimeSpec`, `computerTools`, and the executor + builders selects which action plane(s) the model sees. `computer` is the + pre-modes default and stays byte-compatible. `browser` exposes the new + browser-plane canonical actions; `hybrid` exposes both planes deduplicated + to one tool per capability, with browser actions restricted to element refs + so the OS screenshot is the single coordinate frame. +- New browser-plane canonical actions (`CUA_BROWSER_ACTION_TYPES`): + `browser_snapshot`, `browser_find`, `browser_text`, `browser_click`, + `browser_fill`, `browser_scroll_to`, `browser_navigate`, + `browser_list_tabs`, `browser_new_tab`, `browser_screenshot`, + `browser_evaluate`, and friends, with per-mode tool naming + (`cuaToolNameForAction`), descriptions, schemas, and system prompts. +- New `nativeTool` option drives Anthropic models through their native + computer-use declarations: `computer_20260701` (computer mode, with + `enable_zoom`) and `browser_20260701` (browser mode) behind + `anthropic-beta: computer-use-2026-07-01`. +- Breaking: the `javascriptExec` option is gone. `browser_evaluate` is part + of the default browser/hybrid action sets, and native `browser_20260701` + declarations default `enable_javascript_exec` to true (an explicit value on + the spec wins). Opt out by passing an explicit `actions` list or native + tool spec. +- Breaking: `CUA_DEFAULT_BROWSER_ACTION_TYPES` is removed; the default + browser set is `CUA_BROWSER_ACTION_TYPES`. +- New `zoom` computer action (cropped display inspection). + ## 0.4.0 - 2026-07-07 Breaking: adopts pi-ai 0.80's instance-based `Models` API and drops the diff --git a/packages/ai/package.json b/packages/ai/package.json index 2e5bc4d6..09ed707d 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-ai", - "version": "0.4.0", + "version": "0.5.0", "description": "Kernel-curated computer-use model access built on pi-ai", "license": "MIT", "type": "module", diff --git a/packages/cli/package.json b/packages/cli/package.json index ae251bb8..f8db9380 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-cli", - "version": "0.2.0", + "version": "0.3.0", "description": "Kernel-cloud-browser computer-use TUI built on @onkernel/cua-agent and pi-tui", "license": "MIT", "type": "module", @@ -36,8 +36,8 @@ "dependencies": { "@earendil-works/pi-coding-agent": "0.80.3", "@earendil-works/pi-tui": "0.80.3", - "@onkernel/cua-agent": "0.4.0", - "@onkernel/cua-ai": "0.4.0", + "@onkernel/cua-agent": "0.5.0", + "@onkernel/cua-ai": "0.5.0", "@onkernel/sdk": "0.49.0" }, "devDependencies": { From aa1d034d4a041cd169ca65fef918129ca6ea392c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:28:38 +0000 Subject: [PATCH 30/34] Enable cursor hints by default in browser mode Browser-mode snapshots are the model's only page view, and cursor:pointer hints surface clickable elements with no ARIA role (the common div-soup case). The scan is already noise-bounded (skips native interactive subtrees, dedupes inherited cursors, caps at 100) and browser mode runs page-world JS by default anyway (browser_evaluate, fill), so the opt-in gate bought nothing. cursorHints: false opts out; hybrid/computer modes still never scan. --- packages/agent/CHANGELOG.md | 5 +++-- packages/agent/src/agent.ts | 4 ++-- packages/agent/src/tools.ts | 2 +- packages/agent/src/translator/translator.ts | 8 ++++++-- packages/agent/test/translator-browser.test.ts | 12 +++++++----- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 7d131565..0fb164f8 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -24,8 +24,9 @@ the `javascriptExec` and `computerUseExtra` options. browser mode grounds on the viewport and routes navigation through CDP (browser and hybrid modes both route `computer_use_extra` navigation over the browser plane so refs invalidate correctly). -- New `cursorHints` option marks cursor:pointer elements as clickable hints - in browser-mode snapshots (default off). +- Browser-mode snapshots mark cursor:pointer elements as clickable hints by + default (surfacing clickable div-soup with no ARIA role); pass + `cursorHints: false` to opt out. Never runs in hybrid/computer mode. - Breaking: `computerUseExtra` is removed — the `computer_use_extra` navigation helper is always registered (deduped by name against caller executors). `javascriptExec` is removed — see `@onkernel/cua-ai` 0.5.0. diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 1f9b30d0..e3daaedc 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -76,7 +76,7 @@ export type CuaAgentOptions = Omit & { mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only; default false. */ + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only, on by default there; pass false to opt out. */ cursorHints?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; @@ -114,7 +114,7 @@ export type CuaAgentHarnessOptions< mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only; default false. */ + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only, on by default there; pass false to opt out. */ cursorHints?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 1a10ad2f..453f51f6 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -25,7 +25,7 @@ export interface ComputerToolOptions { screenshot?: CuaScreenshotSpec; /** Action plane(s) in play; controls whether the post-action fallback capture is the OS display or the viewport. Default "computer". */ mode?: CuaMode; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode. Default false. */ + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode, where it is on by default; pass false to opt out. */ cursorHints?: boolean; playwright?: boolean; } diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index f38f156c..bc7714af 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -35,7 +35,7 @@ export interface InternalComputerTranslatorOptions { screenshot?: CuaScreenshotSpec; /** Action plane(s) in play; browser-executor extras like cursor hints are gated to "browser". */ mode?: CuaMode; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode. Default false. */ + /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode, where it is on by default; pass false to opt out. */ cursorHints?: boolean; /** Browser executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ createBrowserExecutor?: (cdpWsUrl: string, options: BrowserExecutorOptions) => BrowserExecutor; @@ -59,7 +59,11 @@ export class InternalComputerTranslator { this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; this.cdpWsUrl = opts.browser.cdp_ws_url; - this.browserExecutorOptions = { cursorHints: opts.cursorHints === true && opts.mode === "browser" }; + // On by default in browser mode: snapshots are the model's only eyes + // there, and cursor hints surface clickable div-soup that has no ARIA + // role. Never in hybrid/computer mode, where the OS screenshot grounds + // clickability and the browser plane stays scan-free. + this.browserExecutorOptions = { cursorHints: opts.cursorHints !== false && opts.mode === "browser" }; this.browserExecutorFactory = opts.createBrowserExecutor ?? ((cdpWsUrl, options) => new BrowserExecutor(cdpWsUrl, options)); } diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 4f437ec2..4221467c 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -598,8 +598,8 @@ describe("BrowserExecutor cursor-pointer hints", () => { expect(sent.some((cmd) => cmd.method === "Runtime.releaseObjectGroup")).toBe(true); }); - it("only enables cursor hints on the executor in browser mode", () => { - const recordedFor = (mode?: "computer" | "browser" | "hybrid") => { + it("enables cursor hints by default in browser mode only, with an explicit opt-out", () => { + const recordedFor = (mode?: "computer" | "browser" | "hybrid", cursorHints?: boolean) => { const recorded: BrowserExecutorOptions[] = []; const { executor } = createFakeBrowserExecutor(); const { client } = createClient(); @@ -607,7 +607,7 @@ describe("BrowserExecutor cursor-pointer hints", () => { browser, client, mode, - cursorHints: true, + cursorHints, createBrowserExecutor: (_cdpWsUrl, options) => { recorded.push(options); return executor; @@ -617,8 +617,10 @@ describe("BrowserExecutor cursor-pointer hints", () => { return recorded[0]!; }; expect(recordedFor("browser").cursorHints).toBe(true); - expect(recordedFor("hybrid").cursorHints).toBe(false); - expect(recordedFor("computer").cursorHints).toBe(false); + expect(recordedFor("browser", true).cursorHints).toBe(true); + expect(recordedFor("browser", false).cursorHints).toBe(false); + expect(recordedFor("hybrid", true).cursorHints).toBe(false); + expect(recordedFor("computer", true).cursorHints).toBe(false); expect(recordedFor(undefined).cursorHints).toBe(false); }); }); From a7ce18c9008f9292a039f99a90dc315121419330 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:36:11 +0000 Subject: [PATCH 31/34] Make cursor hints an unconditional part of browser snapshots Snapshots always mark cursor:pointer elements with no interactive ARIA role as clickable hints; the cursorHints option is gone. The scan is noise-bounded and runs in the same CDP plane snapshots already use, so there was no configuration worth exposing. Also correct the 0.5.0 changelog entries: javascriptExec, cursorHints, and CUA_DEFAULT_BROWSER_ACTION_TYPES never shipped in a release, so their removal is not breaking; only the computerUseExtra removal is. --- packages/agent/CHANGELOG.md | 25 +++++------- packages/agent/src/agent.ts | 10 ----- packages/agent/src/tools.ts | 2 - packages/agent/src/translator/browser.ts | 12 +----- packages/agent/src/translator/translator.ts | 18 +++------ .../agent/test/translator-browser.test.ts | 40 +------------------ packages/ai/CHANGELOG.md | 9 ++--- 7 files changed, 21 insertions(+), 95 deletions(-) diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 0fb164f8..a6e0aa37 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,15 +2,17 @@ ## 0.5.0 - 2026-07-09 -Breaking: adds the browser action plane and runtime mode switching; removes -the `javascriptExec` and `computerUseExtra` options. +Adds the browser action plane and runtime mode switching. Breaking: the +`computerUseExtra` option is removed — the `computer_use_extra` navigation +helper is always registered. - New `BrowserExecutor`: drives the browser plane over CDP. Accessibility - snapshots with element refs (`[e12]`) and node states - (checked/expanded/disabled/value/…), iframe and OOPIF stitching with - per-frame session-aware refs, StaticText dedupe and wrapper collapsing, an - unchanged-snapshot short-circuit, lexical `find`, `fill`, CDP navigation - and tab management, and a JavaScript dialog guard. Refs invalidate on real + snapshots with element refs (`[e12]`), node states + (checked/expanded/disabled/value/…), and cursor:pointer clickable hints + for elements with no interactive ARIA role; iframe and OOPIF stitching + with per-frame session-aware refs; StaticText dedupe and wrapper + collapsing; an unchanged-snapshot short-circuit; lexical `find`, `fill`, + CDP navigation and tab management; and a JavaScript dialog guard. Refs invalidate on real navigations (`Page.frameNavigated`), self-heal via (role, name, nth) when the page changes but the element is still unambiguous, and the ref table is bounded (per-target cap, generation sweeps). `exportRefState()` / @@ -24,15 +26,6 @@ the `javascriptExec` and `computerUseExtra` options. browser mode grounds on the viewport and routes navigation through CDP (browser and hybrid modes both route `computer_use_extra` navigation over the browser plane so refs invalidate correctly). -- Browser-mode snapshots mark cursor:pointer elements as clickable hints by - default (surfacing clickable div-soup with no ARIA role); pass - `cursorHints: false` to opt out. Never runs in hybrid/computer mode. -- Breaking: `computerUseExtra` is removed — the `computer_use_extra` - navigation helper is always registered (deduped by name against caller - executors). `javascriptExec` is removed — see `@onkernel/cua-ai` 0.5.0. -- Breaking: `BrowserExecutor` is constructed from a CDP WebSocket URL and - owns its connection; `createBrowserExecutor` remains the injection seam on - the translator options. - Updated `@onkernel/cua-ai` to 0.5.0. ## 0.4.0 - 2026-07-07 diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index e3daaedc..456d6876 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -76,8 +76,6 @@ export type CuaAgentOptions = Omit & { mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only, on by default there; pass false to opt out. */ - cursorHints?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; }; @@ -114,8 +112,6 @@ export type CuaAgentHarnessOptions< mode?: CuaMode; /** Drive the model through a provider-native tool declaration (validated against `mode`). */ nativeTool?: CuaNativeToolSpec; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Browser mode only, on by default there; pass false to opt out. */ - cursorHints?: boolean; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */ @@ -141,7 +137,6 @@ class CuaRuntimeController { extraTools?: AgentTool[]; mode?: CuaMode; nativeTool?: CuaNativeToolSpec; - cursorHints?: boolean; playwright?: boolean; onPayload?: SimpleStreamOptions["onPayload"]; }, @@ -230,7 +225,6 @@ class CuaRuntimeController { coordinateSystem: this.runtimeSpec.coordinateSystem, screenshot: this.runtimeSpec.screenshot, mode: this.runtimeSpec.mode, - cursorHints: this.options.cursorHints, }); } } @@ -264,7 +258,6 @@ export class CuaAgent extends Agent { extraTools, mode, nativeTool, - cursorHints, playwright, ...agentOptions } = options; @@ -275,7 +268,6 @@ export class CuaAgent extends Agent { extraTools, mode, nativeTool, - cursorHints, playwright, onPayload, }); @@ -411,7 +403,6 @@ export class CuaAgentHarness< extraTools, mode, nativeTool, - cursorHints, playwright, systemPrompt, onPayload, @@ -425,7 +416,6 @@ export class CuaAgentHarness< extraTools, mode, nativeTool, - cursorHints, playwright, onPayload, }); diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 453f51f6..9a227434 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -25,8 +25,6 @@ export interface ComputerToolOptions { screenshot?: CuaScreenshotSpec; /** Action plane(s) in play; controls whether the post-action fallback capture is the OS display or the viewport. Default "computer". */ mode?: CuaMode; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode, where it is on by default; pass false to opt out. */ - cursorHints?: boolean; playwright?: boolean; } diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 4abcfd1c..7f2288b1 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -86,11 +86,6 @@ export interface BrowserFindCandidate { score: number; } -export interface BrowserExecutorOptions { - /** Mark elements whose computed cursor is "pointer" as clickable hints in snapshots. Default false. */ - cursorHints?: boolean; -} - /** * Serializable ref state, so refs minted in one process (e.g. a `cua * snapshot` invocation) can be resolved in a later one against the same @@ -145,12 +140,9 @@ export class BrowserExecutor { private readonly dialogNotes: string[] = []; private refCounter = 0; private activeTargetId?: string; - private readonly cursorHints: boolean; - private readonly cdp: CdpConnection; - constructor(cdp: string | CdpConnection, options: BrowserExecutorOptions = {}) { - this.cursorHints = options.cursorHints ?? false; + constructor(cdp: string | CdpConnection) { this.cdp = typeof cdp === "string" ? new CdpConnection(cdp) : cdp; this.cdp.onEvent((event) => this.handleCdpEvent(event)); } @@ -335,7 +327,7 @@ export class BrowserExecutor { generation: this.generation(frameKey), interactiveOnly, nthIndex: buildNthIndex(nodes), - cursorIds: this.cursorHints && frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, + cursorIds: frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, }; const stitches = frameKey === targetId ? await this.stitchFrames(nodes, targetId, pageSession, interactiveOnly) : new Map(); const lines: RenderedLine[] = []; diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index bc7714af..0a2fdab4 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -22,7 +22,7 @@ import { type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import sharp from "sharp"; -import { BrowserExecutor, type BrowserExecutorOptions } from "./browser"; +import { BrowserExecutor } from "./browser"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult } from "./types"; @@ -35,10 +35,8 @@ export interface InternalComputerTranslatorOptions { screenshot?: CuaScreenshotSpec; /** Action plane(s) in play; browser-executor extras like cursor hints are gated to "browser". */ mode?: CuaMode; - /** Mark cursor:pointer elements as clickable hints in browser snapshots. Only honored in "browser" mode, where it is on by default; pass false to opt out. */ - cursorHints?: boolean; /** Browser executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ - createBrowserExecutor?: (cdpWsUrl: string, options: BrowserExecutorOptions) => BrowserExecutor; + createBrowserExecutor?: (cdpWsUrl: string) => BrowserExecutor; } export class InternalComputerTranslator { @@ -48,8 +46,7 @@ export class InternalComputerTranslator { private readonly screenshotSpec?: CuaScreenshotSpec; private readonly viewport: { width: number; height: number }; private readonly cdpWsUrl?: string; - private readonly browserExecutorOptions: BrowserExecutorOptions; - private readonly browserExecutorFactory: (cdpWsUrl: string, options: BrowserExecutorOptions) => BrowserExecutor; + private readonly browserExecutorFactory: (cdpWsUrl: string) => BrowserExecutor; private browserExecutor?: BrowserExecutor; constructor(opts: InternalComputerTranslatorOptions) { @@ -59,12 +56,7 @@ export class InternalComputerTranslator { this.screenshotSpec = opts.screenshot; this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; this.cdpWsUrl = opts.browser.cdp_ws_url; - // On by default in browser mode: snapshots are the model's only eyes - // there, and cursor hints surface clickable div-soup that has no ARIA - // role. Never in hybrid/computer mode, where the OS screenshot grounds - // clickability and the browser plane stays scan-free. - this.browserExecutorOptions = { cursorHints: opts.cursorHints !== false && opts.mode === "browser" }; - this.browserExecutorFactory = opts.createBrowserExecutor ?? ((cdpWsUrl, options) => new BrowserExecutor(cdpWsUrl, options)); + this.browserExecutorFactory = opts.createBrowserExecutor ?? ((cdpWsUrl) => new BrowserExecutor(cdpWsUrl)); } /** Release held resources: closes the browser executor's CDP connection if one was opened. */ @@ -77,7 +69,7 @@ export class InternalComputerTranslator { browser(): BrowserExecutor { if (!this.browserExecutor) { if (!this.cdpWsUrl) throw new Error("browser has no cdp_ws_url; browser actions are unavailable"); - this.browserExecutor = this.browserExecutorFactory(this.cdpWsUrl, this.browserExecutorOptions); + this.browserExecutor = this.browserExecutorFactory(this.cdpWsUrl); } return this.browserExecutor; } diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 4221467c..df6680a1 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -2,7 +2,7 @@ import type Kernel from "@onkernel/sdk"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; -import { BrowserExecutor, type BrowserExecutorOptions } from "../src/translator/browser"; +import { BrowserExecutor } from "../src/translator/browser"; import type { CdpConnection } from "../src/translator/cdp"; import { buildCuaComputerTools } from "../src/tools"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; @@ -578,51 +578,15 @@ describe("BrowserExecutor cursor-pointer hints", () => { ax({ nodeId: "2", role: "generic", name: "Buy now", backendDOMNodeId: 77, parentId: "1" }), ]; - it("does not run the cursor scan by default", async () => { + it("marks cursor:pointer elements as clickable hints in every snapshot", async () => { const { cdp, sent, setCursorBackendIds } = createFakeCdp(POINTER_TREE); setCursorBackendIds([77]); const executor = new BrowserExecutor(cdp); const text = await snapshotText(executor); - expect(text).toContain('generic "Buy now"'); - expect(text).not.toContain("cursor:pointer"); - expect(sent.some((cmd) => cmd.method === "Runtime.evaluate")).toBe(false); - }); - - it("marks cursor:pointer elements as clickable hints when enabled", async () => { - const { cdp, sent, setCursorBackendIds } = createFakeCdp(POINTER_TREE); - setCursorBackendIds([77]); - const executor = new BrowserExecutor(cdp, { cursorHints: true }); - const text = await snapshotText(executor); expect(text).toContain('generic "Buy now" [e1] [cursor:pointer]'); expect(sent.some((cmd) => cmd.method === "DOM.describeNode")).toBe(true); expect(sent.some((cmd) => cmd.method === "Runtime.releaseObjectGroup")).toBe(true); }); - - it("enables cursor hints by default in browser mode only, with an explicit opt-out", () => { - const recordedFor = (mode?: "computer" | "browser" | "hybrid", cursorHints?: boolean) => { - const recorded: BrowserExecutorOptions[] = []; - const { executor } = createFakeBrowserExecutor(); - const { client } = createClient(); - const translator = new InternalComputerTranslator({ - browser, - client, - mode, - cursorHints, - createBrowserExecutor: (_cdpWsUrl, options) => { - recorded.push(options); - return executor; - }, - }); - translator.browser(); - return recorded[0]!; - }; - expect(recordedFor("browser").cursorHints).toBe(true); - expect(recordedFor("browser", true).cursorHints).toBe(true); - expect(recordedFor("browser", false).cursorHints).toBe(false); - expect(recordedFor("hybrid", true).cursorHints).toBe(false); - expect(recordedFor("computer", true).cursorHints).toBe(false); - expect(recordedFor(undefined).cursorHints).toBe(false); - }); }); describe("BrowserExecutor dialog guard", () => { diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 73a353bd..591be5d5 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,8 +2,7 @@ ## 0.5.0 - 2026-07-09 -Breaking: introduces action planes (modes) and Anthropic native tools; -removes the `javascriptExec` option and `CUA_DEFAULT_BROWSER_ACTION_TYPES`. +Introduces action planes (modes) and Anthropic native computer-use tools. - New `mode` option (`"computer"` | `"browser"` | `"hybrid"`, exported as `CuaMode`) on `resolveCuaRuntimeSpec`, `computerTools`, and the executor @@ -22,13 +21,11 @@ removes the `javascriptExec` option and `CUA_DEFAULT_BROWSER_ACTION_TYPES`. computer-use declarations: `computer_20260701` (computer mode, with `enable_zoom`) and `browser_20260701` (browser mode) behind `anthropic-beta: computer-use-2026-07-01`. -- Breaking: the `javascriptExec` option is gone. `browser_evaluate` is part - of the default browser/hybrid action sets, and native `browser_20260701` +- JavaScript execution is on by default: `browser_evaluate` is part of the + default browser/hybrid action sets, and native `browser_20260701` declarations default `enable_javascript_exec` to true (an explicit value on the spec wins). Opt out by passing an explicit `actions` list or native tool spec. -- Breaking: `CUA_DEFAULT_BROWSER_ACTION_TYPES` is removed; the default - browser set is `CUA_BROWSER_ACTION_TYPES`. - New `zoom` computer action (cropped display inspection). ## 0.4.0 - 2026-07-07 From c53a40b1498d629c06a376cbcbb21164dc36533b Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:56:42 +0000 Subject: [PATCH 32/34] Address bugbot findings on switch rollback, CDP sends, and multi-click - Mode/model switches are two-phase: the outgoing translator stays alive until the new toolset is installed, and a failed switch restores the exact runtime the still-exposed tools are bound to (previously rollback left them wrapping a disposed translator) - CdpConnection.send rejects instead of hanging when the socket is no longer open (spec-silent discard) or send throws - browser_click dispatches one press/release cycle per click with an incrementing clickCount, matching native multi-click input - TUI /mode and /model treat a failed named-session persist as a warning on the successful switch, not as a failed switch --- packages/agent/src/agent.ts | 55 +++++++++++---- packages/agent/src/translator/browser.ts | 27 +++++--- packages/agent/src/translator/cdp.ts | 14 +++- packages/agent/test/agent.test.ts | 35 ++++++++++ packages/agent/test/cdp.test.ts | 69 +++++++++++++++++++ .../agent/test/translator-browser.test.ts | 18 +++++ packages/cli/src/tui/main.ts | 25 ++++++- 7 files changed, 213 insertions(+), 30 deletions(-) create mode 100644 packages/agent/test/cdp.test.ts diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 456d6876..f62f3f98 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -165,14 +165,9 @@ class CuaRuntimeController { // A repeated selection must not replace the translator: disposing it // would drop snapshot refs, tab context, and the CDP connection. if (mode === this.runtimeSpec.mode) return; - this.runtimeSpec = this.resolveSpec(this.runtimeSpec.model, mode); + const spec = this.resolveSpec(this.runtimeSpec.model, mode); + this.beginSwitch(spec); this.currentMode = mode; - this.replaceTranslator(); - } - - private replaceTranslator(): void { - this.translator.dispose(); - this.translator = this.createTranslator(); } get systemPrompt(): string { @@ -180,8 +175,34 @@ class CuaRuntimeController { } setModel(model: CuaRuntimeInput): void { - this.runtimeSpec = this.resolveSpec(model); - this.replaceTranslator(); + this.beginSwitch(this.resolveSpec(model)); + } + + // A mode/model switch is two-phase: the outgoing translator must stay + // alive until the new toolset is actually installed, because on failure + // the still-exposed pre-switch tools keep executing against it. + private previousRuntime?: { spec: CuaRuntimeSpec; translator: InternalComputerTranslator; mode?: CuaMode }; + + private beginSwitch(spec: CuaRuntimeSpec): void { + this.previousRuntime = { spec: this.runtimeSpec, translator: this.translator, mode: this.currentMode }; + this.runtimeSpec = spec; + this.translator = this.createTranslator(); + } + + /** Dispose the pre-switch translator once the new toolset is installed. */ + commitSwitch(): void { + this.previousRuntime?.translator.dispose(); + this.previousRuntime = undefined; + } + + /** Restore the pre-switch runtime; the translator the exposed tools wrap stays live. */ + rollbackSwitch(): void { + if (!this.previousRuntime) return; + this.translator.dispose(); + this.runtimeSpec = this.previousRuntime.spec; + this.translator = this.previousRuntime.translator; + this.currentMode = this.previousRuntime.mode; + this.previousRuntime = undefined; } tools(): AgentTool[] { @@ -360,6 +381,7 @@ export class CuaAgent extends Agent { if (this.ownsSystemPrompt) { state.systemPrompt = this.runtime.systemPrompt; } + this.runtime.commitSwitch(); } /** The action plane(s) currently exposed to the model. */ @@ -376,6 +398,7 @@ export class CuaAgent extends Agent { if (this.ownsSystemPrompt) { state.systemPrompt = this.runtime.systemPrompt; } + this.runtime.commitSwitch(); } } @@ -447,16 +470,17 @@ export class CuaAgentHarness< * concrete model selected by `@onkernel/cua-ai`. */ override async setModel(model: CuaRuntimeInput): Promise { - const previousModel = this.runtime.model; this.runtime.setModel(model); const tools = this.runtime.tools(); try { await super.setTools(tools, this.requestedActiveToolNames ?? tools.map((tool) => tool.name)); } catch (err) { - // Keep the runtime in step with the exposed tools when the switch fails. - this.runtime.setModel(previousModel); + // The pre-switch tools stay exposed, so restore the runtime they are + // bound to — including its still-live translator. + this.runtime.rollbackSwitch(); throw err; } + this.runtime.commitSwitch(); await super.setModel(this.runtime.model); } @@ -472,7 +496,6 @@ export class CuaAgentHarness< */ async setMode(mode: CuaMode): Promise { if (mode === this.runtime.mode) return; - const previousMode = this.runtime.mode; const previousNames = new Set(this.getTools().map((tool) => tool.name)); this.runtime.setMode(mode); const tools = this.runtime.tools(); @@ -485,10 +508,12 @@ export class CuaAgentHarness< try { await super.setTools(tools, active); } catch (err) { - // Keep the runtime in step with the exposed tools when the switch fails. - this.runtime.setMode(previousMode); + // The pre-switch tools stay exposed, so restore the runtime they are + // bound to — including its still-live translator. + this.runtime.rollbackSwitch(); throw err; } + this.runtime.commitSwitch(); // The requested subset now reflects this mode's toolset; without this a // later setModel would restore the pre-switch names. if (requested) this.requestedActiveToolNames = active; diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 7f2288b1..9cb21a14 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -547,18 +547,23 @@ export class BrowserExecutor { const point = await this.resolvePoint(action, targetId, session); const modifiers = modifierBits(action.modifiers); const button = action.button ?? "left"; - const clickCount = action.num_clicks ?? 1; + const clicks = action.num_clicks ?? 1; await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y, modifiers }, point.session); - await this.cdp.send( - "Input.dispatchMouseEvent", - { type: "mousePressed", x: point.x, y: point.y, button, clickCount, modifiers }, - point.session, - ); - await this.cdp.send( - "Input.dispatchMouseEvent", - { type: "mouseReleased", x: point.x, y: point.y, button, clickCount, modifiers }, - point.session, - ); + // Native multi-clicks are separate press/release cycles with an + // incrementing clickCount; a single pair with the final count is not how + // real input arrives and can register as one click. + for (let clickCount = 1; clickCount <= clicks; clickCount++) { + await this.cdp.send( + "Input.dispatchMouseEvent", + { type: "mousePressed", x: point.x, y: point.y, button, clickCount, modifiers }, + point.session, + ); + await this.cdp.send( + "Input.dispatchMouseEvent", + { type: "mouseReleased", x: point.x, y: point.y, button, clickCount, modifiers }, + point.session, + ); + } } private async hover(action: CuaActionBrowserHover): Promise { diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index 546da8f7..c38604ae 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -48,7 +48,19 @@ export class CdpConnection { const message = JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }); return new Promise((resolve, reject) => { this.pending.set(id, { resolve: resolve as (result: unknown) => void, reject }); - socket.send(message); + // A non-OPEN socket silently discards send() per the WebSocket spec, + // which would leave this command pending forever. + if (socket.readyState !== WebSocket.OPEN) { + this.pending.delete(id); + reject(new Error(`CDP connection closed before ${method} could be sent`)); + return; + } + try { + socket.send(message); + } catch (err) { + this.pending.delete(id); + reject(err instanceof Error ? err : new Error(String(err))); + } }); } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index cd10aac1..12b742e6 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -485,6 +485,41 @@ describe("CuaAgentHarness", () => { expect(active).not.toContain("custom"); }); + it("a failed mode switch keeps the pre-switch runtime and its live translator", async () => { + const { env, session } = await createHarnessServices(); + let failWrites = false; + const flakySession = new Proxy(session, { + get(target, prop, receiver) { + if (prop === "appendActiveToolsChange" && failWrites) { + return () => Promise.reject(new Error("session write failed")); + } + return Reflect.get(target, prop, receiver); + }, + }); + const harness = new CuaAgentHarness({ + env, + session: flakySession, + browser, + client, + model: "anthropic:claude-opus-4-5", + }); + const runtime = (harness as unknown as { runtime: { translator: unknown } }).runtime; + const translatorBefore = runtime.translator; + + failWrites = true; + await expect(harness.setMode("browser")).rejects.toThrow("session write failed"); + + // The exposed tools wrap this translator; rollback must not have + // disposed or replaced it. + expect(runtime.translator).toBe(translatorBefore); + expect(harness.getMode()).toBe("computer"); + + failWrites = false; + await harness.setMode("browser"); + expect(harness.getMode()).toBe("browser"); + expect(harness.getTools().map((tool) => tool.name)).toContain("snapshot"); + }); + it("treats a repeated setMode as a no-op", async () => { const harness = new CuaAgentHarness({ ...(await createHarnessServices()), diff --git a/packages/agent/test/cdp.test.ts b/packages/agent/test/cdp.test.ts new file mode 100644 index 00000000..6fe84fd8 --- /dev/null +++ b/packages/agent/test/cdp.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { CdpConnection } from "../src/translator/cdp"; + +class FakeSocket { + static instances: FakeSocket[] = []; + readyState = 0; + sent: string[] = []; + private listeners = new Map void>>(); + + constructor(public url: string) { + FakeSocket.instances.push(this); + queueMicrotask(() => { + this.readyState = 1; + this.emit("open", {}); + }); + } + + addEventListener(type: string, listener: (event: unknown) => void): void { + const list = this.listeners.get(type) ?? []; + list.push(listener); + this.listeners.set(type, list); + } + + emit(type: string, event: unknown): void { + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + + send(message: string): void { + // Mirrors the WebSocket spec: a non-OPEN socket discards silently. + if (this.readyState !== 1) return; + this.sent.push(message); + } + + close(): void { + this.readyState = 3; + this.emit("close", {}); + } +} + +afterEach(() => { + vi.unstubAllGlobals(); + FakeSocket.instances = []; +}); + +describe("CdpConnection send", () => { + it("rejects instead of hanging when the socket closed between connect and send", async () => { + vi.stubGlobal("WebSocket", Object.assign(FakeSocket, { OPEN: 1 })); + const cdp = new CdpConnection("wss://fake.test/cdp"); + + const first = cdp.send("Target.getTargets"); + await vi.waitFor(() => expect(FakeSocket.instances[0]!.sent).toHaveLength(1)); + const socket = FakeSocket.instances[0]!; + socket.emit("message", { data: JSON.stringify({ id: 1, result: { targetInfos: [] } }) }); + await first; + + // Close silently (no close event yet): send() would be discarded. + socket.readyState = 3; + await expect(cdp.send("Target.getTargets")).rejects.toThrow(/closed before/); + }); + + it("rejects in-flight commands when the connection closes", async () => { + vi.stubGlobal("WebSocket", Object.assign(FakeSocket, { OPEN: 1 })); + const cdp = new CdpConnection("wss://fake.test/cdp"); + const pending = cdp.send("Target.getTargets"); + await Promise.resolve(); + FakeSocket.instances[0]!.close(); + await expect(pending).rejects.toThrow(/closed/); + }); +}); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index df6680a1..b345b62f 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -782,6 +782,24 @@ describe("navigation tool grounding frame", () => { }); }); +describe("BrowserExecutor multi-click", () => { + it("dispatches one press/release cycle per click with incrementing clickCount", async () => { + const { cdp, sent } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + await executor.execute({ type: "browser_click", ref: "e1", num_clicks: 2 } as CuaBrowserAction); + + const mouse = sent.filter((cmd) => cmd.method === "Input.dispatchMouseEvent").map((cmd) => cmd.params); + expect(mouse.map((params) => [params.type, params.clickCount])).toEqual([ + ["mouseMoved", undefined], + ["mousePressed", 1], + ["mouseReleased", 1], + ["mousePressed", 2], + ["mouseReleased", 2], + ]); + }); +}); + describe("BrowserExecutor ref state export/import", () => { it("resolves refs imported from a previous executor against the same browser", async () => { const first = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); diff --git a/packages/cli/src/tui/main.ts b/packages/cli/src/tui/main.ts index 25df0a11..fffbf3cd 100644 --- a/packages/cli/src/tui/main.ts +++ b/packages/cli/src/tui/main.ts @@ -484,7 +484,6 @@ async function applyModelCommand( try { const resolved = resolveCuaModelRef(ref); await opts.harness.setModel(resolved); - if (opts.namedSession) await updateNamedSessionRuntime(opts.namedSession, { model: resolved }); const model = opts.harness.getModel(); footer.update({ provider: model.provider, @@ -493,6 +492,7 @@ async function applyModelCommand( }); status.update({ model: modelLabel(model) }); messages.addNotice(`model → ${resolved}`); + await persistNamedSessionRuntime(opts, messages, { model: resolved }); } catch (err) { messages.addError((err as Error).message); } @@ -506,10 +506,29 @@ async function applyModeCommand(opts: InteractiveOptions, messages: MessageList, } try { await opts.harness.setMode(value); - if (opts.namedSession) await updateNamedSessionRuntime(opts.namedSession, { mode: value }); - messages.addNotice(`mode → ${value}`); } catch (err) { messages.addError((err as Error).message); + return; + } + messages.addNotice(`mode → ${value}`); + await persistNamedSessionRuntime(opts, messages, { mode: value }); +} + +// Persistence is best-effort: the live switch already happened, so a failed +// metadata write must not masquerade as a failed switch — warn that resume +// will restore the previous value instead. +async function persistNamedSessionRuntime( + opts: InteractiveOptions, + messages: MessageList, + patch: { model?: string; mode?: string }, +): Promise { + if (!opts.namedSession) return; + try { + await updateNamedSessionRuntime(opts.namedSession, patch); + } catch (err) { + messages.addError( + `switched, but failed to persist to session "${opts.namedSession}" (resume will restore the previous value): ${(err as Error).message}`, + ); } } From e8207b174f289b9ae8cd8817e2c8ce6f4eb5d1e5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:07:54 +0000 Subject: [PATCH 33/34] Fix overlapping-switch translator leak and hold_key duration unit - beginSwitch disposes the superseded pending translator when a second mode/model switch starts before the first commits, instead of overwriting the rollback slot and orphaning the original - Native hold_key maps its seconds duration to the canonical keypress duration in milliseconds, matching how wait already converts --- packages/agent/src/agent.ts | 9 +++- packages/agent/test/agent.test.ts | 46 ++++++++++++++++++- packages/ai/src/providers/anthropic/native.ts | 3 +- packages/ai/test/native-tools.test.ts | 6 +++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index f62f3f98..2ec37185 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -184,7 +184,14 @@ class CuaRuntimeController { private previousRuntime?: { spec: CuaRuntimeSpec; translator: InternalComputerTranslator; mode?: CuaMode }; private beginSwitch(spec: CuaRuntimeSpec): void { - this.previousRuntime = { spec: this.runtimeSpec, translator: this.translator, mode: this.currentMode }; + if (this.previousRuntime) { + // A switch is already pending: its translator was never installed + // into the exposed tools, so dispose it rather than orphaning it. + // previousRuntime keeps pointing at the runtime the tools still wrap. + this.translator.dispose(); + } else { + this.previousRuntime = { spec: this.runtimeSpec, translator: this.translator, mode: this.currentMode }; + } this.runtimeSpec = spec; this.translator = this.createTranslator(); } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 12b742e6..7dff12af 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createAssistantMessageEventStream, type AssistantMessage } from "@earendil-works/pi-ai"; import { resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; @@ -520,6 +520,50 @@ describe("CuaAgentHarness", () => { expect(harness.getTools().map((tool) => tool.name)).toContain("snapshot"); }); + it("an overlapping mode switch disposes the superseded pending translator", async () => { + const { env, session } = await createHarnessServices(); + let gate: Promise | undefined; + const gatedSession = new Proxy(session, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (prop === "appendActiveToolsChange" && gate) { + const pending = gate; + gate = undefined; + return async (...args: unknown[]) => { + await pending; + return (value as (...a: unknown[]) => Promise).apply(target, args); + }; + } + return typeof value === "function" ? (value as (...a: unknown[]) => unknown).bind(target) : value; + }, + }); + const harness = new CuaAgentHarness({ + env, + session: gatedSession, + browser, + client, + model: "anthropic:claude-opus-4-5", + }); + const runtime = (harness as unknown as { runtime: { translator: { dispose(): void } } }).runtime; + const original = vi.spyOn(runtime.translator, "dispose"); + + let release!: () => void; + gate = new Promise((resolve) => { + release = resolve; + }); + const first = harness.setMode("browser"); + const superseded = vi.spyOn(runtime.translator, "dispose"); + + const second = harness.setMode("hybrid"); + release(); + await Promise.all([first, second]); + + // Neither the original nor the superseded pending translator may leak. + expect(original).toHaveBeenCalled(); + expect(superseded).toHaveBeenCalled(); + expect(harness.getMode()).toBe("hybrid"); + }); + it("treats a repeated setMode as a no-op", async () => { const harness = new CuaAgentHarness({ ...(await createHarnessServices()), diff --git a/packages/ai/src/providers/anthropic/native.ts b/packages/ai/src/providers/anthropic/native.ts index 418b2d05..a1f2b18c 100644 --- a/packages/ai/src/providers/anthropic/native.ts +++ b/packages/ai/src/providers/anthropic/native.ts @@ -120,7 +120,8 @@ export function mapNativeComputerInput(input: NativeInput): CuaAction[] { return Array.from({ length: repeat }, () => ({ type: "keypress" as const, keys: [text(input)] })); } case "hold_key": - return [{ type: "keypress", keys: [text(input)], duration: durationSeconds(input) }]; + // Canonical keypress duration is milliseconds; the native tool speaks seconds. + return [{ type: "keypress", keys: [text(input)], duration: durationSeconds(input) * 1000 }]; case "wait": return [{ type: "wait", ms: durationSeconds(input) * 1000 }]; case "cursor_position": diff --git a/packages/ai/test/native-tools.test.ts b/packages/ai/test/native-tools.test.ts index 399bc498..dd118e92 100644 --- a/packages/ai/test/native-tools.test.ts +++ b/packages/ai/test/native-tools.test.ts @@ -96,6 +96,12 @@ describe("computer_20260701 action mapping", () => { expect(mapNativeComputerInput({ action: "left_click" })).toEqual([{ type: "click", button: "left" }]); }); + it("maps hold_key seconds to keypress duration in milliseconds", () => { + expect(mapNativeComputerInput({ action: "hold_key", text: "a", duration: 2 })).toEqual([ + { type: "keypress", keys: ["a"], duration: 2000 }, + ]); + }); + it("expands key repeat into repeated keypresses", () => { expect(mapNativeComputerInput({ action: "key", text: "Down", repeat: 3 })).toEqual([ { type: "keypress", keys: ["Down"] }, From f02a6e6ad8b7678d2359ad206689a0eff1b94890 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:10:43 +0000 Subject: [PATCH 34/34] Keep the translator alive across mode switches The translator only depends on the provider's coordinate system and screenshot transform, both mode-independent, so replacing it on setMode destroyed live CDP state, tabs, and element refs for no reason. Switches now rebuild the translator only when a model change alters that config (e.g. pixel vs normalized coordinates); mode switches keep it. Drop the translator's dead mode option. --- packages/agent/CHANGELOG.md | 8 ++-- packages/agent/src/agent.ts | 47 ++++++++++++--------- packages/agent/src/translator/translator.ts | 3 -- packages/agent/test/agent.test.ts | 42 ++++++++++++++++-- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index a6e0aa37..e094246d 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -19,9 +19,11 @@ helper is always registered. `importRefState()` persist refs across processes against the same browser. - `CuaAgent` and `CuaAgentHarness` accept `mode` (`"computer"` | `"browser"` | `"hybrid"`) and `nativeTool`, and support runtime plane switching via - `setMode()` / `getMode()`; mode switches preserve the requested activation - state of surviving tools, dispose the previous translator's CDP - connection, and roll back cleanly on failure (as does `setModel()`). + `setMode()` / `getMode()`. Mode switches preserve the requested activation + state of surviving tools and keep the translator — CDP connection, tabs, + and element refs — alive; the translator is only rebuilt when a model + switch changes the provider's coordinate system or screenshot transform. + Both switches roll back cleanly on failure. - Post-action grounding captures and the navigation helper are mode-aware: browser mode grounds on the viewport and routes navigation through CDP (browser and hybrid modes both route `computer_use_extra` navigation over diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 2ec37185..38ca37b9 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -162,11 +162,8 @@ class CuaRuntimeController { } setMode(mode: CuaMode): void { - // A repeated selection must not replace the translator: disposing it - // would drop snapshot refs, tab context, and the CDP connection. if (mode === this.runtimeSpec.mode) return; - const spec = this.resolveSpec(this.runtimeSpec.model, mode); - this.beginSwitch(spec); + this.beginSwitch(this.resolveSpec(this.runtimeSpec.model, mode)); this.currentMode = mode; } @@ -178,36 +175,49 @@ class CuaRuntimeController { this.beginSwitch(this.resolveSpec(model)); } - // A mode/model switch is two-phase: the outgoing translator must stay - // alive until the new toolset is actually installed, because on failure - // the still-exposed pre-switch tools keep executing against it. - private previousRuntime?: { spec: CuaRuntimeSpec; translator: InternalComputerTranslator; mode?: CuaMode }; + // A mode/model switch is two-phase: when the new spec needs a different + // translator configuration, the outgoing translator must stay alive until + // the new toolset is actually installed, because on failure the + // still-exposed pre-switch tools keep executing against it. + private previousRuntime?: { spec: CuaRuntimeSpec; translator?: InternalComputerTranslator; mode?: CuaMode }; private beginSwitch(spec: CuaRuntimeSpec): void { - if (this.previousRuntime) { - // A switch is already pending: its translator was never installed - // into the exposed tools, so dispose it rather than orphaning it. - // previousRuntime keeps pointing at the runtime the tools still wrap. + // The translator only cares about the provider's coordinate system and + // screenshot transform. Keep it — and its CDP connection, tabs, and + // refs — whenever those are unchanged (always true for mode switches). + const replaceTranslator = + JSON.stringify([spec.coordinateSystem, spec.screenshot]) !== + JSON.stringify([this.runtimeSpec.coordinateSystem, this.runtimeSpec.screenshot]); + if (!this.previousRuntime) { + this.previousRuntime = { spec: this.runtimeSpec, mode: this.currentMode }; + } + this.runtimeSpec = spec; + if (!replaceTranslator) return; + if (this.previousRuntime.translator) { + // An earlier pending switch already replaced the translator; its + // replacement was never installed into the exposed tools, so + // dispose it rather than orphaning it. this.translator.dispose(); } else { - this.previousRuntime = { spec: this.runtimeSpec, translator: this.translator, mode: this.currentMode }; + this.previousRuntime.translator = this.translator; } - this.runtimeSpec = spec; this.translator = this.createTranslator(); } - /** Dispose the pre-switch translator once the new toolset is installed. */ + /** Dispose the pre-switch translator (when one was replaced) once the new toolset is installed. */ commitSwitch(): void { - this.previousRuntime?.translator.dispose(); + this.previousRuntime?.translator?.dispose(); this.previousRuntime = undefined; } /** Restore the pre-switch runtime; the translator the exposed tools wrap stays live. */ rollbackSwitch(): void { if (!this.previousRuntime) return; - this.translator.dispose(); + if (this.previousRuntime.translator) { + this.translator.dispose(); + this.translator = this.previousRuntime.translator; + } this.runtimeSpec = this.previousRuntime.spec; - this.translator = this.previousRuntime.translator; this.currentMode = this.previousRuntime.mode; this.previousRuntime = undefined; } @@ -252,7 +262,6 @@ class CuaRuntimeController { client: this.options.client, coordinateSystem: this.runtimeSpec.coordinateSystem, screenshot: this.runtimeSpec.screenshot, - mode: this.runtimeSpec.mode, }); } } diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 0a2fdab4..5610d31b 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -17,7 +17,6 @@ import { type CuaActionZoom, type CuaBrowserAction, type CuaDragMouseButton, - type CuaMode, type CuaMouseButton, type CuaScreenshotSpec, } from "@onkernel/cua-ai"; @@ -33,8 +32,6 @@ export interface InternalComputerTranslatorOptions { client: Kernel; coordinateSystem?: ComputerToolCoordinateSystem; screenshot?: CuaScreenshotSpec; - /** Action plane(s) in play; browser-executor extras like cursor hints are gated to "browser". */ - mode?: CuaMode; /** Browser executor factory, overridable for tests. Defaults to a raw-CDP executor on the browser's cdp_ws_url. */ createBrowserExecutor?: (cdpWsUrl: string) => BrowserExecutor; } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 7dff12af..d7d2273e 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -520,7 +520,41 @@ describe("CuaAgentHarness", () => { expect(harness.getTools().map((tool) => tool.name)).toContain("snapshot"); }); - it("an overlapping mode switch disposes the superseded pending translator", async () => { + it("setMode keeps the translator and its CDP-backed state alive", async () => { + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "anthropic:claude-opus-4-5", + }); + const runtime = (harness as unknown as { runtime: { translator: unknown } }).runtime; + const translator = runtime.translator; + + await harness.setMode("browser"); + await harness.setMode("hybrid"); + + expect(runtime.translator).toBe(translator); + }); + + it("setModel keeps the translator when the provider translator config is unchanged", async () => { + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "anthropic:claude-opus-4-5", + }); + const runtime = (harness as unknown as { runtime: { translator: unknown } }).runtime; + const translator = runtime.translator; + + await harness.setModel("anthropic:claude-opus-4-7"); + expect(runtime.translator).toBe(translator); + + // Gemini uses a normalized coordinate system: the translator must be rebuilt. + await harness.setModel("google:gemini-3-flash-preview"); + expect(runtime.translator).not.toBe(translator); + }); + + it("an overlapping model switch disposes the superseded pending translator", async () => { const { env, session } = await createHarnessServices(); let gate: Promise | undefined; const gatedSession = new Proxy(session, { @@ -551,17 +585,17 @@ describe("CuaAgentHarness", () => { gate = new Promise((resolve) => { release = resolve; }); - const first = harness.setMode("browser"); + const first = harness.setModel("google:gemini-3-flash-preview"); const superseded = vi.spyOn(runtime.translator, "dispose"); - const second = harness.setMode("hybrid"); + const second = harness.setModel("openai:gpt-5.5"); release(); await Promise.all([first, second]); // Neither the original nor the superseded pending translator may leak. expect(original).toHaveBeenCalled(); expect(superseded).toHaveBeenCalled(); - expect(harness.getMode()).toBe("hybrid"); + expect(harness.getModel().id).toBe("gpt-5.5"); }); it("treats a repeated setMode as a no-op", async () => {