From 63510982cf93fba1665595007f78bbf7e07d73c8 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 12 Jun 2026 01:21:48 +0000 Subject: [PATCH 1/2] Tighten cua-agent design seams from review - The translator now consumes the canonical CuaAction union with an exhaustive switch instead of re-parsing untyped records; malformed shapes can no longer silently coerce to 0,0 clicks, and drift in the canonical vocabulary becomes a compile error. The documented button-coercion contract is the one remaining runtime guard. - cua-ai exports CuaSimpleStreamOptions so the keepToolNames channel the Yutori/Tzafon adapters consume is a declared type instead of a cast. - CuaRuntimeController holds one translator per runtime, shared by the tools and the payload getScreenshot capability. - prepareNextTurn is pass-through (stock pi behavior) until a user hook returns an update or a mid-run model assignment dirties the runtime; the state proxy re-keys off the underlying state object identity. - keys.ts reframed: the alias table absorbs model nondeterminism in key naming generally, not provider-specific spellings. - README: harness quickstart showcases session-backed turns and mid-session provider switching with direct prompt() result handling; computerUseExtra is introduced with its rationale. - docs/architecture.md encodes the boundary: provider differences arrive as CuaRuntimeSpec data, agent returns capabilities via CuaPayloadContext, no provider conditionals in packages/agent. - New tests: a full fake-stream tool-call turn (toolCall -> Kernel batch -> toolResult fed back), prepareNextTurn pass-through, and the one-shot refresh after mid-run model assignment. Co-Authored-By: Claude Opus 4.7 --- docs/architecture.md | 26 +- packages/agent/README.md | 48 ++- packages/agent/src/agent.ts | 48 +-- packages/agent/src/tools.ts | 17 +- packages/agent/src/translator/keys.ts | 5 + packages/agent/src/translator/translator.ts | 376 +++++++++----------- packages/agent/src/translator/types.ts | 2 - packages/agent/test/agent.test.ts | 76 ++++ packages/agent/test/keys.test.ts | 2 +- packages/ai/src/providers/common.ts | 11 +- 10 files changed, 347 insertions(+), 264 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 90ff7e0c..dc8f512c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,10 +45,28 @@ someone who wants to read the code, contribute, or fork. - executing canonical CUA tool calls against Kernel browsers - typed executor coverage and translator integration -In practice this means any new provider quirk should be implemented in -`@onkernel/cua-ai` and surfaced through provider-neutral runtime specs. -`@onkernel/cua-agent` should consume that spec without explicit -provider-specific conditionals. +The boundary is a single data seam. Every provider difference arrives in +`@onkernel/cua-agent` as data through `CuaRuntimeSpec` — `toolDefinitions`, +`toolExecutors`, `defaultSystemPrompt`, `coordinateSystem`, `screenshot`, and +`onPayload` — resolved per model by `resolveCuaRuntimeSpec()`. In the other +direction, the agent supplies capabilities back to provider middleware through +`CuaPayloadContext` (`keepToolNames`, `getScreenshot`): the provider hook +decides *whether and how* to use a capability (policy), the agent decides +*how it is performed* against the Kernel browser (mechanism). + +The invariant: `packages/agent/src` contains no provider names and no +provider conditionals. A new provider difference is a new or extended +`CuaRuntimeSpec`/`CuaPayloadContext` field plus provider code in +`@onkernel/cua-ai` — never a branch in `@onkernel/cua-agent`. The grep test +is literal: searching agent `src/` for a provider name should only ever hit +doc comments. + +One deliberate exception to "push it upstream": generic model imprecision. +Models of every provider are loose about things like key naming +(`ctrl`/`cmd`/`ArrowLeft`/word-form punctuation), so the agent-side +translator absorbs that nondeterminism when mapping canonical actions to +Kernel's X11 key vocabulary. That is corrective plumbing for model output in +general, not provider policy, and it stays in `@onkernel/cua-agent`. ## Layers diff --git a/packages/agent/README.md b/packages/agent/README.md index 72740899..03b1f205 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -37,11 +37,17 @@ await agent.prompt("Open news.ycombinator.com and summarize the top story."); ## Quick Start (`CuaAgentHarness`) +`prompt()` returns the turn's final assistant message, and every turn is +persisted to the session — later prompts see the full transcript. Runtime +config like the model can change between turns (or even mid-turn, applying at +the next provider request): + ```ts import { CuaAgentHarness, InMemorySessionRepo, NodeExecutionEnv } from "@onkernel/cua-agent"; +import type { AssistantMessage } from "@onkernel/cua-ai"; const sessionRepo = new InMemorySessionRepo(); -const session = await sessionRepo.create({ id: "example" }); +const session = await sessionRepo.create({ id: "research" }); const harness = new CuaAgentHarness({ browser, @@ -51,22 +57,26 @@ const harness = new CuaAgentHarness({ session, }); -const response = await harness.prompt("Open example.com and tell me the current URL."); -const branch = await session.getBranch(); -const lastAssistant = [...branch] - .reverse() - .flatMap((entry) => - entry.type === "message" && entry.message.role === "assistant" ? [entry.message] : [], - )[0]; -const assistant = lastAssistant ?? response; -const assistantText = assistant.content - .flatMap((block) => (block.type === "text" ? [block.text] : [])) - .join("") - .trim(); -console.log("assistant stopReason:", assistant.stopReason); -console.log("assistant text:", assistantText || "(no text)"); +const textOf = (message: AssistantMessage) => + message.content.flatMap((block) => (block.type === "text" ? [block.text] : [])).join("").trim(); + +// Turn 1: a session-backed prompt. +const first = await harness.prompt("Open example.com and describe what you see."); +console.log(textOf(first)); + +// Swap providers mid-session; CUA tools and the default prompt refresh to match. +await harness.setModel("anthropic:claude-opus-4-7"); + +// Turn 2 continues the same transcript on the new model. +const second = await harness.prompt("Open the most relevant link from what you found."); +console.log(textOf(second)); ``` +While a turn is running, `steer()` injects course corrections, `followUp()` +queues the next instruction, and `subscribe()` streams the underlying agent +events. `compact()` and session branching are available for long-running +transcripts — see the pi-agent-core docs for the full harness lifecycle. + Use `CuaAgent` when you want direct pi `Agent` control: raw message state, lifecycle events, custom streaming, and explicit prompt/continue/queue control. Reach for the harness shape when you want an app layer around the loop: @@ -108,9 +118,11 @@ computer-use tools. This is useful when the model needs to call application-specific code, such as looking up a record, writing a database row, or handing off to another service while it also controls the browser. -`computerUseExtra: true` adds the `computer_use_extra` tool. Use it when you -want one compact helper for common browser navigation/read operations: -`goto`, `back`, `forward`, and `url`. +Not every provider's native computer-use vocabulary includes browser +navigation — some models can click and type but have no direct way to open a +URL or go back. `computerUseExtra: true` adds `computer_use_extra`, a +provider-neutral escape hatch exposing `goto`, `back`, `forward`, and `url` +so navigation works uniformly regardless of which model is driving. ### Model Switching diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 2d4208b6..49a4a6b3 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -13,6 +13,8 @@ import { type Api, CUA_NAVIGATION_TOOL_NAME, type CuaModelRef, + type CuaRuntimeSpec, + type CuaSimpleStreamOptions, getCuaEnvApiKey, type Model, resolveCuaRuntimeSpec, @@ -20,14 +22,12 @@ import { streamSimple, } from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; -import { createCuaComputerTools } from "./tools"; +import { buildCuaComputerTools } from "./tools"; import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; /** A CUA model reference string or a concrete pi model object. */ type CuaRuntimeInput = CuaModelRef | Model; -type CuaRuntimeSpec = ReturnType; - /** * Agent state exposed by {@link CuaAgent}. * @@ -132,14 +132,13 @@ class CuaRuntimeController { tools(): AgentTool[] { return [ - ...createCuaComputerTools({ - browser: this.options.browser, - client: this.options.client, - toolExecutors: this.runtimeSpec.toolExecutors, - coordinateSystem: this.runtimeSpec.coordinateSystem, - screenshot: this.runtimeSpec.screenshot, - computerUseExtra: this.options.computerUseExtra, - }), + ...buildCuaComputerTools( + { + toolExecutors: this.runtimeSpec.toolExecutors, + computerUseExtra: this.options.computerUseExtra, + }, + this.translator, + ), ...(this.options.extraTools ?? []), ]; } @@ -190,7 +189,9 @@ async function getCuaEnvApiKeyAndHeaders(model: Model): Promise<{ apiKey: s export class CuaAgent extends Agent { private readonly runtime: CuaRuntimeController; private readonly ownsSystemPrompt: boolean; + private runtimeDirty = false; private stateProxy?: CuaAgentState; + private stateProxyTarget?: AgentState; constructor(options: CuaAgentOptions) { const { @@ -213,11 +214,11 @@ export class CuaAgent extends Agent { onPayload, }); const wrappedStreamFn: StreamFn = (model, context, streamOptions) => { - const optionsWithCuaRuntime = { + const optionsWithCuaRuntime: CuaSimpleStreamOptions = { ...streamOptions, onPayload: runtime.onPayload(), keepToolNames: runtime.keepToolNames(), - } as SimpleStreamOptions & { keepToolNames?: string[] }; + }; return (streamFn ?? streamSimple)(model, context, optionsWithCuaRuntime); }; @@ -236,15 +237,19 @@ export class CuaAgent extends Agent { this.runtime = runtime; this.ownsSystemPrompt = initialState.systemPrompt === undefined; /** - * pi calls `prepareNextTurn` between provider requests. Wrapping it lets CUA - * honor any user-provided turn update while also refreshing provider-specific - * defaults if that update changes the model. + * pi's loop only re-reads model/tools/prompt between provider requests + * through `prepareNextTurn`. The wrapper stays pass-through (returning + * `undefined`, i.e. stock pi behavior) until either the user hook returns + * an update or a mid-run model assignment marks the CUA runtime dirty — + * only then is a turn update built from current state. */ this.prepareNextTurn = async (signal: AbortSignal | undefined) => { const update = await prepareNextTurn?.(signal); if (update?.model) { this.applyRuntime(update.model as CuaRuntimeInput); } + if (!update && !this.runtimeDirty) return undefined; + this.runtimeDirty = false; const state = super.state; const context = update?.context ?? { @@ -271,14 +276,16 @@ export class CuaAgent extends Agent { * and payload hooks for the selected provider. */ override get state(): CuaAgentState { - if (!this.stateProxy) { - this.stateProxy = new Proxy(super.state, { - set: (target, prop, value, receiver) => { + const target = super.state; + if (!this.stateProxy || this.stateProxyTarget !== target) { + this.stateProxyTarget = target; + this.stateProxy = new Proxy(target, { + set: (proxied, prop, value, receiver) => { if (prop === "model") { this.applyRuntime(value as CuaRuntimeInput); return true; } - return Reflect.set(target, prop, value, receiver); + return Reflect.set(proxied, prop, value, receiver); }, }) as CuaAgentState; } @@ -287,6 +294,7 @@ export class CuaAgent extends Agent { private applyRuntime(model: CuaRuntimeInput): void { this.runtime.setModel(model); + this.runtimeDirty = true; const state = super.state; state.model = this.runtime.model; state.tools = this.runtime.tools(); diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 3940ee32..74becaab 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -43,11 +43,18 @@ type NavigationExecutorSpec = { kind: "navigation"; definition: Tool }; type ComputerExecutorSpec = CuaToolExecutorSpec | NavigationExecutorSpec; export function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTool[] { - const translator = new InternalComputerTranslator(args); + return buildCuaComputerTools(args, new InternalComputerTranslator(args)); +} + +/** Build executor tools against an existing translator (internal; not part of the package surface). */ +export function buildCuaComputerTools( + args: Pick, + translator: InternalComputerTranslator, +): CuaExecutorTool[] { return withNavigationTool(args).map((executor) => createExecutorTool(executor, translator)); } -function withNavigationTool(args: ComputerToolOptions): ComputerExecutorSpec[] { +function withNavigationTool(args: Pick): ComputerExecutorSpec[] { const executors: ComputerExecutorSpec[] = [...args.toolExecutors]; const existing = new Set(executors.map((executor) => executor.definition.name)); if (args.computerUseExtra && !existing.has(CUA_NAVIGATION_TOOL_NAME)) { @@ -92,7 +99,7 @@ async function executeBatchTool(translator: InternalComputerTranslator, params: const content: ToolContent = []; const readResults: BatchDetails["readResults"] = []; try { - const result = await translator.executeBatch(params.actions as unknown as Array>); + const result = await translator.executeBatch(params.actions); for (const read of result.readResults) { if (read.type === "url") { readResults.push({ type: "url", url: read.url }); @@ -124,8 +131,10 @@ async function executeNavigationTool(translator: InternalComputerTranslator, par if (action === "url") { url = await translator.currentUrl(); statusText = `Current URL: ${url}`; + } else if (action === "goto") { + await translator.executeBatch([{ type: "goto", url: params.url ?? "" }]); } else { - await translator.executeBatch([{ type: action, url: params.url }]); + await translator.executeBatch([{ type: action }]); } const screenshot = await translator.screenshot(); return { diff --git a/packages/agent/src/translator/keys.ts b/packages/agent/src/translator/keys.ts index bcecc275..bec079b4 100644 --- a/packages/agent/src/translator/keys.ts +++ b/packages/agent/src/translator/keys.ts @@ -1,5 +1,10 @@ export const KERNEL_MODIFIER_KEYSYMS = ["Control_L", "Alt_L", "Shift_L", "Super_L"] as const; +// Models are imprecise about key naming regardless of provider: the same +// model may emit W3C KeyboardEvent names ("ArrowLeft"), shorthand ("ctrl", +// "cmd"), keypad names ("kp_enter"), or word-form punctuation ("plus"). +// This table is the corrective force that absorbs that nondeterminism into +// Kernel's X11 keysym vocabulary. const KEY_ALIASES: Record = { alt: "Alt_L", alt_l: "Alt_L", diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 120511cd..495b19a9 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -1,9 +1,25 @@ import type Kernel from "@onkernel/sdk"; import type { BrowserCreateResponse, BrowserRetrieveResponse } from "@onkernel/sdk/resources/browsers"; -import { normalizeGotoUrl, type ComputerToolCoordinateSystem, type CuaScreenshotSpec } from "@onkernel/cua-ai"; +import { + normalizeGotoUrl, + type ComputerToolCoordinateSystem, + type CuaAction, + type CuaActionClick, + type CuaActionDoubleClick, + type CuaActionDrag, + type CuaActionMouseDown, + type CuaActionMouseUp, + type CuaActionMove, + type CuaActionScroll, + type CuaActionTypeText, + type CuaActionWait, + type CuaDragMouseButton, + type CuaMouseButton, + type CuaScreenshotSpec, +} from "@onkernel/cua-ai"; import sharp from "sharp"; import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; -import type { BatchExecutionResult, ModelAction } from "./types"; +import type { BatchExecutionResult } from "./types"; export type KernelBrowser = BrowserCreateResponse | BrowserRetrieveResponse; @@ -66,10 +82,10 @@ export class InternalComputerTranslator { async currentMousePosition(): Promise<{ x: number; y: number }> { const pos = await this.client.browsers.computer.getMousePosition(this.sessionId); - return { x: toInt(pos.x), y: toInt(pos.y) }; + return { x: Math.trunc(pos.x), y: Math.trunc(pos.y) }; } - async executeBatch(actions: ModelAction[]): Promise { + async executeBatch(actions: CuaAction[]): Promise { const result: BatchExecutionResult = { readResults: [] }; const pending: KernelBatchAction[] = []; @@ -78,253 +94,185 @@ export class InternalComputerTranslator { await this.runKernelBatch(pending.splice(0)); }; - for (let i = 0; i < actions.length; i++) { - const action = actions[i]!; - const type = typeof action.type === "string" ? action.type : ""; - if (type === "screenshot") { - await flush(); - result.readResults.push({ type: "screenshot", ...(await this.screenshot()) }); - continue; + for (const action of actions) { + switch (action.type) { + case "screenshot": + await flush(); + result.readResults.push({ type: "screenshot", ...(await this.screenshot()) }); + break; + case "url": + await flush(); + result.readResults.push({ type: "url", url: await this.currentUrl() }); + break; + case "cursor_position": + await flush(); + result.readResults.push({ type: "cursor_position", ...(await this.currentMousePosition()) }); + break; + case "goto": + pending.push( + keypress(["Control", "l"]), + { type: "type_text", type_text: { text: normalizeGotoUrl(action.url) ?? "" } }, + keypress(["Enter"]), + ); + break; + case "back": + pending.push(keypress(["Alt", "Left"])); + break; + case "forward": + pending.push(keypress(["Alt", "Right"])); + break; + default: + pending.push(this.toSdkAction(action)); + break; } - if (type === "url") { - await flush(); - result.readResults.push({ type: "url", url: await this.currentUrl() }); - continue; - } - if (type === "cursor_position") { - await flush(); - const pos = await this.currentMousePosition(); - result.readResults.push({ type: "cursor_position", ...pos }); - continue; - } - if (type === "goto") { - const url = normalizeGotoUrl(action.url) ?? ""; - pending.push( - keypress(["Control", "l"]), - { type: "type_text", type_text: { text: url } }, - keypress(["Enter"]), - ); - continue; - } - if (type === "back") { - pending.push(keypress(["Alt", "Left"])); - continue; - } - if (type === "forward") { - pending.push(keypress(["Alt", "Right"])); - continue; - } - pending.push(toSdkAction(type, action, this.coordinateSystem, this.viewport)); } await flush(); return result; } - private async runKernelBatch(actions: KernelBatchAction[]): Promise { - await this.client.browsers.computer.batch(this.sessionId, { actions }); + private toSdkAction( + action: Exclude, + ): KernelBatchAction { + switch (action.type) { + case "click": + return this.clickAction(action, { button: mouseButton(action.button) }); + case "double_click": + return this.clickAction(action, { num_clicks: 2 }); + case "mouse_down": + return this.clickAction(action, { button: mouseButton(action.button), click_type: "down" }); + case "mouse_up": + return this.clickAction(action, { button: mouseButton(action.button), click_type: "up" }); + case "type": + return typeText(action); + case "keypress": + return keypress(action.keys, action.duration); + case "scroll": + return this.scrollAction(action); + case "move": + return this.moveAction(action); + case "drag": + return this.dragAction(action); + case "wait": + return waitAction(action); + default: + return unreachable(action); + } } -} -type KernelBatchAction = - Parameters[1]["actions"][number]; + private clickAction( + action: CuaActionClick | CuaActionDoubleClick | CuaActionMouseDown | CuaActionMouseUp, + extra: { button?: CuaMouseButton; num_clicks?: number; click_type?: "down" | "up" }, + ): KernelBatchAction { + const point = this.toViewportPoint(action.x, action.y); + return { + type: "click_mouse", + click_mouse: { + x: point.x, + y: point.y, + ...extra, + ...holdKeys(action.hold_keys), + }, + }; + } -type ClickMouseButton = "back" | "forward" | "left" | "right" | "middle"; -type DragMouseButton = "left" | "right" | "middle"; + private scrollAction(action: CuaActionScroll): KernelBatchAction { + const point = this.toViewportPoint(action.x ?? 0, action.y ?? 0); + return { + type: "scroll", + scroll: { + x: point.x, + y: point.y, + delta_x: Math.trunc(action.scroll_x ?? 0), + delta_y: Math.trunc(action.scroll_y ?? 0), + ...holdKeys(action.hold_keys), + }, + }; + } -function toSdkAction( - type: string, - action: ModelAction, - coordinateSystem: ComputerToolCoordinateSystem, - viewport: { width: number; height: number }, -): KernelBatchAction { - switch (type) { - case "click": { - const clickHoldKeys = readHoldKeys(action.hold_keys); - const point = toViewportPoint(action, coordinateSystem, viewport); - return { - type: "click_mouse", - click_mouse: { - x: point.x, - y: point.y, - button: clickMouseButtonOr(action.button, "left"), - ...(clickHoldKeys.length > 0 ? { hold_keys: clickHoldKeys } : {}), - }, - }; - } - case "double_click": { - const doubleClickHoldKeys = readHoldKeys(action.hold_keys); - const point = toViewportPoint(action, coordinateSystem, viewport); - return { - type: "click_mouse", - click_mouse: { - x: point.x, - y: point.y, - num_clicks: 2, - ...(doubleClickHoldKeys.length > 0 ? { hold_keys: doubleClickHoldKeys } : {}), - }, - }; - } - case "mouse_down": - case "mouse_up": { - const mouseHoldKeys = readHoldKeys(action.hold_keys); - const point = toViewportPoint(action, coordinateSystem, viewport); - return { - type: "click_mouse", - click_mouse: { - x: point.x, - y: point.y, - button: clickMouseButtonOr(action.button, "left"), - click_type: type === "mouse_down" ? "down" : "up", - ...(mouseHoldKeys.length > 0 ? { hold_keys: mouseHoldKeys } : {}), - }, - }; - } - case "type": - return { type: "type_text", type_text: { text: typeof action.text === "string" ? action.text : "" } }; - case "keypress": - return keypress(toStringArray(action.keys), action.duration); - case "scroll": { - const scrollHoldKeys = readHoldKeys(action.hold_keys); - const point = toViewportPoint(action, coordinateSystem, viewport); - return { - type: "scroll", - scroll: { - x: point.x, - y: point.y, - delta_x: toInt(action.scroll_x), - delta_y: toInt(action.scroll_y), - ...(scrollHoldKeys.length > 0 ? { hold_keys: scrollHoldKeys } : {}), - }, - }; - } - case "move": { - const moveHoldKeys = readHoldKeys(action.hold_keys); - const point = toViewportPoint(action, coordinateSystem, viewport); - return { - type: "move_mouse", - move_mouse: { - x: point.x, - y: point.y, - ...(moveHoldKeys.length > 0 ? { hold_keys: moveHoldKeys } : {}), - }, - }; - } - case "drag": { - const dragHoldKeys = readHoldKeys(action.hold_keys); - return { - type: "drag_mouse", - drag_mouse: { - path: toPath(action.path, coordinateSystem, viewport), - button: dragMouseButtonOr(action.button, "left"), - ...(dragHoldKeys.length > 0 ? { hold_keys: dragHoldKeys } : {}), - }, - }; - } - case "wait": - return { type: "sleep", sleep: { duration_ms: typeof action.ms === "number" ? Math.trunc(action.ms) : 1000 } }; - default: - throw new Error(`unknown computer action type: ${type}`); + private moveAction(action: CuaActionMove): KernelBatchAction { + const point = this.toViewportPoint(action.x, action.y); + return { type: "move_mouse", move_mouse: { x: point.x, y: point.y } }; + } + + private dragAction(action: CuaActionDrag): KernelBatchAction { + return { + type: "drag_mouse", + drag_mouse: { + path: action.path.map((point) => { + const transformed = this.toViewportPoint(point.x, point.y); + return [transformed.x, transformed.y] as [number, number]; + }), + button: dragButton(action.button), + ...holdKeys(action.hold_keys), + }, + }; } -} -function toInt(value: unknown): number { - if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value); - if (typeof value === "string" && value.trim()) { - const n = Number(value); - if (Number.isFinite(n)) return Math.trunc(n); + private toViewportPoint(x: number, y: number): { x: number; y: number } { + if (this.coordinateSystem.type === "pixel") return { x: Math.trunc(x), y: Math.trunc(y) }; + const [min, max] = this.coordinateSystem.range; + const scale = max - min; + if (scale <= 0) return { x: Math.trunc(x), y: Math.trunc(y) }; + return { + x: clamp(Math.round(((x - min) / scale) * this.viewport.width), 0, this.viewport.width - 1), + y: clamp(Math.round(((y - min) / scale) * this.viewport.height), 0, this.viewport.height - 1), + }; + } + + private async runKernelBatch(actions: KernelBatchAction[]): Promise { + await this.client.browsers.computer.batch(this.sessionId, { actions }); } - return 0; } -function stringOr(value: unknown, fallback: string): string { - return typeof value === "string" && value.length > 0 ? value : fallback; +type KernelBatchAction = + Parameters[1]["actions"][number]; + +const CLICK_BUTTONS: ReadonlySet = new Set(["left", "right", "middle", "back", "forward"]); +const DRAG_BUTTONS: ReadonlySet = new Set(["left", "right", "middle"]); + +// The wire schemas keep button as an open string for provider compatibility; +// per the documented CuaMouseButton contract, values outside the set coerce +// to "left". +function mouseButton(value: string | undefined): CuaMouseButton { + return value !== undefined && CLICK_BUTTONS.has(value) ? (value as CuaMouseButton) : "left"; } -function clickMouseButtonOr(value: unknown, fallback: ClickMouseButton): ClickMouseButton { - const candidate = stringOr(value, fallback); - if (candidate === "left" || candidate === "right" || candidate === "middle" || candidate === "back" || candidate === "forward") { - return candidate; - } - return fallback; +function dragButton(value: string | undefined): CuaDragMouseButton { + return value !== undefined && DRAG_BUTTONS.has(value) ? (value as CuaDragMouseButton) : "left"; } -function dragMouseButtonOr(value: unknown, fallback: DragMouseButton): DragMouseButton { - const candidate = stringOr(value, fallback); - if (candidate === "left" || candidate === "right" || candidate === "middle") { - return candidate; - } - return fallback; +function typeText(action: CuaActionTypeText): KernelBatchAction { + return { type: "type_text", type_text: { text: action.text } }; } -function toStringArray(value: unknown): string[] { - return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; +function waitAction(action: CuaActionWait): KernelBatchAction { + return { type: "sleep", sleep: { duration_ms: Math.trunc(action.ms ?? 1000) } }; } -function readHoldKeys(value: unknown): string[] { - return toStringArray(value).map(normalizeKernelKey); +function holdKeys(keys: string[] | undefined): { hold_keys?: string[] } { + if (!keys || keys.length === 0) return {}; + return { hold_keys: keys.map(normalizeKernelKey) }; } -function keypress(keys: string[], duration: unknown = undefined): KernelBatchAction { +function keypress(keys: string[], duration?: number): KernelBatchAction { const translated = keys.flatMap(normalizeKernelKeyCombo); const pressedKeys = translated.filter((key) => !isKernelModifierKey(key)); - const holdKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1); + const heldKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1); return { type: "press_key", press_key: { keys: pressedKeys.length > 0 ? pressedKeys : translated.slice(-1), - ...(holdKeys.length > 0 ? { hold_keys: holdKeys } : {}), + ...(heldKeys.length > 0 ? { hold_keys: heldKeys } : {}), ...(typeof duration === "number" && Number.isFinite(duration) && duration > 0 ? { duration: Math.trunc(duration) } : {}), }, }; } -function toPath( - value: unknown, - coordinateSystem: ComputerToolCoordinateSystem = { type: "pixel" }, - viewport: { width: number; height: number } = { width: 1920, height: 1080 }, -): Array<[number, number]> { - if (!Array.isArray(value)) return []; - return value.map((point) => toPathPoint(point, coordinateSystem, viewport)); -} - -function toPathPoint(value: unknown, coordinateSystem: ComputerToolCoordinateSystem, viewport: { width: number; height: number }): [number, number] { - if (Array.isArray(value)) { - const point = transformPoint(toInt(value[0]), toInt(value[1]), coordinateSystem, viewport); - return [point.x, point.y]; - } - if (value && typeof value === "object") { - const point = value as Record; - const transformed = transformPoint(toInt(point.x), toInt(point.y), coordinateSystem, viewport); - return [transformed.x, transformed.y]; - } - return [0, 0]; -} - -function toViewportPoint( - action: Record, - coordinateSystem: ComputerToolCoordinateSystem, - viewport: { width: number; height: number }, -): { x: number; y: number } { - return transformPoint(toInt(action.x), toInt(action.y), coordinateSystem, viewport); -} - -function transformPoint( - x: number, - y: number, - coordinateSystem: ComputerToolCoordinateSystem, - viewport: { width: number; height: number }, -): { x: number; y: number } { - if (coordinateSystem.type === "pixel") return { x, y }; - const [min, max] = coordinateSystem.range; - const scale = max - min; - if (scale <= 0) return { x, y }; - return { - x: clamp(Math.round(((x - min) / scale) * viewport.width), 0, viewport.width - 1), - y: clamp(Math.round(((y - min) / scale) * viewport.height), 0, viewport.height - 1), - }; -} - function clamp(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } + +function unreachable(action: never): never { + throw new Error(`unknown computer action type: ${JSON.stringify(action)}`); +} diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index eea4dfc5..e9aaca37 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -1,5 +1,3 @@ -export type ModelAction = Record; - export type BatchReadResult = | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index e8e6c1dc..73e83eec 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -267,6 +267,82 @@ describe("CuaAgent", () => { ]); expect(payload.messages[0]!.content.at(-1)?.image_url?.url.startsWith("data:image/webp;base64,")).toBe(true); }); + + it("leaves pi turn preparation untouched while the runtime is unchanged", async () => { + const agent = new CuaAgent({ + browser, + client, + initialState: { model: "openai:gpt-5.5" }, + }); + + await expect(agent.prepareNextTurn?.(undefined)).resolves.toBeUndefined(); + }); + + it("builds a one-shot turn update after a mid-run model assignment", async () => { + const runtime = resolveCuaRuntimeSpec("google:gemini-3-flash-preview"); + const agent = new CuaAgent({ + browser, + client, + initialState: { model: "openai:gpt-5.5" }, + }); + + agent.state.model = "google:gemini-3-flash-preview"; + + const update = await agent.prepareNextTurn?.(undefined); + expect(update?.model?.id).toBe(runtime.model.id); + expect(update?.context?.tools).toHaveLength(runtime.toolExecutors.length); + + await expect(agent.prepareNextTurn?.(undefined)).resolves.toBeUndefined(); + }); + + it("executes model tool calls against the Kernel browser and feeds the result back", async () => { + let screenshots = 0; + const screenshotClient = { + browsers: { + computer: { + captureScreenshot: async () => { + screenshots += 1; + return new Response(tinyPng); + }, + }, + }, + } as unknown as Kernel; + const contexts: Array<{ messages: Array<{ role: string; content: Array<{ type: string; mimeType?: string }> }> }> = []; + let providerCalls = 0; + const streamFn: StreamFn = (model, context, _options) => { + contexts.push(context as never); + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + if (providerCalls++ === 0) { + message.content = [{ type: "toolCall", id: "tool-1", name: "screenshot", arguments: {} }]; + message.stopReason = "toolUse"; + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "toolUse", message }); + stream.end(message); + } else { + message.content = [{ type: "text", text: "done" }]; + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + } + return stream; + }; + + const agent = new CuaAgent({ + browser, + client: screenshotClient, + streamFn, + initialState: { model: "openai:gpt-5.5" }, + }); + + await agent.prompt("inspect the page"); + + expect(screenshots).toBe(1); + expect(providerCalls).toBe(2); + const fedBack = contexts[1]!.messages.find((message) => message.role === "toolResult"); + expect(fedBack, "second provider request should carry the tool result").toBeDefined(); + expect(fedBack!.content.some((block) => block.type === "image" && block.mimeType === "image/png")).toBe(true); + }); }); describe("CuaAgentHarness", () => { diff --git a/packages/agent/test/keys.test.ts b/packages/agent/test/keys.test.ts index 701c6ac7..fdc4db46 100644 --- a/packages/agent/test/keys.test.ts +++ b/packages/agent/test/keys.test.ts @@ -15,7 +15,7 @@ describe("Kernel key normalization", () => { expect(normalizeKernelKey("f12")).toBe("F12"); }); - it("covers Yutori's word-form punctuation and sequential key syntax", () => { + it("absorbs word-form punctuation and sequential key syntax models emit", () => { expect(normalizeKernelKeyCombo("ctrl+plus")).toEqual(["Control_L", "plus"]); expect(normalizeKernelKeyCombo("command+backquote")).toEqual(["Super_L", "grave"]); expect(normalizeKernelKeyCombo("option+tab")).toEqual(["Alt_L", "Tab"]); diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 2b06ac37..b31b4799 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -1,4 +1,4 @@ -import { Type, type Api, type Model, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai"; +import { Type, type Api, type Model, type SimpleStreamOptions, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai"; import type { CuaModelRef, CuaProvider } from "../models"; export const CUA_ACTION_TYPES = [ @@ -448,6 +448,15 @@ export interface CuaPayloadContext { export type CuaPayloadHook = (payload: unknown, model: Model, context?: CuaPayloadContext) => unknown | Promise; +/** + * pi-ai `SimpleStreamOptions` plus the CUA extension consumed by the + * Yutori/Tzafon stream adapters. Pass `keepToolNames` for caller tools that + * must survive provider-native tool-set substitution. + */ +export interface CuaSimpleStreamOptions extends SimpleStreamOptions { + keepToolNames?: readonly string[]; +} + /** * Runtime configuration for a supported CUA model. * From 0d0baf869b28fdac8639475c432cd9a78b32449f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 12 Jun 2026 01:33:47 +0000 Subject: [PATCH 2/2] Release metadata: cua-ai 0.3.0, cua-agent 0.3.3 Co-Authored-By: Claude Opus 4.7 --- package-lock.json | 6 +++--- packages/agent/CHANGELOG.md | 16 ++++++++++++++++ packages/agent/package.json | 4 ++-- packages/ai/CHANGELOG.md | 6 ++++++ packages/ai/package.json | 2 +- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7024d6ff..eca44fe7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5428,12 +5428,12 @@ }, "packages/agent": { "name": "@onkernel/cua-agent", - "version": "0.3.2", + "version": "0.3.3", "license": "MIT", "dependencies": { "@earendil-works/pi-agent-core": "0.79.1", "@earendil-works/pi-ai": "0.79.1", - "@onkernel/cua-ai": "0.2.2", + "@onkernel/cua-ai": "0.3.0", "@onkernel/sdk": "0.49.0", "sharp": "^0.34.5" }, @@ -5444,7 +5444,7 @@ }, "packages/ai": { "name": "@onkernel/cua-ai", - "version": "0.2.2", + "version": "0.3.0", "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "0.79.1", diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 977ccb90..affb1ce0 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.3.3 - 2026-06-12 + +- The action translator now consumes the canonical `CuaAction` union with an + exhaustive switch. Malformed action shapes fail loudly instead of silently + coercing (previously e.g. a click at 0,0); the documented mouse-button + coercion to `"left"` is unchanged. +- `prepareNextTurn` no longer rebuilds the turn context on every turn: it + keeps stock pi behavior until a user hook returns an update or a mid-run + model assignment requires a refresh. +- One translator instance per runtime is shared between the executor tools + and the provider screenshot capability. +- The `CuaAgentHarness` README quickstart showcases session-backed turns and + mid-session model switching; `computerUseExtra` is documented with its + rationale. +- Update the `@onkernel/cua-ai` dependency to 0.3.0. + ## 0.3.2 - 2026-06-11 - Update the `@onkernel/cua-ai` dependency to 0.2.2. diff --git a/packages/agent/package.json b/packages/agent/package.json index 64bbfbc0..fed5b841 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-agent", - "version": "0.3.2", + "version": "0.3.3", "description": "Kernel browser computer-use Agent and AgentHarness classes built on pi-agent-core", "license": "MIT", "type": "module", @@ -42,7 +42,7 @@ "dependencies": { "@earendil-works/pi-agent-core": "0.79.1", "@earendil-works/pi-ai": "0.79.1", - "@onkernel/cua-ai": "0.2.2", + "@onkernel/cua-ai": "0.3.0", "@onkernel/sdk": "0.49.0", "sharp": "^0.34.5" }, diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 1cfa0337..fcd36a1b 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.3.0 - 2026-06-12 + +- Add `CuaSimpleStreamOptions`: pi-ai `SimpleStreamOptions` plus the + `keepToolNames` extension the Yutori/Tzafon stream adapters consume, so + callers can pass it through `streamSimple` without a cast. + ## 0.2.2 - 2026-06-11 - Add computer-use support for `gpt-5.4-mini`, `gemini-3.1-flash-lite`, `tzafon.northstar-cua-fast-1.6`, and `tzafon.northstar-cua-fast-1.7-experiment`. diff --git a/packages/ai/package.json b/packages/ai/package.json index 40f7c323..cdfca00c 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-ai", - "version": "0.2.2", + "version": "0.3.0", "description": "Kernel-curated computer-use model access built on pi-ai", "license": "MIT", "type": "module",