From e3d07edabbbff528df63d3f2020efde7857159f6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:16:46 +0000 Subject: [PATCH 1/9] Fix cua-ai ESM packaging and CI test coverage Switch packages/ai to NodeNext module resolution and add explicit .js extensions to relative imports so the published dist loads in plain Node ESM (previously failed with ERR_UNSUPPORTED_DIR_IMPORT). Run the full unit suite in CI and release instead of hard-coded file lists by excluding integration/live tests from the default vitest config, and add a post-pack ESM import smoke test to CI and the release workflow. Co-Authored-By: Claude Opus 4.7 --- .agents/skills/release/SKILL.md | 2 +- .github/workflows/ci.yml | 13 +++++++++-- .github/workflows/release-cua-ai.yml | 23 ++++++++++--------- packages/ai/package.json | 3 ++- packages/ai/src/api-keys.ts | 2 +- packages/ai/src/index.ts | 20 ++++++++-------- packages/ai/src/providers.ts | 4 ++-- .../ai/src/providers/anthropic/actions.ts | 2 +- packages/ai/src/providers/anthropic/index.ts | 8 +++---- packages/ai/src/providers/common.ts | 2 +- packages/ai/src/providers/gemini/index.ts | 8 +++---- packages/ai/src/providers/openai/index.ts | 8 +++---- packages/ai/src/providers/tzafon/index.ts | 10 ++++---- packages/ai/src/providers/tzafon/provider.ts | 2 +- packages/ai/src/providers/yutori/actions.ts | 2 +- packages/ai/src/providers/yutori/index.ts | 12 +++++----- packages/ai/src/providers/yutori/provider.ts | 4 ++-- packages/ai/src/runtime-spec.ts | 16 ++++++------- packages/ai/test/api-keys.test.ts | 2 +- .../ai/test/computer-tool.integration.test.ts | 2 +- packages/ai/test/models.test.ts | 2 +- packages/ai/test/provider-module.test.ts | 4 ++-- packages/ai/test/providers.test.ts | 2 +- packages/ai/test/runtime-spec.test.ts | 2 +- packages/ai/test/tools.test.ts | 2 +- packages/ai/test/tzafon-payload.test.ts | 2 +- packages/ai/test/tzafon-provider.test.ts | 2 +- packages/ai/test/yutori-actions.test.ts | 2 +- packages/ai/test/yutori-payload.test.ts | 2 +- packages/ai/tsconfig.build.json | 2 ++ packages/ai/vitest.config.ts | 5 +++- packages/ai/vitest.integration.config.ts | 13 +++++++++++ 32 files changed, 107 insertions(+), 78 deletions(-) create mode 100644 packages/ai/vitest.integration.config.ts diff --git a/.agents/skills/release/SKILL.md b/.agents/skills/release/SKILL.md index 0e1bbf9f..50df2eea 100644 --- a/.agents/skills/release/SKILL.md +++ b/.agents/skills/release/SKILL.md @@ -130,7 +130,7 @@ Run the checks for each package being released: ```bash npm ci npm run build --workspace @onkernel/cua-ai -npm test --workspace @onkernel/cua-ai -- test/api-keys.test.ts test/models.test.ts test/providers.test.ts test/runtime-spec.test.ts test/tools.test.ts test/yutori-payload.test.ts +npm test --workspace @onkernel/cua-ai npm pack --workspace @onkernel/cua-ai --dry-run ``` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5a3ddc0..3c1fef7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,16 @@ jobs: - run: npm ci - run: npm run build --workspace @onkernel/cua-ai - name: Unit tests - run: npm test --workspace @onkernel/cua-ai -- test/models.test.ts test/tools.test.ts + run: npm test --workspace @onkernel/cua-ai + - name: Pack tarball + run: npm pack --workspace @onkernel/cua-ai --pack-destination "$RUNNER_TEMP" + - name: ESM import smoke test + run: | + mkdir -p "$RUNNER_TEMP/esm-smoke" + cd "$RUNNER_TEMP/esm-smoke" + npm init -y + npm install "$RUNNER_TEMP"/onkernel-cua-ai-*.tgz + node --input-type=module -e "import('@onkernel/cua-ai').then((m) => { if (typeof m.getCuaModel !== 'function') process.exit(1); })" integration: runs-on: ubuntu-latest @@ -45,7 +54,7 @@ jobs: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} - run: npm test --workspace @onkernel/cua-ai -- test/computer-tool.integration.test.ts + run: npm run test:integration --workspace @onkernel/cua-ai agent-e2e: runs-on: ubuntu-latest diff --git a/.github/workflows/release-cua-ai.yml b/.github/workflows/release-cua-ai.yml index ae1e34aa..34586460 100644 --- a/.github/workflows/release-cua-ai.yml +++ b/.github/workflows/release-cua-ai.yml @@ -60,17 +60,18 @@ jobs: - run: npm run build --workspace @onkernel/cua-ai - name: Unit tests - run: > - npm test --workspace @onkernel/cua-ai -- - test/api-keys.test.ts - test/models.test.ts - test/providers.test.ts - test/runtime-spec.test.ts - test/tools.test.ts - test/yutori-payload.test.ts - - - name: Pack dry run - run: npm pack --workspace @onkernel/cua-ai --dry-run + run: npm test --workspace @onkernel/cua-ai + + - name: Pack tarball + run: npm pack --workspace @onkernel/cua-ai --pack-destination "$RUNNER_TEMP" + + - name: ESM import smoke test + run: | + mkdir -p "$RUNNER_TEMP/esm-smoke" + cd "$RUNNER_TEMP/esm-smoke" + npm init -y + npm install "$RUNNER_TEMP"/onkernel-cua-ai-*.tgz + node --input-type=module -e "import('@onkernel/cua-ai').then((m) => { if (typeof m.getCuaModel !== 'function') process.exit(1); })" - name: Publish to npm run: npm publish --workspace @onkernel/cua-ai --access public diff --git a/packages/ai/package.json b/packages/ai/package.json index d6dd1ede..7a8a6844 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -35,7 +35,8 @@ "build": "tsc -b", "clean": "tsc -b --clean", "example:quickstart": "tsx examples/quickstart.ts", - "test": "vitest --run" + "test": "vitest --run", + "test:integration": "vitest --run --config vitest.integration.config.ts" }, "dependencies": { "@earendil-works/pi-ai": "^0.74.0", diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index edf04be3..2c3578b4 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -1,5 +1,5 @@ import type { Api, Model } from "@earendil-works/pi-ai"; -import { parseCuaModelRef, providerForModel, type CuaModelRef, type CuaProvider } from "./models"; +import { parseCuaModelRef, providerForModel, type CuaModelRef, type CuaProvider } from "./models.js"; /** * Environment variables accepted for each CUA provider. diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 11ce4b5f..dafd7dcc 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,15 +1,15 @@ -import { registerCuaProviders } from "./providers"; +import { registerCuaProviders } from "./providers.js"; export * from "@earendil-works/pi-ai"; -export * from "./models"; -export * from "./api-keys"; -export * from "./runtime-spec"; -export * from "./providers/common"; -export * as anthropic from "./providers/anthropic/index"; -export * as gemini from "./providers/gemini/index"; -export * as openai from "./providers/openai/index"; -export * as tzafon from "./providers/tzafon/index"; -export * as yutori from "./providers/yutori/index"; +export * from "./models.js"; +export * from "./api-keys.js"; +export * from "./runtime-spec.js"; +export * from "./providers/common.js"; +export * as anthropic from "./providers/anthropic/index.js"; +export * as gemini from "./providers/gemini/index.js"; +export * as openai from "./providers/openai/index.js"; +export * as tzafon from "./providers/tzafon/index.js"; +export * as yutori from "./providers/yutori/index.js"; registerCuaProviders(); diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index 9a7962f1..69cacb9d 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -1,6 +1,6 @@ import { registerApiProvider } from "@earendil-works/pi-ai"; -import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; -import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; +import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider.js"; +import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider.js"; let registered = false; diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts index 4e3c72af..76cb8833 100644 --- a/packages/ai/src/providers/anthropic/actions.ts +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -10,7 +10,7 @@ import { type CuaAction, type CuaActionType, type CuaToolExecutorSpec, -} from "../common"; +} from "../common.js"; // Source of truth: Anthropic's computer-use best-practices quickstart // computer/browser tool action enums. These are the browser actions Anthropic diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 4d41dac1..7fa7c233 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -1,16 +1,16 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; -import { computerToolExecutors, computerTools } from "./actions"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; +import { computerToolExecutors, computerTools } from "./actions.js"; export { computerToolExecutors, computerTools, createActionSchema, -} from "./actions"; +} from "./actions.js"; export type { AnthropicAction, AnthropicComputerToolsOptions, AnthropicComputerToolsOptions as ComputerToolsOptions, -} from "./actions"; +} from "./actions.js"; // Anthropic's quickstart uses pixel coordinates for both its computer and // browser tools. diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index e2efa1df..2c27c59f 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -1,5 +1,5 @@ import { Type, type Api, type Model, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai"; -import type { CuaModelRef, CuaProvider } from "../models"; +import type { CuaModelRef, CuaProvider } from "../models.js"; export const CUA_ACTION_TYPES = [ "click", diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index 0770fc37..cd319ee5 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -1,16 +1,16 @@ -import { computerToolExecutors, computerTools } from "../common"; -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors, computerTools } from "../common.js"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, -} from "../common"; +} from "../common.js"; export type { CuaAction as GeminiAction, ComputerToolsOptions, -} from "../common"; +} from "../common.js"; // Provider-native function names emitted on `functionCall.name` (PREDEFINED_COMPUTER_USE_FUNCTIONS): // open_web_browser, click_at, hover_at, type_text_at, scroll_document, diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 1b1a7a1f..1c61779b 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -1,5 +1,5 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; -import { computerToolExecutors, computerTools } from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; +import { computerToolExecutors, computerTools } from "../common.js"; export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, @@ -9,12 +9,12 @@ export { computerTools, createCuaActionSchema as createActionSchema, CuaNavigationSchema as OpenAIExtraSchema, -} from "../common"; +} from "../common.js"; export type { CuaAction as OpenAIAction, ComputerToolsOptions, CuaNavigationInput as OpenAIExtraInput, -} from "../common"; +} from "../common.js"; // Provider-native action vocabulary emitted on `computer_call.action.type`: // click, double_click, drag, move, scroll, type, keypress, wait, screenshot diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 0445f92e..7b3bf2c1 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -1,23 +1,23 @@ -import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, type CuaProviderModule } from "../common"; -import { tzafonComputerUseOnPayload } from "./provider"; +import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, type CuaProviderModule } from "../common.js"; +import { tzafonComputerUseOnPayload } from "./provider.js"; export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, -} from "../common"; +} from "../common.js"; export type { CuaAction as TzafonAction, ComputerToolsOptions, -} from "../common"; +} from "../common.js"; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses, tzafonComputerUseOnPayload, tzafonToolCallId, -} from "./provider"; +} from "./provider.js"; // Provider-native action vocabulary. The model card lists supported actions; // the Responses API loop dispatches on `action.type` and adds terminal control diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index dc78f349..81b62d7b 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -13,7 +13,7 @@ import { type ToolCall, } from "@earendil-works/pi-ai"; import Lightcone from "@tzafon/lightcone"; -import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction, type CuaPayloadContext } from "../common"; +import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction, type CuaPayloadContext } from "../common.js"; export const TZAFON_RESPONSES_API = "tzafon-responses"; const TZAFON_COMPUTER_USE_TOOL = { diff --git a/packages/ai/src/providers/yutori/actions.ts b/packages/ai/src/providers/yutori/actions.ts index 93b32ba8..f39271d6 100644 --- a/packages/ai/src/providers/yutori/actions.ts +++ b/packages/ai/src/providers/yutori/actions.ts @@ -5,7 +5,7 @@ import { type CuaAction, type CuaToolExecutorSpec, type CuaActionType, -} from "../common"; +} from "../common.js"; /** * Native Yutori Navigator n1.5 tool-set ids. diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index aa07e3c1..3840aef0 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,6 +1,6 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; -import { computerToolExecutors } from "./actions"; -import { yutoriNativeToolSetOnPayload } from "./provider"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; +import { computerToolExecutors } from "./actions.js"; +import { yutoriNativeToolSetOnPayload } from "./provider.js"; export { computerToolExecutors, @@ -15,14 +15,14 @@ export { YUTORI_N15_CORE_TOOL_SET, YUTORI_N15_EXPANDED_ACTION_TYPES, YUTORI_N15_EXPANDED_TOOL_SET, -} from "./actions"; -export type { YutoriN1ActionType, YutoriN15CoreActionType, YutoriN15ExpandedActionType, YutoriNativeActionType } from "./actions"; +} from "./actions.js"; +export type { YutoriN1ActionType, YutoriN15CoreActionType, YutoriN15ExpandedActionType, YutoriNativeActionType } from "./actions.js"; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori, yutoriNativeToolSetOnPayload, -} from "./provider"; +} from "./provider.js"; // Provider-native action vocabulary differs between Navigator versions: // n1 (fixed tool set): diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index 35d8adaf..f79d324e 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -18,8 +18,8 @@ import { toCanonicalActions, yutoriToolSetForModel, YUTORI_N15_EXPANDED_ACTION_TYPES, -} from "./actions"; -import { canonicalToolCallArguments, canonicalToolCallName, type CuaPayloadContext } from "../common"; +} from "./actions.js"; +import { canonicalToolCallArguments, canonicalToolCallName, type CuaPayloadContext } from "../common.js"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index d8469ace..148f639c 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -1,15 +1,15 @@ -import type { CuaProvider } from "./models"; -import { getCuaModel, providerForModel } from "./models"; -import { providerModule as anthropic } from "./providers/anthropic/index"; -import { providerModule as gemini } from "./providers/gemini/index"; -import { providerModule as openai } from "./providers/openai/index"; -import { providerModule as tzafon } from "./providers/tzafon/index"; -import { providerModule as yutori } from "./providers/yutori/index"; +import type { CuaProvider } from "./models.js"; +import { getCuaModel, providerForModel } from "./models.js"; +import { providerModule as anthropic } from "./providers/anthropic/index.js"; +import { providerModule as gemini } from "./providers/gemini/index.js"; +import { providerModule as openai } from "./providers/openai/index.js"; +import { providerModule as tzafon } from "./providers/tzafon/index.js"; +import { providerModule as yutori } from "./providers/yutori/index.js"; import type { CuaProviderModule, CuaRuntimeSpec, CuaRuntimeSpecInput, -} from "./providers/common"; +} from "./providers/common.js"; const PROVIDERS = { openai, diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts index 501b0106..8711dc7e 100644 --- a/packages/ai/test/api-keys.test.ts +++ b/packages/ai/test/api-keys.test.ts @@ -4,7 +4,7 @@ import { getCuaEnvApiKey, getCuaEnvApiKeyForModel, requireCuaEnvApiKey, -} from "../src/index"; +} from "../src/index.js"; const ENV_KEYS = [ "OPENAI_API_KEY", diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index 7eb53588..35200817 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -14,7 +14,7 @@ import { openai, tzafon, yutori, -} from "../src/index"; +} from "../src/index.js"; const here = dirname(fileURLToPath(import.meta.url)); const screenshotPath = join(here, "..", "examples", "screenshot.png"); diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index 1b9fd07e..214c5920 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -6,7 +6,7 @@ import { findCuaAnnotation, formatCuaModelRef, parseCuaModelRef, -} from "../src/index"; +} from "../src/index.js"; describe("CUA model refs", () => { it("parses and formats provider-qualified refs", () => { diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index b78a1458..8c527d82 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; -import type { CuaProviderModule } from "../src/providers/common"; +import { anthropic, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index.js"; +import type { CuaProviderModule } from "../src/providers/common.js"; const MODULES: Record = { openai, diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index 3903ba16..c39b322c 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getApiProvider, tzafon, yutori } from "../src/index"; +import { getApiProvider, tzafon, yutori } from "../src/index.js"; const TZAFON_RESPONSES_API = tzafon.TZAFON_RESPONSES_API; const YUTORI_CHAT_COMPLETIONS_API = yutori.YUTORI_CHAT_COMPLETIONS_API; diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index 2a88de3b..525e0222 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index"; +import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index.js"; describe("resolveCuaRuntimeSpec", () => { it("resolves a runtime spec for every CUA provider", () => { diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index b5b9f5fd..77d0ef1c 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -11,7 +11,7 @@ import { openai, tzafon, yutori, -} from "../src/index"; +} from "../src/index.js"; const providers = { openai, gemini, tzafon }; const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; diff --git a/packages/ai/test/tzafon-payload.test.ts b/packages/ai/test/tzafon-payload.test.ts index 5c26f4ed..620992c2 100644 --- a/packages/ai/test/tzafon-payload.test.ts +++ b/packages/ai/test/tzafon-payload.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { tzafon } from "../src/index"; +import { tzafon } from "../src/index.js"; describe("tzafonComputerUseOnPayload", () => { it("replaces local CUA action tools with the native computer_use tool", () => { diff --git a/packages/ai/test/tzafon-provider.test.ts b/packages/ai/test/tzafon-provider.test.ts index fd81451c..b33a2e47 100644 --- a/packages/ai/test/tzafon-provider.test.ts +++ b/packages/ai/test/tzafon-provider.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { tzafon } from "../src/index"; +import { tzafon } from "../src/index.js"; describe("streamTzafonResponses", () => { it("derives unique ids when one computer_call expands to multiple actions", () => { diff --git a/packages/ai/test/yutori-actions.test.ts b/packages/ai/test/yutori-actions.test.ts index 5ec89566..e26c6714 100644 --- a/packages/ai/test/yutori-actions.test.ts +++ b/packages/ai/test/yutori-actions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { type CuaAction, yutori } from "../src/index"; +import { type CuaAction, yutori } from "../src/index.js"; const n15CoreActionArgs = { left_click: { coordinates: [500, 250] }, diff --git a/packages/ai/test/yutori-payload.test.ts b/packages/ai/test/yutori-payload.test.ts index 5f399ad0..145e20cc 100644 --- a/packages/ai/test/yutori-payload.test.ts +++ b/packages/ai/test/yutori-payload.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { yutori } from "../src/index"; +import { yutori } from "../src/index.js"; describe("yutoriNativeToolSetOnPayload", () => { it("removes local Yutori action tool definitions from payload.tools", () => { diff --git a/packages/ai/tsconfig.build.json b/packages/ai/tsconfig.build.json index 450f9ec1..efe350d7 100644 --- a/packages/ai/tsconfig.build.json +++ b/packages/ai/tsconfig.build.json @@ -1,6 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", "outDir": "./dist", "rootDir": "./src" }, diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 039b5887..219048aa 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -1,4 +1,4 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ server: { @@ -8,5 +8,8 @@ export default defineConfig({ globals: true, environment: "node", testTimeout: 30000, + // Unit runs cover every test file except the opt-in suites; use + // vitest.integration.config.ts to run those. + exclude: [...configDefaults.exclude, "**/*.integration.test.ts", "**/*.live.test.ts"], }, }); diff --git a/packages/ai/vitest.integration.config.ts b/packages/ai/vitest.integration.config.ts new file mode 100644 index 00000000..01bc9e51 --- /dev/null +++ b/packages/ai/vitest.integration.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + server: { + host: "127.0.0.1", + }, + test: { + globals: true, + environment: "node", + testTimeout: 30000, + include: ["test/**/*.integration.test.ts", "test/**/*.live.test.ts"], + }, +}); From 522399f2fb9d4e221afa5dc59df6c488a3178585 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:21:53 +0000 Subject: [PATCH 2/9] Tighten CUA model catalog matching and ref parsing Family annotations now cover only the root id plus numeric revision or dated-snapshot suffixes, so named sibling variants like gpt-5.4-mini no longer list as CUA-capable. Drop gemini-2.5-computer-use-preview-10-2025 from the catalog: it rejects the function-declaration tools this package sends. Accept "gemini:" refs as an alias for the canonical "google:" prefix, name the valid providers in the unsupported-provider error, and add JSDoc to the models and api-keys entry points. Co-Authored-By: Claude Opus 4.7 --- packages/ai/src/api-keys.ts | 9 +++ packages/ai/src/models.ts | 103 ++++++++++++++++++++++++-------- packages/ai/test/models.test.ts | 44 +++++++++++++- 3 files changed, 128 insertions(+), 28 deletions(-) diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index 2c3578b4..6cc634fe 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -16,6 +16,11 @@ const CUA_PROVIDER_API_KEY_ENV_VARS: Record = { yutori: ["YUTORI_API_KEY"], }; +/** + * List the environment variables checked for a provider's API key, in + * precedence order. Accepts `"gemini"` as an alias for `"google"`; returns an + * empty list for unknown providers. + */ export function cuaApiKeyEnvVarsForProvider(provider: string): readonly string[] { if (provider === "gemini") { return CUA_PROVIDER_API_KEY_ENV_VARS.google; @@ -23,6 +28,7 @@ export function cuaApiKeyEnvVarsForProvider(provider: string): readonly string[] return CUA_PROVIDER_API_KEY_ENV_VARS[provider as keyof typeof CUA_PROVIDER_API_KEY_ENV_VARS] ?? []; } +/** Read a provider's API key from the environment, or return undefined when unset. */ export function getCuaEnvApiKey(provider: string): string | undefined { for (const envVar of cuaApiKeyEnvVarsForProvider(provider)) { const value = process.env[envVar]; @@ -31,6 +37,7 @@ export function getCuaEnvApiKey(provider: string): string | undefined { return undefined; } +/** Read a provider's API key from the environment, or throw naming the variables to set. */ export function requireCuaEnvApiKey(provider: string): string { const apiKey = getCuaEnvApiKey(provider); if (apiKey) return apiKey; @@ -41,11 +48,13 @@ export function requireCuaEnvApiKey(provider: string): string { throw new Error(`Missing API key for "${provider}". Set one of: ${envVars.join(", ")}`); } +/** {@link getCuaEnvApiKey} keyed by a model ref or concrete model instead of a provider name. */ export function getCuaEnvApiKeyForModel(input: CuaModelRef | Model): string | undefined { const provider = typeof input === "string" ? parseCuaModelRef(input).provider : providerForModel(input); return getCuaEnvApiKey(provider); } +/** {@link requireCuaEnvApiKey} keyed by a model ref or concrete model instead of a provider name. */ export function requireCuaEnvApiKeyForModel(input: CuaModelRef | Model): string { const provider = typeof input === "string" ? parseCuaModelRef(input).provider : providerForModel(input); return requireCuaEnvApiKey(provider); diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index ce4b0dca..232057e0 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -5,42 +5,58 @@ import { getModels, } from "@earendil-works/pi-ai"; +/** Providers with curated computer-use model support. */ export type CuaProvider = "openai" | "anthropic" | "google" | "tzafon" | "yutori"; + +/** Provider-qualified model reference, e.g. `"openai:gpt-5.5"` or `"google:gemini-3-flash-preview"`. */ export type CuaModelRef = `${CuaProvider}:${string}`; +/** One entry returned by {@link listCuaModels}. */ export interface CuaModelInfo { + /** Provider-qualified ref accepted by {@link getCuaModel}. */ ref: CuaModelRef; provider: CuaProvider; + /** Provider-native model id (the part after the colon). */ model: string; + /** Human-readable model name. */ name: string; } +/** All providers this package curates computer-use models for. */ export const CUA_PROVIDERS: readonly CuaProvider[] = ["openai", "anthropic", "google", "tzafon", "yutori"]; -// CUA support annotations. -// -// pi-ai's model registry is generated from models.dev (see -// node_modules/@earendil-works/pi-ai/scripts/generate-models.ts) and lists every -// model a provider offers. Only some of those models support computer-use, so -// this table layers per-provider CUA-support annotations on top of the -// registry. Each entry cites the official source documenting CUA support. -// -// Match kinds: -// - exact: id === match.id -// - family: id === match.family || id.startsWith(match.family + "-") -// -// To verify support and add new entries, follow the `update-models` skill at -// .agents/skills/update-models/SKILL.md. - +/** + * How a {@link CuaModelAnnotation} matches model ids. + * + * - `exact`: `id === match.id` + * - `family`: `id === match.family`, or `match.family` plus hyphen-separated + * numeric segments (revisions and dated snapshots, e.g. "claude-opus-4-7", + * "gpt-5.5-2026-04-23"). Named variants like "gpt-5.4-mini" are distinct + * models and need their own entry. + */ export type CuaModelMatch = | { readonly kind: "exact"; readonly id: string } | { readonly kind: "family"; readonly family: string }; +/** One CUA-support annotation: a model-id match plus the official source documenting support. */ export interface CuaModelAnnotation { readonly match: CuaModelMatch; + /** URL of the provider documentation establishing computer-use support. */ readonly source: string; } +/** + * Per-provider computer-use support annotations. + * + * pi-ai's model registry is generated from models.dev (see + * node_modules/@earendil-works/pi-ai/scripts/generate-models.ts) and lists every + * model a provider offers. Only some of those models support computer-use, so + * this table layers per-provider CUA-support annotations on top of the + * registry. Each entry cites the official source documenting CUA support. + * + * To verify support and add new entries, follow the `update-models` skill at + * .agents/skills/update-models/SKILL.md. + */ export const CUA_MODEL_ANNOTATIONS: Record = { openai: [ { match: { kind: "family", family: "gpt-5.4" }, source: "https://developers.openai.com/api/docs/models/gpt-5.4" }, @@ -52,10 +68,12 @@ export const CUA_MODEL_ANNOTATIONS: Record[]> = { cuaModel("openai", "gpt-5.5-2026-04-23", "GPT-5.5 (2026-04-23)"), ], anthropic: [], - google: [ - cuaModel("google", "gemini-2.5-computer-use-preview-10-2025", "Gemini 2.5 Computer Use Preview"), - ], + google: [], tzafon: [ cuaModel("tzafon", "tzafon.northstar-cua-fast", "Tzafon Northstar CUA Fast"), ], @@ -93,23 +109,37 @@ const CUA_MODEL_OVERRIDES: Record[]> = { ], }; +/** + * Split a provider-qualified ref like `"openai:gpt-5.5"` into its parts. + * + * `"gemini:"` is accepted as an alias for the canonical `"google:"` prefix + * and normalizes to provider `"google"`. Throws when the ref is unqualified + * or names an unsupported provider. + */ export function parseCuaModelRef(ref: string): { provider: CuaProvider; model: string } { const idx = ref.indexOf(":"); if (idx <= 0 || idx === ref.length - 1) { throw new Error(`CUA model ref must be provider-qualified as ":"; got "${ref}"`); } - const provider = ref.slice(0, idx); + const prefix = ref.slice(0, idx); + const provider = prefix === "gemini" ? "google" : prefix; const model = ref.slice(idx + 1); if (!isCuaProvider(provider)) { - throw new Error(`unsupported CUA provider "${provider}"`); + throw new Error(`unsupported CUA provider "${prefix}" (expected one of: ${CUA_PROVIDERS.join(", ")})`); } return { provider, model }; } +/** Join a provider and model id into a {@link CuaModelRef}. */ export function formatCuaModelRef(provider: CuaProvider, model: string): CuaModelRef { return `${provider}:${model}` as CuaModelRef; } +/** + * List the computer-use-capable models this package curates, optionally + * filtered to one provider. Merges pi-ai's registry with local overrides and + * keeps only models annotated in {@link CUA_MODEL_ANNOTATIONS}. + */ export function listCuaModels(provider?: CuaProvider): CuaModelInfo[] { const providers = provider ? [provider] : [...CUA_PROVIDERS]; const byRef = new Map(); @@ -135,6 +165,13 @@ export function listCuaModels(provider?: CuaProvider): CuaModelInfo[] { return [...byRef.values()].sort(compareCuaModels); } +/** + * Resolve a {@link CuaModelRef} to a concrete pi-ai model. + * + * Throws when the ref is unqualified, names an unsupported provider, or names + * a model without a CUA-support annotation. `"gemini:"` refs are accepted as + * an alias for `"google:"` (see {@link parseCuaModelRef}). + */ export function getCuaModel(ref: CuaModelRef): Model { const { provider, model: modelId } = parseCuaModelRef(ref); if (!supportsCuaProvider(provider, modelId)) { @@ -147,13 +184,15 @@ export function getCuaModel(ref: CuaModelRef): Model { throw new Error(`CUA model "${ref}" is supported but not registered. Add it to pi-ai (models.dev) or CUA_MODEL_OVERRIDES.`); } +/** Return the {@link CuaProvider} for a concrete model, or throw when it is not a CUA provider. */ export function providerForModel(model: Model): CuaProvider { if (!isCuaProvider(model.provider)) { - throw new Error(`unsupported CUA model provider "${model.provider}"`); + throw new Error(`unsupported CUA model provider "${model.provider}" (expected one of: ${CUA_PROVIDERS.join(", ")})`); } return model.provider; } +/** Narrow an arbitrary string to {@link CuaProvider}. */ export function isCuaProvider(value: string): value is CuaProvider { return (CUA_PROVIDERS as readonly string[]).includes(value); } @@ -162,19 +201,33 @@ function supportsCuaProvider(provider: CuaProvider, modelId: string): boolean { return findCuaAnnotation(provider, modelId) !== undefined; } +/** Find the CUA-support annotation covering a model id, if any. */ export function findCuaAnnotation(provider: CuaProvider, modelId: string): CuaModelAnnotation | undefined { const id = modelId.toLowerCase(); for (const annotation of CUA_MODEL_ANNOTATIONS[provider]) { if (annotation.match.kind === "exact") { if (id === annotation.match.id.toLowerCase()) return annotation; - } else { - const family = annotation.match.family.toLowerCase(); - if (id === family || id.startsWith(`${family}-`)) return annotation; + } else if (isCuaFamilyMatch(id, annotation.match.family.toLowerCase())) { + return annotation; } } return undefined; } +// A family annotation covers its root id plus suffixes made of +// hyphen-separated numeric segments: revisions like "claude-opus-4-7" and +// dated snapshots like "gpt-5.5-2026-04-23" or "claude-3-7-sonnet-20250219". +// Named sibling variants ("gpt-5.4-mini") may not support computer use and +// must be annotated explicitly. +function isCuaFamilyMatch(id: string, family: string): boolean { + if (id === family) return true; + if (!id.startsWith(`${family}-`)) return false; + return id + .slice(family.length + 1) + .split("-") + .every((segment) => /^\d+$/.test(segment)); +} + function cuaModel(provider: CuaProvider, id: string, name: string): Model { const base = { id, diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index 214c5920..edd0cbad 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest"; -import { getCuaModel, listCuaModels } from "../src/index.js"; import { CUA_MODEL_ANNOTATIONS, CUA_PROVIDERS, findCuaAnnotation, formatCuaModelRef, + getCuaModel, + listCuaModels, parseCuaModelRef, } from "../src/index.js"; @@ -20,6 +21,22 @@ describe("CUA model refs", () => { expect(() => getCuaModel("openai:gpt-3.5" as never)).toThrow(/unsupported CUA model/); }); + it("names the valid providers in the unsupported-provider error", () => { + expect(() => parseCuaModelRef("bogus:model")).toThrow( + 'unsupported CUA provider "bogus" (expected one of: openai, anthropic, google, tzafon, yutori)', + ); + }); + + it("accepts gemini: as an alias for google:", () => { + expect(parseCuaModelRef("gemini:gemini-3-flash-preview")).toEqual({ + provider: "google", + model: "gemini-3-flash-preview", + }); + const model = getCuaModel("gemini:gemini-3-flash-preview" as never); + expect(model.provider).toBe("google"); + expect(model.id).toBe("gemini-3-flash-preview"); + }); + it("lists curated model refs without a default", () => { const models = listCuaModels(); expect(models.some((model) => model.ref === "openai:gpt-5.5")).toBe(true); @@ -41,7 +58,7 @@ describe("CUA model refs", () => { it("rejects supported model IDs that are not in pi-ai or overrides", () => { // Matches the openai allowlist but has no pi-ai or override entry. - expect(() => getCuaModel("openai:gpt-5.4-fake-snapshot")).toThrow( + expect(() => getCuaModel("openai:gpt-5.4-2099-01-01")).toThrow( /not registered/, ); }); @@ -62,10 +79,11 @@ describe("CUA support annotations", () => { } }); - it("matches family roots and dated snapshots", () => { + it("matches family roots, dated snapshots, and numeric revisions", () => { expect(findCuaAnnotation("openai", "gpt-5.5")?.match).toEqual({ kind: "family", family: "gpt-5.5" }); expect(findCuaAnnotation("openai", "gpt-5.5-2026-04-23")?.match).toEqual({ kind: "family", family: "gpt-5.5" }); expect(findCuaAnnotation("anthropic", "claude-opus-4-7")).toBeDefined(); + expect(findCuaAnnotation("anthropic", "claude-3-7-sonnet-20250219")).toBeDefined(); }); it("does not match adjacent families", () => { @@ -74,10 +92,30 @@ describe("CUA support annotations", () => { expect(findCuaAnnotation("anthropic", "claude-3-5-sonnet")).toBeUndefined(); }); + it("does not match named sibling variants of a family", () => { + expect(findCuaAnnotation("openai", "gpt-5.4-mini")).toBeUndefined(); + expect(findCuaAnnotation("openai", "gpt-5.4-nano")).toBeUndefined(); + expect(findCuaAnnotation("openai", "gpt-5.4-pro")).toBeUndefined(); + expect(findCuaAnnotation("openai", "gpt-5.5-pro")).toBeUndefined(); + const openaiModels = listCuaModels("openai").map((model) => model.model); + expect(openaiModels).not.toContain("gpt-5.4-mini"); + expect(openaiModels).not.toContain("gpt-5.4-nano"); + expect(openaiModels).not.toContain("gpt-5.4-pro"); + expect(openaiModels).toContain("gpt-5.5"); + }); + it("matches exact-id annotations", () => { expect(findCuaAnnotation("google", "gemini-3-flash-preview")).toBeDefined(); expect(findCuaAnnotation("google", "gemini-3-pro-preview")).toBeDefined(); expect(findCuaAnnotation("yutori", "n1.5-latest")).toBeDefined(); expect(findCuaAnnotation("tzafon", "tzafon.northstar-cua-fast")).toBeDefined(); }); + + it("no longer advertises the Gemini 2.5 computer-use preview", () => { + // The model rejects the function-declaration tools this package sends; + // it needs Google's native tools.computer_use wrapper. + expect(findCuaAnnotation("google", "gemini-2.5-computer-use-preview-10-2025")).toBeUndefined(); + expect(listCuaModels("google").map((model) => model.model)).not.toContain("gemini-2.5-computer-use-preview-10-2025"); + expect(() => getCuaModel("google:gemini-2.5-computer-use-preview-10-2025")).toThrow(/unsupported CUA model/); + }); }); From 74bc78cdc0ecedb402d9f9793c59baadf4632836 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:35:35 +0000 Subject: [PATCH 3/9] Unify cua-ai provider namespaces and harden stream parsing Standardize every provider namespace on _CUA_ACTION_TYPES and _COMPUTER_INSTRUCTIONS, restore anthropic's action-type export, and enforce the convention in tests. Align CUA_BATCH_TOOL_NAME with the computer_batch tool Anthropic actually ships and export ANTHROPIC_BATCH_TOOL_NAME. Make yutori.computerTools honor and validate its options instead of silently ignoring them, and document that the definitions are local mirrors stripped from the wire payload. Export and align the Yutori/Tzafon stream option interfaces (both now carry keepToolNames; redundant temperature/maxOutputTokens dropped), type mouse buttons as closed unions, export registerCuaProviders and make it re-register after pi-ai registry mutators clobber it, and thread tool options through resolveCuaRuntimeSpec. Harden the Yutori stream so one malformed tool call degrades to empty args instead of erroring the whole turn, restore api-literal typing on the Tzafon stream, and add unit coverage for Tzafon action normalization and argument unwrapping plus mocked-stream tests for both custom providers. Co-Authored-By: Claude Opus 4.7 --- packages/ai/src/index.ts | 1 + packages/ai/src/providers.ts | 14 +++- .../ai/src/providers/anthropic/actions.ts | 28 ++++--- packages/ai/src/providers/anthropic/index.ts | 2 + packages/ai/src/providers/common.ts | 27 +++++-- packages/ai/src/providers/gemini/index.ts | 4 +- packages/ai/src/providers/tzafon/index.ts | 8 +- packages/ai/src/providers/tzafon/provider.ts | 18 +++-- packages/ai/src/providers/yutori/actions.ts | 56 ++++++++++--- packages/ai/src/providers/yutori/index.ts | 17 +++- packages/ai/src/providers/yutori/provider.ts | 12 ++- packages/ai/src/runtime-spec.ts | 11 ++- packages/ai/test/provider-module.test.ts | 34 +++++++- packages/ai/test/providers.test.ts | 13 ++- packages/ai/test/runtime-spec.test.ts | 13 +++ packages/ai/test/tools.test.ts | 23 +++++- packages/ai/test/tzafon-actions.test.ts | 80 +++++++++++++++++++ packages/ai/test/tzafon-provider.test.ts | 78 +++++++++++++++++- packages/ai/test/yutori-provider.test.ts | 78 ++++++++++++++++++ 19 files changed, 461 insertions(+), 56 deletions(-) create mode 100644 packages/ai/test/tzafon-actions.test.ts create mode 100644 packages/ai/test/yutori-provider.test.ts diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index dafd7dcc..59132081 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -2,6 +2,7 @@ import { registerCuaProviders } from "./providers.js"; export * from "@earendil-works/pi-ai"; +export { registerCuaProviders } from "./providers.js"; export * from "./models.js"; export * from "./api-keys.js"; export * from "./runtime-spec.js"; diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index 69cacb9d..a0f0edd5 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -2,14 +2,21 @@ import { registerApiProvider } from "@earendil-works/pi-ai"; import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider.js"; import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider.js"; -let registered = false; - // pi-ai eagerly registers openai-responses, anthropic-messages, and // google-generative-ai when its index module loads (see // node_modules/@earendil-works/pi-ai/dist/providers/register-builtins.js). // CUA only needs to add the providers pi-ai does not ship: Tzafon and Yutori. + +/** + * Register the Yutori and Tzafon stream providers with pi-ai's global API + * registry. Importing `@onkernel/cua-ai` calls this automatically. + * + * The pi-ai registry mutators this package re-exports (`clearApiProviders`, + * `resetApiProviders`, `unregisterApiProviders`) deregister these providers, + * after which Yutori/Tzafon streaming fails until they are registered again. + * Call this to restore them; it is idempotent and safe to call repeatedly. + */ export function registerCuaProviders(): void { - if (registered) return; registerApiProvider({ api: YUTORI_CHAT_COMPLETIONS_API, stream: streamYutori, @@ -20,7 +27,6 @@ export function registerCuaProviders(): void { stream: streamTzafonResponses, streamSimple: streamSimpleTzafonResponses, }); - registered = true; } export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses }; diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts index 76cb8833..933b79bd 100644 --- a/packages/ai/src/providers/anthropic/actions.ts +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -1,6 +1,7 @@ import type { Tool, TSchema } from "@earendil-works/pi-ai"; import { CUA_BATCH_TOOL_DESCRIPTION, + CUA_BATCH_TOOL_NAME, createCuaActionSchema, createCuaActionToolExecutors, createCuaActionToolDefinitions, @@ -12,12 +13,16 @@ import { type CuaToolExecutorSpec, } from "../common.js"; -// Source of truth: Anthropic's computer-use best-practices quickstart -// computer/browser tool action enums. These are the browser actions Anthropic -// currently accepts under CUA's canonical individual tool names. -// https://github.com/anthropics/claude-quickstarts/blob/main/computer-use-best-practices/computer_use/tools/computer.py -// https://github.com/anthropics/claude-quickstarts/blob/main/computer-use-best-practices/computer_use/tools/browser.py -const ANTHROPIC_CANONICAL_ACTION_TYPES = [ +/** + * Canonical CUA action types Anthropic browser computer-use tools support. + * + * Source of truth: Anthropic's computer-use best-practices quickstart + * computer/browser tool action enums. These are the browser actions Anthropic + * currently accepts under CUA's canonical individual tool names. + * https://github.com/anthropics/claude-quickstarts/blob/main/computer-use-best-practices/computer_use/tools/computer.py + * https://github.com/anthropics/claude-quickstarts/blob/main/computer-use-best-practices/computer_use/tools/browser.py + */ +export const ANTHROPIC_CUA_ACTION_TYPES = [ "click", "double_click", "mouse_down", @@ -33,10 +38,13 @@ const ANTHROPIC_CANONICAL_ACTION_TYPES = [ "cursor_position", ] as const satisfies readonly CuaActionType[]; -type AnthropicCanonicalActionType = (typeof ANTHROPIC_CANONICAL_ACTION_TYPES)[number]; +type AnthropicCanonicalActionType = (typeof ANTHROPIC_CUA_ACTION_TYPES)[number]; + +const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set(ANTHROPIC_CUA_ACTION_TYPES); + +/** Name of the batch tool included by default in Anthropic computer-use tools. */ +export const ANTHROPIC_BATCH_TOOL_NAME = CUA_BATCH_TOOL_NAME; -const ANTHROPIC_CANONICAL_ACTION_TYPE_SET: ReadonlySet = new Set(ANTHROPIC_CANONICAL_ACTION_TYPES); -const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; const ANTHROPIC_BATCH_TOOL_DESCRIPTION = [ CUA_BATCH_TOOL_DESCRIPTION, "Coordinates in a batch refer to the screenshot taken before the batch call.", @@ -52,7 +60,7 @@ export interface AnthropicComputerToolsOptions extends ComputerToolsOptions { export type AnthropicAction = Extract; function resolveAnthropicActions(actions: readonly CuaActionType[] | undefined): readonly AnthropicCanonicalActionType[] { - const resolved = actions ?? ANTHROPIC_CANONICAL_ACTION_TYPES; + const resolved = actions ?? ANTHROPIC_CUA_ACTION_TYPES; const supported: AnthropicCanonicalActionType[] = []; const unsupported: CuaActionType[] = []; for (const action of resolved) { diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 7fa7c233..72ad5c30 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -2,6 +2,8 @@ import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common. import { computerToolExecutors, computerTools } from "./actions.js"; export { + ANTHROPIC_BATCH_TOOL_NAME, + ANTHROPIC_CUA_ACTION_TYPES, computerToolExecutors, computerTools, createActionSchema, diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 2c27c59f..0dc743c8 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -22,11 +22,23 @@ export const CUA_ACTION_TYPES = [ export type CuaActionType = (typeof CUA_ACTION_TYPES)[number]; +/** + * Mouse buttons accepted by click, mouse_down, and mouse_up actions. The + * executor coerces anything outside this set to "left". + */ +export type CuaMouseButton = "left" | "right" | "middle" | "back" | "forward"; + +/** + * Mouse buttons accepted by drag actions. The executor coerces anything + * outside this set to "left". + */ +export type CuaDragMouseButton = "left" | "right" | "middle"; + export interface CuaActionClick { type: "click"; x: number; y: number; - button?: string; + button?: CuaMouseButton; hold_keys?: string[]; } @@ -41,7 +53,7 @@ export interface CuaActionMouseDown { type: "mouse_down"; x: number; y: number; - button?: string; + button?: CuaMouseButton; hold_keys?: string[]; } @@ -49,7 +61,7 @@ export interface CuaActionMouseUp { type: "mouse_up"; x: number; y: number; - button?: string; + button?: CuaMouseButton; hold_keys?: string[]; } @@ -82,7 +94,7 @@ export interface CuaActionMove { export interface CuaActionDrag { type: "drag"; path: Array<{ x: number; y: number }>; - button?: string; + button?: CuaDragMouseButton; hold_keys?: string[]; } @@ -298,7 +310,12 @@ export interface CuaToolExecutorSpec { toActions(args: unknown): CuaAction[]; } -export const CUA_BATCH_TOOL_NAME = "batch_computer_actions"; +/** + * Default name for batch computer-action tools created by + * {@link createCuaBatchToolDefinition} and the name Anthropic's batch tool + * ships under (the only provider that includes one by default). + */ +export const CUA_BATCH_TOOL_NAME = "computer_batch"; export const CUA_NAVIGATION_TOOL_NAME = "computer_use_extra"; export const CUA_BATCH_TOOL_DESCRIPTION = [ diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index cd319ee5..8e801256 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -23,10 +23,10 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { return { type: "normalized", range: [0, 999] }; } -export const GEMINI_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Use the provider coordinate system for tool calls, and request screenshots or URL reads when state changes.`; +export const GEMINI_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use the provider coordinate system for tool calls, and request screenshots or URL reads when state changes.`; export function buildGeminiSystemPrompt(opts: { suffix?: string } = {}): string { - return [GEMINI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); + return [GEMINI_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } export const providerModule = { diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 7b3bf2c1..57be34b4 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -2,7 +2,7 @@ import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem import { tzafonComputerUseOnPayload } from "./provider.js"; export { - CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, + CUA_ACTION_TYPES as TZAFON_CUA_ACTION_TYPES, computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, @@ -15,9 +15,11 @@ export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses, + toCanonicalActions, tzafonComputerUseOnPayload, tzafonToolCallId, } from "./provider.js"; +export type { TzafonCanonicalAction, TzafonResponsesOptions } from "./provider.js"; // Provider-native action vocabulary. The model card lists supported actions; // the Responses API loop dispatches on `action.type` and adds terminal control @@ -33,11 +35,11 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { return { type: "normalized", range: [0, 999] }; } -export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Include screenshot or URL reads when you need updated state.`; +export const TZAFON_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Include screenshot or URL reads when you need updated state.`; /** Build the default system prompt used with Tzafon CUA models. */ export function buildTzafonSystemPrompt(opts: { suffix?: string } = {}): string { - return [TZAFON_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); + return [TZAFON_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } export const providerModule = { diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index 81b62d7b..ea275922 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -24,15 +24,17 @@ const TZAFON_COMPUTER_USE_TOOL = { } as const; const TZAFON_LOCAL_ACTION_TOOL_NAMES = new Set(CUA_ACTION_TYPES); +/** Stream options accepted by {@link streamTzafonResponses}. */ export interface TzafonResponsesOptions extends StreamOptions { - maxOutputTokens?: number; + /** Tool names to keep in the outbound payload even though they collide with local CUA action tool names. */ + keepToolNames?: readonly string[]; } -export const streamSimpleTzafonResponses: StreamFunction = (model, context, options) => { +export const streamSimpleTzafonResponses: StreamFunction = (model, context, options) => { return streamTzafonResponses(model, context, options); }; -export const streamTzafonResponses: StreamFunction = (model, context, options) => { +export const streamTzafonResponses: StreamFunction = (model, context, options) => { const stream = createAssistantMessageEventStream(); const output = initialAssistantMessage(model); @@ -47,10 +49,10 @@ export const streamTzafonResponses: StreamFunction, { - keepToolNames: keepToolNamesFromContext(context), + keepToolNames: [...keepToolNamesFromContext(context), ...(options?.keepToolNames ?? [])], }); const nextPayload = await options?.onPayload?.(tzafonPayload ?? payload, model as Model); if (options?.signal?.aborted) throw new Error("Request was aborted"); @@ -175,9 +177,11 @@ function emitToolCall( stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); } -type TzafonCanonicalAction = CuaAction | { type: "answer"; text: string }; +/** A canonical CUA action, or the terminal `answer` text Tzafon emits when it is done. */ +export type TzafonCanonicalAction = CuaAction | { type: "answer"; text: string }; -function toCanonicalActions(action: unknown): TzafonCanonicalAction[] { +/** Normalize one Tzafon `computer_call.action` payload into canonical CUA actions. */ +export function toCanonicalActions(action: unknown): TzafonCanonicalAction[] { if (!action || typeof action !== "object") return []; const current = action as Record; const type = getString(current, "type"); diff --git a/packages/ai/src/providers/yutori/actions.ts b/packages/ai/src/providers/yutori/actions.ts index f39271d6..7b74c71c 100644 --- a/packages/ai/src/providers/yutori/actions.ts +++ b/packages/ai/src/providers/yutori/actions.ts @@ -1,7 +1,10 @@ +import type { Tool, TSchema } from "@earendil-works/pi-ai"; import { + createCuaActionSchema, createCuaActionToolDefinitions, createCuaActionToolExecutors, normalizeGotoUrl, + type ComputerToolsOptions, type CuaAction, type CuaToolExecutorSpec, type CuaActionType, @@ -83,7 +86,12 @@ export const YUTORI_N15_ACTION_TYPES = [ ...YUTORI_N15_EXPANDED_ACTION_TYPES, ] as const; -export const YUTORI_CANONICAL_ACTION_TYPES = [ +/** + * Canonical CUA action types Yutori's native actions normalize into. These are + * the tool-call names {@link streamYutori} emits and the local executors + * CuaAgent installs for Yutori models. + */ +export const YUTORI_CUA_ACTION_TYPES = [ "click", "double_click", "mouse_down", @@ -99,6 +107,11 @@ export const YUTORI_CANONICAL_ACTION_TYPES = [ "forward", ] as const satisfies readonly CuaActionType[]; +type YutoriCanonicalActionType = (typeof YUTORI_CUA_ACTION_TYPES)[number]; + +/** Canonical CUA action shape emitted for Yutori models. */ +export type YutoriAction = Extract; + export type YutoriN1ActionType = (typeof YUTORI_N1_ACTION_TYPES)[number]; export type YutoriN15CoreActionType = (typeof YUTORI_N15_CORE_ACTION_TYPES)[number]; export type YutoriN15ExpandedActionType = (typeof YUTORI_N15_EXPANDED_ACTION_TYPES)[number]; @@ -110,19 +123,44 @@ const DEFAULT_WAIT_MS = 2000; const NAVIGATION_WAIT_MS = 1500; const GOTO_WAIT_MS = 2000; +function resolveYutoriActions(actions: readonly CuaActionType[] | undefined): readonly YutoriCanonicalActionType[] { + const resolved = actions ?? YUTORI_CUA_ACTION_TYPES; + const supported: YutoriCanonicalActionType[] = []; + const unsupported: CuaActionType[] = []; + for (const action of resolved) { + if (isYutoriCanonicalAction(action)) supported.push(action); + else unsupported.push(action); + } + if (unsupported.length > 0) throw new Error(`unsupported Yutori canonical action(s): ${unsupported.join(", ")}`); + return supported; +} + +function isYutoriCanonicalAction(action: CuaActionType): action is YutoriCanonicalActionType { + return (YUTORI_CUA_ACTION_TYPES as readonly string[]).includes(action); +} + +/** Build the TypeBox schema for Yutori-supported canonical browser actions. */ +export function createActionSchema(actions?: readonly CuaActionType[]): TSchema { + return createCuaActionSchema(resolveYutoriActions(actions)); +} + /** - * Build Yutori CUA computer-use tools. + * Build local mirrors of the canonical action tools Yutori models call. * - * Use this when calling `complete()` or `stream()` directly and you need an - * array of `Tool` objects for Yutori browser actions. + * These definitions are never sent to the API: `streamYutori` strips them from + * the outbound payload and selects Yutori's native `tool_set` instead, then + * normalizes the model's native tool calls back into these canonical names. + * Install them locally so the normalized calls have matching executors — + * `providerModule.toolDefinitions()` is intentionally `[]`. Pass `actions` to + * mirror only a supported subset, such as `["click"]`. */ -export function computerTools(_options?: unknown) { - return createCuaActionToolDefinitions(YUTORI_CANONICAL_ACTION_TYPES); +export function computerTools(options: ComputerToolsOptions = {}): Tool[] { + return createCuaActionToolDefinitions(resolveYutoriActions(options.actions)); } /** Build the local execution adapters used by CuaAgent and CuaAgentHarness. */ -export function computerToolExecutors(_options?: unknown): CuaToolExecutorSpec[] { - return createCuaActionToolExecutors(YUTORI_CANONICAL_ACTION_TYPES); +export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaToolExecutorSpec[] { + return createCuaActionToolExecutors(resolveYutoriActions(options.actions)); } export function yutoriToolSetForModel(modelId: string): typeof YUTORI_N15_CORE_TOOL_SET | undefined { @@ -134,7 +172,7 @@ export function yutoriNativeActionsForModel(modelId: string): readonly YutoriNat } export function isYutoriLocalActionToolName(name: string): boolean { - return (YUTORI_CANONICAL_ACTION_TYPES as readonly string[]).includes(name); + return (YUTORI_CUA_ACTION_TYPES as readonly string[]).includes(name); } export function toCanonicalActions(name: string, args: Record): CuaAction[] | undefined { diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index 3840aef0..3675805d 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -5,10 +5,11 @@ import { yutoriNativeToolSetOnPayload } from "./provider.js"; export { computerToolExecutors, computerTools, + createActionSchema, toCanonicalActions, yutoriNativeActionsForModel, yutoriToolSetForModel, - YUTORI_CANONICAL_ACTION_TYPES, + YUTORI_CUA_ACTION_TYPES, YUTORI_N1_ACTION_TYPES, YUTORI_N15_ACTION_TYPES, YUTORI_N15_CORE_ACTION_TYPES, @@ -16,13 +17,21 @@ export { YUTORI_N15_EXPANDED_ACTION_TYPES, YUTORI_N15_EXPANDED_TOOL_SET, } from "./actions.js"; -export type { YutoriN1ActionType, YutoriN15CoreActionType, YutoriN15ExpandedActionType, YutoriNativeActionType } from "./actions.js"; +export type { + YutoriAction, + YutoriN1ActionType, + YutoriN15CoreActionType, + YutoriN15ExpandedActionType, + YutoriNativeActionType, +} from "./actions.js"; +export type { ComputerToolsOptions } from "../common.js"; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori, yutoriNativeToolSetOnPayload, } from "./provider.js"; +export type { YutoriOptions } from "./provider.js"; // Provider-native action vocabulary differs between Navigator versions: // n1 (fixed tool set): @@ -46,10 +55,10 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { // Yutori's Navigator quickstart recommends putting extra instructions in the // first user message instead of supplying a custom system prompt. // Source: https://docs.yutori.com/llm-quickstart.md -export const YUTORI_INSTRUCTIONS_RAW = ""; +export const YUTORI_COMPUTER_INSTRUCTIONS = ""; export function buildYutoriSystemPrompt(opts: { suffix?: string } = {}): string { - return [YUTORI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); + return [YUTORI_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } export const providerModule = { diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index f79d324e..d8ef70f2 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -23,8 +23,9 @@ import { canonicalToolCallArguments, canonicalToolCallName, type CuaPayloadConte export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; +/** Stream options accepted by {@link streamYutori}. */ export interface YutoriOptions extends StreamOptions { - temperature?: number; + /** Tool names to keep in the outbound payload even though they collide with local CUA action tool names. */ keepToolNames?: readonly string[]; } @@ -238,9 +239,16 @@ function toOpenAIContentPart(part: TextContent | ImageContent): Record { if (!value?.trim()) return {}; - return JSON.parse(value) as Record; + try { + const parsed = JSON.parse(value) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record) : {}; + } catch { + return {}; + } } function usageFromYutori(usage: unknown): AssistantMessage["usage"] { diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 148f639c..e5a0ad7b 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -6,6 +6,7 @@ import { providerModule as openai } from "./providers/openai/index.js"; import { providerModule as tzafon } from "./providers/tzafon/index.js"; import { providerModule as yutori } from "./providers/yutori/index.js"; import type { + ComputerToolsOptions, CuaProviderModule, CuaRuntimeSpec, CuaRuntimeSpecInput, @@ -23,17 +24,19 @@ const PROVIDERS = { * Resolve provider defaults from either a CUA model ref or a concrete model. * * Use the returned spec to build computer-use requests without hard-coding - * model-provider rules in your application. + * model-provider rules in your application. Pass `options` (e.g. + * `{ actions: ["click"] }`) to narrow the resolved tool definitions and + * executors to a supported subset. */ -export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpec { +export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput, options?: ComputerToolsOptions): CuaRuntimeSpec { const model = typeof input === "string" ? getCuaModel(input) : input; const provider = providerForModel(model); const mod: CuaProviderModule = PROVIDERS[provider]; return { model, provider, - toolDefinitions: mod.toolDefinitions(), - toolExecutors: mod.toolExecutors(), + toolDefinitions: mod.toolDefinitions(options), + toolExecutors: mod.toolExecutors(options), defaultSystemPrompt: mod.buildSystemPrompt(), coordinateSystem: mod.coordinateSystem(), screenshot: mod.screenshot, diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index 8c527d82..a8910d81 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index.js"; +import { anthropic, CUA_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index.js"; import type { CuaProviderModule } from "../src/providers/common.js"; const MODULES: Record = { @@ -10,6 +10,14 @@ const MODULES: Record = { yutori, }; +const NAMESPACES: Record; prefix: string }> = { + openai: { namespace: openai, prefix: "OPENAI" }, + anthropic: { namespace: anthropic, prefix: "ANTHROPIC" }, + google: { namespace: gemini, prefix: "GEMINI" }, + tzafon: { namespace: tzafon, prefix: "TZAFON" }, + yutori: { namespace: yutori, prefix: "YUTORI" }, +}; + describe("provider modules satisfy the uniform contract", () => { for (const provider of CUA_PROVIDERS) { it(`${provider} conforms to CuaProviderModule`, () => { @@ -44,3 +52,27 @@ describe("provider modules satisfy the uniform contract", () => { expect(yutori.providerModule.toolDefinitions()).toEqual([]); }); }); + +describe("provider namespaces export a uniform surface", () => { + for (const provider of CUA_PROVIDERS) { + it(`${provider} follows the namespace export conventions`, () => { + const { namespace, prefix } = NAMESPACES[provider]; + + const actionTypes = namespace[`${prefix}_CUA_ACTION_TYPES`]; + expect(Array.isArray(actionTypes), `${prefix}_CUA_ACTION_TYPES must be exported`).toBe(true); + expect((actionTypes as unknown[]).length).toBeGreaterThan(0); + for (const action of actionTypes as string[]) { + expect(CUA_ACTION_TYPES).toContain(action); + } + + expect(namespace[`${prefix}_COMPUTER_INSTRUCTIONS`], `${prefix}_COMPUTER_INSTRUCTIONS must be exported`).toBeTypeOf( + "string", + ); + expect(namespace.computerTools).toBeTypeOf("function"); + expect(namespace.computerToolExecutors).toBeTypeOf("function"); + expect(namespace.createActionSchema).toBeTypeOf("function"); + expect(namespace.coordinateSystem).toBeTypeOf("function"); + expect(namespace.providerModule).toBeTypeOf("object"); + }); + } +}); diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index c39b322c..a80bfd8c 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getApiProvider, tzafon, yutori } from "../src/index.js"; +import { getApiProvider, registerCuaProviders, resetApiProviders, tzafon, yutori } from "../src/index.js"; const TZAFON_RESPONSES_API = tzafon.TZAFON_RESPONSES_API; const YUTORI_CHAT_COMPLETIONS_API = yutori.YUTORI_CHAT_COMPLETIONS_API; @@ -29,4 +29,15 @@ describe("CUA provider registration", () => { expect(provider?.streamSimple).toBeTypeOf("function"); } }); + + it("restores CUA providers after pi-ai registry mutators clobber them", () => { + resetApiProviders(); + expect(getApiProvider(YUTORI_CHAT_COMPLETIONS_API)).toBeUndefined(); + expect(getApiProvider(TZAFON_RESPONSES_API)).toBeUndefined(); + + registerCuaProviders(); + registerCuaProviders(); + expect(getApiProvider(YUTORI_CHAT_COMPLETIONS_API)).toBeDefined(); + expect(getApiProvider(TZAFON_RESPONSES_API)).toBeDefined(); + }); }); diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index 525e0222..fd5c4c4d 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -41,4 +41,17 @@ describe("resolveCuaRuntimeSpec", () => { expect(tzafonSpec.onPayload).toBeTypeOf("function"); expect(anthropicSpec.onPayload).toBeUndefined(); }); + + it("threads tool options through to the provider module", () => { + const openaiSpec = resolveCuaRuntimeSpec("openai:gpt-5.5", { actions: ["click"] }); + expect(openaiSpec.toolDefinitions.map((tool) => tool.name)).toEqual(["click"]); + expect(openaiSpec.toolExecutors.map((executor) => executor.definition.name)).toEqual(["click"]); + + const anthropicSpec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-7", { actions: ["click"] }); + expect(anthropicSpec.toolDefinitions.map((tool) => tool.name)).toEqual(["click", "computer_batch"]); + + const yutoriSpec = resolveCuaRuntimeSpec("yutori:n1.5-latest", { actions: ["click"] }); + expect(yutoriSpec.toolDefinitions).toEqual([]); + expect(yutoriSpec.toolExecutors.map((executor) => executor.definition.name)).toEqual(["click"]); + }); }); diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index 77d0ef1c..667fa712 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -14,7 +14,7 @@ import { } from "../src/index.js"; const providers = { openai, gemini, tzafon }; -const ANTHROPIC_BATCH_TOOL_NAME = "computer_batch"; +const ANTHROPIC_BATCH_TOOL_NAME = anthropic.ANTHROPIC_BATCH_TOOL_NAME; function batchActionVariants(tool: { parameters: any }): any[] { const items = tool.parameters.properties.actions.items; @@ -51,6 +51,11 @@ describe("computer tool definitions", () => { expect(batchActionVariants(tool).map((v) => v.properties.type.const)).toEqual(subset); }); + it("names the default batch tool after the one Anthropic ships", () => { + expect(CUA_BATCH_TOOL_NAME).toBe("computer_batch"); + expect(anthropic.ANTHROPIC_BATCH_TOOL_NAME).toBe(CUA_BATCH_TOOL_NAME); + }); + it("emits a single-variant batch schema when narrowed to one action", () => { const tool = createCuaBatchToolDefinition(["click"]); const items = tool.parameters.properties.actions.items; @@ -66,11 +71,25 @@ describe("computer tool definitions", () => { it("exposes local canonical executor definitions for Yutori", () => { const tools = yutori.computerTools(); - expect(tools.map((tool) => tool.name)).toEqual([...yutori.YUTORI_CANONICAL_ACTION_TYPES]); + expect(tools.map((tool) => tool.name)).toEqual([...yutori.YUTORI_CUA_ACTION_TYPES]); expect(tools.map((tool) => tool.name)).not.toContain(CUA_BATCH_TOOL_NAME); expect(tools.map((tool) => tool.name)).not.toContain(CUA_NAVIGATION_TOOL_NAME); }); + it("narrows Yutori tools and executors to a supported subset", () => { + const tools = yutori.computerTools({ actions: ["click"] }); + expect(tools.map((tool) => tool.name)).toEqual(["click"]); + const executors = yutori.computerToolExecutors({ actions: ["click", "type"] }); + expect(executors.map((executor) => executor.definition.name)).toEqual(["click", "type"]); + }); + + it("rejects unsupported Yutori action narrowing", () => { + expect(() => yutori.computerTools({ actions: ["url"] })).toThrow("unsupported Yutori canonical action(s): url"); + expect(() => yutori.createActionSchema(["screenshot"])).toThrow( + "unsupported Yutori canonical action(s): screenshot", + ); + }); + it("exports Yutori native action sets by model family", () => { expect(yutori.yutoriNativeActionsForModel("n1-latest")).toEqual(yutori.YUTORI_N1_ACTION_TYPES); expect(yutori.yutoriNativeActionsForModel("n1.5-latest")).toEqual(yutori.YUTORI_N15_CORE_ACTION_TYPES); diff --git a/packages/ai/test/tzafon-actions.test.ts b/packages/ai/test/tzafon-actions.test.ts new file mode 100644 index 00000000..ff516c19 --- /dev/null +++ b/packages/ai/test/tzafon-actions.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { tzafon } from "../src/index.js"; + +describe("Tzafon native action normalization", () => { + it("normalizes click variants", () => { + expect(tzafon.toCanonicalActions({ type: "click", x: 10, y: 20 })).toEqual([{ type: "click", x: 10, y: 20 }]); + expect(tzafon.toCanonicalActions({ type: "left_click", x: 10, y: 20 })).toEqual([{ type: "click", x: 10, y: 20 }]); + expect(tzafon.toCanonicalActions({ type: "right_click", x: 10, y: 20 })).toEqual([ + { type: "click", x: 10, y: 20, button: "right" }, + ]); + expect(tzafon.toCanonicalActions({ type: "double_click", x: 10, y: 20 })).toEqual([ + { type: "double_click", x: 10, y: 20 }, + ]); + expect(tzafon.toCanonicalActions({ type: "triple_click", x: 10, y: 20 })).toEqual([ + { type: "double_click", x: 10, y: 20 }, + { type: "click", x: 10, y: 20 }, + ]); + }); + + it("coerces string coordinates to numbers", () => { + expect(tzafon.toCanonicalActions({ type: "click", x: "10", y: "20" })).toEqual([{ type: "click", x: 10, y: 20 }]); + }); + + it("drops pointer actions without usable coordinates", () => { + expect(tzafon.toCanonicalActions({ type: "click" })).toEqual([]); + expect(tzafon.toCanonicalActions({ type: "hover", x: 5 })).toEqual([]); + expect(tzafon.toCanonicalActions(undefined)).toEqual([]); + expect(tzafon.toCanonicalActions({ type: "unknown_action" })).toEqual([]); + }); + + it("normalizes move, hover, and drag", () => { + expect(tzafon.toCanonicalActions({ type: "move", x: 1, y: 2 })).toEqual([{ type: "move", x: 1, y: 2 }]); + expect(tzafon.toCanonicalActions({ type: "hover", x: 1, y: 2 })).toEqual([{ type: "move", x: 1, y: 2 }]); + expect(tzafon.toCanonicalActions({ type: "drag", path: [{ x: 1, y: 2 }, { x: 3, y: 4 }] })).toEqual([ + { type: "drag", path: [{ x: 1, y: 2 }, { x: 3, y: 4 }] }, + ]); + expect(tzafon.toCanonicalActions({ type: "drag", x: 1, y: 2, end_x: 3, end_y: 4 })).toEqual([ + { type: "drag", path: [{ x: 1, y: 2 }, { x: 3, y: 4 }] }, + ]); + expect(tzafon.toCanonicalActions({ type: "drag", x: 1, y: 2, x2: 3, y2: 4 })).toEqual([ + { type: "drag", path: [{ x: 1, y: 2 }, { x: 3, y: 4 }] }, + ]); + }); + + it("normalizes typing and keyboard actions", () => { + expect(tzafon.toCanonicalActions({ type: "type", text: "hello" })).toEqual([{ type: "type", text: "hello" }]); + expect(tzafon.toCanonicalActions({ type: "key", key: "enter" })).toEqual([{ type: "keypress", keys: ["enter"] }]); + expect(tzafon.toCanonicalActions({ type: "key", text: "esc" })).toEqual([{ type: "keypress", keys: ["esc"] }]); + expect(tzafon.toCanonicalActions({ type: "keypress", keys: ["ctrl", "a"] })).toEqual([ + { type: "keypress", keys: ["ctrl", "a"] }, + ]); + expect(tzafon.toCanonicalActions({ type: "keypress" })).toEqual([]); + }); + + it("normalizes scroll variants", () => { + expect(tzafon.toCanonicalActions({ type: "scroll", x: 5, y: 6, scroll_y: 120 })).toEqual([ + { type: "scroll", x: 5, y: 6, scroll_y: 120 }, + ]); + expect(tzafon.toCanonicalActions({ type: "scroll", amount: 240 })).toEqual([{ type: "scroll", scroll_y: 240 }]); + expect(tzafon.toCanonicalActions({ type: "hscroll", amount: 120 })).toEqual([{ type: "scroll", scroll_x: 120 }]); + expect(tzafon.toCanonicalActions({ type: "hscroll" })).toEqual([{ type: "scroll", scroll_x: 0 }]); + }); + + it("normalizes navigation, waits, and screenshots", () => { + expect(tzafon.toCanonicalActions({ type: "navigate", url: "https://example.com" })).toEqual([ + { type: "goto", url: "https://example.com" }, + ]); + expect(tzafon.toCanonicalActions({ type: "wait", ms: 500 })).toEqual([{ type: "wait", ms: 500 }]); + expect(tzafon.toCanonicalActions({ type: "wait", seconds: 2 })).toEqual([{ type: "wait", ms: 2000 }]); + expect(tzafon.toCanonicalActions({ type: "screenshot" })).toEqual([{ type: "screenshot" }]); + }); + + it("maps terminal actions to answer text", () => { + expect(tzafon.toCanonicalActions({ type: "answer", text: "done!" })).toEqual([{ type: "answer", text: "done!" }]); + expect(tzafon.toCanonicalActions({ type: "done", result: "ok" })).toEqual([{ type: "answer", text: "ok" }]); + expect(tzafon.toCanonicalActions({ type: "terminate", status: "success" })).toEqual([ + { type: "answer", text: "success" }, + ]); + }); +}); diff --git a/packages/ai/test/tzafon-provider.test.ts b/packages/ai/test/tzafon-provider.test.ts index b33a2e47..c678142d 100644 --- a/packages/ai/test/tzafon-provider.test.ts +++ b/packages/ai/test/tzafon-provider.test.ts @@ -1,5 +1,20 @@ -import { describe, expect, it } from "vitest"; -import { tzafon } from "../src/index.js"; +import { describe, expect, it, vi } from "vitest"; +import type { Model, ToolCall } from "@earendil-works/pi-ai"; +import { getCuaModel, tzafon } from "../src/index.js"; + +const { responsesCreate } = vi.hoisted(() => ({ responsesCreate: vi.fn() })); + +vi.mock("@tzafon/lightcone", () => ({ + default: class { + responses = { create: responsesCreate }; + }, +})); + +const model = getCuaModel("tzafon:tzafon.northstar-cua-fast") as Model; + +function toolCalls(content: Array<{ type: string }>): ToolCall[] { + return content.filter((part): part is ToolCall => part.type === "toolCall"); +} describe("streamTzafonResponses", () => { it("derives unique ids when one computer_call expands to multiple actions", () => { @@ -7,4 +22,63 @@ describe("streamTzafonResponses", () => { expect(tzafon.tzafonToolCallId("call_1", 1)).toBe("call_1:1"); expect(tzafon.tzafonToolCallId("call_1", 2)).toBe("call_1:2"); }); + + it("unwraps stringified nested arguments and coerces numeric strings on function calls", async () => { + responsesCreate.mockResolvedValueOnce({ + id: "resp_1", + usage: { input_tokens: 1, output_tokens: 2 }, + output: [ + { + type: "function_call", + call_id: "call_1", + name: "computer_batch", + // Observed Tzafon shape: the actions array arrives JSON-encoded + // inside the argument object, with stringified coordinates. + arguments: JSON.stringify({ actions: JSON.stringify([{ type: "click", x: "10", y: "20" }]) }), + }, + ], + }); + + const message = await tzafon.streamTzafonResponses(model, { messages: [] }, { apiKey: "test" }).result(); + expect(message.stopReason).toBe("toolUse"); + expect(message.errorMessage).toBeUndefined(); + const calls = toolCalls(message.content); + expect(calls).toHaveLength(1); + expect(calls[0]!.name).toBe("computer_batch"); + expect(calls[0]!.arguments).toEqual({ actions: [{ type: "click", x: 10, y: 20 }] }); + }); + + it("normalizes computer_call actions with string coordinates", async () => { + responsesCreate.mockResolvedValueOnce({ + id: "resp_2", + usage: {}, + output: [{ type: "computer_call", call_id: "call_2", action: { type: "left_click", x: "500", y: "250" } }], + }); + + const message = await tzafon.streamTzafonResponses(model, { messages: [] }, { apiKey: "test" }).result(); + expect(message.stopReason).toBe("toolUse"); + const calls = toolCalls(message.content); + expect(calls).toHaveLength(1); + expect(calls[0]!.name).toBe("click"); + expect(calls[0]!.arguments).toEqual({ x: 500, y: 250 }); + }); + + it("degrades malformed function-call arguments to empty args instead of failing the turn", async () => { + responsesCreate.mockResolvedValueOnce({ + id: "resp_3", + usage: {}, + output: [ + { type: "function_call", call_id: "call_bad", name: "custom_tool", arguments: "{not json" }, + { type: "computer_call", call_id: "call_good", action: { type: "left_click", x: 1, y: 2 } }, + ], + }); + + const message = await tzafon.streamTzafonResponses(model, { messages: [] }, { apiKey: "test" }).result(); + expect(message.stopReason).toBe("toolUse"); + expect(message.errorMessage).toBeUndefined(); + const calls = toolCalls(message.content); + expect(calls).toHaveLength(2); + expect(calls[0]!).toMatchObject({ name: "custom_tool", arguments: {} }); + expect(calls[1]!).toMatchObject({ name: "click", arguments: { x: 1, y: 2 } }); + }); }); diff --git a/packages/ai/test/yutori-provider.test.ts b/packages/ai/test/yutori-provider.test.ts new file mode 100644 index 00000000..e8c2b994 --- /dev/null +++ b/packages/ai/test/yutori-provider.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Model, ToolCall } from "@earendil-works/pi-ai"; +import { getCuaModel, yutori } from "../src/index.js"; + +const { completionsCreate } = vi.hoisted(() => ({ completionsCreate: vi.fn() })); + +vi.mock("openai", () => ({ + default: class { + chat = { + completions: { + create: (...args: unknown[]) => ({ + withResponse: async () => ({ data: completionsCreate(...args), response: { status: 200, headers: new Headers() } }), + }), + }, + }; + }, +})); + +const model = getCuaModel("yutori:n1.5-latest") as Model; + +function toolCalls(content: Array<{ type: string }>): ToolCall[] { + return content.filter((part): part is ToolCall => part.type === "toolCall"); +} + +describe("streamYutori", () => { + it("normalizes native tool calls into canonical actions", async () => { + completionsCreate.mockReturnValueOnce({ + id: "chatcmpl_1", + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + choices: [ + { + finish_reason: "tool_calls", + message: { + content: "", + tool_calls: [ + { type: "function", id: "call_1", function: { name: "left_click", arguments: JSON.stringify({ coordinates: [100, 200] }) } }, + ], + }, + }, + ], + }); + + const message = await yutori.streamYutori(model, { messages: [] }, { apiKey: "test" }).result(); + expect(message.stopReason).toBe("toolUse"); + const calls = toolCalls(message.content); + expect(calls).toHaveLength(1); + expect(calls[0]!).toMatchObject({ id: "call_1", name: "click", arguments: { x: 100, y: 200 } }); + }); + + it("degrades one malformed tool call to empty args instead of erroring the whole response", async () => { + completionsCreate.mockReturnValueOnce({ + id: "chatcmpl_2", + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + choices: [ + { + finish_reason: "tool_calls", + message: { + content: "", + tool_calls: [ + { type: "function", id: "call_bad", function: { name: "left_click", arguments: "{not json" } }, + { type: "function", id: "call_good", function: { name: "left_click", arguments: JSON.stringify({ coordinates: [100, 200] }) } }, + ], + }, + }, + ], + }); + + const message = await yutori.streamYutori(model, { messages: [] }, { apiKey: "test" }).result(); + expect(message.stopReason).toBe("toolUse"); + expect(message.errorMessage).toBeUndefined(); + const calls = toolCalls(message.content); + expect(calls).toHaveLength(2); + // The malformed call degrades to its raw name with empty arguments. + expect(calls[0]!).toMatchObject({ id: "call_bad", name: "left_click", arguments: {} }); + // The well-formed call still normalizes to a canonical click. + expect(calls[1]!).toMatchObject({ id: "call_good", name: "click", arguments: { x: 100, y: 200 } }); + }); +}); From f1e712b1ec337813e6993020f7b9a69260558da5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:55:15 +0000 Subject: [PATCH 4/9] Overhaul cua-ai docs and quickstart example Document API key prerequisites and helpers, stopReason error handling, a multi-turn tool-result loop, the complete export surface, and per-provider canonical action subsets. Align the family-matcher docs with the tightened semantics and drop the removed Gemini 2.5 CUA preview model. Make the shipped quickstart import the package name (resolvable from the npm tarball), resolve its API key loudly, check stopReason, and switch providers via CUA_MODEL. Add packages/ai and packages/agent to the root README. Co-Authored-By: Claude Opus 4.7 --- README.md | 21 +- packages/ai/README.md | 389 +++++++++++++++++++++++---- packages/ai/docs/supported-models.md | 22 +- packages/ai/examples/quickstart.ts | 51 ++-- 4 files changed, 402 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 837ef8a9..f46d985d 100644 --- a/README.md +++ b/README.md @@ -42,14 +42,24 @@ All of them expect you to: ``` packages/ +├── ai/ # @onkernel/cua-ai - CUA model catalog + tool schemas + provider adapters (on npm) +├── agent/ # @onkernel/cua-agent - CuaAgent/CuaAgentHarness Kernel-browser execution loop (on npm) ├── cua-translator/ # @onkernel/cua-translator - shared SDK types + translator + browser-session ├── cua-openai/ # @onkernel/cua-openai - gpt-* (batch_computer_actions + computer_use_extra) ├── cua-anthropic/ # @onkernel/cua-anthropic - claude-* (computer_20251124 + batch_computer_actions + onPayload) ├── cua-gemini/ # @onkernel/cua-gemini - gemini-* (predefined functions + batch_computer_actions) ├── cua-yutori/ # @onkernel/cua-yutori - n1* (Navigator browser actions) -└── cua-cli/ # @onkernel/cua-cli - the CLI; depends on all providers above +└── cua-cli/ # @onkernel/cua-cli - the CLI; depends on the cua-* providers above ``` +**Building your own agent? Start here:** [`packages/ai`](packages/ai) +(`@onkernel/cua-ai`) is the model layer — the curated computer-use model +catalog, canonical tool schemas, and per-provider adapters on top of pi-ai. +[`packages/agent`](packages/agent) (`@onkernel/cua-agent`) is the execution +layer — `CuaAgent`/`CuaAgentHarness` run those tool calls against a Kernel +browser. Both are published to npm. The `cua-*` packages below back the `cua` +CLI. + ```mermaid flowchart LR trans[("@onkernel/cua-translator")] @@ -74,6 +84,8 @@ flowchart LR | Package | What it ships | | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| [`@onkernel/cua-ai`](packages/ai) | Computer-use model catalog (`getCuaModel`/`listCuaModels`), canonical CUA tool schemas, and provider adapters/runtime specs built on pi-ai. On npm. | +| [`@onkernel/cua-agent`](packages/agent) | `CuaAgent`/`CuaAgentHarness` classes that execute cua-ai tool calls against a Kernel browser, screenshot loop included. On npm. | | [`@onkernel/cua-translator`](packages/cua-translator) | Provider-agnostic `ComputerTranslator`, key/scroll/drag math, `goto`/`back`/`forward`/`url` builders, browser-session helper. | | [`@onkernel/cua-openai`](packages/cua-openai) | `batch_computer_actions` + `computer_use_extra` AgentTools and JSON Schemas for OpenAI computer-use models. | | [`@onkernel/cua-anthropic`](packages/cua-anthropic) | `computer` (built-in `computer_20251124`) + `batch_computer_actions` AgentTools, beta-header stream wrapper, payload hook. | @@ -364,6 +376,8 @@ bin/ skills/ └── cua-cli/SKILL.md # skill aimed at OTHER agents driving cua via shell packages/ +├── ai/ # @onkernel/cua-ai — model layer (see packages/ai/README.md) +├── agent/ # @onkernel/cua-agent — Kernel-browser execution layer (see packages/agent/README.md) ├── cua-translator/ │ └── src/ │ ├── types.ts # ModelAction / BatchAction / errors @@ -420,8 +434,9 @@ packages/ ## Roadmap -- Publish each `@onkernel/cua-*` package to npm so `kernel/cli` - templates and other consumers can depend on them directly. +- Publish the remaining per-provider `@onkernel/cua-*` packages to npm + (`@onkernel/cua-ai` and `@onkernel/cua-agent` are already published) so + `kernel/cli` templates and other consumers can depend on them directly. - Auto-respawn dead Kernel sessions when `-s ` is used (today we refuse with a clear error and ask the user to re-`session start`). - `--local` Docker-backed browser as an alternative to Kernel cloud. diff --git a/packages/ai/README.md b/packages/ai/README.md index 3163d82f..aa8b88b5 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -10,30 +10,72 @@ for building CUA agents on Kernel. npm install @onkernel/cua-ai ``` +## Prerequisites + +You need an API key for each provider you call. The helpers in this package +check these environment variables, in order: + +| Provider | Environment variables (checked in order) | +| ----------- | ------------------------------------------- | +| `openai` | `OPENAI_API_KEY` | +| `anthropic` | `ANTHROPIC_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` | +| `google` | `GOOGLE_API_KEY`, `GEMINI_API_KEY` | +| `tzafon` | `TZAFON_API_KEY` | +| `yutori` | `YUTORI_API_KEY` | + +The exported helpers wrap this table: + +- `cuaApiKeyEnvVarsForProvider(provider)` — the env var names for a provider + (accepts `"gemini"` as an alias for `"google"`). +- `getCuaEnvApiKey(provider)` — read the key, or `undefined` when unset. +- `requireCuaEnvApiKey(provider)` — read the key, or throw naming the + variables to set. +- `getCuaEnvApiKeyForModel(refOrModel)` / `requireCuaEnvApiKeyForModel(refOrModel)` + — the same, keyed by a model ref like `"openai:gpt-5.5"` or a concrete + `Model`. + +Pass the resolved key as the `apiKey` stream option (as in the Quick Start +below) so a missing key fails loudly before any request is made. If you omit +`apiKey`, pi-ai's built-in providers fall back to their own env lookup +(`OPENAI_API_KEY`; `ANTHROPIC_OAUTH_TOKEN`/`ANTHROPIC_API_KEY`; for `google` +only `GEMINI_API_KEY`, not `GOOGLE_API_KEY`), and this package's Tzafon/Yutori +stream adapters read `TZAFON_API_KEY`/`YUTORI_API_KEY`. + ## Quick Start ```ts import { readFile } from "node:fs/promises"; -import { complete, getCuaModel, openai } from "@onkernel/cua-ai"; - -const screenshot = await readFile("examples/screenshot.png"); +import { complete, getCuaModel, openai, requireCuaEnvApiKeyForModel } from "@onkernel/cua-ai"; const model = getCuaModel("openai:gpt-5.5"); - -const response = await complete(model, { - systemPrompt: "You are a browser automation agent.", - messages: [ - { - role: "user", - content: [ - { type: "text", text: "Click the Login button in this screenshot." }, - { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, - ], - timestamp: Date.now(), - }, - ], - tools: openai.computerTools({ actions: ["click"] }), -}); +const apiKey = requireCuaEnvApiKeyForModel("openai:gpt-5.5"); // throws unless OPENAI_API_KEY is set + +// Any screenshot of the page you want to act on, resolved relative to this +// module so the snippet does not depend on the process working directory. +const screenshot = await readFile(new URL("./screenshot.png", import.meta.url)); + +const response = await complete( + model, + { + systemPrompt: "You are a browser automation agent.", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Click the sign in / up link in this screenshot." }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, + ], + tools: openai.computerTools({ actions: ["click"] }), + }, + { apiKey }, +); + +if (response.stopReason === "error" || response.stopReason === "aborted") { + throw new Error(response.errorMessage ?? `request ended with stopReason "${response.stopReason}"`); +} for (const block of response.content) { if (block.type === "toolCall" && block.name === "click") { @@ -42,14 +84,147 @@ for (const block of response.content) { } ``` +A runnable version ships at [`examples/quickstart.ts`](./examples/quickstart.ts) +(with a sample screenshot). In this repo, run it from `packages/ai` with +`npm run example:quickstart`; switch providers with the `CUA_MODEL` env var, +e.g. `CUA_MODEL=anthropic:claude-opus-4-7`. + +## Error Handling + +pi-ai's `complete()` and `stream()` **resolve instead of throwing** when a +request fails. The returned `AssistantMessage` carries the outcome on +`stopReason`: + +- `"stop"`, `"length"`, `"toolUse"` — success; `content` holds the response. +- `"error"` — the provider call failed (bad API key, no model access, network + error, …). `content` is empty and `errorMessage` holds the provider error. +- `"aborted"` — the request was cancelled via the `signal` stream option. + +Always check `stopReason` before reading `content` — otherwise a typo'd API +key looks like a successful run that produced nothing: + +```ts +if (response.stopReason === "error" || response.stopReason === "aborted") { + throw new Error(response.errorMessage ?? `request ended with stopReason "${response.stopReason}"`); +} +``` + +`getCuaModel()`, `requireCuaEnvApiKey*()`, and `computerTools({ actions })` +validate eagerly and throw regular errors. + +## Continuing the Loop + +A computer-use session is a loop: the model calls a tool, you execute it +against a real browser, and you send the result (with a fresh screenshot) back +so the model can plan the next step. Tool results are pi-ai +`ToolResultMessage`s: + +```ts +type ToolResultMessage = { + role: "toolResult"; + toolCallId: string; // ToolCall.id from the assistant message + toolName: string; // ToolCall.name + content: (TextContent | ImageContent)[]; + details?: unknown; // optional executor metadata, not sent to the model + isError: boolean; + timestamp: number; +}; +``` + +A minimal two-turn loop: + +```ts +import { complete, getCuaModel, openai, requireCuaEnvApiKeyForModel, type Message } from "@onkernel/cua-ai"; + +const model = getCuaModel("openai:gpt-5.5"); +const apiKey = requireCuaEnvApiKeyForModel("openai:gpt-5.5"); +const tools = openai.computerTools({ actions: ["click", "type", "screenshot"] }); + +const messages: Message[] = [ + { + role: "user", + content: [ + { type: "text", text: "Click the sign in / up link in this screenshot." }, + { type: "image", data: screenshotBase64, mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, +]; + +// Turn 1: the model responds with tool calls. +const first = await complete(model, { messages, tools }, { apiKey }); +if (first.stopReason === "error" || first.stopReason === "aborted") { + throw new Error(first.errorMessage); +} +messages.push(first); // the AssistantMessage joins the transcript as-is + +// Execute each tool call against your browser stack, then append a +// toolResult message carrying a fresh screenshot. +for (const block of first.content) { + if (block.type !== "toolCall") continue; + const freshScreenshotBase64 = await runInYourBrowser(block.name, block.arguments); + messages.push({ + role: "toolResult", + toolCallId: block.id, + toolName: block.name, + content: [ + { type: "text", text: "done" }, + { type: "image", data: freshScreenshotBase64, mimeType: "image/png" }, + ], + isError: false, + timestamp: Date.now(), + }); +} + +// Turn 2: the model sees the results and plans the next action. +const second = await complete(model, { messages, tools }, { apiKey }); +``` + +The provider namespaces' `computerToolExecutors()` return +`CuaToolExecutorSpec`s whose `toActions(args)` converts a tool call's +arguments into canonical `CuaAction`s — use them to drive your own executor +instead of switching on tool names by hand. + +Yutori is the one provider where the wire differs: `streamYutori` strips +caller-supplied tools whose names collide with Yutori's canonical action names +(`yutori.YUTORI_CUA_ACTION_TYPES`) from the outbound payload and selects +Yutori's documented native `tool_set` instead, +then normalizes the native tool calls back to canonical names — so the loop +above still works unchanged. Tools that must stay on the payload can be listed +in the `keepToolNames` stream option (`yutori.YutoriOptions`). The same +`keepToolNames` option exists for Tzafon (`tzafon.TzafonResponsesOptions`), +whose adapter likewise replaces canonical action tools with Tzafon's native +`computer_use` tool on the wire. + +This package does not execute browser actions. Use +[`@onkernel/cua-agent`](https://www.npmjs.com/package/@onkernel/cua-agent) +when you want this loop run for you against a Kernel browser. + ## Core Concepts -`@onkernel/cua-ai` re-exports the core primitives of -[`@earendil-works/pi-ai`](https://github.com/earendil-works/pi/tree/main/packages/ai): +`@onkernel/cua-ai` re-exports the full surface of +[`@earendil-works/pi-ai`](https://github.com/earendil-works/pi/tree/main/packages/ai) +(`export * from "@earendil-works/pi-ai"`), including the core primitives: `Model`, `Context`, `Message`, `Tool`, `complete`, `stream`, `completeSimple`, -`streamSimple`, `Type`, `Static`, `TSchema`, and the event/validation helpers -that pi-ai exposes. Some familiarity with pi-ai is assumed; Kernel adds the -computer-use model catalog and provider/tool metadata. +`streamSimple`, `Type`, `Static`, `TSchema`, and the event/validation helpers. +Some familiarity with pi-ai is assumed; Kernel adds the computer-use model +catalog and provider/tool metadata. + +**Versioning note**: because of the wildcard re-export, pi-ai's API is part of +this package's public surface. This release is developed and tested against +`@earendil-works/pi-ai` `^0.74.0`; pi-ai exports can change within that semver +range independently of `@onkernel/cua-ai` releases. If you depend on pi-ai +symbols beyond the core primitives above, consider pinning pi-ai in your own +project. + +**Import side effect**: importing `@onkernel/cua-ai` registers the +`yutori-chat-completions` and `tzafon-responses` stream providers with pi-ai's +global API registry. The re-exported pi-ai registry mutators +(`clearApiProviders`, `resetApiProviders`, `unregisterApiProviders`) +deregister them — call the exported `registerCuaProviders()` to restore. For +Yutori/Tzafon models to stream, `complete`/`stream` must come from the same +pi-ai copy that saw the registration, so make sure your install dedupes to a +single `@earendil-works/pi-ai`. ### Model Refs @@ -59,13 +234,19 @@ computer-use model catalog and provider/tool metadata. ```ts getCuaModel("openai:gpt-5.5"); getCuaModel("anthropic:claude-opus-4-7"); -getCuaModel("google:gemini-2.5-computer-use-preview-10-2025"); +getCuaModel("google:gemini-3-flash-preview"); getCuaModel("tzafon:tzafon.northstar-cua-fast"); getCuaModel("yutori:n1.5-latest"); ``` `getCuaModel(ref)` returns a pi-ai `Model` you can pass to `complete()` -or `stream()`. +or `stream()`. It throws when the ref names a model without a CUA-support +annotation. + +The Google provider id is `google` (matching pi-ai's `Model.provider`), while +the namespace export is `gemini` — so refs are `google:…` but tools come from +`gemini.computerTools()`. `"gemini:"` refs are accepted as an alias and +normalize to `google`. See [`docs/supported-models.md`](./docs/supported-models.md) for the current list of CUA-supporting models per provider. @@ -95,17 +276,41 @@ interface CuaModelInfo { } ``` -### Exports +## Exports + +Everything below is importable from the package root. pi-ai's full surface is +re-exported alongside (see [Core Concepts](#core-concepts)). -Top-level exports: +### Models and refs - `getCuaModel(ref: CuaModelRef): Model` - `listCuaModels(provider?: CuaProvider): CuaModelInfo[]` +- `parseCuaModelRef(ref: string): { provider: CuaProvider; model: string }` — + accepts the `"gemini:"` alias +- `formatCuaModelRef(provider, model): CuaModelRef` - `providerForModel(model: Model): CuaProvider` -- `resolveCuaRuntimeSpec(input: CuaModelRef | Model): CuaRuntimeSpec` +- `isCuaProvider(value: string): value is CuaProvider` +- `findCuaAnnotation(provider, modelId): CuaModelAnnotation | undefined` - `CUA_PROVIDERS: readonly CuaProvider[]` -- `CuaBatchSchema`, `CuaActionSchema`, `CuaNavigationSchema` TypeBox schemas -- `createCuaActionSchema(actions?)`, `createCuaBatchSchema(actions?)` +- `CUA_MODEL_ANNOTATIONS: Record` — + the source-cited support table +- Types: `CuaProvider`, `CuaModelRef`, `CuaModelInfo`, `CuaModelAnnotation`, + `CuaModelMatch` + +### API keys + +- `cuaApiKeyEnvVarsForProvider(provider): readonly string[]` +- `getCuaEnvApiKey(provider): string | undefined` +- `requireCuaEnvApiKey(provider): string` +- `getCuaEnvApiKeyForModel(refOrModel): string | undefined` +- `requireCuaEnvApiKeyForModel(refOrModel): string` + +### Runtime specs + +- `resolveCuaRuntimeSpec(input: CuaModelRef | Model, options?: ComputerToolsOptions): CuaRuntimeSpec` +- Types: `CuaRuntimeSpec`, `CuaRuntimeSpecInput`, `CuaProviderModule`, + `CuaScreenshotSpec`, `CuaScreenshotTransformSpec`, `CuaPayloadHook`, + `CuaPayloadContext` `resolveCuaRuntimeSpec()` centralizes provider-specific defaults for runtime consumers: @@ -118,6 +323,44 @@ runtime consumers: - optional provider screenshot input policy - optional provider payload middleware (for protocol quirks) +Pass `options` (e.g. `{ actions: ["click"] }`) to narrow the resolved tool +definitions and executors; it is forwarded to the provider module's +`toolDefinitions()`/`toolExecutors()`, so providers with a restricted subset +(Anthropic, Yutori) throw on unsupported actions. + +### Canonical actions and tools + +- `CUA_ACTION_TYPES: readonly CuaActionType[]` — the 16 canonical action names +- `computerTools(options?: ComputerToolsOptions): Tool[]` / + `createCuaActionToolDefinitions(actions?)` — one `Tool` per canonical action + (the full canonical superset; provider namespaces apply provider defaults + and validation on top) +- `computerToolExecutors(options?)` / `createCuaActionToolExecutors(actions?)` + — matching `CuaToolExecutorSpec[]` execution adapters +- `createCuaActionSchema(actions?)`, `CuaActionSchema` — TypeBox union schema +- `createCuaBatchSchema(actions?)`, `CuaBatchSchema`, + `createCuaBatchToolDefinition(actions?, options?)`, + `createCuaBatchToolExecutor(actions?, options?)`, + `CUA_BATCH_TOOL_NAME` (`"computer_batch"`), `CUA_BATCH_TOOL_DESCRIPTION` +- `createCuaNavigationToolDefinition()`, `CuaNavigationSchema`, + `CUA_NAVIGATION_TOOL_NAME` (`"computer_use_extra"`), + `CUA_NAVIGATION_TOOL_DESCRIPTION` +- `canonicalToolCallName(action)`, `canonicalToolCallArguments(action)` — map + a normalized `CuaAction` back to its tool-call name/arguments +- `normalizeGotoUrl(value)` — prefix bare hostnames with `https://` +- Types: `CuaAction` (plus the 16 per-action interfaces), `CuaActionType`, + `CuaMouseButton`, `CuaDragMouseButton`, `CuaBatchInput`, + `CuaNavigationInput`, `CuaToolExecutorSpec`, `ComputerToolsOptions`, + `ComputerToolCoordinateSystem` + +### Provider registration + +- `registerCuaProviders(): void` — re-register the Yutori/Tzafon stream + providers with pi-ai's global registry (runs automatically on import; + idempotent; call it after any pi-ai registry mutator) + +## Provider Tools + Provider namespaces expose `computerTools({ actions? })` for building the provider's default CUA `Tool[]` definitions. These are the tools sent to the model when you call `complete()` or `stream()` directly. The @@ -135,7 +378,20 @@ const clickOnlyTools = openai.computerTools({ actions: ["click"] }); ``` When `actions` is provided, it must be a subset of that provider's supported -canonical action set. +canonical action set; unsupported actions throw (e.g. +`anthropic.computerTools({ actions: ["back"] })` throws +`unsupported Anthropic canonical action(s): back`). + +Per-provider canonical action subsets (each namespace exports its list as +`_CUA_ACTION_TYPES`): + +| Namespace | Canonical actions | +| ----------- | ---------------------------------------------------------------------------------- | +| `openai` | all 16 | +| `anthropic` | 13 — everything except `back`, `forward`, `url`; adds `computer_batch` by default | +| `gemini` | all 16 | +| `tzafon` | all 16 (replaced on the wire by Tzafon's native `computer_use` tool) | +| `yutori` | 13 — everything except `screenshot`, `url`, `cursor_position` (local mirrors only) | Runtime specs also include `toolExecutors`: provider-owned adapters that use the same tool-call names as the model-facing tools and translate their @@ -143,8 +399,12 @@ arguments into canonical CUA actions for `@onkernel/cua-agent`. For most providers, `toolDefinitions` and `toolExecutors` line up one-for-one. Some providers are different on the wire: Yutori exposes browser actions through its documented `tool_set` request field, so its runtime spec has no model-facing -`toolDefinitions` but still provides local `toolExecutors` for the canonical -actions emitted after Yutori's native tool calls are normalized. Caller-provided +`toolDefinitions` (`yutori.providerModule.toolDefinitions()` is `[]`) but +still provides local `toolExecutors` for the canonical actions emitted after +Yutori's native tool calls are normalized. `yutori.computerTools()` builds +local mirrors of those canonical tools — they are never sent to the API +(`streamYutori` strips them from the outbound payload) and exist so the +normalized tool calls have matching local definitions/executors. Caller-provided tools that should remain on the provider payload can be preserved by payload middleware via `CuaPayloadContext.keepToolNames`. @@ -197,7 +457,7 @@ type CuaActionClick = { type: "click"; x: number; y: number; - button?: string; + button?: CuaMouseButton; // "left" | "right" | "middle" | "back" | "forward" hold_keys?: string[]; }; @@ -207,6 +467,11 @@ type CuaActionGoto = { }; ``` +Mouse buttons are closed unions: `CuaMouseButton` for `click`/`mouse_down`/ +`mouse_up` and `CuaDragMouseButton` (`"left" | "right" | "middle"`) for +`drag`. Executors coerce anything outside the set to `"left"`. `keys` stays +`string[]` — the agent-side key-alias table passes unrecognized keys through. + `createCuaBatchToolDefinition(actions?, options?)` builds a batch tool schema whose input is: @@ -217,7 +482,9 @@ type CuaBatchInput = { ``` Providers can include a batch tool when their model is expected to use one. -Anthropic does this by default with `computer_batch`; Yutori does not. +Anthropic does this by default with `computer_batch` (also exported as +`anthropic.ANTHROPIC_BATCH_TOOL_NAME`, equal to the top-level +`CUA_BATCH_TOOL_NAME`); Yutori does not. `createCuaBatchToolExecutor()` is the matching execution adapter for turning that provider-defined batch input into canonical CUA actions. @@ -231,20 +498,44 @@ type CuaNavigationInput = { }; ``` -Provider namespaces: - -Each provider namespace exposes `computerTools`, `computerToolExecutors`, -`coordinateSystem`, a `buildSystemPrompt` helper, and a -`providerModule` object wiring those functions to the uniform -`CuaProviderModule` contract that `resolveCuaRuntimeSpec` looks up: - -- `openai`: `buildOpenAISystemPrompt`, OpenAI CUA action schemas -- `anthropic`: `buildAnthropicSystemPrompt`, CUA action schema aliases -- `gemini`: `buildGeminiSystemPrompt`, CUA action schema aliases -- `tzafon`: `buildTzafonSystemPrompt`, local `tzafon-responses` stream adapter -- `yutori`: `buildYutoriSystemPrompt`, native Navigator action sets, - native-to-canonical action helpers, local `yutori-chat-completions` stream - adapter, and `yutoriNativeToolSetOnPayload` +## Provider Namespaces + +Every provider namespace (`openai`, `anthropic`, `gemini`, `tzafon`, +`yutori`) follows one convention: + +- `computerTools(options?)` and `computerToolExecutors(options?)` +- `createActionSchema(actions?)` — TypeBox schema for the provider's subset +- `coordinateSystem()` +- `buildSystemPrompt({ suffix? })` and + `_COMPUTER_INSTRUCTIONS` (the prompt text) +- `_CUA_ACTION_TYPES` — the supported canonical action subset +- `Action` type — the canonical action union for that subset +- `ComputerToolsOptions` type (Anthropic's adds `excludeBatch`, also exported + as `AnthropicComputerToolsOptions`) +- `providerModule` — the uniform `CuaProviderModule` object that + `resolveCuaRuntimeSpec` looks up + +Provider-specific extras: + +- `openai`: `openaiResponsesStoreOnPayload` payload hook, plus the + `computer_use_extra` navigation aliases `OPENAI_EXTRA_TOOL_NAME`, + `OPENAI_EXTRA_TOOL_DESCRIPTION`, `OpenAIExtraSchema`, `OpenAIExtraInput` +- `anthropic`: `ANTHROPIC_BATCH_TOOL_NAME` (`"computer_batch"`) +- `tzafon`: the `tzafon-responses` stream adapter (`TZAFON_RESPONSES_API`, + `streamTzafonResponses`, `streamSimpleTzafonResponses`, + `TzafonResponsesOptions` with `keepToolNames`), `tzafonComputerUseOnPayload` + payload middleware, `tzafonToolCallId`, and the native-to-canonical + normalizer `toCanonicalActions` (+ `TzafonCanonicalAction`) +- `yutori`: the `yutori-chat-completions` stream adapter + (`YUTORI_CHAT_COMPLETIONS_API`, `streamYutori`, `streamSimpleYutori`, + `YutoriOptions` with `keepToolNames`), `yutoriNativeToolSetOnPayload` + payload middleware, the native Navigator action sets + (`YUTORI_N1_ACTION_TYPES`, `YUTORI_N15_CORE_ACTION_TYPES`, + `YUTORI_N15_EXPANDED_ACTION_TYPES`, `YUTORI_N15_ACTION_TYPES`, the + `YUTORI_N15_CORE_TOOL_SET`/`YUTORI_N15_EXPANDED_TOOL_SET` tool-set ids, and + the matching `Yutori*ActionType` types), `yutoriToolSetForModel`, + `yutoriNativeActionsForModel`, and the native-to-canonical normalizer + `toCanonicalActions` This package does not execute browser actions. Use `@onkernel/cua-agent` when you want model tool calls executed against a Kernel browser. diff --git a/packages/ai/docs/supported-models.md b/packages/ai/docs/supported-models.md index 705b661e..23c4a998 100644 --- a/packages/ai/docs/supported-models.md +++ b/packages/ai/docs/supported-models.md @@ -2,9 +2,14 @@ `@onkernel/cua-ai` accepts any pi-ai model whose ID is annotated as CUA-supporting in `CUA_MODEL_ANNOTATIONS` (see -[`src/models.ts`](../src/models.ts)). Annotations are either a `family` -match (root + dated snapshots) or an `exact` ID match. Each annotation -cites the provider's CUA docs. +[`src/models.ts`](https://github.com/kernel/cua/blob/main/packages/ai/src/models.ts)). +Annotations are either a `family` +match or an `exact` ID match. A family match covers the family root plus +suffixes made of hyphen-separated numeric segments — revisions and dated +snapshots such as `claude-opus-4-7`, `gpt-5.5-2026-04-23`, or +`claude-3-7-sonnet-20250219`. Named sibling variants like `gpt-5.4-mini` +are distinct models that may not support computer use, so they need their +own annotation. Each annotation cites the provider's CUA docs. The list below is the current snapshot. Run `listCuaModels(provider?)` for the live list — it merges pi-ai's registry @@ -14,7 +19,7 @@ with CUA-only entries that pi-ai does not ship yet. API: `openai-responses` · coordinates: pixel -Family matches (all dated snapshots accepted): +Family matches (root + numeric revision/dated-snapshot suffixes): - `gpt-5.4` ([docs](https://developers.openai.com/api/docs/models/gpt-5.4)) - `gpt-5.5` ([docs](https://developers.openai.com/api/docs/models/gpt-5.5)) @@ -23,7 +28,7 @@ Family matches (all dated snapshots accepted): API: `anthropic-messages` · coordinates: pixel -Family matches (all dated snapshots accepted): +Family matches (root + numeric revision/dated-snapshot suffixes): - `claude-3-7-sonnet` - `claude-opus-4` @@ -36,11 +41,16 @@ Source: [Anthropic computer use docs](https://docs.anthropic.com/en/docs/build-w API: `google-generative-ai` · coordinates: normalized 0–999 +Model refs use the `google:` prefix; `gemini:` is accepted as an alias. + Exact IDs: - `gemini-3-flash-preview` - `gemini-3-pro-preview` -- `gemini-2.5-computer-use-preview-10-2025` + +`gemini-2.5-computer-use-preview-10-2025` is deliberately not annotated: it +rejects the standard function declarations this package sends and requires +Google's native `tools.computer_use` request wrapper instead. Source: [Gemini computer use docs](https://ai.google.dev/gemini-api/docs/computer-use). diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index b48ac465..5c0e64fd 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -1,35 +1,34 @@ import { readFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { complete, getCuaModel, openai } from "../src/index"; +import { + complete, + getCuaModel, + requireCuaEnvApiKeyForModel, + resolveCuaRuntimeSpec, + type CuaModelRef, +} from "@onkernel/cua-ai"; -const here = dirname(fileURLToPath(import.meta.url)); -const screenshotPath = join(here, "screenshot.png"); -const apiKey = process.env.OPENAI_API_KEY; -if (!apiKey) throw new Error("Set OPENAI_API_KEY to run this example."); - -const modelRef = "openai:gpt-5.5"; +// Switch providers by setting CUA_MODEL (and the matching API key env var): +// anthropic:claude-opus-4-7 ANTHROPIC_API_KEY +// google:gemini-3-flash-preview GOOGLE_API_KEY +// tzafon:tzafon.northstar-cua-fast TZAFON_API_KEY +// yutori:n1.5-latest YUTORI_API_KEY +const modelRef = (process.env.CUA_MODEL ?? "openai:gpt-5.5") as CuaModelRef; const model = getCuaModel(modelRef); -const screenshot = await readFile(screenshotPath); +const apiKey = requireCuaEnvApiKeyForModel(modelRef); + +// resolveCuaRuntimeSpec returns the provider's default tool definitions +// (narrowed here to click-only). For a single provider you can use its +// namespace directly, e.g. openai.computerTools({ actions: ["click"] }). +const spec = resolveCuaRuntimeSpec(modelRef, { actions: ["click"] }); -// Other provider examples. Add the provider namespace to the top-level import -// before switching these values. -// const apiKey = process.env.ANTHROPIC_API_KEY; -// const modelRef = "anthropic:claude-opus-4-7"; -// const model = getCuaModel(modelRef); -// const tools = anthropic.computerTools({ actions: ["click"] }); -// -// const apiKey = process.env.GOOGLE_API_KEY; -// const modelRef = "google:gemini-2.5-computer-use-preview-10-2025"; -// const model = getCuaModel(modelRef); -// const tools = gemini.computerTools({ actions: ["click"] }); +const screenshot = await readFile(new URL("./screenshot.png", import.meta.url)); const response = await complete( model, { systemPrompt: [ "You are controlling a browser from a screenshot.", - "Call the computer tool with the pixel coordinates of the target. Do not describe the click in prose unless you cannot identify the target.", + "Call the computer tool with the coordinates of the target. Do not describe the click in prose unless you cannot identify the target.", ].join("\n"), messages: [ { @@ -41,7 +40,7 @@ const response = await complete( timestamp: Date.now(), }, ], - tools: openai.computerTools({ actions: ["click"] }), + tools: spec.toolDefinitions, }, { apiKey, @@ -49,6 +48,12 @@ const response = await complete( }, ); +// complete() resolves instead of throwing on provider errors; always check +// stopReason before reading content. +if (response.stopReason === "error" || response.stopReason === "aborted") { + throw new Error(response.errorMessage ?? `request ended with stopReason "${response.stopReason}"`); +} + console.log(`model: ${modelRef}`); for (const block of response.content) { if (block.type === "text") { From 416d31bc99d062246d538866df286a3f64528f1b Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:55:23 +0000 Subject: [PATCH 5/9] Bump @onkernel/cua-ai to 0.2.0 and refresh release skill Add a 0.2.0 changelog entry covering the unreleased breaking changes since the 0.1.0 publish plus this release's fixes. Ship docs/ in the tarball and run the in-repo example through the source export condition. Update the release skill to full-suite test invocations and add a post-publish step that installs and imports the published package. Co-Authored-By: Claude Opus 4.7 --- .agents/skills/release/SKILL.md | 20 ++++++- package-lock.json | 13 ++++- packages/ai/CHANGELOG.md | 96 +++++++++++++++++++++++++++++++++ packages/ai/package.json | 5 +- 4 files changed, 130 insertions(+), 4 deletions(-) diff --git a/.agents/skills/release/SKILL.md b/.agents/skills/release/SKILL.md index 50df2eea..8b24e696 100644 --- a/.agents/skills/release/SKILL.md +++ b/.agents/skills/release/SKILL.md @@ -139,10 +139,15 @@ For `@onkernel/cua-agent`: ```bash npm run build --workspace @onkernel/cua-ai npm run build --workspace @onkernel/cua-agent -npm test --workspace @onkernel/cua-agent -- test/agent.test.ts test/tool-exhaustiveness.test.ts +npm test --workspace @onkernel/cua-agent npm pack --workspace @onkernel/cua-agent --dry-run ``` +Run the full unit suites — do not pass individual test files. `cua-ai` +excludes integration/live tests by default (`npm run test:integration +--workspace @onkernel/cua-ai` runs them separately), and the `cua-agent` live +e2e tests skip unless `CUA_E2E_LIVE=1` is set. + Do not push release tags if build, tests, or pack dry-runs fail. ## Commit To Main @@ -201,6 +206,19 @@ npm view @onkernel/cua-agent@ version npm dist-tag ls @onkernel/cua-agent ``` +Then verify the published artifact actually imports — `npm view` only proves +the version exists, not that the tarball is loadable: + +```bash +cd "$(mktemp -d)" +npm init -y +npm install @onkernel/cua-ai@ +node --input-type=module -e "import('@onkernel/cua-ai').then((m) => { if (typeof m.getCuaModel !== 'function') process.exit(1); })" +``` + +For `@onkernel/cua-agent`, install `@onkernel/cua-agent@` the same +way and check `typeof m.CuaAgent === "function"`. + If a workflow fails after a tag is pushed, do not reuse the same package version unless npm did not publish it. Fix forward with a new commit and a new patch version when a package version has reached npm. diff --git a/package-lock.json b/package-lock.json index d6ab979d..d98ea888 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5170,9 +5170,20 @@ "vitest": "^3.2.4" } }, + "packages/agent/node_modules/@onkernel/cua-ai": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@onkernel/cua-ai/-/cua-ai-0.1.0.tgz", + "integrity": "sha512-S+bhPUwMIgppvUWa5dojSNHOnxYy361ENT5OckCyJxwdbwtworop4CbUr3I67LF/d9xcLhMd4+keMejGgz9PHg==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "^0.74.0", + "@tzafon/lightcone": "^0.7.0", + "openai": "^6.26.0" + } + }, "packages/ai": { "name": "@onkernel/cua-ai", - "version": "0.1.0", + "version": "0.2.0", "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "^0.74.0", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 4c2a3cdd..79a01ebe 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,101 @@ # Changelog +## 0.2.0 - 2026-06-10 + +### Fixed + +- The published package is now importable under plain Node ESM. 0.1.0 shipped + extensionless relative imports in `dist/`, so `import "@onkernel/cua-ai"` + failed outside bundlers; compiled output now uses explicit `.js` specifiers. +- The shipped `examples/quickstart.ts` imports `@onkernel/cua-ai` instead of a + `../src` path that does not exist in the tarball, checks `stopReason` so + provider errors are no longer silent, resolves its API key via + `requireCuaEnvApiKeyForModel`, and switches providers with the `CUA_MODEL` + env var. +- `docs/` (the supported-models list the README links to) is now included in + the npm tarball. +- A malformed Yutori tool call now degrades to an empty-arguments call instead + of failing the entire response, matching the existing Tzafon hardening. + +### Breaking changes + +- Provider namespaces follow one convention. Every namespace now exports + `computerTools({ actions? })` / `computerToolExecutors({ actions? })`, + `createActionSchema`, `coordinateSystem()`, `providerModule`, + `_CUA_ACTION_TYPES`, `_COMPUTER_INSTRUCTIONS`, a + `Action` type, and `ComputerToolsOptions`. This replaces 0.1.0's + `createComputerToolDefinitions(options)` / + `CreateComputerToolDefinitionsOptions`, the per-namespace + `COMPUTER_TOOL_COORDINATES` constants, `TZAFON_ACTION_TYPES` / + `YUTORI_ACTION_TYPES`, and the `OPENAI_BATCH_INSTRUCTIONS` / + `GEMINI_INSTRUCTIONS_RAW` / `TZAFON_INSTRUCTIONS_RAW` / + `YUTORI_INSTRUCTIONS_RAW` prompt constants. +- `CUA_BATCH_TOOL_NAME` is now `"computer_batch"` (was + `"batch_computer_actions"`), matching the batch tool Anthropic ships by + default. `anthropic.ANTHROPIC_BATCH_TOOL_NAME` carries the same new value; + the other per-namespace batch aliases (`TZAFON_BATCH_TOOL_NAME`, + `YUTORI_BATCH_TOOL_NAME`, `*_BATCH_DESCRIPTION`, `*BatchSchema`, + `*BatchInput`) were removed — use `CUA_BATCH_TOOL_NAME`, + `CUA_BATCH_TOOL_DESCRIPTION`, `CuaBatchSchema`, and `CuaBatchInput`. +- Anthropic tools are now the 13 canonical browser actions Anthropic supports + (no `back`/`forward`/`url`) plus a `computer_batch` batch tool by default; + pass `excludeBatch: true` to omit it. Unsupported `actions` entries throw. + `anthropic.ANTHROPIC_CUA_ACTION_TYPES` reflects the supported subset rather + than aliasing the full canonical list. +- Yutori models now use Yutori's documented native `tool_set` request field. + `streamYutori` strips canonical action tools from the outbound payload + (preserve specific tools via the `keepToolNames` stream option), selects the + n1.5 core tool set where applicable, and normalizes native tool calls back + to canonical names. `yutori.providerModule.toolDefinitions()` is `[]`; + `yutori.computerTools()` builds local mirrors for executor lookup, validates + `{ actions }` against the supported subset, and throws on unsupported + actions. `yutoriBuiltinToolsOnPayload` was replaced by + `yutoriNativeToolSetOnPayload`. The Yutori runtime spec also carries a + screenshot policy (append a 1280x800 webp screenshot to the latest message). +- Family model annotations now match only the family root plus numeric + revision or dated-snapshot suffixes (`claude-opus-4-7`, + `gpt-5.5-2026-04-23`). Named sibling variants such as `gpt-5.4-mini` are no + longer listed by `listCuaModels()` or accepted by `getCuaModel()` without + their own annotation. +- `google:gemini-2.5-computer-use-preview-10-2025` was removed from the + catalog: it rejects the standard function declarations this package sends + and requires Google's native `tools.computer_use` wrapper. Use + `google:gemini-3-flash-preview` or `google:gemini-3-pro-preview`. +- `streamTzafonResponses` no longer accepts a `maxOutputTokens` option — use + the standard `maxTokens` stream option. + +### Added + +- `CuaProviderModule` contract plus a `providerModule` export per namespace, + and a richer `CuaRuntimeSpec`: `toolExecutors` (local adapters that turn + provider tool calls into canonical `CuaAction`s via `CuaToolExecutorSpec`), + `coordinateSystem`, and optional `screenshot` policy alongside the existing + tool definitions, default prompt, and payload middleware. +- `resolveCuaRuntimeSpec(input, options?)` accepts `ComputerToolsOptions` and + forwards it to the provider module, so runtime consumers can narrow tool + definitions and executors (e.g. `{ actions: ["click"] }`). +- `registerCuaProviders()` is exported: importing the package still registers + the Yutori/Tzafon stream providers automatically, and this restores them + after pi-ai registry mutators (`clearApiProviders`, `resetApiProviders`, + `unregisterApiProviders`). +- `parseCuaModelRef` / `getCuaModel` accept `"gemini:"` refs as an alias for + `"google:"`, and unsupported-provider errors now list the valid providers. +- `CuaMouseButton` and `CuaDragMouseButton` closed unions type the `button` + field on click/mouse_down/mouse_up and drag actions. +- `yutori.YutoriOptions` and `tzafon.TzafonResponsesOptions` are exported and + aligned; both support `keepToolNames` to preserve caller tools that collide + with canonical action names on the wire. +- Yutori native action vocabulary exports: `YUTORI_N1_ACTION_TYPES`, + `YUTORI_N15_CORE_ACTION_TYPES`, `YUTORI_N15_EXPANDED_ACTION_TYPES`, + tool-set ids, `yutoriToolSetForModel`, `yutoriNativeActionsForModel`, and + `toCanonicalActions`; Tzafon exports `toCanonicalActions`, + `TzafonCanonicalAction`, `tzafonComputerUseOnPayload`, and + `tzafonToolCallId`. +- README and JSDoc coverage across the public surface: API key prerequisites + and helpers, error handling (`stopReason` semantics), a multi-turn + tool-result example, the complete export list, and per-provider canonical + action subsets. + ## 0.1.0 - Provider-qualified CUA model catalog with support annotations and curated overrides. diff --git a/packages/ai/package.json b/packages/ai/package.json index 7a8a6844..14140905 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-ai", - "version": "0.1.0", + "version": "0.2.0", "description": "Kernel-curated computer-use model access built on pi-ai", "license": "MIT", "type": "module", @@ -24,6 +24,7 @@ }, "files": [ "dist", + "docs", "examples", "README.md", "CHANGELOG.md" @@ -34,7 +35,7 @@ "scripts": { "build": "tsc -b", "clean": "tsc -b --clean", - "example:quickstart": "tsx examples/quickstart.ts", + "example:quickstart": "NODE_OPTIONS=--conditions=source tsx examples/quickstart.ts", "test": "vitest --run", "test:integration": "vitest --run --config vitest.integration.config.ts" }, From ec12fbaaf697236e4736b2a8b0fbee8054049f46 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 18:45:29 +0000 Subject: [PATCH 6/9] Bundle cua-ai dist with tsdown and restore extensionless imports Replace the NodeNext + .js-specifier approach with a tsdown bundle: dist/ is a single ESM file with bundled-in relative modules, deps stay external, and src/test go back to extensionless imports. tsc -b remains for typechecking only, emitting declarations to a gitignored dist-tsc/ so project references keep working. Root build now builds cua-ai before tsc -b since dependents resolve its types from dist/. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 1 + package-lock.json | 811 +++++++++++++++++- package.json | 4 +- packages/ai/CHANGELOG.md | 2 +- packages/ai/package.json | 6 +- packages/ai/src/api-keys.ts | 2 +- packages/ai/src/index.ts | 22 +- packages/ai/src/providers.ts | 4 +- .../ai/src/providers/anthropic/actions.ts | 2 +- packages/ai/src/providers/anthropic/index.ts | 8 +- packages/ai/src/providers/common.ts | 2 +- packages/ai/src/providers/gemini/index.ts | 8 +- packages/ai/src/providers/openai/index.ts | 8 +- packages/ai/src/providers/tzafon/index.ts | 12 +- packages/ai/src/providers/tzafon/provider.ts | 2 +- packages/ai/src/providers/yutori/actions.ts | 2 +- packages/ai/src/providers/yutori/index.ts | 16 +- packages/ai/src/providers/yutori/provider.ts | 4 +- packages/ai/src/runtime-spec.ts | 16 +- packages/ai/test/api-keys.test.ts | 2 +- .../ai/test/computer-tool.integration.test.ts | 2 +- packages/ai/test/models.test.ts | 2 +- packages/ai/test/provider-module.test.ts | 4 +- packages/ai/test/providers.test.ts | 2 +- packages/ai/test/runtime-spec.test.ts | 2 +- packages/ai/test/tools.test.ts | 2 +- packages/ai/test/tzafon-actions.test.ts | 2 +- packages/ai/test/tzafon-payload.test.ts | 2 +- packages/ai/test/tzafon-provider.test.ts | 2 +- packages/ai/test/yutori-actions.test.ts | 2 +- packages/ai/test/yutori-payload.test.ts | 2 +- packages/ai/test/yutori-provider.test.ts | 2 +- packages/ai/tsconfig.build.json | 9 +- packages/ai/tsdown.config.ts | 11 + 34 files changed, 897 insertions(+), 83 deletions(-) create mode 100644 packages/ai/tsdown.config.ts diff --git a/.gitignore b/.gitignore index 5bcb1de6..7fb0578f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ dist/ .DS_Store *.tsbuildinfo packages/*/dist/ +packages/*/dist-tsc/ packages/*/*.tsbuildinfo packages/*/native/build/ packages/ptywright/.cache/ diff --git a/package-lock.json b/package-lock.json index d98ea888..817951e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -665,6 +665,60 @@ "node": ">=18.0.0" } }, + "node_modules/@babel/generator": { + "version": "8.0.0-rc.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.0-rc.6.tgz", + "integrity": "sha512-6mIzgVK8DgEzvIapoQwhXTMnnkuE4STQmVv9H03i/tZ2ml8oev3TRvZJgTenK2Bsq0YWNtzOrFdTyNzCMFtjJQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^8.0.0-rc.6", + "@babel/types": "^8.0.0-rc.6", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "@types/jsesc": "^2.5.0", + "jsesc": "^3.0.2" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "8.0.0-rc.6", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0-rc.6.tgz", + "integrity": "sha512-BCkFy+zN6kXQed3YOT7aJl93NfDSzQc3pBfsvTVPs9gU9X3V0aefEF5kwBT0E+mDWH9QgKaZstYUQN9VdQZT4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "8.0.0-rc.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.0-rc.6.tgz", + "integrity": "sha512-nVJ+1JcCgntv8d78rRo++o2wuODT0Irknx2BF8Np4Ft2CRgjLqIs4qzSZ8b66yGbBdMWGmZBO9WEZv1hhNiSpg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/parser": { + "version": "8.0.0-rc.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.0-rc.6.tgz", + "integrity": "sha512-rOS8IpdO7mQELkTPlCsTgPejO0bFuZdEDCGQJouYbYf9e1FLTym7Fei2pEjq8q7MWbX0ravcd7QQYKs1TxOuog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^8.0.0-rc.6" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, "node_modules/@babel/runtime": { "version": "7.29.2", "license": "MIT", @@ -672,6 +726,20 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/types": { + "version": "8.0.0-rc.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.0-rc.6.tgz", + "integrity": "sha512-p7/ABylAYlexb31wtRdIfH9L9A0Z2T/9H6zAqzqndkY2PLkvNNc580wGhp/gGKN4Sp9sQvSkhc6Oga8/O+wTyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^8.0.0-rc.6", + "@babel/helper-validator-identifier": "^8.0.0-rc.6" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, "node_modules/@borewit/text-codec": { "version": "0.2.2", "license": "MIT", @@ -736,6 +804,18 @@ "zod-to-json-schema": "^3.25.0" } }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/runtime": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", @@ -746,6 +826,17 @@ "tslib": "^2.4.0" } }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", @@ -1676,6 +1767,27 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -1683,6 +1795,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@mariozechner/clipboard": { "version": "0.3.2", "license": "MIT", @@ -1824,6 +1947,25 @@ "zod-to-json-schema": "^3.24.1" } }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", + "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, "node_modules/@onkernel/cua-agent": { "resolved": "packages/agent", "link": true @@ -1868,6 +2010,16 @@ "version": "0.49.0", "license": "Apache-2.0" }, + "node_modules/@oxc-project/types": { + "version": "0.134.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.134.0.tgz", + "integrity": "sha512-T0xuRRKrQFmocH8y+jGfpmSkGcheaJExY9lEihmR1Gm2aH+75B8CzgU2rABRQSzzDxLjZ15Sc0bRVLj5lVeNXQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "license": "BSD-3-Clause" @@ -1912,6 +2064,283 @@ "version": "1.1.0", "license": "BSD-3-Clause" }, + "node_modules/@quansync/fs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@quansync/fs/-/fs-1.0.0.tgz", + "integrity": "sha512-4TJ3DFtlf1L5LDMaM6CanJ/0lckGNtJcMjQ1NAV6zDmA0tEHKZtxNKin8EgPaVX1YzljbxckyT2tJrpQKAtngQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.0.tgz", + "integrity": "sha512-gCYzGOSkYY6Z034suzd20euvds7lPzMEEla62DJGE/ZAlR4OMBnNbvnBSsIGUCAr52gaWMsloGxP4tVGtN5aCA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.0.tgz", + "integrity": "sha512-JQBD77MNgu+4Z6RAyg69acugdrhhVoWesr3l47zohYZ2YV2fwkWMArkN/2p4l6Ei+Sno7W5q+UsKdVWq5Ens0w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.0.tgz", + "integrity": "sha512-p/8cXUTK4Sob604e+xxPhVSbDFf29E6J0l/xESM9rdCfn3aDai3nEs6TnMHUsdD5aNlFz0+gDbiGlozLKGa2YA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.0.tgz", + "integrity": "sha512-KbtOSlVv6fElujiZWMcC3aQYhEwLVVf073RcwlSmpGQvIsKZFUqc0ef4sjUuurRwfbiI6JJXji9DQn+86hawmQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.0.tgz", + "integrity": "sha512-9fZ9i0o0/MQaw7om6Z6TsT7tfCk0jtbEFtC+aPqZL5RNsGWNcHvn6EHgL3dAprjq+AZzPTAQjg2JtpJaMt+6pg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.0.tgz", + "integrity": "sha512-+tog7T66i+yFyIuuAnjL6xmW182W/qTBOUt6BtQ6lBIM1Eikh/fSMz4HGgvuCp5uU0zuIVWng7kDYthjCMOHcg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.0.tgz", + "integrity": "sha512-4b7yruLIIj/oZ3GpcLOvxcLCLDMraohn3IhQfN2hBP4w9UekG0DTIajWguJosRGfySf/+h/NwRUiMKoCpxCrqQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.0.tgz", + "integrity": "sha512-QRDOVZd0bhQ5jLsUsCC3dUxDWdTSVY9WMznowZgCGOrZfLLgctWpelhUASEiBwsXfat/JwYnVd1EaxMhqyT+UQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.0.tgz", + "integrity": "sha512-ypxT+Hq76NFG7woFbNbySnGEajFuYuIXeKz/jfCU+lXUoxfi3zLE6OG/ZQNeK3RpZSYJlAe2bokpsQ046CaieQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.0.tgz", + "integrity": "sha512-IdovCmfROFmpTLahdecTDFL74aLERVYN68F/mLZjfVh6LfoplPfI6deyHNMTcVujbokDV5k05XrFO22zfv+qjg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.0.tgz", + "integrity": "sha512-pcA8xlFp2tyk9T2R6Fi/rPe3bQ1MA+sSMDNUU5Ogu80GHOatkE4P8YCreGAvZErm5Ho2YRXnyvNrWiRncfVysQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.0.tgz", + "integrity": "sha512-4+fexHayrLCWpriPh4c6dNvL4an34DEZCG7zOM/FD5QNF6h8DT+bDXzyB/kfC8lDJbaFb7jKShtnjDQFXVQEjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.0.tgz", + "integrity": "sha512-SbL++MNmOw6QamrwIGDMSSfM4ceTzFr+RjbOExJSLLBinScU4WI5OdA413h1qwPw2yH7lVF1+H4svQ+6mSXKTQ==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.0.tgz", + "integrity": "sha512-+xTE6XC7wBgk0VKRXGG+QAnyW5S9b8vfsFpiMjf0waQTmSQSU8onsH/beyZ8X4aXVveJnotiy7VDjLOaW8bTrg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.0.tgz", + "integrity": "sha512-Ogji1TQNqH3ACLnYr+1Ns1nyrJ0CO2P585u9Hsh02pXvtFiFpgtgT2b3P4PnCOU86VVCvqtAeCN4OftMT8KU4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.60.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz", @@ -2843,6 +3272,17 @@ "version": "0.23.0", "license": "MIT" }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2868,6 +3308,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/jsesc": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@types/jsesc/-/jsesc-2.5.1.tgz", + "integrity": "sha512-9VN+6yxLOPLOav+7PwjZbxiID2bVaeq0ED4qSQmdQTdjnXJSaCVKTR58t15oqH1H5t8Ng2ZX1SabJVoN9Q34bw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/mime-types": { "version": "2.1.4", "license": "MIT" @@ -3071,6 +3518,16 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ansis": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ansis/-/ansis-4.3.1.tgz", + "integrity": "sha512-BJ8/l4R5LRE7hW9WdSuGYrLSHi2ynxeFpDFbH0K/CgNeY/tyhk+vO6TYxXC5r5CpUhNVX310xzPsN/H9lCdfOA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, "node_modules/any-promise": { "version": "1.3.0", "license": "MIT" @@ -3085,6 +3542,24 @@ "node": ">=12" } }, + "node_modules/ast-kit": { + "version": "3.0.0-beta.1", + "resolved": "https://registry.npmjs.org/ast-kit/-/ast-kit-3.0.0-beta.1.tgz", + "integrity": "sha512-trmleAnZ2PxN/loHWVhhx1qeOHSRXq4TDsBBxq3GqeJitfk3+jTQ+v/C1km/KYq9M7wKqCewMh+/NAvVH7m+bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^8.0.0-beta.4", + "estree-walker": "^3.0.3", + "pathe": "^2.0.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, "node_modules/ast-types": { "version": "0.13.4", "license": "MIT", @@ -3134,6 +3609,16 @@ "node": "*" } }, + "node_modules/birpc": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/birpc/-/birpc-4.0.0.tgz", + "integrity": "sha512-LShSxJP0KTmd101b6DRyGBj57LZxSDYWKitQNW/mi8GRMvZb078Uf9+pveax1DrVL89vm7mWe+TovdI/UDOuPw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, "node_modules/bowser": { "version": "2.14.1", "license": "MIT" @@ -3311,6 +3796,13 @@ "node": ">=6" } }, + "node_modules/defu": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", + "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", + "dev": true, + "license": "MIT" + }, "node_modules/degenerator": { "version": "5.0.1", "license": "MIT", @@ -3339,6 +3831,27 @@ "node": ">=0.3.1" } }, + "node_modules/dts-resolver": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/dts-resolver/-/dts-resolver-3.0.0.tgz", + "integrity": "sha512-1T1f+z+4tl9XD+m+0HBgWoL/nm0bOIffyWaUuUSBlFg/86IWvfx+wjNaO/ybU0AJzG9/Mi5hBUgGV6zCmWEN7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "oxc-resolver": ">=11.0.0" + }, + "peerDependenciesMeta": { + "oxc-resolver": { + "optional": true + } + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "license": "Apache-2.0", @@ -3350,6 +3863,16 @@ "version": "8.0.0", "license": "MIT" }, + "node_modules/empathic": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.1.tgz", + "integrity": "sha512-YGRs8knHhKHVShLkFET/rWAU8kmHbOV5LwN938RHI0pljAJ1Gf6SzXsSmRaEzcXTtOOmVqJ5+WtQPL5uigY50Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "license": "MIT", @@ -3776,6 +4299,13 @@ "node": "*" } }, + "node_modules/hookable": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/hookable/-/hookable-6.1.1.tgz", + "integrity": "sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ==", + "dev": true, + "license": "MIT" + }, "node_modules/hosted-git-info": { "version": "9.0.2", "license": "ISC", @@ -3840,6 +4370,19 @@ "node": ">= 4" } }, + "node_modules/import-without-cache": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/import-without-cache/-/import-without-cache-0.4.0.tgz", + "integrity": "sha512-NkJQA7oZ4YHQhd2+H3BoRFKF3d/XNsiKpHZCQEMH9pDX27hQQLsTyOocyRgaIVtf8gHX3Nt3LPkR4e5EdtPAGQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, "node_modules/ip-address": { "version": "10.1.0", "license": "MIT", @@ -3861,6 +4404,19 @@ "dev": true, "license": "MIT" }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "license": "MIT", @@ -4092,6 +4648,20 @@ "node": ">=0.10.0" } }, + "node_modules/obug": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.2.tgz", + "integrity": "sha512-AWGB9WFcRXOQs48Z/udjI5ZcZMHXwX8XPByNpOydgcGsDLIzjGizhoMWJyKAWze7AVW/2W1i+/gPX4YtKe5cyg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/once": { "version": "1.4.0", "license": "ISC", @@ -4347,6 +4917,23 @@ "once": "^1.3.1" } }, + "node_modules/quansync": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-1.0.0.tgz", + "integrity": "sha512-5xZacEEufv3HSTPQuchrvV6soaiACMFnq1H8wkVioctoH3TRha9Sz66lOxRwPK/qZj7HPiSveih9yAyh98gvqA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, "node_modules/require-directory": { "version": "2.1.1", "license": "MIT", @@ -4378,6 +4965,100 @@ "node": ">= 4" } }, + "node_modules/rolldown": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.0.tgz", + "integrity": "sha512-zpMvlJhs5PkXRTtKc0CaLBVI9AR/VDiJFpM+kx//hgToEca7FgMlGjaRIisXBcb19T76LswgmKECSQ96hjWr5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.134.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.0", + "@rolldown/binding-darwin-arm64": "1.1.0", + "@rolldown/binding-darwin-x64": "1.1.0", + "@rolldown/binding-freebsd-x64": "1.1.0", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.0", + "@rolldown/binding-linux-arm64-gnu": "1.1.0", + "@rolldown/binding-linux-arm64-musl": "1.1.0", + "@rolldown/binding-linux-ppc64-gnu": "1.1.0", + "@rolldown/binding-linux-s390x-gnu": "1.1.0", + "@rolldown/binding-linux-x64-gnu": "1.1.0", + "@rolldown/binding-linux-x64-musl": "1.1.0", + "@rolldown/binding-openharmony-arm64": "1.1.0", + "@rolldown/binding-wasm32-wasi": "1.1.0", + "@rolldown/binding-win32-arm64-msvc": "1.1.0", + "@rolldown/binding-win32-x64-msvc": "1.1.0" + } + }, + "node_modules/rolldown-plugin-dts": { + "version": "0.25.2", + "resolved": "https://registry.npmjs.org/rolldown-plugin-dts/-/rolldown-plugin-dts-0.25.2.tgz", + "integrity": "sha512-nMhN/R+vmR8GM45ZW1FWMSjRTSDDn/6w4GTf8RNrEFCBdl8B1kySWrU1ixPtbwzXoRlcO+R/S88VgXuJQwfdDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "8.0.0-rc.6", + "@babel/helper-validator-identifier": "8.0.0-rc.6", + "@babel/parser": "8.0.0-rc.6", + "ast-kit": "^3.0.0-beta.1", + "birpc": "^4.0.0", + "dts-resolver": "^3.0.0", + "get-tsconfig": "5.0.0-beta.5", + "obug": "^2.1.1" + }, + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@ts-macro/tsc": "^0.3.6", + "@typescript/native-preview": ">=7.0.0-dev.20260325.1", + "rolldown": "^1.0.0", + "typescript": "^5.0.0 || ^6.0.0", + "vue-tsc": "~3.2.0" + }, + "peerDependenciesMeta": { + "@ts-macro/tsc": { + "optional": true + }, + "@typescript/native-preview": { + "optional": true + }, + "typescript": { + "optional": true + }, + "vue-tsc": { + "optional": true + } + } + }, + "node_modules/rolldown-plugin-dts/node_modules/get-tsconfig": { + "version": "5.0.0-beta.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-5.0.0-beta.5.tgz", + "integrity": "sha512-/6gFNr0N04nob252sTQxyFLi3eKFRqIg1I87YcqAMT1i6SQrSF6KujUEQrtrjMV0H/eejTCltLdDSTEMzHbnsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "engines": { + "node": ">=20.20.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/rollup": { "version": "4.60.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", @@ -4442,9 +5123,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", - "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz", + "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -4690,9 +5371,9 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.16", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", - "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { @@ -4752,10 +5433,113 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "license": "MIT", + "bin": { + "tree-kill": "cli.js" + } + }, "node_modules/ts-algebra": { "version": "2.0.0", "license": "MIT" }, + "node_modules/tsdown": { + "version": "0.22.2", + "resolved": "https://registry.npmjs.org/tsdown/-/tsdown-0.22.2.tgz", + "integrity": "sha512-VX9gsyKXsTnBZjnIM4jsHl9aRv+GfgkE/k1hQslilaBfZMlaw3JuGR+6yhiU0QxWBtOCDnTjwOSoXzgB7Rr50g==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansis": "^4.3.1", + "cac": "^7.0.0", + "defu": "^6.1.7", + "empathic": "^2.0.1", + "hookable": "^6.1.1", + "import-without-cache": "^0.4.0", + "obug": "^2.1.1", + "picomatch": "^4.0.4", + "rolldown": "~1.1.0", + "rolldown-plugin-dts": "^0.25.2", + "semver": "^7.8.1", + "tinyexec": "^1.2.4", + "tinyglobby": "^0.2.17", + "tree-kill": "^1.2.2", + "unconfig-core": "^7.5.0" + }, + "bin": { + "tsdown": "dist/run.mjs" + }, + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@arethetypeswrong/core": "^0.18.1", + "@tsdown/css": "0.22.2", + "@tsdown/exe": "0.22.2", + "@vitejs/devtools": "*", + "publint": "^0.3.8", + "tsx": "*", + "typescript": "^5.0.0 || ^6.0.0", + "unplugin-unused": "^0.5.0", + "unrun": "*" + }, + "peerDependenciesMeta": { + "@arethetypeswrong/core": { + "optional": true + }, + "@tsdown/css": { + "optional": true + }, + "@tsdown/exe": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "publint": { + "optional": true + }, + "tsx": { + "optional": true + }, + "typescript": { + "optional": true + }, + "unplugin-unused": { + "optional": true + }, + "unrun": { + "optional": true + } + } + }, + "node_modules/tsdown/node_modules/cac": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cac/-/cac-7.0.0.tgz", + "integrity": "sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/tsdown/node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tslib": { "version": "2.8.1", "license": "0BSD" @@ -4808,6 +5592,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/unconfig-core": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/unconfig-core/-/unconfig-core-7.5.0.tgz", + "integrity": "sha512-Su3FauozOGP44ZmKdHy2oE6LPjk51M/TRRjHv2HNCWiDvfvCoxC2lno6jevMA91MYAdCdwP05QnWdWpSbncX/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, "node_modules/undici": { "version": "7.25.0", "license": "MIT", @@ -5191,6 +5989,7 @@ "openai": "^6.26.0" }, "devDependencies": { + "tsdown": "^0.22.2", "vitest": "^3.2.4" } }, diff --git a/package.json b/package.json index 1f6f2345..461da3e0 100644 --- a/package.json +++ b/package.json @@ -17,10 +17,10 @@ "packages/cua-cli" ], "scripts": { - "build": "tsc -b && npm run build:native --workspace @onkernel/ptywright --if-present", + "build": "npm run build --workspace @onkernel/cua-ai && tsc -b && npm run build --workspace @onkernel/cua-agent && npm run build:native --workspace @onkernel/ptywright --if-present", "build:cli": "npm run build --workspace @onkernel/cua-cli", "dev": "tsc -b --watch", - "typecheck": "tsc -b", + "typecheck": "npm run build --workspace @onkernel/cua-ai && tsc -b", "clean": "tsc -b --clean && npm run clean:native --workspace @onkernel/ptywright --if-present" }, "engines": { diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 79a01ebe..a967cd7b 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -6,7 +6,7 @@ - The published package is now importable under plain Node ESM. 0.1.0 shipped extensionless relative imports in `dist/`, so `import "@onkernel/cua-ai"` - failed outside bundlers; compiled output now uses explicit `.js` specifiers. + failed outside bundlers; `dist/` is now bundled with tsdown. - The shipped `examples/quickstart.ts` imports `@onkernel/cua-ai` instead of a `../src` path that does not exist in the tarball, checks `stopReason` so provider errors are no longer silent, resolves its API key via diff --git a/packages/ai/package.json b/packages/ai/package.json index 14140905..bf7d3384 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -33,8 +33,9 @@ "access": "public" }, "scripts": { - "build": "tsc -b", - "clean": "tsc -b --clean", + "build": "tsdown", + "typecheck": "tsc -b", + "clean": "tsc -b --clean && rm -rf dist dist-tsc", "example:quickstart": "NODE_OPTIONS=--conditions=source tsx examples/quickstart.ts", "test": "vitest --run", "test:integration": "vitest --run --config vitest.integration.config.ts" @@ -45,6 +46,7 @@ "openai": "^6.26.0" }, "devDependencies": { + "tsdown": "^0.22.2", "vitest": "^3.2.4" } } diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index 6cc634fe..cfbbed74 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -1,5 +1,5 @@ import type { Api, Model } from "@earendil-works/pi-ai"; -import { parseCuaModelRef, providerForModel, type CuaModelRef, type CuaProvider } from "./models.js"; +import { parseCuaModelRef, providerForModel, type CuaModelRef, type CuaProvider } from "./models"; /** * Environment variables accepted for each CUA provider. diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 59132081..4bed13fb 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,16 +1,16 @@ -import { registerCuaProviders } from "./providers.js"; +import { registerCuaProviders } from "./providers"; export * from "@earendil-works/pi-ai"; -export { registerCuaProviders } from "./providers.js"; -export * from "./models.js"; -export * from "./api-keys.js"; -export * from "./runtime-spec.js"; -export * from "./providers/common.js"; -export * as anthropic from "./providers/anthropic/index.js"; -export * as gemini from "./providers/gemini/index.js"; -export * as openai from "./providers/openai/index.js"; -export * as tzafon from "./providers/tzafon/index.js"; -export * as yutori from "./providers/yutori/index.js"; +export { registerCuaProviders } from "./providers"; +export * from "./models"; +export * from "./api-keys"; +export * from "./runtime-spec"; +export * from "./providers/common"; +export * as anthropic from "./providers/anthropic/index"; +export * as gemini from "./providers/gemini/index"; +export * as openai from "./providers/openai/index"; +export * as tzafon from "./providers/tzafon/index"; +export * as yutori from "./providers/yutori/index"; registerCuaProviders(); diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index a0f0edd5..ba47b2ae 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -1,6 +1,6 @@ import { registerApiProvider } from "@earendil-works/pi-ai"; -import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider.js"; -import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider.js"; +import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; +import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; // pi-ai eagerly registers openai-responses, anthropic-messages, and // google-generative-ai when its index module loads (see diff --git a/packages/ai/src/providers/anthropic/actions.ts b/packages/ai/src/providers/anthropic/actions.ts index 933b79bd..af0a1fc4 100644 --- a/packages/ai/src/providers/anthropic/actions.ts +++ b/packages/ai/src/providers/anthropic/actions.ts @@ -11,7 +11,7 @@ import { type CuaAction, type CuaActionType, type CuaToolExecutorSpec, -} from "../common.js"; +} from "../common"; /** * Canonical CUA action types Anthropic browser computer-use tools support. diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 72ad5c30..d69b14bd 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -1,5 +1,5 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; -import { computerToolExecutors, computerTools } from "./actions.js"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors, computerTools } from "./actions"; export { ANTHROPIC_BATCH_TOOL_NAME, @@ -7,12 +7,12 @@ export { computerToolExecutors, computerTools, createActionSchema, -} from "./actions.js"; +} from "./actions"; export type { AnthropicAction, AnthropicComputerToolsOptions, AnthropicComputerToolsOptions as ComputerToolsOptions, -} from "./actions.js"; +} from "./actions"; // Anthropic's quickstart uses pixel coordinates for both its computer and // browser tools. diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 0dc743c8..6fdcbdaa 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -1,5 +1,5 @@ import { Type, type Api, type Model, type Static, type TSchema, type Tool } from "@earendil-works/pi-ai"; -import type { CuaModelRef, CuaProvider } from "../models.js"; +import type { CuaModelRef, CuaProvider } from "../models"; export const CUA_ACTION_TYPES = [ "click", diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index 8e801256..1a337c7e 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -1,16 +1,16 @@ -import { computerToolExecutors, computerTools } from "../common.js"; -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; +import { computerToolExecutors, computerTools } from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as GeminiAction, ComputerToolsOptions, -} from "../common.js"; +} from "../common"; // Provider-native function names emitted on `functionCall.name` (PREDEFINED_COMPUTER_USE_FUNCTIONS): // open_web_browser, click_at, hover_at, type_text_at, scroll_document, diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 1c61779b..1b1a7a1f 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -1,5 +1,5 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; -import { computerToolExecutors, computerTools } from "../common.js"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors, computerTools } from "../common"; export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, @@ -9,12 +9,12 @@ export { computerTools, createCuaActionSchema as createActionSchema, CuaNavigationSchema as OpenAIExtraSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as OpenAIAction, ComputerToolsOptions, CuaNavigationInput as OpenAIExtraInput, -} from "../common.js"; +} from "../common"; // Provider-native action vocabulary emitted on `computer_call.action.type`: // click, double_click, drag, move, scroll, type, keypress, wait, screenshot diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 57be34b4..11f3bfcc 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -1,16 +1,16 @@ -import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, type CuaProviderModule } from "../common.js"; -import { tzafonComputerUseOnPayload } from "./provider.js"; +import { computerToolExecutors, computerTools, type ComputerToolCoordinateSystem, type CuaProviderModule } from "../common"; +import { tzafonComputerUseOnPayload } from "./provider"; export { CUA_ACTION_TYPES as TZAFON_CUA_ACTION_TYPES, computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as TzafonAction, ComputerToolsOptions, -} from "../common.js"; +} from "../common"; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, @@ -18,8 +18,8 @@ export { toCanonicalActions, tzafonComputerUseOnPayload, tzafonToolCallId, -} from "./provider.js"; -export type { TzafonCanonicalAction, TzafonResponsesOptions } from "./provider.js"; +} from "./provider"; +export type { TzafonCanonicalAction, TzafonResponsesOptions } from "./provider"; // Provider-native action vocabulary. The model card lists supported actions; // the Responses API loop dispatches on `action.type` and adds terminal control diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index ea275922..f39b511f 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -13,7 +13,7 @@ import { type ToolCall, } from "@earendil-works/pi-ai"; import Lightcone from "@tzafon/lightcone"; -import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction, type CuaPayloadContext } from "../common.js"; +import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction, type CuaPayloadContext } from "../common"; export const TZAFON_RESPONSES_API = "tzafon-responses"; const TZAFON_COMPUTER_USE_TOOL = { diff --git a/packages/ai/src/providers/yutori/actions.ts b/packages/ai/src/providers/yutori/actions.ts index 7b74c71c..0558548e 100644 --- a/packages/ai/src/providers/yutori/actions.ts +++ b/packages/ai/src/providers/yutori/actions.ts @@ -8,7 +8,7 @@ import { type CuaAction, type CuaToolExecutorSpec, type CuaActionType, -} from "../common.js"; +} from "../common"; /** * Native Yutori Navigator n1.5 tool-set ids. diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index 3675805d..a6e2bf8f 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,6 +1,6 @@ -import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common.js"; -import { computerToolExecutors } from "./actions.js"; -import { yutoriNativeToolSetOnPayload } from "./provider.js"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; +import { computerToolExecutors } from "./actions"; +import { yutoriNativeToolSetOnPayload } from "./provider"; export { computerToolExecutors, @@ -16,22 +16,22 @@ export { YUTORI_N15_CORE_TOOL_SET, YUTORI_N15_EXPANDED_ACTION_TYPES, YUTORI_N15_EXPANDED_TOOL_SET, -} from "./actions.js"; +} from "./actions"; export type { YutoriAction, YutoriN1ActionType, YutoriN15CoreActionType, YutoriN15ExpandedActionType, YutoriNativeActionType, -} from "./actions.js"; -export type { ComputerToolsOptions } from "../common.js"; +} from "./actions"; +export type { ComputerToolsOptions } from "../common"; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori, yutoriNativeToolSetOnPayload, -} from "./provider.js"; -export type { YutoriOptions } from "./provider.js"; +} from "./provider"; +export type { YutoriOptions } from "./provider"; // Provider-native action vocabulary differs between Navigator versions: // n1 (fixed tool set): diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index d8ef70f2..3a002960 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -18,8 +18,8 @@ import { toCanonicalActions, yutoriToolSetForModel, YUTORI_N15_EXPANDED_ACTION_TYPES, -} from "./actions.js"; -import { canonicalToolCallArguments, canonicalToolCallName, type CuaPayloadContext } from "../common.js"; +} from "./actions"; +import { canonicalToolCallArguments, canonicalToolCallName, type CuaPayloadContext } from "../common"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index e5a0ad7b..fc9fdb5e 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -1,16 +1,16 @@ -import type { CuaProvider } from "./models.js"; -import { getCuaModel, providerForModel } from "./models.js"; -import { providerModule as anthropic } from "./providers/anthropic/index.js"; -import { providerModule as gemini } from "./providers/gemini/index.js"; -import { providerModule as openai } from "./providers/openai/index.js"; -import { providerModule as tzafon } from "./providers/tzafon/index.js"; -import { providerModule as yutori } from "./providers/yutori/index.js"; +import type { CuaProvider } from "./models"; +import { getCuaModel, providerForModel } from "./models"; +import { providerModule as anthropic } from "./providers/anthropic/index"; +import { providerModule as gemini } from "./providers/gemini/index"; +import { providerModule as openai } from "./providers/openai/index"; +import { providerModule as tzafon } from "./providers/tzafon/index"; +import { providerModule as yutori } from "./providers/yutori/index"; import type { ComputerToolsOptions, CuaProviderModule, CuaRuntimeSpec, CuaRuntimeSpecInput, -} from "./providers/common.js"; +} from "./providers/common"; const PROVIDERS = { openai, diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts index 8711dc7e..501b0106 100644 --- a/packages/ai/test/api-keys.test.ts +++ b/packages/ai/test/api-keys.test.ts @@ -4,7 +4,7 @@ import { getCuaEnvApiKey, getCuaEnvApiKeyForModel, requireCuaEnvApiKey, -} from "../src/index.js"; +} from "../src/index"; const ENV_KEYS = [ "OPENAI_API_KEY", diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index 35200817..7eb53588 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -14,7 +14,7 @@ import { openai, tzafon, yutori, -} from "../src/index.js"; +} from "../src/index"; const here = dirname(fileURLToPath(import.meta.url)); const screenshotPath = join(here, "..", "examples", "screenshot.png"); diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index edd0cbad..399abaa7 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -7,7 +7,7 @@ import { getCuaModel, listCuaModels, parseCuaModelRef, -} from "../src/index.js"; +} from "../src/index"; describe("CUA model refs", () => { it("parses and formats provider-qualified refs", () => { diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index a8910d81..17b17a16 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index.js"; -import type { CuaProviderModule } from "../src/providers/common.js"; +import { anthropic, CUA_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; +import type { CuaProviderModule } from "../src/providers/common"; const MODULES: Record = { openai, diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index a80bfd8c..7804afff 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getApiProvider, registerCuaProviders, resetApiProviders, tzafon, yutori } from "../src/index.js"; +import { getApiProvider, registerCuaProviders, resetApiProviders, tzafon, yutori } from "../src/index"; const TZAFON_RESPONSES_API = tzafon.TZAFON_RESPONSES_API; const YUTORI_CHAT_COMPLETIONS_API = yutori.YUTORI_CHAT_COMPLETIONS_API; diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index fd5c4c4d..8be6fcf3 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index.js"; +import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index"; describe("resolveCuaRuntimeSpec", () => { it("resolves a runtime spec for every CUA provider", () => { diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index 667fa712..6658eaa4 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -11,7 +11,7 @@ import { openai, tzafon, yutori, -} from "../src/index.js"; +} from "../src/index"; const providers = { openai, gemini, tzafon }; const ANTHROPIC_BATCH_TOOL_NAME = anthropic.ANTHROPIC_BATCH_TOOL_NAME; diff --git a/packages/ai/test/tzafon-actions.test.ts b/packages/ai/test/tzafon-actions.test.ts index ff516c19..1c584c2e 100644 --- a/packages/ai/test/tzafon-actions.test.ts +++ b/packages/ai/test/tzafon-actions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { tzafon } from "../src/index.js"; +import { tzafon } from "../src/index"; describe("Tzafon native action normalization", () => { it("normalizes click variants", () => { diff --git a/packages/ai/test/tzafon-payload.test.ts b/packages/ai/test/tzafon-payload.test.ts index 620992c2..5c26f4ed 100644 --- a/packages/ai/test/tzafon-payload.test.ts +++ b/packages/ai/test/tzafon-payload.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { tzafon } from "../src/index.js"; +import { tzafon } from "../src/index"; describe("tzafonComputerUseOnPayload", () => { it("replaces local CUA action tools with the native computer_use tool", () => { diff --git a/packages/ai/test/tzafon-provider.test.ts b/packages/ai/test/tzafon-provider.test.ts index c678142d..e0aef59d 100644 --- a/packages/ai/test/tzafon-provider.test.ts +++ b/packages/ai/test/tzafon-provider.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import type { Model, ToolCall } from "@earendil-works/pi-ai"; -import { getCuaModel, tzafon } from "../src/index.js"; +import { getCuaModel, tzafon } from "../src/index"; const { responsesCreate } = vi.hoisted(() => ({ responsesCreate: vi.fn() })); diff --git a/packages/ai/test/yutori-actions.test.ts b/packages/ai/test/yutori-actions.test.ts index e26c6714..5ec89566 100644 --- a/packages/ai/test/yutori-actions.test.ts +++ b/packages/ai/test/yutori-actions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { type CuaAction, yutori } from "../src/index.js"; +import { type CuaAction, yutori } from "../src/index"; const n15CoreActionArgs = { left_click: { coordinates: [500, 250] }, diff --git a/packages/ai/test/yutori-payload.test.ts b/packages/ai/test/yutori-payload.test.ts index 145e20cc..5f399ad0 100644 --- a/packages/ai/test/yutori-payload.test.ts +++ b/packages/ai/test/yutori-payload.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { yutori } from "../src/index.js"; +import { yutori } from "../src/index"; describe("yutoriNativeToolSetOnPayload", () => { it("removes local Yutori action tool definitions from payload.tools", () => { diff --git a/packages/ai/test/yutori-provider.test.ts b/packages/ai/test/yutori-provider.test.ts index e8c2b994..6a6e161c 100644 --- a/packages/ai/test/yutori-provider.test.ts +++ b/packages/ai/test/yutori-provider.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import type { Model, ToolCall } from "@earendil-works/pi-ai"; -import { getCuaModel, yutori } from "../src/index.js"; +import { getCuaModel, yutori } from "../src/index"; const { completionsCreate } = vi.hoisted(() => ({ completionsCreate: vi.fn() })); diff --git a/packages/ai/tsconfig.build.json b/packages/ai/tsconfig.build.json index efe350d7..386602f1 100644 --- a/packages/ai/tsconfig.build.json +++ b/packages/ai/tsconfig.build.json @@ -1,10 +1,11 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "module": "NodeNext", - "moduleResolution": "NodeNext", - "outDir": "./dist", - "rootDir": "./src" + "outDir": "./dist-tsc", + "rootDir": "./src", + "emitDeclarationOnly": true, + "sourceMap": false, + "declarationMap": false }, "include": ["src/**/*.ts"], "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] diff --git a/packages/ai/tsdown.config.ts b/packages/ai/tsdown.config.ts new file mode 100644 index 00000000..f377486b --- /dev/null +++ b/packages/ai/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + platform: "node", + dts: true, + sourcemap: false, + clean: true, + outExtensions: () => ({ js: ".js", dts: ".d.ts" }), +}); From c00c7c4ef1cc414f15dca9eb23dd6ed49f0ede5f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 20:43:19 +0000 Subject: [PATCH 7/9] Trim cua-ai README internals and point newcomers at cua-agent Lead the loop section with the cua-agent pointer, drop the executor/ Yutori wire-format internals, the pi-ai versioning and registry side-effect notes, and the google-vs-gemini naming aside. Mark the legacy cua-* packages deprecated in the root README and make the "start here" pointer lead with cua-agent. Co-Authored-By: Claude Opus 4.7 --- README.md | 36 +++++++++++++++++---------------- packages/ai/README.md | 46 +++++-------------------------------------- 2 files changed, 24 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index f46d985d..20e2928b 100644 --- a/README.md +++ b/README.md @@ -44,21 +44,23 @@ All of them expect you to: packages/ ├── ai/ # @onkernel/cua-ai - CUA model catalog + tool schemas + provider adapters (on npm) ├── agent/ # @onkernel/cua-agent - CuaAgent/CuaAgentHarness Kernel-browser execution loop (on npm) -├── cua-translator/ # @onkernel/cua-translator - shared SDK types + translator + browser-session -├── cua-openai/ # @onkernel/cua-openai - gpt-* (batch_computer_actions + computer_use_extra) -├── cua-anthropic/ # @onkernel/cua-anthropic - claude-* (computer_20251124 + batch_computer_actions + onPayload) -├── cua-gemini/ # @onkernel/cua-gemini - gemini-* (predefined functions + batch_computer_actions) -├── cua-yutori/ # @onkernel/cua-yutori - n1* (Navigator browser actions) +├── cua-translator/ # @onkernel/cua-translator - [deprecated] shared SDK types + translator + browser-session +├── cua-openai/ # @onkernel/cua-openai - [deprecated] gpt-* (batch_computer_actions + computer_use_extra) +├── cua-anthropic/ # @onkernel/cua-anthropic - [deprecated] claude-* (computer_20251124 + batch_computer_actions + onPayload) +├── cua-gemini/ # @onkernel/cua-gemini - [deprecated] gemini-* (predefined functions + batch_computer_actions) +├── cua-yutori/ # @onkernel/cua-yutori - [deprecated] n1* (Navigator browser actions) └── cua-cli/ # @onkernel/cua-cli - the CLI; depends on the cua-* providers above ``` -**Building your own agent? Start here:** [`packages/ai`](packages/ai) -(`@onkernel/cua-ai`) is the model layer — the curated computer-use model -catalog, canonical tool schemas, and per-provider adapters on top of pi-ai. -[`packages/agent`](packages/agent) (`@onkernel/cua-agent`) is the execution -layer — `CuaAgent`/`CuaAgentHarness` run those tool calls against a Kernel -browser. Both are published to npm. The `cua-*` packages below back the `cua` -CLI. +**Building your own agent? Start here:** [`packages/agent`](packages/agent) +(`@onkernel/cua-agent`) — `CuaAgent`/`CuaAgentHarness` run the full +computer-use loop against a Kernel browser. It sits on +[`packages/ai`](packages/ai) (`@onkernel/cua-ai`), the model layer with the +curated computer-use model catalog, canonical tool schemas, and per-provider +adapters on top of pi-ai; reach for cua-ai directly only when you bring your +own execution. Both are published to npm. The `cua-*` packages below are +deprecated — cua-ai/cua-agent are the canonical source now — and remain only +to back the `cua` CLI. ```mermaid flowchart LR @@ -86,11 +88,11 @@ flowchart LR | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | | [`@onkernel/cua-ai`](packages/ai) | Computer-use model catalog (`getCuaModel`/`listCuaModels`), canonical CUA tool schemas, and provider adapters/runtime specs built on pi-ai. On npm. | | [`@onkernel/cua-agent`](packages/agent) | `CuaAgent`/`CuaAgentHarness` classes that execute cua-ai tool calls against a Kernel browser, screenshot loop included. On npm. | -| [`@onkernel/cua-translator`](packages/cua-translator) | Provider-agnostic `ComputerTranslator`, key/scroll/drag math, `goto`/`back`/`forward`/`url` builders, browser-session helper. | -| [`@onkernel/cua-openai`](packages/cua-openai) | `batch_computer_actions` + `computer_use_extra` AgentTools and JSON Schemas for OpenAI computer-use models. | -| [`@onkernel/cua-anthropic`](packages/cua-anthropic) | `computer` (built-in `computer_20251124`) + `batch_computer_actions` AgentTools, beta-header stream wrapper, payload hook. | -| [`@onkernel/cua-gemini`](packages/cua-gemini) | 13 per-action AgentTools matching Gemini's predefined computer-use functions, plus `batch_computer_actions`. Coordinate denormalization. | -| [`@onkernel/cua-yutori`](packages/cua-yutori) | AgentTools matching Yutori Navigator browser actions, with outbound payload filtering so Yutori uses its built-in action set. | +| [`@onkernel/cua-translator`](packages/cua-translator) | **Deprecated** (use `cua-ai`/`cua-agent`). Provider-agnostic `ComputerTranslator`, key/scroll/drag math, `goto`/`back`/`forward`/`url` builders, browser-session helper. | +| [`@onkernel/cua-openai`](packages/cua-openai) | **Deprecated** (use `cua-ai`/`cua-agent`). `batch_computer_actions` + `computer_use_extra` AgentTools and JSON Schemas for OpenAI computer-use models. | +| [`@onkernel/cua-anthropic`](packages/cua-anthropic) | **Deprecated** (use `cua-ai`/`cua-agent`). `computer` (built-in `computer_20251124`) + `batch_computer_actions` AgentTools, beta-header stream wrapper, payload hook. | +| [`@onkernel/cua-gemini`](packages/cua-gemini) | **Deprecated** (use `cua-ai`/`cua-agent`). 13 per-action AgentTools matching Gemini's predefined computer-use functions, plus `batch_computer_actions`. Coordinate denormalization. | +| [`@onkernel/cua-yutori`](packages/cua-yutori) | **Deprecated** (use `cua-ai`/`cua-agent`). AgentTools matching Yutori Navigator browser actions, with outbound payload filtering so Yutori uses its built-in action set. | | [`@onkernel/cua-cli`](packages/cua-cli) | The `cua` binary: argv parsing, config, sessions, skills, JSONL output, pi-tui front-end. | --- diff --git a/packages/ai/README.md b/packages/ai/README.md index aa8b88b5..5bf15863 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -114,6 +114,11 @@ validate eagerly and throw regular errors. ## Continuing the Loop +[`@onkernel/cua-agent`](https://www.npmjs.com/package/@onkernel/cua-agent) +runs this loop for you — `CuaAgent`/`CuaAgentHarness` classes with browser +execution against a Kernel browser. Reach for it first; the rest of this +section is for driving the loop yourself against your own browser stack. + A computer-use session is a loop: the model calls a tool, you execute it against a real browser, and you send the result (with a fresh screenshot) back so the model can plan the next step. Tool results are pi-ai @@ -180,26 +185,6 @@ for (const block of first.content) { const second = await complete(model, { messages, tools }, { apiKey }); ``` -The provider namespaces' `computerToolExecutors()` return -`CuaToolExecutorSpec`s whose `toActions(args)` converts a tool call's -arguments into canonical `CuaAction`s — use them to drive your own executor -instead of switching on tool names by hand. - -Yutori is the one provider where the wire differs: `streamYutori` strips -caller-supplied tools whose names collide with Yutori's canonical action names -(`yutori.YUTORI_CUA_ACTION_TYPES`) from the outbound payload and selects -Yutori's documented native `tool_set` instead, -then normalizes the native tool calls back to canonical names — so the loop -above still works unchanged. Tools that must stay on the payload can be listed -in the `keepToolNames` stream option (`yutori.YutoriOptions`). The same -`keepToolNames` option exists for Tzafon (`tzafon.TzafonResponsesOptions`), -whose adapter likewise replaces canonical action tools with Tzafon's native -`computer_use` tool on the wire. - -This package does not execute browser actions. Use -[`@onkernel/cua-agent`](https://www.npmjs.com/package/@onkernel/cua-agent) -when you want this loop run for you against a Kernel browser. - ## Core Concepts `@onkernel/cua-ai` re-exports the full surface of @@ -210,22 +195,6 @@ when you want this loop run for you against a Kernel browser. Some familiarity with pi-ai is assumed; Kernel adds the computer-use model catalog and provider/tool metadata. -**Versioning note**: because of the wildcard re-export, pi-ai's API is part of -this package's public surface. This release is developed and tested against -`@earendil-works/pi-ai` `^0.74.0`; pi-ai exports can change within that semver -range independently of `@onkernel/cua-ai` releases. If you depend on pi-ai -symbols beyond the core primitives above, consider pinning pi-ai in your own -project. - -**Import side effect**: importing `@onkernel/cua-ai` registers the -`yutori-chat-completions` and `tzafon-responses` stream providers with pi-ai's -global API registry. The re-exported pi-ai registry mutators -(`clearApiProviders`, `resetApiProviders`, `unregisterApiProviders`) -deregister them — call the exported `registerCuaProviders()` to restore. For -Yutori/Tzafon models to stream, `complete`/`stream` must come from the same -pi-ai copy that saw the registration, so make sure your install dedupes to a -single `@earendil-works/pi-ai`. - ### Model Refs `getCuaModel()` accepts only provider-qualified model refs of the form @@ -243,11 +212,6 @@ getCuaModel("yutori:n1.5-latest"); or `stream()`. It throws when the ref names a model without a CUA-support annotation. -The Google provider id is `google` (matching pi-ai's `Model.provider`), while -the namespace export is `gemini` — so refs are `google:…` but tools come from -`gemini.computerTools()`. `"gemini:"` refs are accepted as an alias and -normalize to `google`. - See [`docs/supported-models.md`](./docs/supported-models.md) for the current list of CUA-supporting models per provider. From 2fbe7eebd2d2d4b793621730b952b9de70c9e8a7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 20:48:24 +0000 Subject: [PATCH 8/9] Run CI on Node 22 tsdown evaluates its TypeScript config via Node's native type stripping (node >=22.18); on Node 20 it falls back to the optional unrun loader, which is not installed, so every job failed at the cua-ai build step. Node 22 also matches the >=22.19.0 engines floor the de-vendor PR introduces. Release workflows already run Node 24. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c1fef7c..bd3d6da1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm - run: npm ci - run: npm run build --workspace @onkernel/cua-ai @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm - run: npm ci - run: npm run build --workspace @onkernel/cua-ai @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm - run: npm ci - run: npm run build --workspace @onkernel/cua-ai From 7880147f5eaa0160b527e365394a6e399cff7f73 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 10 Jun 2026 20:53:50 +0000 Subject: [PATCH 9/9] Bump cua-agent's cua-ai pin to 0.2.0 The exact 0.1.0 pin plus the workspace version bump made npm ci nest the published 0.1.0 (stale API, ESM-broken) under packages/agent, which broke the agent-e2e job at the build step. Pinning 0.2.0 resolves the workspace package again. The de-vendor PR sets the same value, so the branches still merge cleanly. Co-Authored-By: Claude Opus 4.7 --- package-lock.json | 13 +------------ packages/agent/package.json | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 817951e2..393ad16d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5959,7 +5959,7 @@ "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "^0.74.0", - "@onkernel/cua-ai": "0.1.0", + "@onkernel/cua-ai": "0.2.0", "@onkernel/sdk": "0.49.0", "sharp": "^0.34.5", "typebox": "^1.1.38" @@ -5968,17 +5968,6 @@ "vitest": "^3.2.4" } }, - "packages/agent/node_modules/@onkernel/cua-ai": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@onkernel/cua-ai/-/cua-ai-0.1.0.tgz", - "integrity": "sha512-S+bhPUwMIgppvUWa5dojSNHOnxYy361ENT5OckCyJxwdbwtworop4CbUr3I67LF/d9xcLhMd4+keMejGgz9PHg==", - "license": "MIT", - "dependencies": { - "@earendil-works/pi-ai": "^0.74.0", - "@tzafon/lightcone": "^0.7.0", - "openai": "^6.26.0" - } - }, "packages/ai": { "name": "@onkernel/cua-ai", "version": "0.2.0", diff --git a/packages/agent/package.json b/packages/agent/package.json index d7915b6c..defa2a9f 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -42,7 +42,7 @@ }, "dependencies": { "@earendil-works/pi-ai": "^0.74.0", - "@onkernel/cua-ai": "0.1.0", + "@onkernel/cua-ai": "0.2.0", "@onkernel/sdk": "0.49.0", "sharp": "^0.34.5", "typebox": "^1.1.38"