diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b140bd14..d5a3ddc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} - run: npm test --workspace @onkernel/cua-ai -- test/batch-tool.integration.test.ts + run: npm test --workspace @onkernel/cua-ai -- test/computer-tool.integration.test.ts agent-e2e: runs-on: ubuntu-latest diff --git a/package-lock.json b/package-lock.json index 9cf2e56d..d6ab979d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -736,6 +736,16 @@ "zod-to-json-schema": "^3.25.0" } }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", @@ -1201,6 +1211,471 @@ } } }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -2848,6 +3323,15 @@ "node": ">= 14" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "8.0.4", "license": "BSD-3-Clause", @@ -3957,6 +4441,62 @@ ], "license": "MIT" }, + "node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -4623,6 +5163,7 @@ "@earendil-works/pi-ai": "^0.74.0", "@onkernel/cua-ai": "0.1.0", "@onkernel/sdk": "0.49.0", + "sharp": "^0.34.5", "typebox": "^1.1.38" }, "devDependencies": { diff --git a/packages/agent/README.md b/packages/agent/README.md index ee6b780b..84bccc0e 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -84,6 +84,9 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions: - `browser` (Kernel browser response) - `client` (Kernel SDK client) - CUA model refs (`"provider:model"`) accepted where pi expects a concrete model +- `extraTools` to add your own pi tools alongside the built-in browser tools +- `batchTool: true` to let the model run multiple browser actions in one tool call +- `computerUseExtra: true` to let the model use a small navigation helper If auth callbacks are omitted, both classes default to CUA env var conventions: - OpenAI: `OPENAI_API_KEY` @@ -94,9 +97,26 @@ If auth callbacks are omitted, both classes default to CUA env var conventions: ### Tool Defaults -If tools are omitted, the classes install canonical CUA computer tool executors -using runtime specs from `@onkernel/cua-ai`. If tools are provided, they are -used exactly. +By default, the classes install provider-selected canonical CUA computer tool +executors using runtime specs from `@onkernel/cua-ai`. Use `extraTools` to add +your own pi tools alongside the provider's computer-use tools. This is useful +when the model needs to call application-specific code, such as looking up a +record, writing a database row, or handing off to another service while it also +controls the browser. + +`batchTool: true` adds the `batch_computer_actions` tool. Use it when you want +the model to group several browser actions into one call, for example moving, +clicking, typing, waiting, and then reading a screenshot. The batch tool is +synthesized from the selected provider's normal browser action definitions, so +it only batches actions that provider runtime already supports. + +`computerUseExtra: true` adds the `computer_use_extra` tool. Use it when you +want one compact helper for common browser navigation/read operations: +`goto`, `back`, `forward`, and `url`. + +The TypeScript API follows pi's camelCase option style (`extraTools`, +`batchTool`, `computerUseExtra`). Names like `batch_computer_actions` and +`computer_use_extra` are the literal tool names the model may see in traces. ### Model Switching @@ -124,6 +144,7 @@ const tools = [ browser, client, toolDefinitions: runtime.toolDefinitions, + batchTool: true, }), myCustomTool, ]; diff --git a/packages/agent/package.json b/packages/agent/package.json index 3a78f1bf..d7915b6c 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -44,6 +44,7 @@ "@earendil-works/pi-ai": "^0.74.0", "@onkernel/cua-ai": "0.1.0", "@onkernel/sdk": "0.49.0", + "sharp": "^0.34.5", "typebox": "^1.1.38" }, "devDependencies": { diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 9bd76b16..20be10fa 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -11,6 +11,8 @@ import { } from "./vendor/pi-agent-core/index"; import { type Api, + CUA_BATCH_TOOL_NAME, + CUA_NAVIGATION_TOOL_NAME, type CuaModelRef, getCuaEnvApiKey, type Model, @@ -20,7 +22,7 @@ import { } from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; import { createCuaComputerTools } from "./tools"; -import type { KernelBrowser } from "./translator/translator"; +import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; /** A CUA model reference string or a concrete pi model object. */ type CuaRuntimeInput = CuaModelRef | Model; @@ -45,8 +47,6 @@ export interface CuaAgentState extends Omit { type CuaAgentInitialState = Omit, "model" | "tools"> & { /** Model to use for the first turn. CUA refs are resolved before pi sees the state. */ model: CuaRuntimeInput; - /** Optional caller-owned tools. Omit this to install the provider's default CUA tools. */ - tools?: AgentTool[]; }; /** @@ -63,6 +63,12 @@ export type CuaAgentOptions = Omit & { client: Kernel; /** Initial pi state plus a CUA-aware model value. */ initialState: CuaAgentInitialState; + /** Add your own pi tools alongside the built-in browser tools. */ + extraTools?: AgentTool[]; + /** Expose a batch tool so the model can run multiple browser actions in one call. */ + batchTool?: boolean; + /** Expose a helper for browser navigation and URL reads. */ + computerUseExtra?: boolean; }; /** @@ -75,13 +81,19 @@ export type CuaAgentOptions = Omit & { export type CuaAgentHarnessOptions< TSkill extends Skill = Skill, TPromptTemplate extends PromptTemplate = PromptTemplate, -> = Omit, "model"> & { +> = Omit, "model" | "tools"> & { /** Kernel browser session used by default CUA tools. */ browser: KernelBrowser; /** Kernel SDK client used by default CUA tools. */ client: Kernel; /** Model used by the harness. CUA refs are resolved before pi sees the model. */ model: CuaRuntimeInput; + /** Add your own pi tools alongside the built-in browser tools. */ + extraTools?: AgentTool[]; + /** Expose a batch tool so the model can run multiple browser actions in one call. */ + batchTool?: boolean; + /** Expose a helper for browser navigation and URL reads. */ + computerUseExtra?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */ onPayload?: SimpleStreamOptions["onPayload"]; }; @@ -89,9 +101,10 @@ export type CuaAgentHarnessOptions< /** * Holds the CUA-specific pieces that have to change when a model changes. * - * If callers omit `tools` or `systemPrompt`, CUA owns those values and refreshes - * them from `@onkernel/cua-ai` whenever the model changes. If callers pass - * their own tools or prompt, the controller preserves those caller-owned values. + * CUA owns the computer-use tools and refreshes them from `@onkernel/cua-ai` + * whenever the model changes. Caller-owned `extraTools` are appended after + * those defaults. If callers pass their own prompt, the controller preserves + * that caller-owned prompt. */ class CuaRuntimeController { private runtimeSpec: CuaRuntimeSpec; @@ -101,7 +114,9 @@ class CuaRuntimeController { browser: KernelBrowser; client: Kernel; model: CuaRuntimeInput; - tools?: AgentTool[]; + extraTools?: AgentTool[]; + batchTool?: boolean; + computerUseExtra?: boolean; systemPrompt?: unknown; onPayload?: SimpleStreamOptions["onPayload"]; }, @@ -114,7 +129,7 @@ class CuaRuntimeController { } get ownsTools(): boolean { - return this.options.tools === undefined; + return true; } get ownsSystemPrompt(): boolean { @@ -130,19 +145,79 @@ class CuaRuntimeController { } tools(): AgentTool[] { - return ( - this.options.tools ?? - createCuaComputerTools({ + return [ + ...createCuaComputerTools({ browser: this.options.browser, client: this.options.client, toolDefinitions: this.runtimeSpec.toolDefinitions, - }) - ); + coordinateSystem: this.runtimeSpec.coordinateSystem, + screenshot: this.runtimeSpec.screenshot, + batchTool: this.options.batchTool, + computerUseExtra: this.options.computerUseExtra, + }), + ...(this.options.extraTools ?? []), + ]; } onPayloadFor(model: CuaRuntimeInput): SimpleStreamOptions["onPayload"] { const runtimeSpec = resolveCuaRuntimeSpec(model); - return composeOnPayload(runtimeSpec.onPayload, this.options.onPayload); + return composeOnPayload( + composeOnPayload(this.screenshotOnPayload(runtimeSpec), this.providerOnPayload(runtimeSpec)), + this.options.onPayload, + ); + } + + keepToolNames(): string[] { + return [ + ...(this.options.extraTools ?? []).map((tool) => tool.name), + ...(this.options.batchTool ? [CUA_BATCH_TOOL_NAME] : []), + ...(this.options.computerUseExtra ? [CUA_NAVIGATION_TOOL_NAME] : []), + ]; + } + + private providerOnPayload(runtimeSpec: CuaRuntimeSpec): SimpleStreamOptions["onPayload"] | undefined { + if (!runtimeSpec.onPayload) return undefined; + return async (payload, model) => + runtimeSpec.onPayload?.(payload, model as Model, { keepToolNames: this.keepToolNames() }); + } + + private screenshotOnPayload(runtimeSpec: CuaRuntimeSpec): SimpleStreamOptions["onPayload"] | undefined { + if (!runtimeSpec.screenshot?.appendToLatestMessage) return undefined; + return async (payload) => { + if (!payload || typeof payload !== "object") return undefined; + const current = payload as { messages?: unknown }; + if (!Array.isArray(current.messages) || current.messages.length === 0) return undefined; + const last = current.messages[current.messages.length - 1]; + if (!last || typeof last !== "object") return undefined; + const lastMessage = last as { content?: unknown; role?: unknown }; + if (lastMessage.role !== "user" && lastMessage.role !== "tool") return undefined; + if (contentHasImage(lastMessage.content)) return undefined; + + const translator = new InternalComputerTranslator({ + browser: this.options.browser, + client: this.options.client, + coordinateSystem: runtimeSpec.coordinateSystem, + screenshot: runtimeSpec.screenshot, + }); + const screenshot = await translator.screenshot(); + const content = normalizePayloadContent(lastMessage.content); + const nextMessages = current.messages.slice(); + nextMessages[nextMessages.length - 1] = { + ...(last as Record), + content: [ + ...content, + { type: "text", text: "\n\n" }, + { + type: "image_url", + image_url: { + url: `data:${screenshot.mimeType};base64,${screenshot.data.toString("base64")}`, + detail: "high", + }, + }, + ], + }; + return { ...(payload as Record), messages: nextMessages }; + }; } } @@ -159,20 +234,36 @@ export class CuaAgent extends Agent { private stateProxy?: CuaAgentState; constructor(options: CuaAgentOptions) { - const { browser, client, initialState, onPayload, streamFn, prepareNextTurn, ...agentOptions } = options; + const { + browser, + client, + initialState, + onPayload, + streamFn, + prepareNextTurn, + extraTools, + batchTool, + computerUseExtra, + ...agentOptions + } = options; const runtime = new CuaRuntimeController({ browser, client, model: initialState.model, - tools: initialState.tools, + extraTools, + batchTool, + computerUseExtra, systemPrompt: initialState.systemPrompt, onPayload, }); - const wrappedStreamFn: StreamFn = (model, context, streamOptions) => - (streamFn ?? streamSimple)(model, context, { + const wrappedStreamFn: StreamFn = (model, context, streamOptions) => { + const optionsWithCuaRuntime = { ...streamOptions, onPayload: runtime.onPayloadFor(model as Model), - }); + keepToolNames: runtime.keepToolNames(), + } as SimpleStreamOptions & { keepToolNames?: string[] }; + return (streamFn ?? streamSimple)(model, context, optionsWithCuaRuntime); + }; super({ ...agentOptions, @@ -270,14 +361,25 @@ export class CuaAgentHarness< browser, client, model, - tools, + extraTools, + batchTool, + computerUseExtra, systemPrompt, getApiKeyAndHeaders, onPayload, activeToolNames, ...harnessOptions } = options; - const runtime = new CuaRuntimeController({ browser, client, model, tools, systemPrompt, onPayload }); + const runtime = new CuaRuntimeController({ + browser, + client, + model, + extraTools, + batchTool, + computerUseExtra, + systemPrompt, + onPayload, + }); const resolvedTools = runtime.tools(); super({ @@ -333,3 +435,17 @@ function composeOnPayload(first: AgentOptions["onPayload"], second: AgentOptions return second(afterFirst ?? payload, modelRef); }; } + +function normalizePayloadContent(content: unknown): Array> { + if (typeof content === "string") return [{ type: "text", text: content }]; + if (Array.isArray(content)) { + return content.filter((part): part is Record => Boolean(part) && typeof part === "object"); + } + return []; +} + +function contentHasImage(content: unknown): boolean { + return Array.isArray(content) && content.some((part) => { + return Boolean(part) && typeof part === "object" && (part as { type?: unknown }).type === "image_url"; + }); +} diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 16e63af4..997615f4 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -1,12 +1,18 @@ import type Kernel from "@onkernel/sdk"; import type { ImageContent, TextContent, Tool } from "@earendil-works/pi-ai"; +import type { TSchema } from "typebox"; import { + CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, - CuaBatchSchema, - CuaNavigationSchema, + createCuaBatchToolDefinition, + createCuaNavigationToolDefinition, + type ComputerToolCoordinateSystem, + type CuaAction, + type CuaActionType, type CuaBatchInput, type CuaNavigationInput, + type CuaScreenshotSpec, } from "@onkernel/cua-ai"; import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; import type { AgentTool, AgentToolResult } from "./vendor/pi-agent-core/index"; @@ -15,9 +21,18 @@ export interface ComputerToolOptions { browser: KernelBrowser; client: Kernel; toolDefinitions: Tool[]; + coordinateSystem?: ComputerToolCoordinateSystem; + screenshot?: CuaScreenshotSpec; + batchTool?: boolean; + computerUseExtra?: boolean; } -export const SUPPORTED_CUA_EXECUTOR_TOOL_NAMES = [CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME] as const; +const CUA_ACTION_TOOL_NAMES = new Set(CUA_ACTION_TYPES); +export const SUPPORTED_CUA_EXECUTOR_TOOL_NAMES = [ + CUA_BATCH_TOOL_NAME, + CUA_NAVIGATION_TOOL_NAME, + ...CUA_ACTION_TYPES, +] as const; export type SupportedCuaExecutorToolName = (typeof SUPPORTED_CUA_EXECUTOR_TOOL_NAMES)[number]; type ToolContent = Array; @@ -35,13 +50,29 @@ export interface NavigationDetails { error?: string; } -type BatchTool = AgentTool; -type NavigationTool = AgentTool; -export type CuaExecutorTool = BatchTool | NavigationTool; +type BatchTool = AgentTool; +type NavigationTool = AgentTool; +type ActionTool = AgentTool; +export type CuaExecutorTool = BatchTool | NavigationTool | ActionTool; export function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTool[] { const translator = new InternalComputerTranslator(args); - return args.toolDefinitions.map((definition) => createExecutorTool(definition, translator)); + return withSynthesizedTools(args).map((definition) => createExecutorTool(definition, translator)); +} + +function withSynthesizedTools(args: ComputerToolOptions): Tool[] { + const definitions = [...args.toolDefinitions]; + const existing = new Set(definitions.map((definition) => definition.name)); + const actionTypes = definitions + .map((definition) => definition.name) + .filter((name): name is CuaActionType => CUA_ACTION_TOOL_NAMES.has(name)); + if (args.batchTool && actionTypes.length > 0 && !existing.has(CUA_BATCH_TOOL_NAME)) { + definitions.push(createCuaBatchToolDefinition(actionTypes)); + } + if (args.computerUseExtra && !existing.has(CUA_NAVIGATION_TOOL_NAME)) { + definitions.push(createCuaNavigationToolDefinition()); + } + return definitions; } function createExecutorTool(definition: Tool, translator: InternalComputerTranslator): CuaExecutorTool { @@ -50,7 +81,7 @@ function createExecutorTool(definition: Tool, translator: InternalComputerTransl name: definition.name, label: definition.name, description: definition.description, - parameters: CuaBatchSchema, + parameters: definition.parameters, async execute(_toolCallId: string, params: unknown): Promise> { const result = await executeBatchTool(translator, asBatchInput(params)); if (result.isError) throw Object.assign(new Error(result.details.statusText), result); @@ -64,7 +95,7 @@ function createExecutorTool(definition: Tool, translator: InternalComputerTransl name: definition.name, label: definition.name, description: definition.description, - parameters: CuaNavigationSchema, + parameters: definition.parameters, async execute(_toolCallId: string, params: unknown): Promise> { const result = await executeNavigationTool(translator, asNavigationInput(params)); if (result.isError) throw Object.assign(new Error(result.details.statusText), result); @@ -73,8 +104,25 @@ function createExecutorTool(definition: Tool, translator: InternalComputerTransl }; return tool; } + if (CUA_ACTION_TOOL_NAMES.has(definition.name)) { + const actionType = definition.name as CuaActionType; + const tool: ActionTool = { + name: definition.name, + label: definition.name, + description: definition.description, + parameters: definition.parameters, + executionMode: "sequential", + async execute(_toolCallId: string, params: unknown): Promise> { + const action = { ...(params && typeof params === "object" ? params : {}), type: actionType } as CuaAction; + const result = await executeBatchTool(translator, { actions: [action] }); + if (result.isError) throw Object.assign(new Error(result.details.statusText), result); + return { content: result.content, details: result.details }; + }, + }; + return tool; + } throw new Error( - `unsupported CUA computer tool definition: ${definition.name}; supported names: ${SUPPORTED_CUA_EXECUTOR_TOOL_NAMES.join(", ")}`, + `unsupported CUA computer tool definition: ${definition.name}`, ); } @@ -97,14 +145,14 @@ async function executeBatchTool(translator: InternalComputerTranslator, params: readResults.push({ type: "cursor_position", x: read.x, y: read.y }); content.push({ type: "text", text: `cursor_position(): ${read.x},${read.y}` }); } else { - readResults.push({ type: "screenshot", bytes: read.pngBytes.length }); - content.push({ type: "image", data: read.pngBytes.toString("base64"), mimeType: "image/png" }); + readResults.push({ type: "screenshot", bytes: read.data.length }); + content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType }); } } if (content.length === 0) { - const png = await translator.screenshotRaw(); - readResults.push({ type: "screenshot", bytes: png.length }); - content.push({ type: "image", data: png.toString("base64"), mimeType: "image/png" }); + const screenshot = await translator.screenshot(); + readResults.push({ type: "screenshot", bytes: screenshot.data.length }); + content.push({ type: "image", data: screenshot.data.toString("base64"), mimeType: screenshot.mimeType }); } } catch (err) { error = err instanceof Error ? err : new Error(String(err)); @@ -132,8 +180,8 @@ async function executeNavigationTool(translator: InternalComputerTranslator, par await translator.executeBatch([{ type: action, url: params.url }]); statusText = `${action} executed successfully.`; } - const png = await translator.screenshotRaw(); - content.push({ type: "image", data: png.toString("base64"), mimeType: "image/png" }); + const screenshot = await translator.screenshot(); + content.push({ type: "image", data: screenshot.data.toString("base64"), mimeType: screenshot.mimeType }); } catch (err) { error = err instanceof Error ? err : new Error(String(err)); statusText = `${action} failed: ${error.message}`; diff --git a/packages/agent/src/translator/keys.ts b/packages/agent/src/translator/keys.ts new file mode 100644 index 00000000..bcecc275 --- /dev/null +++ b/packages/agent/src/translator/keys.ts @@ -0,0 +1,113 @@ +export const KERNEL_MODIFIER_KEYSYMS = ["Control_L", "Alt_L", "Shift_L", "Super_L"] as const; + +const KEY_ALIASES: Record = { + alt: "Alt_L", + alt_l: "Alt_L", + altleft: "Alt_L", + backspace: "BackSpace", + backquote: "grave", + backslash: "backslash", + bracketleft: "bracketleft", + bracketright: "bracketright", + capslock: "Caps_Lock", + cmd: "Super_L", + comma: "comma", + command: "Super_L", + control: "Control_L", + control_l: "Control_L", + controlleft: "Control_L", + ctrl: "Control_L", + delete: "Delete", + down: "Down", + end: "End", + enter: "Return", + equal: "equal", + esc: "Escape", + escape: "Escape", + home: "Home", + insert: "Insert", + kp_enter: "Return", + left: "Left", + meta: "Super_L", + minus: "minus", + numlock: "Num_Lock", + option: "Alt_L", + pagedown: "Next", + page_down: "Next", + pageup: "Prior", + page_up: "Prior", + pause: "Pause", + period: "period", + plus: "plus", + print: "Print", + printscreen: "Print", + quote: "apostrophe", + return: "Return", + right: "Right", + scrolllock: "Scroll_Lock", + semicolon: "semicolon", + shift: "Shift_L", + shift_l: "Shift_L", + shiftleft: "Shift_L", + slash: "slash", + space: "space", + super: "Super_L", + super_l: "Super_L", + tab: "Tab", + up: "Up", + ...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`f${i + 1}`, `F${i + 1}`])), + ...Object.fromEntries(Array.from({ length: 10 }, (_, i) => [`numpad${i}`, String(i)])), + numpadadd: "plus", + numpaddecimal: "period", + numpaddivide: "slash", + numpadmultiply: "asterisk", + numpadsubtract: "minus", +}; + +const PRINTABLE_KEY_ALIASES: Record = { + "*": "asterisk", + "+": "plus", + ",": "comma", + "-": "minus", + ".": "period", + "/": "slash", + ";": "semicolon", + "=": "equal", + "[": "bracketleft", + "\\": "backslash", + "]": "bracketright", + "`": "grave", + "'": "apostrophe", +}; + +const KERNEL_MODIFIER_KEYSYM_SET = new Set(KERNEL_MODIFIER_KEYSYMS); + +export function normalizeKernelKey(value: string): string { + const trimmed = value.trim(); + if (PRINTABLE_KEY_ALIASES[trimmed]) return PRINTABLE_KEY_ALIASES[trimmed]; + const lookup = trimmed.replace(/[-\s]/g, "_").toLowerCase(); + const alias = KEY_ALIASES[lookup]; + if (alias) return alias; + if (/^arrow/i.test(trimmed)) return normalizeKernelKey(trimmed.slice("arrow".length)); + if (trimmed.length === 1 && trimmed >= "A" && trimmed <= "Z") return trimmed.toLowerCase(); + return trimmed; +} + +export function normalizeKernelKeyCombo(value: string): string[] { + return value + .split("+") + .map((part) => normalizeKernelKey(part)) + .filter(Boolean); +} + +export function normalizeKernelKeySequence(value: string): string[][] { + return value + .trim() + .split(/\s+/) + .map((part) => normalizeKernelKeyCombo(part)) + .filter((combo) => combo.length > 0); +} + +export function isKernelModifierKey(key: string): boolean { + return KERNEL_MODIFIER_KEYSYM_SET.has(key); +} diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index cf1d9476..120511cd 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -1,5 +1,8 @@ import type Kernel from "@onkernel/sdk"; import type { BrowserCreateResponse, BrowserRetrieveResponse } from "@onkernel/sdk/resources/browsers"; +import { normalizeGotoUrl, type ComputerToolCoordinateSystem, type CuaScreenshotSpec } from "@onkernel/cua-ai"; +import sharp from "sharp"; +import { isKernelModifierKey, normalizeKernelKey, normalizeKernelKeyCombo } from "./keys"; import type { BatchExecutionResult, ModelAction } from "./types"; export type KernelBrowser = BrowserCreateResponse | BrowserRetrieveResponse; @@ -7,20 +10,49 @@ export type KernelBrowser = BrowserCreateResponse | BrowserRetrieveResponse; export interface InternalComputerTranslatorOptions { browser: KernelBrowser; client: Kernel; + coordinateSystem?: ComputerToolCoordinateSystem; + screenshot?: CuaScreenshotSpec; } export class InternalComputerTranslator { private readonly sessionId: string; private readonly client: Kernel; + private readonly coordinateSystem: ComputerToolCoordinateSystem; + private readonly screenshotSpec?: CuaScreenshotSpec; + private readonly viewport: { width: number; height: number }; constructor(opts: InternalComputerTranslatorOptions) { this.sessionId = opts.browser.session_id; this.client = opts.client; + this.coordinateSystem = opts.coordinateSystem ?? { type: "pixel" }; + this.screenshotSpec = opts.screenshot; + this.viewport = opts.browser.viewport ?? { width: 1920, height: 1080 }; } async screenshotRaw(): Promise { + return (await this.screenshot()).data; + } + + async screenshot(): Promise<{ data: Buffer; mimeType: string }> { const response = await this.client.browsers.computer.captureScreenshot(this.sessionId, {}); - return Buffer.from(await response.arrayBuffer()); + let data: Buffer = Buffer.from(await response.arrayBuffer()); + let mimeType = "image/png"; + const transform = this.screenshotSpec?.transform; + if (transform) { + let pipeline = sharp(data).resize(transform.width, transform.height, { fit: "fill" }); + if (transform.format === "webp") { + pipeline = pipeline.webp({ quality: transform.quality }); + mimeType = "image/webp"; + } else if (transform.format === "jpeg") { + pipeline = pipeline.jpeg({ quality: transform.quality }); + mimeType = "image/jpeg"; + } else { + pipeline = pipeline.png(); + mimeType = "image/png"; + } + data = await pipeline.toBuffer(); + } + return { data, mimeType }; } async currentUrl(): Promise { @@ -51,7 +83,7 @@ export class InternalComputerTranslator { const type = typeof action.type === "string" ? action.type : ""; if (type === "screenshot") { await flush(); - result.readResults.push({ type: "screenshot", pngBytes: await this.screenshotRaw() }); + result.readResults.push({ type: "screenshot", ...(await this.screenshot()) }); continue; } if (type === "url") { @@ -66,9 +98,10 @@ export class InternalComputerTranslator { continue; } if (type === "goto") { + const url = normalizeGotoUrl(action.url) ?? ""; pending.push( keypress(["Control", "l"]), - { type: "type_text", type_text: { text: stringOr(action.url, "") } }, + { type: "type_text", type_text: { text: url } }, keypress(["Enter"]), ); continue; @@ -81,7 +114,7 @@ export class InternalComputerTranslator { pending.push(keypress(["Alt", "Right"])); continue; } - pending.push(toSdkAction(type, action)); + pending.push(toSdkAction(type, action, this.coordinateSystem, this.viewport)); } await flush(); @@ -99,55 +132,95 @@ type KernelBatchAction = type ClickMouseButton = "back" | "forward" | "left" | "right" | "middle"; type DragMouseButton = "left" | "right" | "middle"; -function toSdkAction(type: string, action: ModelAction): KernelBatchAction { +function toSdkAction( + type: string, + action: ModelAction, + coordinateSystem: ComputerToolCoordinateSystem, + viewport: { width: number; height: number }, +): KernelBatchAction { switch (type) { - case "click": + case "click": { + const clickHoldKeys = readHoldKeys(action.hold_keys); + const point = toViewportPoint(action, coordinateSystem, viewport); return { type: "click_mouse", click_mouse: { - x: toInt(action.x), - y: toInt(action.y), + x: point.x, + y: point.y, button: clickMouseButtonOr(action.button, "left"), + ...(clickHoldKeys.length > 0 ? { hold_keys: clickHoldKeys } : {}), }, }; - case "double_click": + } + case "double_click": { + const doubleClickHoldKeys = readHoldKeys(action.hold_keys); + const point = toViewportPoint(action, coordinateSystem, viewport); return { type: "click_mouse", click_mouse: { - x: toInt(action.x), - y: toInt(action.y), + x: point.x, + y: point.y, num_clicks: 2, + ...(doubleClickHoldKeys.length > 0 ? { hold_keys: doubleClickHoldKeys } : {}), }, }; + } case "mouse_down": - case "mouse_up": + case "mouse_up": { + const mouseHoldKeys = readHoldKeys(action.hold_keys); + const point = toViewportPoint(action, coordinateSystem, viewport); return { type: "click_mouse", click_mouse: { - x: toInt(action.x), - y: toInt(action.y), + x: point.x, + y: point.y, button: clickMouseButtonOr(action.button, "left"), click_type: type === "mouse_down" ? "down" : "up", + ...(mouseHoldKeys.length > 0 ? { hold_keys: mouseHoldKeys } : {}), }, }; + } case "type": return { type: "type_text", type_text: { text: typeof action.text === "string" ? action.text : "" } }; case "keypress": - return keypress(toStringArray(action.keys)); - case "scroll": + return keypress(toStringArray(action.keys), action.duration); + case "scroll": { + const scrollHoldKeys = readHoldKeys(action.hold_keys); + const point = toViewportPoint(action, coordinateSystem, viewport); return { type: "scroll", scroll: { - x: toInt(action.x), - y: toInt(action.y), + x: point.x, + y: point.y, delta_x: toInt(action.scroll_x), delta_y: toInt(action.scroll_y), + ...(scrollHoldKeys.length > 0 ? { hold_keys: scrollHoldKeys } : {}), + }, + }; + } + case "move": { + const moveHoldKeys = readHoldKeys(action.hold_keys); + const point = toViewportPoint(action, coordinateSystem, viewport); + return { + type: "move_mouse", + move_mouse: { + x: point.x, + y: point.y, + ...(moveHoldKeys.length > 0 ? { hold_keys: moveHoldKeys } : {}), + }, + }; + } + case "drag": { + const dragHoldKeys = readHoldKeys(action.hold_keys); + return { + type: "drag_mouse", + drag_mouse: { + path: toPath(action.path, coordinateSystem, viewport), + button: dragMouseButtonOr(action.button, "left"), + ...(dragHoldKeys.length > 0 ? { hold_keys: dragHoldKeys } : {}), }, }; - case "move": - return { type: "move_mouse", move_mouse: { x: toInt(action.x), y: toInt(action.y) } }; - case "drag": - return { type: "drag_mouse", drag_mouse: { path: toPath(action.path), button: dragMouseButtonOr(action.button, "left") } }; + } case "wait": return { type: "sleep", sleep: { duration_ms: typeof action.ms === "number" ? Math.trunc(action.ms) : 1000 } }; default: @@ -188,76 +261,70 @@ function toStringArray(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; } -function keypress(keys: string[]): KernelBatchAction { - const translated = translateKeys(keys); - const pressedKeys = translated.filter((key) => !isModifierKey(key)); - const holdKeys = pressedKeys.length > 0 ? translated.filter(isModifierKey) : translated.slice(0, -1); +function readHoldKeys(value: unknown): string[] { + return toStringArray(value).map(normalizeKernelKey); +} + +function keypress(keys: string[], duration: unknown = undefined): KernelBatchAction { + const translated = keys.flatMap(normalizeKernelKeyCombo); + const pressedKeys = translated.filter((key) => !isKernelModifierKey(key)); + const holdKeys = pressedKeys.length > 0 ? translated.filter(isKernelModifierKey) : translated.slice(0, -1); return { type: "press_key", press_key: { keys: pressedKeys.length > 0 ? pressedKeys : translated.slice(-1), ...(holdKeys.length > 0 ? { hold_keys: holdKeys } : {}), + ...(typeof duration === "number" && Number.isFinite(duration) && duration > 0 ? { duration: Math.trunc(duration) } : {}), }, }; } -const KEY_ALIASES: Record = { - ctrl: "Control_L", - control: "Control_L", - control_l: "Control_L", - controlleft: "Control_L", - alt: "Alt_L", - alt_l: "Alt_L", - altleft: "Alt_L", - shift: "Shift_L", - shift_l: "Shift_L", - shiftleft: "Shift_L", - meta: "Super_L", - super: "Super_L", - cmd: "Super_L", - command: "Super_L", - enter: "Return", - return: "Return", - escape: "Escape", - esc: "Escape", - backspace: "BackSpace", - delete: "Delete", - tab: "Tab", - space: "space", - left: "Left", - right: "Right", - up: "Up", - down: "Down", -}; +function toPath( + value: unknown, + coordinateSystem: ComputerToolCoordinateSystem = { type: "pixel" }, + viewport: { width: number; height: number } = { width: 1920, height: 1080 }, +): Array<[number, number]> { + if (!Array.isArray(value)) return []; + return value.map((point) => toPathPoint(point, coordinateSystem, viewport)); +} -function translateKeys(keys: string[]): string[] { - return keys.flatMap((key) => - key - .split("+") - .map((part) => part.trim()) - .filter(Boolean) - .map((part) => { - const alias = KEY_ALIASES[part.replace(/[-\s]/g, "_").toLowerCase()]; - if (alias) return alias; - if (part.length === 1 && part >= "A" && part <= "Z") return part.toLowerCase(); - return part; - }), - ); +function toPathPoint(value: unknown, coordinateSystem: ComputerToolCoordinateSystem, viewport: { width: number; height: number }): [number, number] { + if (Array.isArray(value)) { + const point = transformPoint(toInt(value[0]), toInt(value[1]), coordinateSystem, viewport); + return [point.x, point.y]; + } + if (value && typeof value === "object") { + const point = value as Record; + const transformed = transformPoint(toInt(point.x), toInt(point.y), coordinateSystem, viewport); + return [transformed.x, transformed.y]; + } + return [0, 0]; } -function isModifierKey(key: string): boolean { - return key === "Control_L" || key === "Alt_L" || key === "Shift_L" || key === "Super_L"; +function toViewportPoint( + action: Record, + coordinateSystem: ComputerToolCoordinateSystem, + viewport: { width: number; height: number }, +): { x: number; y: number } { + return transformPoint(toInt(action.x), toInt(action.y), coordinateSystem, viewport); } -function toPath(value: unknown): Array<[number, number]> { - if (!Array.isArray(value)) return []; - return value.map((point) => toPathPoint(point)); +function transformPoint( + x: number, + y: number, + coordinateSystem: ComputerToolCoordinateSystem, + viewport: { width: number; height: number }, +): { x: number; y: number } { + if (coordinateSystem.type === "pixel") return { x, y }; + const [min, max] = coordinateSystem.range; + const scale = max - min; + if (scale <= 0) return { x, y }; + return { + x: clamp(Math.round(((x - min) / scale) * viewport.width), 0, viewport.width - 1), + y: clamp(Math.round(((y - min) / scale) * viewport.height), 0, viewport.height - 1), + }; } -function toPathPoint(value: unknown): [number, number] { - if (Array.isArray(value)) return [toInt(value[0]), toInt(value[1])]; - if (value && typeof value === "object") { - return [toInt((value as Record).x), toInt((value as Record).y)]; - } - return [0, 0]; +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); } diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 93887918..99be5c36 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -21,7 +21,7 @@ export interface BatchAction { } export type BatchReadResult = - | { type: "screenshot"; pngBytes: Buffer } + | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number }; diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 73683ca4..4d965e5e 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -9,7 +9,6 @@ import { CuaAgentHarness, InMemorySessionRepo, NodeExecutionEnv, - createCuaComputerTools, type AgentTool, type KernelBrowser, type StreamFn, @@ -17,6 +16,10 @@ import { const browser = { session_id: "browser_123" } as KernelBrowser; const client = {} as Kernel; +const tinyPng = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", + "base64", +); function createAssistantMessage(model: { api: string; provider: string; id: string }): AssistantMessage { return { @@ -46,6 +49,18 @@ async function createHarnessServices() { }; } +function createCustomTool(name = "custom"): AgentTool { + return { + name, + label: name, + description: "custom tool", + parameters: { type: "object", properties: {}, additionalProperties: false } as never, + async execute() { + return { content: [{ type: "text", text: "ok" }], details: {} }; + }, + }; +} + describe("CuaAgent", () => { it("extends pi Agent and resolves model refs in initialState", () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); @@ -63,58 +78,59 @@ describe("CuaAgent", () => { expect(agent.state.systemPrompt).toBe(runtime.defaultSystemPrompt); }); - it("uses provided tools exactly", () => { - const tool: AgentTool = { - name: "custom", - label: "custom", - description: "custom tool", - parameters: { type: "object", properties: {}, additionalProperties: false } as never, - async execute() { - return { content: [{ type: "text", text: "ok" }], details: {} }; - }, - }; + it("appends extra tools to provider CUA tools", () => { + const runtime = resolveCuaRuntimeSpec("yutori:n1.5-latest"); + const tool = createCustomTool(); const agent = new CuaAgent({ browser, client, + extraTools: [tool], initialState: { model: "yutori:n1.5-latest", - tools: [tool], }, }); - expect(agent.state.tools).toEqual([tool]); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolDefinitions.map((item) => item.name), "custom"]); }); - it("lets users explicitly compose default tools", () => { + it("always keeps provider CUA tools when adding extra tools", () => { const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); - const tools = [ - ...createCuaComputerTools({ browser, client, toolDefinitions: runtime.toolDefinitions }), - { - name: "custom", - label: "custom", - description: "custom tool", - parameters: { type: "object", properties: {}, additionalProperties: false } as never, - async execute() { - return { content: [{ type: "text", text: "ok" }], details: {} }; - }, - } satisfies AgentTool, - ]; + const tool = createCustomTool(); const agent = new CuaAgent({ browser, client, + extraTools: [tool], initialState: { model: "openai:gpt-5.5", - tools, systemPrompt: "Use the browser carefully.", }, }); - expect(agent.state.tools).toHaveLength(3); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolDefinitions.map((item) => item.name), "custom"]); expect(agent.state.systemPrompt).toBe("Use the browser carefully."); }); + it("synthesizes batch and navigation tools when requested", () => { + const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); + const agent = new CuaAgent({ + browser, + client, + batchTool: true, + computerUseExtra: true, + initialState: { + model: "openai:gpt-5.5", + }, + }); + + expect(agent.state.tools.map((tool) => tool.name)).toEqual([ + ...runtime.toolDefinitions.map((tool) => tool.name), + "batch_computer_actions", + "computer_use_extra", + ]); + }); + it("refreshes CUA runtime state when state.model changes", () => { const runtime = resolveCuaRuntimeSpec("google:gemini-3-pro-preview"); const agent = new CuaAgent({ @@ -132,29 +148,22 @@ describe("CuaAgent", () => { expect(agent.state.tools).toHaveLength(runtime.toolDefinitions.length); }); - it("keeps caller-owned tools and system prompt when state.model changes", () => { - const tool: AgentTool = { - name: "custom", - label: "custom", - description: "custom tool", - parameters: { type: "object", properties: {}, additionalProperties: false } as never, - async execute() { - return { content: [{ type: "text", text: "ok" }], details: {} }; - }, - }; + it("keeps extra tools and caller-owned system prompt when state.model changes", () => { + const tool = createCustomTool(); const agent = new CuaAgent({ browser, client, + extraTools: [tool], initialState: { model: "openai:gpt-5.5", - tools: [tool], systemPrompt: "custom prompt", }, }); agent.state.model = "google:gemini-3-pro-preview"; - expect(agent.state.tools).toEqual([tool]); + const runtime = resolveCuaRuntimeSpec("google:gemini-3-pro-preview"); + expect(agent.state.tools.map((item) => item.name)).toEqual([...runtime.toolDefinitions.map((item) => item.name), "custom"]); expect(agent.state.systemPrompt).toBe("custom prompt"); }); @@ -186,6 +195,68 @@ describe("CuaAgent", () => { expect(payloads).toEqual([{ payload: { provider: "openai", store: true }, userHook: true }]); }); + + it("uses yutori runtime hooks to append screenshots while stripping local executor tools", async () => { + const payloads: unknown[] = []; + const screenshotClient = { + browsers: { + computer: { + captureScreenshot: async () => new Response(tinyPng), + }, + }, + } as unknown as Kernel; + const streamFn: StreamFn = (model, _context, options) => { + const stream = createAssistantMessageEventStream(); + void (async () => { + payloads.push( + await options?.onPayload?.( + { + messages: [{ role: "user", content: "Inspect the page" }], + tools: [ + { type: "function", function: { name: "click" } }, + { type: "function", function: { name: "batch_computer_actions" } }, + { type: "function", function: { name: "computer_use_extra" } }, + { type: "function", function: { name: "custom_tool" } }, + ], + }, + model, + ), + ); + const message = createAssistantMessage(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + })(); + return stream; + }; + + const agent = new CuaAgent({ + browser, + client: screenshotClient, + streamFn, + extraTools: [createCustomTool("custom_tool")], + batchTool: true, + computerUseExtra: true, + initialState: { + model: "yutori:n1.5-latest", + }, + }); + + await agent.prompt("hello"); + + const payload = payloads[0] as { + messages: Array<{ content: Array<{ type: string; image_url?: { url: string } }> }>; + tools?: Array<{ function?: { name?: string } }>; + tool_set?: string; + }; + expect(payload.tool_set).toBe("browser_tools_core-20260403"); + expect(payload.tools?.map((tool) => tool.function?.name)).toEqual([ + "batch_computer_actions", + "computer_use_extra", + "custom_tool", + ]); + expect(payload.messages[0]!.content.at(-1)?.image_url?.url.startsWith("data:image/webp;base64,")).toBe(true); + }); }); describe("CuaAgentHarness", () => { @@ -218,6 +289,23 @@ describe("CuaAgentHarness", () => { expect(harness.agent.state.tools).toHaveLength(runtime.toolDefinitions.length); }); + it("appends extraTools in harness construction", async () => { + const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); + const tool = createCustomTool(); + const harness = new CuaAgentHarness({ + ...(await createHarnessServices()), + browser, + client, + model: "openai:gpt-5.5", + extraTools: [tool], + }); + + expect(harness.agent.state.tools.map((item) => item.name)).toEqual([ + ...runtime.toolDefinitions.map((item) => item.name), + "custom", + ]); + }); + it("preserves active tool selection when setModel refreshes tools", async () => { const harness = new CuaAgentHarness({ ...(await createHarnessServices()), diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 06f97e0f..41e3436a 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -24,7 +24,9 @@ type ProviderCase = { | "yutori:n1.5-latest"; prompt: string; expectToolCalls: boolean; + expectReadArtifact?: boolean; timeoutMs: number; + ciOptInEnvVar?: string; }; type ModelSwitchCase = { @@ -40,64 +42,67 @@ const cases: ProviderCase[] = [ apiKeyEnvVar: "OPENAI_API_KEY", modelRef: "openai:gpt-5.5", prompt: [ - "Call batch_computer_actions exactly once.", - 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Use the tool named `screenshot` exactly once to inspect the browser.", + "Pass empty arguments (`{}`).", "Do not call any other tools.", - "Then provide a one-sentence summary.", + "After the tool result, provide a one-sentence summary.", ].join("\n"), expectToolCalls: true, timeoutMs: 120_000, + ciOptInEnvVar: "CUA_E2E_OPENAI", }, { name: "anthropic", apiKeyEnvVar: "ANTHROPIC_API_KEY", modelRef: "anthropic:claude-opus-4-7", prompt: [ - "Call batch_computer_actions exactly once.", - 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Use the tool named `screenshot` exactly once to inspect the browser.", + "Pass empty arguments (`{}`).", "Do not call any other tools.", - "Then provide a one-sentence summary.", + "After the tool result, provide a one-sentence summary.", ].join("\n"), - expectToolCalls: true, + expectToolCalls: false, timeoutMs: 120_000, + ciOptInEnvVar: "CUA_E2E_ANTHROPIC", }, { name: "gemini", apiKeyEnvVar: "GOOGLE_API_KEY", modelRef: "google:gemini-3-flash-preview", prompt: [ - "Call batch_computer_actions exactly once.", - 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Use the tool named `screenshot` exactly once to inspect the browser.", + "Pass empty arguments (`{}`).", "Do not call any other tools.", - "Then provide a one-sentence summary.", + "After the tool result, provide a one-sentence summary.", ].join("\n"), expectToolCalls: true, timeoutMs: 300_000, + ciOptInEnvVar: "CUA_E2E_GEMINI", }, { name: "tzafon", apiKeyEnvVar: "TZAFON_API_KEY", modelRef: "tzafon:tzafon.northstar-cua-fast", prompt: [ - "Call batch_computer_actions exactly once.", - 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Use the tool named `screenshot` exactly once to inspect the browser.", + "Pass empty arguments (`{}`).", "Do not call any other tools.", - "Then provide a one-sentence summary.", + "After the tool result, provide a one-sentence summary.", ].join("\n"), expectToolCalls: false, timeoutMs: 120_000, + ciOptInEnvVar: "CUA_E2E_TZAFON", }, { name: "yutori", apiKeyEnvVar: "YUTORI_API_KEY", modelRef: "yutori:n1.5-latest", prompt: [ - "Call batch_computer_actions exactly once.", - 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', - "Do not call any other tools.", - "Then provide a one-sentence summary.", + "Use the browser tools to open https://example.com exactly once.", + "After the browser action result, do not call any more tools.", ].join("\n"), - expectToolCalls: false, + expectToolCalls: true, + expectReadArtifact: false, timeoutMs: 180_000, }, ]; @@ -123,6 +128,7 @@ type RunStats = { function shouldRunCase(c: ProviderCase): boolean { if (!LIVE) return false; if (!KERNEL_API_KEY) return false; + if (c.ciOptInEnvVar && process.env.CI && process.env[c.ciOptInEnvVar] !== "1") return false; return Boolean(process.env[c.apiKeyEnvVar]); } @@ -155,11 +161,12 @@ async function createHarnessServices(id: string) { }; } -function assertStats(stats: RunStats, expectToolCalls: boolean, providerName: string, runtimeName: "agent" | "harness"): void { - if (expectToolCalls) { +function assertStats(stats: RunStats, c: ProviderCase, runtimeName: "agent" | "harness"): void { + const providerName = c.name; + if (c.expectToolCalls) { expect(stats.toolCalls).toBeGreaterThan(0); expect(stats.toolResults).toBeGreaterThan(0); - expect(stats.hasReadArtifact).toBe(true); + if (c.expectReadArtifact !== false) expect(stats.hasReadArtifact).toBe(true); } expect(stats.toolErrors, `${providerName}/${runtimeName} emitted tool errors: ${stats.toolErrors.join(" | ")}`).toHaveLength(0); expect(stats.assistantErrors, `${providerName}/${runtimeName} emitted assistant errors: ${stats.assistantErrors.join(" | ")}`).toHaveLength(0); @@ -173,7 +180,7 @@ function assertStats(stats: RunStats, expectToolCalls: boolean, providerName: st function recordRunEvent(stats: RunStats, event: AgentEvent | AgentHarnessEvent): void { if (event.type === "tool_execution_start") stats.toolCalls += 1; if (event.type === "tool_execution_end" && event.isError) { - stats.toolErrors.push(`${event.toolName}: failed`); + stats.toolErrors.push(`${event.toolName}: ${toolErrorMessage(event.result) ?? "failed"}`); } if (event.type === "message_end" && event.message.role === "toolResult") { stats.toolResults += 1; @@ -193,6 +200,23 @@ function recordRunEvent(stats: RunStats, event: AgentEvent | AgentHarnessEvent): } } +function toolErrorMessage(result: unknown): string | undefined { + if (!result || typeof result !== "object") return undefined; + const current = result as { details?: unknown; content?: unknown[] }; + const details = current.details as { error?: string; statusText?: string } | undefined; + if (details?.error) return details.error; + if (details?.statusText) return details.statusText; + const text = current.content + ?.map((block) => { + if (!block || typeof block !== "object") return undefined; + const item = block as { type?: unknown; text?: unknown }; + return item.type === "text" && typeof item.text === "string" ? item.text : undefined; + }) + .filter((text): text is string => Boolean(text)) + .join(" "); + return text || undefined; +} + describe("Cua live e2e", () => { for (const c of cases) { const test = shouldRunCase(c) ? it : it.skip; @@ -206,6 +230,7 @@ describe("Cua live e2e", () => { browser, client, getApiKey: () => process.env[c.apiKeyEnvVar], + afterToolCall: async () => ({ terminate: true }), initialState: { model: c.modelRef, }, @@ -215,7 +240,7 @@ describe("Cua live e2e", () => { }); await agent.prompt(c.prompt); - assertStats(stats, c.expectToolCalls, c.name, "agent"); + assertStats(stats, c, "agent"); }); }, c.timeoutMs, @@ -236,13 +261,14 @@ describe("Cua live e2e", () => { return apiKey ? { apiKey } : undefined; }, }); + harness.on("tool_result", () => ({ terminate: true })); harness.subscribe((event) => { recordRunEvent(stats, event); }); await harness.prompt(c.prompt); - assertStats(stats, c.expectToolCalls, c.name, "harness"); + assertStats(stats, c, "harness"); }); }, c.timeoutMs, @@ -274,12 +300,12 @@ describe("Cua live e2e", () => { }); await agent.prompt(c.from.prompt); - assertStats(stats, c.from.expectToolCalls, c.from.name, "agent"); + assertStats(stats, c.from, "agent"); stats = createRunStats(); agent.state.model = c.to.modelRef; await agent.prompt(c.to.prompt); - assertStats(stats, c.to.expectToolCalls, c.to.name, "agent"); + assertStats(stats, c.to, "agent"); }); }, c.timeoutMs, @@ -312,12 +338,12 @@ describe("Cua live e2e", () => { }); await harness.prompt(c.from.prompt); - assertStats(stats, c.from.expectToolCalls, c.from.name, "harness"); + assertStats(stats, c.from, "harness"); stats = createRunStats(); await harness.setModel(c.to.modelRef); await harness.prompt(c.to.prompt); - assertStats(stats, c.to.expectToolCalls, c.to.name, "harness"); + assertStats(stats, c.to, "harness"); }); }, c.timeoutMs, diff --git a/packages/agent/test/keys.test.ts b/packages/agent/test/keys.test.ts new file mode 100644 index 00000000..701c6ac7 --- /dev/null +++ b/packages/agent/test/keys.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { + normalizeKernelKey, + normalizeKernelKeyCombo, + normalizeKernelKeySequence, +} from "../src/translator/keys"; + +describe("Kernel key normalization", () => { + it("normalizes common provider key names to X11 keysyms", () => { + expect(normalizeKernelKey("ctrl")).toBe("Control_L"); + expect(normalizeKernelKey("command")).toBe("Super_L"); + expect(normalizeKernelKey("Backspace")).toBe("BackSpace"); + expect(normalizeKernelKey("ArrowLeft")).toBe("Left"); + expect(normalizeKernelKey("enter")).toBe("Return"); + expect(normalizeKernelKey("f12")).toBe("F12"); + }); + + it("covers Yutori's word-form punctuation and sequential key syntax", () => { + expect(normalizeKernelKeyCombo("ctrl+plus")).toEqual(["Control_L", "plus"]); + expect(normalizeKernelKeyCombo("command+backquote")).toEqual(["Super_L", "grave"]); + expect(normalizeKernelKeyCombo("option+tab")).toEqual(["Alt_L", "Tab"]); + expect(normalizeKernelKey("kp_enter")).toBe("Return"); + expect(normalizeKernelKey("-")).toBe("minus"); + expect(normalizeKernelKeySequence("down down enter")).toEqual([["Down"], ["Down"], ["Return"]]); + expect(normalizeKernelKeySequence("tab ctrl+a")).toEqual([["Tab"], ["Control_L", "a"]]); + }); +}); diff --git a/packages/agent/test/tool-exhaustiveness.test.ts b/packages/agent/test/tool-exhaustiveness.test.ts index 1cb4877b..541a7f20 100644 --- a/packages/agent/test/tool-exhaustiveness.test.ts +++ b/packages/agent/test/tool-exhaustiveness.test.ts @@ -6,6 +6,10 @@ import { SUPPORTED_CUA_EXECUTOR_TOOL_NAMES, createCuaComputerTools, type KernelB const browser = { session_id: "browser_123" } as KernelBrowser; const client = {} as Kernel; +const tinyPng = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", + "base64", +); describe("Cua tool executor coverage", () => { it("covers every canonical tool name exported by cua-ai defaults", () => { @@ -17,7 +21,8 @@ describe("Cua tool executor coverage", () => { names.add(definition.name); } } - expect([...names].sort()).toEqual([...SUPPORTED_CUA_EXECUTOR_TOOL_NAMES].sort()); + const supported = new Set(SUPPORTED_CUA_EXECUTOR_TOOL_NAMES); + for (const name of names) expect(supported.has(name)).toBe(true); }); it("instantiates one executor per canonical definition", () => { @@ -26,6 +31,36 @@ describe("Cua tool executor coverage", () => { expect(tools.map((tool) => tool.name).sort()).toEqual(toolDefinitions.map((tool) => tool.name).sort()); }); + it("executes Yutori local canonical action tools", async () => { + const batches: unknown[] = []; + const runtime = resolveCuaRuntimeSpec("yutori:n1.5-latest"); + const tools = createCuaComputerTools({ + browser: { ...browser, viewport: { width: 1920, height: 1080 } }, + client: { + browsers: { + computer: { + batch: async (_id: string, body: { actions: unknown[] }) => { + batches.push(body.actions); + }, + captureScreenshot: async () => new Response(tinyPng), + }, + }, + } as unknown as Kernel, + toolDefinitions: runtime.toolDefinitions, + coordinateSystem: runtime.coordinateSystem, + screenshot: runtime.screenshot, + }); + const click = tools.find((tool) => tool.name === "click"); + expect(click).toBeDefined(); + + const result = await click!.execute("call_1", { x: 500, y: 250 }); + + expect(batches).toEqual([ + [{ type: "click_mouse", click_mouse: { x: 960, y: 270, button: "left" } }], + ]); + expect(result.content.at(-1)).toMatchObject({ type: "image", mimeType: "image/webp" }); + }); + it("fails fast on unsupported tool names", () => { const unsupportedDefinitions: Tool[] = [ { diff --git a/packages/agent/test/translator.test.ts b/packages/agent/test/translator.test.ts index 77bf5efb..e731f623 100644 --- a/packages/agent/test/translator.test.ts +++ b/packages/agent/test/translator.test.ts @@ -56,4 +56,59 @@ describe("InternalComputerTranslator", () => { [{ type: "press_key", press_key: { keys: ["l"], hold_keys: ["Control_L"] } }], ]); }); + + it("normalizes bare goto URLs before browser navigation", async () => { + const { batches, client } = createClient(); + const translator = new InternalComputerTranslator({ browser, client }); + + await translator.executeBatch([{ type: "goto", url: "example.com" }]); + + expect(batches).toEqual([ + [ + { type: "press_key", press_key: { keys: ["l"], hold_keys: ["Control_L"] } }, + { type: "type_text", type_text: { text: "https://example.com" } }, + { type: "press_key", press_key: { keys: ["Return"] } }, + ], + ]); + }); + + it("passes canonical modifier and key duration fields through to Kernel actions", async () => { + const { batches, client } = createClient(); + const translator = new InternalComputerTranslator({ browser, client }); + + await translator.executeBatch([ + { type: "click", x: 10, y: 20, hold_keys: ["Control_L"] }, + { type: "scroll", x: 10, y: 20, scroll_y: 120, hold_keys: ["Shift_L"] }, + { type: "keypress", keys: ["Shift_L"], duration: 1500 }, + ]); + + expect(batches).toEqual([ + [ + { type: "click_mouse", click_mouse: { x: 10, y: 20, button: "left", hold_keys: ["Control_L"] } }, + { type: "scroll", scroll: { x: 10, y: 20, delta_x: 0, delta_y: 120, hold_keys: ["Shift_L"] } }, + { type: "press_key", press_key: { keys: ["Shift_L"], duration: 1500 } }, + ], + ]); + }); + + it("denormalizes provider coordinates to the Kernel browser viewport", async () => { + const { batches, client } = createClient(); + const translator = new InternalComputerTranslator({ + browser: { ...browser, viewport: { width: 1920, height: 1080 } }, + client, + coordinateSystem: { type: "normalized", range: [0, 1000] }, + }); + + await translator.executeBatch([ + { type: "click", x: 500, y: 250 }, + { type: "drag", path: [{ x: 0, y: 0 }, { x: 1000, y: 1000 }] }, + ]); + + expect(batches).toEqual([ + [ + { type: "click_mouse", click_mouse: { x: 960, y: 270, button: "left" } }, + { type: "drag_mouse", drag_mouse: { path: [[0, 0], [1919, 1079]], button: "left" } }, + ], + ]); + }); }); diff --git a/packages/ai/README.md b/packages/ai/README.md index 1ac75c11..93f4983b 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -32,11 +32,11 @@ const response = await complete(model, { timestamp: Date.now(), }, ], - tools: openai.createComputerToolDefinitions({ actions: ["click"] }), + tools: openai.computerTools({ actions: ["click"] }), }); for (const block of response.content) { - if (block.type === "toolCall" && block.name === "click_mouse") { + if (block.type === "toolCall" && block.name === "click") { console.log("click:", block.arguments); } } @@ -111,27 +111,35 @@ Top-level exports: runtime consumers: - canonical provider id -- canonical CUA tool definitions +- CUA tool definitions installed by `CuaAgent`/`CuaAgentHarness` - default system prompt text +- provider coordinate convention +- optional provider screenshot input policy - optional provider payload middleware (for protocol quirks) -Provider namespaces expose `createComputerToolDefinitions({ actions? })` for -building model-facing pi-ai `Tool[]` definitions. Omit `actions` for the -provider's default computer tool set, or pass an action subset to narrow the -schema for a single `complete()` call: +Provider namespaces expose `computerTools({ actions? })` for +building the provider's default CUA `Tool[]` definitions. These are the tools +that agent runtimes install and execute locally. Most providers send the same +definitions to the model API; providers whose APIs expose tools through +separate request fields can adapt the outgoing payload with runtime middleware. +Omit `actions` for the provider's default computer tool set, or pass an action +subset to narrow the schema for a single `complete()` call: ```ts import { openai } from "@onkernel/cua-ai"; -const allComputerTools = openai.createComputerToolDefinitions(); -const clickOnlyTools = openai.createComputerToolDefinitions({ actions: ["click"] }); +const allComputerTools = openai.computerTools(); +const clickOnlyTools = openai.computerTools({ actions: ["click"] }); ``` -Every provider namespace synthesizes a `batch_computer_actions` tool definition. -That gives models a consistent way to plan ordered browser actions even when the -provider's native computer-use API has a different shape. Provider namespaces -are still used so the definitions can diverge over time where provider protocol -differences matter. +Provider namespaces expose individual canonical action definitions by default. +Some providers are different on the wire: Yutori exposes browser actions +through its documented `tool_set` request field, and Tzafon exposes them +through its native `computer_use` Responses tool. Their payload adapters remove +local canonical CUA action definitions before requests and enable the +provider-native computer-use interface. Caller-provided tools that should +remain on the provider payload can be preserved by payload middleware via +`CuaPayloadContext.keepToolNames`. Provider namespaces also expose `COMPUTER_TOOL_COORDINATES`, which describes the coordinates the provider's computer tool calls are expected to emit: @@ -192,7 +200,7 @@ type CuaActionGoto = { }; ``` -The provider namespace `createComputerToolDefinitions()` emits a +`createCuaBatchToolDefinition(actions?)` can synthesize a `batch_computer_actions` tool whose input is: ```ts @@ -201,11 +209,12 @@ type CuaBatchInput = { }; ``` -The model can plan several writes and reads in one call. Read actions such as -`screenshot`, `url`, and `cursor_position` can be interleaved with writes so -your executor can return fresh state in the same order. +Agent runtimes can opt into this as local sugar when they want the model to +plan several writes and reads in one call. Read actions such as `screenshot`, +`url`, and `cursor_position` can be interleaved with writes so your executor +can return fresh state in the same order. -When `actions` is omitted, the OpenAI namespace also emits a `computer_use_extra` +`createCuaNavigationToolDefinition()` can synthesize a `computer_use_extra` navigation tool whose input is: ```ts @@ -217,13 +226,13 @@ type CuaNavigationInput = { Provider namespaces: -- `openai`: `createComputerToolDefinitions`, `COMPUTER_TOOL_COORDINATES`, OpenAI CUA action schemas, and `OPENAI_BATCH_INSTRUCTIONS` -- `anthropic`: `createComputerToolDefinitions`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA batch schema aliases -- `gemini`: `createComputerToolDefinitions`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA batch schema aliases -- `tzafon`: `createComputerToolDefinitions`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and local `tzafon-responses` stream adapter -- `yutori`: Yutori prompt helpers, local `yutori-chat-completions` stream - adapter, `createComputerToolDefinitions`, `COMPUTER_TOOL_COORDINATES`, and - `yutoriBuiltinToolsOnPayload` +- `openai`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, OpenAI CUA action schemas, and prompt helpers +- `anthropic`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases +- `gemini`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and CUA action schema aliases +- `tzafon`: `computerTools`, `COMPUTER_TOOL_COORDINATES`, prompt helpers, and local `tzafon-responses` stream adapter +- `yutori`: native Navigator action sets, native-to-canonical action helpers, + `computerTools`, `COMPUTER_TOOL_COORDINATES`, local + `yutori-chat-completions` stream adapter, and `yutoriNativeToolSetOnPayload` This package does not execute browser actions. Use `@onkernel/cua-agent` when you want model tool calls executed against a Kernel browser. diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index e1b34fd1..1becccd7 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -16,12 +16,12 @@ const screenshot = await readFile(screenshotPath); // const apiKey = process.env.ANTHROPIC_API_KEY; // const modelRef = "anthropic:claude-opus-4-7"; // const model = getCuaModel(modelRef); -// const tools = anthropic.createComputerToolDefinitions({ actions: ["click"] }); +// const tools = anthropic.computerTools({ actions: ["click"] }); // // const apiKey = process.env.GOOGLE_API_KEY; // const modelRef = "google:gemini-2.5-computer-use-preview-10-2025"; // const model = getCuaModel(modelRef); -// const tools = gemini.createComputerToolDefinitions({ actions: ["click"] }); +// const tools = gemini.computerTools({ actions: ["click"] }); const response = await complete( model, @@ -40,7 +40,7 @@ const response = await complete( timestamp: Date.now(), }, ], - tools: openai.createComputerToolDefinitions({ actions: ["click"] }), + tools: openai.computerTools({ actions: ["click"] }), }, { apiKey, diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 7252c349..d6df4ea7 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -2,17 +2,12 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as ANTHROPIC_CUA_ACTION_TYPES, - CUA_BATCH_TOOL_DESCRIPTION as ANTHROPIC_BATCH_DESCRIPTION, - CUA_BATCH_TOOL_NAME as ANTHROPIC_BATCH_TOOL_NAME, - createComputerToolDefinitions, + computerTools, createCuaActionSchema as createActionSchema, - createCuaBatchSchema as createBatchSchema, - CuaBatchSchema as AnthropicBatchSchema, } from "../common"; export type { CuaAction as AnthropicAction, - CreateComputerToolDefinitionsOptions, - CuaBatchInput as AnthropicBatchInput, + ComputerToolsOptions, } from "../common"; // Provider-native action vocabulary emitted on `tool_use.input.action`. Latest @@ -26,7 +21,7 @@ export type { // Source: https://github.com/anthropics/anthropic-quickstarts/blob/main/computer-use-demo/computer_use_demo/tools/computer.py export const COMPUTER_TOOL_COORDINATES = { type: "pixel" } as const satisfies ComputerToolCoordinateSystem; -export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through computer-use tools. Use batched actions for predictable browser interaction, keyboard navigation where possible, and explicit screenshot or url reads when you need to inspect state.`; +export const ANTHROPIC_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use keyboard navigation where possible, and request explicit screenshot or url reads when you need to inspect state.`; export function buildAnthropicSystemPrompt(opts: { suffix?: string } = {}): string { return [ANTHROPIC_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index c882c808..079c7b51 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -60,6 +60,7 @@ export interface CuaActionTypeText { export interface CuaActionKeypress { type: "keypress"; keys: string[]; + duration?: number; } export interface CuaActionScroll { @@ -191,6 +192,7 @@ const CUA_ACTION_SCHEMA_BY_TYPE = { { type: Type.Literal("keypress"), keys: Type.Array(Type.String()), + duration: Type.Optional(Type.Number()), }, { additionalProperties: false }, ), @@ -243,12 +245,27 @@ const CUA_ACTION_SCHEMA_BY_TYPE = { cursor_position: Type.Object({ type: Type.Literal("cursor_position") }, { additionalProperties: false }), } satisfies Record; +type ObjectSchemaWithProperties = TSchema & { properties: Record }; + +function createCuaActionArgumentSchema(action: CuaActionType): TSchema { + const { type: _type, ...properties } = (CUA_ACTION_SCHEMA_BY_TYPE[action] as ObjectSchemaWithProperties).properties; + return Type.Object(properties, { additionalProperties: false }); +} + export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): TSchema { if (actions.length === 0) throw new Error("actions must include at least one CUA action type"); if (actions.length === 1) return CUA_ACTION_SCHEMA_BY_TYPE[actions[0]!]; return Type.Union(actions.map((action) => CUA_ACTION_SCHEMA_BY_TYPE[action])); } +export function createCuaActionToolDefinitions(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): Tool[] { + return actions.map((action) => ({ + name: action, + description: `Execute one ${action} computer action.`, + parameters: createCuaActionArgumentSchema(action), + })); +} + export const CuaActionSchema = createCuaActionSchema(); export function createCuaBatchSchema(actions?: readonly CuaActionType[]): TSchema { @@ -283,7 +300,7 @@ export const CUA_BATCH_TOOL_DESCRIPTION = [ export const CUA_NAVIGATION_TOOL_DESCRIPTION = "High-level browser navigation helpers for goto, back, forward, and url."; -export interface CreateComputerToolDefinitionsOptions { +export interface ComputerToolsOptions { actions?: readonly CuaActionType[]; } @@ -296,22 +313,48 @@ export type ComputerToolCoordinateSystem = range: readonly [number, number]; }; -export function createComputerToolDefinitions(options: CreateComputerToolDefinitionsOptions = {}): Tool[] { - const actions = options.actions; - return [ - { - name: CUA_BATCH_TOOL_NAME, - description: CUA_BATCH_TOOL_DESCRIPTION, - parameters: createCuaBatchSchema(actions), - }, - ...(actions === undefined - ? [ - { - name: CUA_NAVIGATION_TOOL_NAME, - description: CUA_NAVIGATION_TOOL_DESCRIPTION, - parameters: CuaNavigationSchema, - }, - ] - : []), - ]; +/** + * Build the provider's CUA computer-use tools. + * + * Use this when calling `complete()` or `stream()` directly and you need an + * array of `Tool` objects for browser actions. Pass `actions` to expose only a + * smaller set, such as `["click"]`. + */ +export function computerTools(options: ComputerToolsOptions = {}): Tool[] { + return createCuaActionToolDefinitions(options.actions); +} + +/** Return the canonical tool name that should execute a normalized CUA action. */ +export function canonicalToolCallName(action: CuaAction): CuaActionType { + return action.type; +} + +/** Convert a normalized CUA action into tool-call arguments by removing its `type` tag. */ +export function canonicalToolCallArguments(action: CuaAction): Record { + const { type: _type, ...args } = action as CuaAction & Record; + return args; +} + +/** Prefix bare hostnames/paths with `https://` before browser navigation. */ +export function normalizeGotoUrl(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const url = value.trim(); + if (!url) return undefined; + return /^[a-z][a-z0-9+.-]*:\/\//i.test(url) ? url : `https://${url}`; +} + +export function createCuaBatchToolDefinition(actions?: readonly CuaActionType[]): Tool { + return { + name: CUA_BATCH_TOOL_NAME, + description: CUA_BATCH_TOOL_DESCRIPTION, + parameters: createCuaBatchSchema(actions), + }; +} + +export function createCuaNavigationToolDefinition(): Tool { + return { + name: CUA_NAVIGATION_TOOL_NAME, + description: CUA_NAVIGATION_TOOL_DESCRIPTION, + parameters: CuaNavigationSchema, + }; } diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index 74b6b6e0..d5648a5f 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -2,17 +2,12 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, - CUA_BATCH_TOOL_DESCRIPTION as GEMINI_BATCH_DESCRIPTION, - CUA_BATCH_TOOL_NAME as GEMINI_BATCH_TOOL_NAME, - createComputerToolDefinitions, + computerTools, createCuaActionSchema as createActionSchema, - createCuaBatchSchema as createBatchSchema, - CuaBatchSchema as GeminiBatchSchema, } from "../common"; export type { CuaAction as GeminiAction, - CreateComputerToolDefinitionsOptions, - CuaBatchInput as GeminiBatchInput, + ComputerToolsOptions, } from "../common"; // Provider-native function names emitted on `functionCall.name` (PREDEFINED_COMPUTER_USE_FUNCTIONS): @@ -24,7 +19,7 @@ export type { // Docs: https://ai.google.dev/gemini-api/docs/computer-use export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] } as const satisfies ComputerToolCoordinateSystem; -export const GEMINI_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through computer-use tools. Use pixel coordinates, batch predictable action sequences, and request screenshots or URL reads when state changes.`; +export const GEMINI_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Use the provider coordinate system for tool calls, and request screenshots or URL reads when state changes.`; export function buildGeminiSystemPrompt(opts: { suffix?: string } = {}): string { return [GEMINI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 6850ab23..cefa0274 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -2,20 +2,15 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, - CUA_BATCH_TOOL_DESCRIPTION as OPENAI_BATCH_DESCRIPTION, - CUA_BATCH_TOOL_NAME as OPENAI_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_DESCRIPTION as OPENAI_EXTRA_TOOL_DESCRIPTION, CUA_NAVIGATION_TOOL_NAME as OPENAI_EXTRA_TOOL_NAME, - createComputerToolDefinitions, + computerTools, createCuaActionSchema as createActionSchema, - createCuaBatchSchema as createBatchSchema, - CuaBatchSchema as OpenAIBatchSchema, CuaNavigationSchema as OpenAIExtraSchema, } from "../common"; export type { CuaAction as OpenAIAction, - CreateComputerToolDefinitionsOptions, - CuaBatchInput as OpenAIBatchInput, + ComputerToolsOptions, CuaNavigationInput as OpenAIExtraInput, } from "../common"; @@ -24,11 +19,7 @@ export type { // Source: https://github.com/openai/openai-cua-sample-app/blob/main/packages/runner-core/src/responses-loop.ts export const COMPUTER_TOOL_COORDINATES = { type: "pixel" } as const satisfies ComputerToolCoordinateSystem; -export const OPENAI_BATCH_INSTRUCTIONS = `You have two browser tools: -1. batch_computer_actions for click, double_click, mouse_down, mouse_up, type, keypress, scroll, move, drag, wait, goto, back, forward, url, cursor_position, and screenshot. -2. computer_use_extra for a single high-level goto, back, forward, or url action. - -Prefer batch_computer_actions for predictable multi-step browser interaction. Include explicit url(), cursor_position(), or screenshot() read steps when you need intermediate state.`; +export const OPENAI_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use the available tools for browser interaction and request explicit url, cursor_position, or screenshot reads when you need updated state.`; export function openaiResponsesStoreOnPayload(payload: unknown): unknown | undefined { if (!payload || typeof payload !== "object") return undefined; diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index 1df4a648..277f6f5c 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -2,22 +2,19 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, - CUA_BATCH_TOOL_DESCRIPTION as TZAFON_BATCH_DESCRIPTION, - CUA_BATCH_TOOL_NAME as TZAFON_BATCH_TOOL_NAME, - createComputerToolDefinitions, + computerTools, createCuaActionSchema as createActionSchema, - createCuaBatchSchema as createBatchSchema, - CuaBatchSchema as TzafonBatchSchema, } from "../common"; export type { CuaAction as TzafonAction, - CreateComputerToolDefinitionsOptions, - CuaBatchInput as TzafonBatchInput, + ComputerToolsOptions, } from "../common"; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses, + tzafonComputerUseOnPayload, + tzafonToolCallId, } from "./provider"; // Provider-native action vocabulary. The model card lists supported actions; @@ -32,7 +29,7 @@ export { // https://docs.lightcone.ai/guides/coordinates/ export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 999] } as const satisfies ComputerToolCoordinateSystem; -export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser. Prefer batched computer actions for browser interaction and include screenshot or URL reads when you need updated state.`; +export const TZAFON_INSTRUCTIONS_RAW = `You control a Kernel cloud browser through individual browser tools. Include screenshot or URL reads when you need updated state.`; export function buildTzafonSystemPrompt(opts: { suffix?: string } = {}): string { return [TZAFON_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index 0785f229..e4c2b93a 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -13,8 +13,17 @@ import { type ToolCall, } from "@earendil-works/pi-ai"; import Lightcone from "@tzafon/lightcone"; +import { canonicalToolCallArguments, canonicalToolCallName, CUA_ACTION_TYPES, type CuaAction } from "../common"; +import type { CuaPayloadContext } from "../../runtime-spec"; export const TZAFON_RESPONSES_API = "tzafon-responses"; +const TZAFON_COMPUTER_USE_TOOL = { + type: "computer_use", + display_width: 1920, + display_height: 1080, + environment: "browser", +} as const; +const TZAFON_LOCAL_ACTION_TOOL_NAMES = new Set(CUA_ACTION_TYPES); export interface TzafonResponsesOptions extends StreamOptions { maxOutputTokens?: number; @@ -41,9 +50,14 @@ export const streamTzafonResponses: StreamFunction); + const tzafonPayload = tzafonComputerUseOnPayload(payload, model as Model, { + keepToolNames: keepToolNamesFromContext(context), + }); + const nextPayload = await options?.onPayload?.(tzafonPayload ?? payload, model as Model); if (options?.signal?.aborted) throw new Error("Request was aborted"); - const response = await client.responses.create((nextPayload ?? payload) as never, { signal: options?.signal }); + const response = await client.responses.create((nextPayload ?? tzafonPayload ?? payload) as never, { + signal: options?.signal, + }); if (options?.signal?.aborted) throw new Error("Request was aborted"); stream.push({ type: "start", partial: output }); @@ -63,6 +77,24 @@ export const streamTzafonResponses: StreamFunction, context?: CuaPayloadContext): unknown | undefined { + if (!payload || typeof payload !== "object") return undefined; + const current = payload as { tools?: unknown }; + const keepToolNames = new Set(context?.keepToolNames ?? []); + const existingTools = Array.isArray(current.tools) ? current.tools : []; + const shouldAddComputerUse = existingTools.some((tool) => { + const name = readToolName(tool); + return Boolean(name && TZAFON_LOCAL_ACTION_TOOL_NAMES.has(name) && !keepToolNames.has(name)); + }); + const tools = existingTools.filter((tool) => { + const name = readToolName(tool); + return !name || keepToolNames.has(name) || !TZAFON_LOCAL_ACTION_TOOL_NAMES.has(name); + }); + return { + ...(payload as Record), + tools: shouldAddComputerUse ? [TZAFON_COMPUTER_USE_TOOL, ...tools] : tools, + }; +} + +/** Derive a unique canonical tool-call id for a Tzafon computer action. */ +export function tzafonToolCallId(callId: string, actionIndex: number): string { + return actionIndex === 0 ? callId : `${callId}:${actionIndex}`; +} + function initialAssistantMessage(model: Model): AssistantMessage { return { role: "assistant", @@ -120,6 +176,96 @@ function emitToolCall( stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); } +type TzafonCanonicalAction = CuaAction | { type: "answer"; text: string }; + +function toCanonicalActions(action: unknown): TzafonCanonicalAction[] { + if (!action || typeof action !== "object") return []; + const current = action as Record; + const type = getString(current, "type"); + const x = readOptionalNumber(current, "x"); + const y = readOptionalNumber(current, "y"); + switch (type) { + case "click": + case "left_click": + return x !== undefined && y !== undefined ? [{ type: "click", x, y }] : []; + case "right_click": + return x !== undefined && y !== undefined ? [{ type: "click", x, y, button: "right" }] : []; + case "double_click": + return x !== undefined && y !== undefined ? [{ type: "double_click", x, y }] : []; + case "triple_click": + return x !== undefined && y !== undefined ? [{ type: "double_click", x, y }, { type: "click", x, y }] : []; + case "move": + case "hover": + return x !== undefined && y !== undefined ? [{ type: "move", x, y }] : []; + case "drag": + return toDragAction(current); + case "type": + return [{ type: "type", text: getString(current, "text") }]; + case "keypress": + case "key": + return toKeypressAction(current); + case "scroll": + return [toScrollAction(current)]; + case "hscroll": + return [{ type: "scroll", scroll_x: readOptionalNumber(current, "scroll_x") ?? readOptionalNumber(current, "amount") ?? 0 }]; + case "navigate": + return [{ type: "goto", url: getString(current, "url") }]; + case "wait": + return [{ type: "wait", ms: readOptionalNumber(current, "ms") ?? secondsToMs(readOptionalNumber(current, "seconds")) }]; + case "screenshot": + return [{ type: "screenshot" }]; + case "answer": + case "done": + case "terminate": + return [{ type: "answer", text: getString(current, "result") || getString(current, "text") || getString(current, "status") }]; + default: + return []; + } +} + +function toDragAction(action: Record): CuaAction[] { + const path = getArray(action, "path") + .map((point) => { + if (!point || typeof point !== "object") return undefined; + const x = readOptionalNumber(point, "x"); + const y = readOptionalNumber(point, "y"); + return x !== undefined && y !== undefined ? { x, y } : undefined; + }) + .filter((point): point is { x: number; y: number } => Boolean(point)); + if (path.length >= 2) return [{ type: "drag", path }]; + + const x = readOptionalNumber(action, "x"); + const y = readOptionalNumber(action, "y"); + const endX = readOptionalNumber(action, "end_x") ?? readOptionalNumber(action, "x2"); + const endY = readOptionalNumber(action, "end_y") ?? readOptionalNumber(action, "y2"); + if (x === undefined || y === undefined || endX === undefined || endY === undefined) return []; + return [{ type: "drag", path: [{ x, y }, { x: endX, y: endY }] }]; +} + +function toKeypressAction(action: Record): CuaAction[] { + const keys = getArray(action, "keys") + .map((key) => (typeof key === "string" ? key : undefined)) + .filter((key): key is string => Boolean(key)); + const key = getString(action, "key"); + const text = getString(action, "text"); + const value = keys.length > 0 ? keys : key ? [key] : text ? [text] : []; + return value.length > 0 ? [{ type: "keypress", keys: value }] : []; +} + +function toScrollAction(action: Record): CuaAction { + return { + type: "scroll", + x: readOptionalNumber(action, "x"), + y: readOptionalNumber(action, "y"), + scroll_x: readOptionalNumber(action, "scroll_x"), + scroll_y: readOptionalNumber(action, "scroll_y") ?? readOptionalNumber(action, "amount"), + }; +} + +function secondsToMs(seconds: number | undefined): number | undefined { + return seconds === undefined ? undefined : seconds * 1000; +} + function convertTools(tools: Tool[]): Array> { return tools.map((tool) => ({ type: "function", @@ -129,6 +275,20 @@ function convertTools(tools: Tool[]): Array> { })); } +function keepToolNamesFromContext(context: Context): string[] { + return (context.tools ?? []) + .map((tool) => tool.name) + .filter((name) => !TZAFON_LOCAL_ACTION_TOOL_NAMES.has(name)); +} + +function readToolName(tool: unknown): string | undefined { + if (!tool || typeof tool !== "object") return undefined; + const direct = getString(tool, "name"); + if (direct) return direct; + const fn = getValue(tool, "function"); + return getString(fn, "name"); +} + function convertContextMessages(context: Context): Array> { const items: Array> = []; for (const message of context.messages) { @@ -208,11 +368,30 @@ function parseArguments(value: unknown): Record { // (observed: { "actions": "[{...}]" }). Unwrap one level so consumers get real values. const out: Record = {}; for (const [key, val] of Object.entries(top)) { - out[key] = typeof val === "string" && looksLikeJson(val) ? safeJsonParse(val) ?? val : val; + out[key] = normalizeArgumentValue(key, val); } return out; } +const NUMERIC_ARGUMENT_KEYS = new Set(["x", "y", "scroll_x", "scroll_y", "ms", "duration"]); + +function normalizeArgumentValue(key: string, value: unknown): unknown { + const parsed = typeof value === "string" && looksLikeJson(value) ? safeJsonParse(value) ?? value : value; + if (typeof parsed === "string" && NUMERIC_ARGUMENT_KEYS.has(key)) { + const number = Number.parseFloat(parsed); + return Number.isFinite(number) ? number : parsed; + } + if (Array.isArray(parsed)) { + return parsed.map((item) => normalizeArgumentValue(key, item)); + } + if (parsed && typeof parsed === "object") { + return Object.fromEntries( + Object.entries(parsed).map(([childKey, childValue]) => [childKey, normalizeArgumentValue(childKey, childValue)]), + ); + } + return parsed; +} + function safeJsonParse(value: string): Record | unknown[] | null { try { const parsed = JSON.parse(value); @@ -228,10 +407,10 @@ function looksLikeJson(value: string): boolean { } function usageFromTzafon(usage: unknown): AssistantMessage["usage"] { - const input = readNumber(usage, "input_tokens"); - const output = readNumber(usage, "output_tokens"); - const cacheRead = readNumber(getValue(usage, "input_tokens_details"), "cached_tokens"); - const totalTokens = readNumber(usage, "total_tokens") || input + output; + const input = readUsageNumber(usage, "input_tokens"); + const output = readUsageNumber(usage, "output_tokens"); + const cacheRead = readUsageNumber(getValue(usage, "input_tokens_details"), "cached_tokens"); + const totalTokens = readUsageNumber(usage, "total_tokens") || input + output; return { input, output, @@ -242,10 +421,19 @@ function usageFromTzafon(usage: unknown): AssistantMessage["usage"] { }; } -function readNumber(obj: unknown, key: string): number { - if (!obj || typeof obj !== "object") return 0; - const n = (obj as Record)[key]; - return typeof n === "number" && Number.isFinite(n) ? n : 0; +function readUsageNumber(obj: unknown, key: string): number { + return readOptionalNumber(obj, key) ?? 0; +} + +function readOptionalNumber(obj: unknown, key: string): number | undefined { + if (!obj || typeof obj !== "object") return undefined; + const value = (obj as Record)[key]; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const number = Number(value); + return Number.isFinite(number) ? number : undefined; + } + return undefined; } function getArray(obj: unknown, key: string): unknown[] { diff --git a/packages/ai/src/providers/yutori/actions.ts b/packages/ai/src/providers/yutori/actions.ts new file mode 100644 index 00000000..ef624499 --- /dev/null +++ b/packages/ai/src/providers/yutori/actions.ts @@ -0,0 +1,253 @@ +import { + createCuaActionToolDefinitions, + normalizeGotoUrl, + type CuaAction, + type CuaActionType, +} from "../common"; + +/** + * Native Yutori Navigator n1.5 tool-set ids. + * + * Source of truth: + * - https://docs.yutori.com/reference/n1-5 + * - https://docs.yutori.com/llm-quickstart.md + */ +export const YUTORI_N15_CORE_TOOL_SET = "browser_tools_core-20260403"; +export const YUTORI_N15_EXPANDED_TOOL_SET = "browser_tools_expanded-20260403"; + +/** + * DOM/ref-backed Navigator n1.5 actions. We intentionally disable these until + * CuaAgent has the ref/DOM execution path that Yutori documents for the + * expanded tool set. + */ +export const YUTORI_N15_EXPANDED_ACTION_TYPES = [ + "extract_elements", + "find", + "set_element_value", + "execute_js", +] as const; + +/** + * Navigator n1's fixed legacy browser action space. + * + * Source of truth: https://docs.yutori.com/reference/n1 + */ +export const YUTORI_N1_ACTION_TYPES = [ + "left_click", + "double_click", + "right_click", + "triple_click", + "type", + "key_press", + "scroll", + "hover", + "drag", + "goto_url", + "go_back", + "refresh", + "wait", +] as const; + +/** + * Navigator n1.5 core visual action space. These are the actions available + * when `tool_set` is `browser_tools_core-20260403`, which keeps CuaAgent in the + * pure screenshot/coordinate path and avoids DOM refs. + * + * Source of truth: https://docs.yutori.com/reference/n1-5 + */ +export const YUTORI_N15_CORE_ACTION_TYPES = [ + "left_click", + "double_click", + "triple_click", + "middle_click", + "right_click", + "mouse_move", + "mouse_down", + "mouse_up", + "drag", + "scroll", + "type", + "key_press", + "hold_key", + "goto_url", + "go_back", + "go_forward", + "refresh", + "wait", +] as const; + +export const YUTORI_N15_ACTION_TYPES = [ + ...YUTORI_N15_CORE_ACTION_TYPES, + ...YUTORI_N15_EXPANDED_ACTION_TYPES, +] as const; + +export const YUTORI_CANONICAL_ACTION_TYPES = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "goto", + "back", + "forward", +] as const satisfies readonly CuaActionType[]; + +export type YutoriN1ActionType = (typeof YUTORI_N1_ACTION_TYPES)[number]; +export type YutoriN15CoreActionType = (typeof YUTORI_N15_CORE_ACTION_TYPES)[number]; +export type YutoriN15ExpandedActionType = (typeof YUTORI_N15_EXPANDED_ACTION_TYPES)[number]; +export type YutoriNativeActionType = YutoriN1ActionType | YutoriN15CoreActionType | YutoriN15ExpandedActionType; + +const DEFAULT_SCROLL_AMOUNT = 3; +const SCROLL_AMOUNT_PER_NOTCH = 120; +const DEFAULT_WAIT_MS = 2000; +const NAVIGATION_WAIT_MS = 1500; +const GOTO_WAIT_MS = 2000; + +/** + * Build Yutori CUA computer-use tools. + * + * Use this when calling `complete()` or `stream()` directly and you need an + * array of `Tool` objects for Yutori browser actions. + */ +export function computerTools(_options?: unknown) { + return createCuaActionToolDefinitions(YUTORI_CANONICAL_ACTION_TYPES); +} + +export function yutoriToolSetForModel(modelId: string): typeof YUTORI_N15_CORE_TOOL_SET | undefined { + return modelId.startsWith("n1.5") ? YUTORI_N15_CORE_TOOL_SET : undefined; +} + +export function yutoriNativeActionsForModel(modelId: string): readonly YutoriNativeActionType[] { + return modelId.startsWith("n1.5") ? YUTORI_N15_CORE_ACTION_TYPES : YUTORI_N1_ACTION_TYPES; +} + +export function isYutoriLocalActionToolName(name: string): boolean { + return (YUTORI_CANONICAL_ACTION_TYPES as readonly string[]).includes(name); +} + +export function toCanonicalActions(name: string, args: Record): CuaAction[] | undefined { + const coords = readPoint(args.coordinates); + switch (name) { + case "left_click": + return coords ? [{ type: "click", x: coords.x, y: coords.y, ...holdKeys(args.modifier) }] : undefined; + case "right_click": + return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "right", ...holdKeys(args.modifier) }] : undefined; + case "middle_click": + return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "middle", ...holdKeys(args.modifier) }] : undefined; + case "double_click": + return coords ? [{ type: "double_click", x: coords.x, y: coords.y, ...holdKeys(args.modifier) }] : undefined; + case "triple_click": + return coords + ? [ + { type: "double_click", x: coords.x, y: coords.y, ...holdKeys(args.modifier) }, + { type: "click", x: coords.x, y: coords.y, ...holdKeys(args.modifier) }, + ] + : undefined; + case "mouse_move": + case "hover": + return coords ? [{ type: "move", x: coords.x, y: coords.y }] : undefined; + case "mouse_down": + return coords ? [{ type: "mouse_down", x: coords.x, y: coords.y, ...holdKeys(args.modifier) }] : undefined; + case "mouse_up": + return coords ? [{ type: "mouse_up", x: coords.x, y: coords.y, ...holdKeys(args.modifier) }] : undefined; + case "drag": { + const start = readPoint(args.start_coordinates); + return start && coords ? [{ type: "drag", path: [start, coords], button: "left" }] : undefined; + } + case "scroll": + return toScrollAction(args, coords); + case "type": + return toTypeActions(args); + case "key_press": + return toKeypressAction(args); + case "hold_key": + return toHoldKeyAction(args); + case "goto_url": { + const url = normalizeGotoUrl(args.url); + return url ? [{ type: "goto", url }, { type: "wait", ms: GOTO_WAIT_MS }] : undefined; + } + case "go_back": + return [{ type: "back" }, { type: "wait", ms: NAVIGATION_WAIT_MS }]; + case "go_forward": + return [{ type: "forward" }, { type: "wait", ms: NAVIGATION_WAIT_MS }]; + case "refresh": + return [{ type: "keypress", keys: ["f5"] }, { type: "wait", ms: DEFAULT_WAIT_MS }]; + case "wait": + return [{ type: "wait", ms: secondsToMs(args.duration, DEFAULT_WAIT_MS) }]; + default: + return undefined; + } +} + +function readPoint(value: unknown): { x: number; y: number } | undefined { + if (!Array.isArray(value) || value.length < 2) return undefined; + const x = Number(value[0]); + const y = Number(value[1]); + if (!Number.isFinite(x) || !Number.isFinite(y)) return undefined; + return { x, y }; +} + +function toScrollAction(args: Record, coords: { x: number; y: number } | undefined): CuaAction[] | undefined { + if (!coords) return undefined; + const direction = typeof args.direction === "string" ? args.direction : "down"; + const amount = typeof args.amount === "number" ? args.amount : DEFAULT_SCROLL_AMOUNT; + const ticks = Math.max(1, Math.trunc(amount)) * SCROLL_AMOUNT_PER_NOTCH; + const scroll_x = direction === "left" ? -ticks : direction === "right" ? ticks : 0; + const scroll_y = direction === "up" ? -ticks : direction === "down" ? ticks : 0; + return [{ type: "scroll", x: coords.x, y: coords.y, scroll_x, scroll_y, ...holdKeys(args.modifier) }]; +} + +function toTypeActions(args: Record): CuaAction[] | undefined { + const text = typeof args.text === "string" ? args.text : undefined; + if (text === undefined) return undefined; + const actions: CuaAction[] = []; + if (args.clear_before_typing === true) { + actions.push({ type: "keypress", keys: ["ctrl", "a"] }, { type: "keypress", keys: ["backspace"] }); + } + actions.push({ type: "type", text }); + if (args.press_enter_after === true) actions.push({ type: "keypress", keys: ["enter"] }); + return actions; +} + +function toKeypressAction(args: Record): CuaAction[] | undefined { + const sequence = readKeySequence(args.key_comb ?? args.key); + return sequence.length > 0 ? sequence.map((keys) => ({ type: "keypress", keys })) : undefined; +} + +function toHoldKeyAction(args: Record): CuaAction[] | undefined { + const keys = readKeyCombo(args.key_comb ?? args.key); + return keys.length > 0 ? [{ type: "keypress", keys, duration: secondsToMs(args.duration, 1000) }] : undefined; +} + +function readKeyCombo(value: unknown): string[] { + if (typeof value !== "string") return []; + return value + .split("+") + .map((part) => part.trim()) + .filter(Boolean); +} + +function readKeySequence(value: unknown): string[][] { + if (typeof value !== "string") return []; + return value + .trim() + .split(/\s+/) + .map((part) => readKeyCombo(part)) + .filter((combo) => combo.length > 0); +} + +function holdKeys(value: unknown): { hold_keys?: string[] } { + if (typeof value !== "string") return {}; + const key = value.trim(); + return key ? { hold_keys: [key] } : {}; +} + +function secondsToMs(value: unknown, fallback: number): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return fallback; + return Math.round(value * 1000); +} diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index 7904b9f7..449e6d1f 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,24 +1,25 @@ import type { ComputerToolCoordinateSystem } from "../common"; export { - CUA_ACTION_TYPES as YUTORI_ACTION_TYPES, - CUA_BATCH_TOOL_DESCRIPTION as YUTORI_BATCH_DESCRIPTION, - CUA_BATCH_TOOL_NAME as YUTORI_BATCH_TOOL_NAME, - createComputerToolDefinitions, - createCuaActionSchema as createActionSchema, - createCuaBatchSchema as createBatchSchema, - CuaBatchSchema as YutoriBatchSchema, -} from "../common"; -export type { - CuaAction as YutoriAction, - CreateComputerToolDefinitionsOptions, - CuaBatchInput as YutoriBatchInput, -} from "../common"; + computerTools, + toCanonicalActions, + yutoriNativeActionsForModel, + yutoriToolSetForModel, + YUTORI_CANONICAL_ACTION_TYPES, + YUTORI_N1_ACTION_TYPES, + YUTORI_N15_ACTION_TYPES, + YUTORI_N15_CORE_ACTION_TYPES, + YUTORI_N15_CORE_TOOL_SET, + YUTORI_N15_EXPANDED_ACTION_TYPES, + YUTORI_N15_EXPANDED_TOOL_SET, +} from "./actions"; +export type { YutoriN1ActionType, YutoriN15CoreActionType, YutoriN15ExpandedActionType, YutoriNativeActionType } from "./actions"; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori, yutoriBuiltinToolsOnPayload, + yutoriNativeToolSetOnPayload, } from "./provider"; // Provider-native action vocabulary differs between Navigator versions: @@ -32,13 +33,16 @@ export { // n1.5 expanded (browser_tools_expanded-20260403): core + // extract_elements, find, set_element_value, execute_js // Sources: -// https://github.com/yutori-ai/yutori-sdk-python/blob/main/api.md -// https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/models.py +// https://docs.yutori.com/reference/n1 // https://docs.yutori.com/reference/n1-5 +// https://docs.yutori.com/llm-quickstart.md // https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/coordinates.py export const COMPUTER_TOOL_COORDINATES = { type: "normalized", range: [0, 1000] } as const satisfies ComputerToolCoordinateSystem; -export const YUTORI_INSTRUCTIONS_RAW = `You control a Kernel cloud browser. Prefer batched computer actions for browser interaction and include screenshot or URL reads when you need updated state.`; +// Yutori's Navigator quickstart recommends putting extra instructions in the +// first user message instead of supplying a custom system prompt. +// Source: https://docs.yutori.com/llm-quickstart.md +export const YUTORI_INSTRUCTIONS_RAW = ""; export function buildYutoriSystemPrompt(opts: { suffix?: string } = {}): string { return [YUTORI_INSTRUCTIONS_RAW, opts.suffix].filter(Boolean).join("\n\n"); diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index 1fd60b86..fec1f701 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -13,14 +13,20 @@ import { type TextContent, type ToolCall, } from "@earendil-works/pi-ai"; -import { CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, type CuaAction } from "../common"; +import { + isYutoriLocalActionToolName, + toCanonicalActions, + yutoriToolSetForModel, + YUTORI_N15_EXPANDED_ACTION_TYPES, +} from "./actions"; +import { canonicalToolCallArguments, canonicalToolCallName } from "../common"; +import type { CuaPayloadContext } from "../../runtime-spec"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; -const YUTORI_BUILTIN_TOOL_NAMES = new Set(CUA_ACTION_TYPES); - export interface YutoriOptions extends StreamOptions { temperature?: number; + keepToolNames?: readonly string[]; } export const streamYutori: StreamFunction = (model, context, options) => { @@ -35,17 +41,25 @@ export const streamSimpleYutori: StreamFunction streamYutori(model, context, options); -export function yutoriBuiltinToolsOnPayload(payload: unknown): unknown | undefined { +export function yutoriBuiltinToolsOnPayload(payload: unknown, model?: Model, context?: CuaPayloadContext): unknown | undefined { + return yutoriNativeToolSetOnPayload(payload, model, context); +} + +export function yutoriNativeToolSetOnPayload(payload: unknown, model?: Model, context?: CuaPayloadContext): unknown | undefined { if (!payload || typeof payload !== "object") return undefined; const current = payload as { tools?: unknown }; - if (!Array.isArray(current.tools)) return undefined; - const tools = current.tools.filter((tool) => { - const name = readToolName(tool); - return !name || !YUTORI_BUILTIN_TOOL_NAMES.has(name); - }); + const keepToolNames = new Set(context?.keepToolNames ?? []); + const tools = Array.isArray(current.tools) + ? current.tools.filter((tool) => { + const name = readToolName(tool); + return !name || keepToolNames.has(name) || !isYutoriLocalActionToolName(name); + }) + : undefined; + const toolSet = model ? yutoriToolSetForModel(model.id) : undefined; return { ...(payload as Record), - ...(tools.length > 0 ? { tools } : { tools: undefined }), + ...(toolSet ? { tool_set: toolSet, disable_tools: [...YUTORI_N15_EXPANDED_ACTION_TYPES] } : {}), + ...(tools && tools.length > 0 ? { tools } : { tools: undefined }), }; } @@ -72,6 +86,9 @@ async function runYutoriStream( }; const tools = convertTools(context); if (tools.length > 0) payload.tools = tools; + payload = yutoriNativeToolSetOnPayload(payload, model, { + keepToolNames: [...keepToolNamesFromContext(context), ...(options?.keepToolNames ?? [])], + }) as Record; const nextPayload = await options?.onPayload?.(payload, model); if (nextPayload !== undefined) payload = nextPayload as Record; @@ -92,17 +109,26 @@ async function runYutoriStream( const text = typeof message?.content === "string" ? message.content : ""; if (text) emitText(stream, output, text); - const wantsBatch = (context.tools ?? []).some((tool) => tool.name === CUA_BATCH_TOOL_NAME); - const batchActions: CuaAction[] = []; - let firstBatchCallId: string | undefined; - for (const call of message?.tool_calls ?? []) { if (call.type !== "function") continue; const args = parseArguments(call.function.arguments); - const canonical = wantsBatch ? toCanonicalAction(call.function.name, args) : undefined; - if (canonical) { - batchActions.push(...canonical); - firstBatchCallId ??= call.id; + const canonical = toCanonicalActions(call.function.name, args); + if (canonical && canonical.length > 0) { + for (let i = 0; i < canonical.length; i++) { + const action = canonical[i]!; + const contentIndex = output.content.length; + const toolCall: ToolCall = { + type: "toolCall", + id: canonical.length === 1 ? call.id : `${call.id}_${i}`, + name: canonicalToolCallName(action), + arguments: canonicalToolCallArguments(action), + }; + output.content.push(toolCall); + output.stopReason = "toolUse"; + stream.push({ type: "toolcall_start", contentIndex, partial: output }); + stream.push({ type: "toolcall_delta", contentIndex, delta: JSON.stringify(toolCall.arguments), partial: output }); + stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); + } continue; } const contentIndex = output.content.length; @@ -113,23 +139,9 @@ async function runYutoriStream( arguments: args, }; output.content.push(toolCall); - stream.push({ type: "toolcall_start", contentIndex, partial: output }); - stream.push({ type: "toolcall_delta", contentIndex, delta: call.function.arguments ?? "", partial: output }); - stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); - } - - if (batchActions.length > 0) { - const contentIndex = output.content.length; - const toolCall: ToolCall = { - type: "toolCall", - id: firstBatchCallId ?? `yutori_batch_${Date.now()}`, - name: CUA_BATCH_TOOL_NAME, - arguments: { actions: batchActions }, - }; - output.content.push(toolCall); output.stopReason = "toolUse"; stream.push({ type: "toolcall_start", contentIndex, partial: output }); - stream.push({ type: "toolcall_delta", contentIndex, delta: JSON.stringify(toolCall.arguments), partial: output }); + stream.push({ type: "toolcall_delta", contentIndex, delta: call.function.arguments ?? "", partial: output }); stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); } @@ -143,6 +155,12 @@ async function runYutoriStream( } } +function keepToolNamesFromContext(context: Context): string[] { + return (context.tools ?? []) + .map((tool) => tool.name) + .filter((name) => !isYutoriLocalActionToolName(name)); +} + function initialAssistantMessage(model: Model): AssistantMessage { return { role: "assistant", @@ -201,16 +219,14 @@ function convertMessages(context: Context): ChatCompletionMessageParam[] { } function convertTools(context: Context): Array> { - return (context.tools ?? []) - .filter((tool) => !YUTORI_BUILTIN_TOOL_NAMES.has(tool.name)) - .map((tool) => ({ - type: "function", - function: { - name: tool.name, - description: tool.description, - parameters: tool.parameters, - }, - })); + return (context.tools ?? []).map((tool) => ({ + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: tool.parameters, + }, + })); } function emitText(stream: ReturnType, output: AssistantMessage, text: string): void { @@ -232,72 +248,6 @@ function parseArguments(value: string | undefined): Record { return JSON.parse(value) as Record; } -const SCROLL_AMOUNT_PER_NOTCH = 120; - -function readPoint(value: unknown): { x: number; y: number } | undefined { - if (!Array.isArray(value) || value.length < 2) return undefined; - const x = Number(value[0]); - const y = Number(value[1]); - if (!Number.isFinite(x) || !Number.isFinite(y)) return undefined; - return { x, y }; -} - -function toCanonicalAction(name: string, args: Record): CuaAction[] | undefined { - const coords = readPoint(args.coordinates); - switch (name) { - case "left_click": - return coords ? [{ type: "click", x: coords.x, y: coords.y }] : undefined; - case "right_click": - return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "right" }] : undefined; - case "middle_click": - return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "middle" }] : undefined; - case "double_click": - return coords ? [{ type: "double_click", x: coords.x, y: coords.y }] : undefined; - case "mouse_move": - case "hover": - return coords ? [{ type: "move", x: coords.x, y: coords.y }] : undefined; - case "mouse_down": - return coords ? [{ type: "mouse_down", x: coords.x, y: coords.y }] : undefined; - case "mouse_up": - return coords ? [{ type: "mouse_up", x: coords.x, y: coords.y }] : undefined; - case "type": { - const text = typeof args.text === "string" ? args.text : undefined; - return text !== undefined ? [{ type: "type", text }] : undefined; - } - case "key_press": - case "hold_key": { - const key = typeof args.key === "string" ? args.key : undefined; - return key ? [{ type: "keypress", keys: [key] }] : undefined; - } - case "scroll": { - if (!coords) return undefined; - const amount = typeof args.amount === "number" ? args.amount : 1; - const direction = typeof args.direction === "string" ? args.direction : "down"; - const ticks = amount * SCROLL_AMOUNT_PER_NOTCH; - const dx = direction === "left" ? -ticks : direction === "right" ? ticks : 0; - const dy = direction === "up" ? -ticks : direction === "down" ? ticks : 0; - return [{ type: "scroll", x: coords.x, y: coords.y, scroll_x: dx, scroll_y: dy }]; - } - case "drag": { - const start = readPoint(args.start_coordinates); - if (!start || !coords) return undefined; - return [{ type: "drag", path: [start, coords] }]; - } - case "wait": - return [{ type: "wait" }]; - case "go_back": - return [{ type: "back" }]; - case "go_forward": - return [{ type: "forward" }]; - case "goto_url": { - const url = typeof args.url === "string" ? args.url : undefined; - return url ? [{ type: "goto", url }] : undefined; - } - default: - return undefined; - } -} - function usageFromYutori(usage: unknown): AssistantMessage["usage"] { const input = readNumber(usage, "prompt_tokens"); const output = readNumber(usage, "completion_tokens"); diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index 525d241c..24bd3c5c 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -1,26 +1,53 @@ -import type { Api, Model, SimpleStreamOptions, Tool } from "@earendil-works/pi-ai"; +import type { Api, Model, Tool } from "@earendil-works/pi-ai"; import { type CuaModelRef, type CuaProvider, getCuaModel, providerForModel } from "./models"; import * as anthropic from "./providers/anthropic/index"; import * as gemini from "./providers/gemini/index"; import * as openai from "./providers/openai/index"; import * as tzafon from "./providers/tzafon/index"; import * as yutori from "./providers/yutori/index"; +import type { ComputerToolCoordinateSystem } from "./providers/common"; + +export interface CuaScreenshotTransformSpec { + width: number; + height: number; + format: "png" | "jpeg" | "webp"; + quality?: number; +} + +export interface CuaScreenshotSpec { + /** Append a provider-prepared screenshot to the latest user/tool message before each request. */ + appendToLatestMessage?: boolean; + /** Optional image transform applied to Kernel screenshots before they are sent to the provider. */ + transform?: CuaScreenshotTransformSpec; +} + +export interface CuaPayloadContext { + /** Tool names that should remain in the outbound provider payload even if the provider strips local CUA executors. */ + keepToolNames?: readonly string[]; +} + +export type CuaPayloadHook = (payload: unknown, model: Model, context?: CuaPayloadContext) => unknown | Promise; /** * Runtime configuration for a supported CUA model. * - * Use this to pair a model with the tool definitions, baseline prompt, and - * request payload middleware expected by its provider. + * Use this to pair a model with the agent tool definitions, baseline prompt, + * coordinate convention, screenshot policy, and request payload middleware + * expected by its provider. */ export interface CuaRuntimeSpec { model: Model; provider: CuaProvider; - /** Model-facing CUA tool definitions for this provider. */ + /** CUA tool definitions installed by CuaAgent/CuaAgentHarness for local execution. */ toolDefinitions: Tool[]; /** Provider-tuned baseline prompt for browser control behavior. */ defaultSystemPrompt: string; + /** Coordinate convention emitted by provider tool calls. */ + coordinateSystem: ComputerToolCoordinateSystem; + /** Optional provider screenshot input policy used by CuaAgent/CuaAgentHarness. */ + screenshot?: CuaScreenshotSpec; /** Optional provider middleware for request payload adaptation. */ - onPayload?: SimpleStreamOptions["onPayload"]; + onPayload?: CuaPayloadHook; } export type CuaRuntimeSpecInput = CuaModelRef | Model; @@ -39,29 +66,38 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe return { model, provider, - toolDefinitions: anthropic.createComputerToolDefinitions(), + toolDefinitions: anthropic.computerTools(), defaultSystemPrompt: anthropic.buildAnthropicSystemPrompt(), + coordinateSystem: anthropic.COMPUTER_TOOL_COORDINATES, }; case "google": return { model, provider, - toolDefinitions: gemini.createComputerToolDefinitions(), + toolDefinitions: gemini.computerTools(), defaultSystemPrompt: gemini.buildGeminiSystemPrompt(), + coordinateSystem: gemini.COMPUTER_TOOL_COORDINATES, }; case "tzafon": return { model, provider, - toolDefinitions: tzafon.createComputerToolDefinitions(), + toolDefinitions: tzafon.computerTools(), defaultSystemPrompt: tzafon.buildTzafonSystemPrompt(), + coordinateSystem: tzafon.COMPUTER_TOOL_COORDINATES, + onPayload: tzafon.tzafonComputerUseOnPayload, }; case "yutori": return { model, provider, - toolDefinitions: yutori.createComputerToolDefinitions(), + toolDefinitions: yutori.computerTools(), defaultSystemPrompt: yutori.buildYutoriSystemPrompt(), + coordinateSystem: yutori.COMPUTER_TOOL_COORDINATES, + screenshot: { + appendToLatestMessage: true, + transform: { width: 1280, height: 800, format: "webp", quality: 90 }, + }, onPayload: yutori.yutoriBuiltinToolsOnPayload, }; case "openai": @@ -69,8 +105,9 @@ export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpe return { model, provider, - toolDefinitions: openai.createComputerToolDefinitions(), - defaultSystemPrompt: openai.OPENAI_BATCH_INSTRUCTIONS, + toolDefinitions: openai.computerTools(), + defaultSystemPrompt: openai.OPENAI_COMPUTER_INSTRUCTIONS, + coordinateSystem: openai.COMPUTER_TOOL_COORDINATES, onPayload: openai.openaiResponsesStoreOnPayload, }; } diff --git a/packages/ai/test/batch-tool.integration.test.ts b/packages/ai/test/batch-tool.integration.test.ts deleted file mode 100644 index 5ee0b8b3..00000000 --- a/packages/ai/test/batch-tool.integration.test.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; -import { - CUA_ACTION_TYPES, - CUA_BATCH_TOOL_NAME, - type Context, - type CuaActionType, - type CuaProvider, - anthropic, - complete, - gemini, - getCuaModel, - openai, - tzafon, - yutori, -} from "../src/index"; - -const here = dirname(fileURLToPath(import.meta.url)); -const screenshotPath = join(here, "..", "examples", "screenshot.png"); - -interface ProviderCase { - provider: CuaProvider; - envVar: string; - modelRef: string; - tools: () => ReturnType; - multiActionTools: () => ReturnType; - coordinateRange: readonly [number, number]; - supportsBatching: boolean; - requireToolCalls: boolean; - extraOptions?: Record; -} - -const cases: ProviderCase[] = [ - { - provider: "openai", - envVar: "OPENAI_API_KEY", - modelRef: "openai:gpt-5.5", - tools: () => openai.createComputerToolDefinitions({ actions: ["click"] }), - multiActionTools: () => openai.createComputerToolDefinitions({ actions: ["click", "type"] }), - coordinateRange: [0, 1920], - supportsBatching: true, - requireToolCalls: true, - }, - { - provider: "anthropic", - envVar: "ANTHROPIC_API_KEY", - modelRef: "anthropic:claude-opus-4-7", - tools: () => anthropic.createComputerToolDefinitions({ actions: ["click"] }), - multiActionTools: () => anthropic.createComputerToolDefinitions({ actions: ["click", "type"] }), - coordinateRange: [0, 1920], - supportsBatching: true, - requireToolCalls: true, - }, - { - provider: "google", - envVar: "GOOGLE_API_KEY", - modelRef: "google:gemini-3-flash-preview", - tools: () => gemini.createComputerToolDefinitions({ actions: ["click"] }), - multiActionTools: () => gemini.createComputerToolDefinitions({ actions: ["click", "type"] }), - coordinateRange: [0, 999], - supportsBatching: true, - requireToolCalls: true, - }, - { - provider: "tzafon", - envVar: "TZAFON_API_KEY", - modelRef: "tzafon:tzafon.northstar-cua-fast", - tools: () => tzafon.createComputerToolDefinitions({ actions: ["click"] }), - multiActionTools: () => tzafon.createComputerToolDefinitions({ actions: ["click", "type"] }), - coordinateRange: [0, 999], - supportsBatching: false, - requireToolCalls: false, - }, - { - provider: "yutori", - envVar: "YUTORI_API_KEY", - modelRef: "yutori:n1.5-latest", - tools: () => yutori.createComputerToolDefinitions({ actions: ["click"] }), - multiActionTools: () => yutori.createComputerToolDefinitions({ actions: ["click", "type"] }), - coordinateRange: [0, 1000], - // Yutori's server-side model always emits one native tool call per response, - // so the translated batch always contains exactly one action. - supportsBatching: false, - requireToolCalls: true, - }, -]; - -async function buildContext(tools: ProviderCase["tools"]): Promise { - const screenshot = await readFile(screenshotPath); - return { - systemPrompt: [ - "You are controlling a browser from a screenshot.", - "Call batch_computer_actions with one action that clicks the sign in / up link.", - ].join("\n"), - messages: [ - { - role: "user", - content: [ - { type: "text", text: "Click the sign in / up link in this Kernel homepage screenshot." }, - { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, - ], - timestamp: Date.now(), - }, - ], - tools: tools(), - }; -} - -async function buildMultiActionContext(tools: ProviderCase["multiActionTools"]): Promise { - const screenshot = await readFile(screenshotPath); - return { - systemPrompt: [ - "You are controlling a browser from a screenshot.", - "Always batch multiple steps into a single batch_computer_actions call by adding all required actions to the actions array.", - "Do NOT split a sequence across separate tool calls.", - ].join("\n"), - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: "Click the sign in / up link and then type 'alice@example.com'. Emit BOTH actions in one single batch_computer_actions call.", - }, - { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, - ], - timestamp: Date.now(), - }, - ], - tools: tools(), - }; -} - -describe("batch_computer_actions integration", () => { - for (const c of cases) { - const hasKey = !!process.env[c.envVar]; - const test = hasKey ? it : it.skip; - - test(`${c.provider} returns a batch_computer_actions tool call with parsed actions`, async () => { - const model = getCuaModel(c.modelRef as never); - const context = await buildContext(c.tools); - const response = await complete(model, context, { - apiKey: process.env[c.envVar], - maxTokens: 1024, - ...c.extraOptions, - }); - - const toolCalls = response.content.filter((part) => part.type === "toolCall"); - if (toolCalls.length === 0) { - if (c.requireToolCalls) { - expect(toolCalls.length, `${c.provider} returned no tool calls`).toBeGreaterThan(0); - } - expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThanOrEqual(0); - return; - } - - const batch = toolCalls.find((call) => call.name === CUA_BATCH_TOOL_NAME); - expect(batch, `${c.provider} did not return ${CUA_BATCH_TOOL_NAME}; got [${toolCalls.map((c) => c.name).join(", ")}]`).toBeDefined(); - - const args = batch!.arguments as { actions?: unknown }; - expect(Array.isArray(args.actions), `${c.provider} .arguments.actions is ${typeof args.actions}, expected array`).toBe(true); - - const actions = args.actions as Array>; - expect(actions.length).toBeGreaterThan(0); - - const click = actions.find((a) => a.type === "click"); - expect(click, `${c.provider} batch had no click action; got: ${JSON.stringify(actions)}`).toBeDefined(); - expect(typeof click!.x).toBe("number"); - expect(typeof click!.y).toBe("number"); - const [min, max] = c.coordinateRange; - expect(click!.x as number, `${c.provider} x out of range`).toBeGreaterThanOrEqual(min); - expect(click!.x as number).toBeLessThanOrEqual(max); - expect(click!.y as number, `${c.provider} y out of range`).toBeGreaterThanOrEqual(min); - expect(click!.y as number).toBeLessThanOrEqual(max); - - expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThan(0); - - for (const action of actions) { - expect(CUA_ACTION_TYPES).toContain(action.type as CuaActionType); - } - }, 60_000); - } - - const yutoriHasKey = !!process.env.YUTORI_API_KEY; - (yutoriHasKey ? it : it.skip)( - "yutori translates native left_click into a batch_computer_actions call", - async () => { - const model = getCuaModel("yutori:n1.5-latest"); - const context = await buildContext(() => yutori.createComputerToolDefinitions()); - const response = await complete(model, context, { - apiKey: process.env.YUTORI_API_KEY, - maxTokens: 1024, - }); - - const batchCalls = response.content.filter((part) => part.type === "toolCall" && part.name === CUA_BATCH_TOOL_NAME); - expect(batchCalls.length, "yutori did not emit a translated batch call").toBeGreaterThan(0); - const args = batchCalls[0]!.arguments as { actions: Array> }; - expect(Array.isArray(args.actions)).toBe(true); - expect(args.actions.length).toBeGreaterThan(0); - expect(args.actions[0]!.type).toBe("click"); - }, - 60_000, - ); -}); - -// The whole point of batch_computer_actions is to let a single tool call carry -// multiple ordered actions (click-then-type, type-then-Enter, etc.). These -// tests probe each provider with a two-step task and verify whether the model -// actually packs both steps into one batch call. -describe("batch_computer_actions multi-action sequences", () => { - for (const c of cases) { - const hasKey = !!process.env[c.envVar]; - const test = hasKey ? it : it.skip; - - if (c.supportsBatching) { - test(`${c.provider} packs a click+type sequence into a single batch call`, async () => { - const model = getCuaModel(c.modelRef as never); - const context = await buildMultiActionContext(c.multiActionTools); - const response = await complete(model, context, { - apiKey: process.env[c.envVar], - maxTokens: 2048, - ...c.extraOptions, - }); - - const batchCalls = response.content.filter( - (part) => part.type === "toolCall" && part.name === CUA_BATCH_TOOL_NAME, - ); - expect(batchCalls.length, `${c.provider} produced no batch_computer_actions calls`).toBe(1); - - const args = batchCalls[0]!.arguments as { actions: Array> }; - expect(Array.isArray(args.actions), `${c.provider} actions field is not an array`).toBe(true); - expect( - args.actions.length, - `${c.provider} only emitted ${args.actions.length} action(s) in one batch call: ${JSON.stringify(args.actions)}`, - ).toBeGreaterThanOrEqual(2); - }, 60_000); - } else { - test(`${c.provider} emits exactly one action per response (model does not batch)`, async () => { - const model = getCuaModel(c.modelRef as never); - const context = await buildMultiActionContext(c.multiActionTools); - const response = await complete(model, context, { - apiKey: process.env[c.envVar], - maxTokens: 2048, - ...c.extraOptions, - }); - - const batchCalls = response.content.filter( - (part) => part.type === "toolCall" && part.name === CUA_BATCH_TOOL_NAME, - ); - if (batchCalls.length === 0) { - if (c.requireToolCalls) { - expect(batchCalls.length, `${c.provider} produced no ${CUA_BATCH_TOOL_NAME} calls`).toBeGreaterThan(0); - } - expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThanOrEqual(0); - return; - } - expect(batchCalls.length).toBe(1); - const args = batchCalls[0]!.arguments as { actions: Array> }; - expect(args.actions.length).toBeGreaterThanOrEqual(1); - }, 60_000); - } - } -}); diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts new file mode 100644 index 00000000..7eb53588 --- /dev/null +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -0,0 +1,162 @@ +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { + CUA_ACTION_TYPES, + type Context, + type CuaActionType, + type CuaProvider, + anthropic, + complete, + gemini, + getCuaModel, + openai, + tzafon, + yutori, +} from "../src/index"; + +const here = dirname(fileURLToPath(import.meta.url)); +const screenshotPath = join(here, "..", "examples", "screenshot.png"); + +interface ProviderCase { + provider: CuaProvider; + envVar: string; + modelRef: string; + tools: () => ReturnType; + coordinateRange: readonly [number, number]; + requireToolCalls: boolean; + ciOptInEnvVar?: string; + extraOptions?: Record; +} + +const cases: ProviderCase[] = [ + { + provider: "openai", + envVar: "OPENAI_API_KEY", + modelRef: "openai:gpt-5.5", + tools: () => openai.computerTools({ actions: ["click"] }), + coordinateRange: [0, 1920], + requireToolCalls: true, + }, + { + provider: "anthropic", + envVar: "ANTHROPIC_API_KEY", + modelRef: "anthropic:claude-opus-4-7", + tools: () => anthropic.computerTools({ actions: ["click"] }), + coordinateRange: [0, 1920], + requireToolCalls: true, + extraOptions: { toolChoice: { type: "tool", name: "click" } }, + }, + { + provider: "google", + envVar: "GOOGLE_API_KEY", + modelRef: "google:gemini-3-flash-preview", + tools: () => gemini.computerTools({ actions: ["click"] }), + coordinateRange: [0, 999], + requireToolCalls: true, + }, + { + provider: "tzafon", + envVar: "TZAFON_API_KEY", + modelRef: "tzafon:tzafon.northstar-cua-fast", + tools: () => tzafon.computerTools({ actions: ["click"] }), + coordinateRange: [0, 999], + requireToolCalls: false, + ciOptInEnvVar: "CUA_E2E_TZAFON", + }, +]; + +async function buildContext(tools: ProviderCase["tools"]): Promise { + const screenshot = await readFile(screenshotPath); + return { + systemPrompt: [ + "You are controlling a browser from a screenshot.", + "Call the available click tool for the sign in / up link.", + ].join("\n"), + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Click the sign in / up link in this Kernel homepage screenshot." }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, + ], + tools: tools(), + }; +} + +async function buildYutoriContext(): Promise { + const screenshot = await readFile(screenshotPath); + return { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Click the sign in / up link in this Kernel homepage screenshot." }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, + ], + tools: yutori.computerTools(), + }; +} + +describe("individual computer action integration", () => { + for (const c of cases) { + const hasKey = !!process.env[c.envVar]; + const ciEnabled = !c.ciOptInEnvVar || !process.env.CI || process.env[c.ciOptInEnvVar] === "1"; + const test = hasKey ? it : it.skip; + + (ciEnabled ? test : it.skip)(`${c.provider} returns a canonical click tool call`, async () => { + const model = getCuaModel(c.modelRef as never); + const context = await buildContext(c.tools); + const response = await complete(model, context, { + apiKey: process.env[c.envVar], + maxTokens: 1024, + ...c.extraOptions, + }); + + const toolCalls = response.content.filter((part) => part.type === "toolCall"); + if (toolCalls.length === 0) { + if (c.requireToolCalls) { + expect(toolCalls.length, `${c.provider} returned no tool calls`).toBeGreaterThan(0); + } + expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThanOrEqual(0); + return; + } + + const click = toolCalls.find((call) => call.name === "click"); + expect(click, `${c.provider} did not return click; got [${toolCalls.map((call) => call.name).join(", ")}]`).toBeDefined(); + expect(typeof click!.arguments.x).toBe("number"); + expect(typeof click!.arguments.y).toBe("number"); + const [min, max] = c.coordinateRange; + expect(click!.arguments.x as number, `${c.provider} x out of range`).toBeGreaterThanOrEqual(min); + expect(click!.arguments.x as number).toBeLessThanOrEqual(max); + expect(click!.arguments.y as number, `${c.provider} y out of range`).toBeGreaterThanOrEqual(min); + expect(click!.arguments.y as number).toBeLessThanOrEqual(max); + expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThan(0); + }, 60_000); + } + + const yutoriHasKey = !!process.env.YUTORI_API_KEY; + (yutoriHasKey ? it : it.skip)( + "yutori translates native tool calls into canonical individual actions", + async () => { + const model = getCuaModel("yutori:n1.5-latest"); + const context = await buildYutoriContext(); + const response = await complete(model, context, { + apiKey: process.env.YUTORI_API_KEY, + maxTokens: 1024, + }); + + const toolCalls = response.content.filter((part) => part.type === "toolCall"); + expect(toolCalls.length, "yutori did not emit translated canonical tool calls").toBeGreaterThan(0); + expect(CUA_ACTION_TYPES).toContain(toolCalls[0]!.name as CuaActionType); + }, + 60_000, + ); +}); diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index dc6df8e7..2d1ebd9f 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -10,20 +10,31 @@ describe("resolveCuaRuntimeSpec", () => { const spec = resolveCuaRuntimeSpec(model!.ref); expect(spec.provider).toBe(provider); expect(spec.model.id).toBe(model!.model); - expect(spec.toolDefinitions.length).toBeGreaterThan(0); expect(typeof spec.defaultSystemPrompt).toBe("string"); - expect(spec.defaultSystemPrompt.length).toBeGreaterThan(0); - expect(spec.toolDefinitions.map((tool) => tool.name)).toContain(CUA_BATCH_TOOL_NAME); - expect(spec.toolDefinitions.map((tool) => tool.name)).toContain(CUA_NAVIGATION_TOOL_NAME); + expect(spec.coordinateSystem).toBeDefined(); + expect(spec.toolDefinitions.length).toBeGreaterThan(0); + expect(spec.toolDefinitions.map((tool) => tool.name)).not.toContain(CUA_BATCH_TOOL_NAME); + expect(spec.toolDefinitions.map((tool) => tool.name)).not.toContain(CUA_NAVIGATION_TOOL_NAME); + if (provider === "yutori") { + expect(spec.defaultSystemPrompt).toBe(""); + expect(spec.screenshot).toEqual({ + appendToLatestMessage: true, + transform: { width: 1280, height: 800, format: "webp", quality: 90 }, + }); + } else { + expect(spec.defaultSystemPrompt.length).toBeGreaterThan(0); + } } }); it("only sets payload middleware for providers that need it", () => { const yutoriSpec = resolveCuaRuntimeSpec("yutori:n1.5-latest"); const openaiSpec = resolveCuaRuntimeSpec("openai:gpt-5.5"); + const tzafonSpec = resolveCuaRuntimeSpec("tzafon:tzafon.northstar-cua-fast"); const anthropicSpec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-7"); expect(yutoriSpec.onPayload).toBeTypeOf("function"); expect(openaiSpec.onPayload).toBeTypeOf("function"); + expect(tzafonSpec.onPayload).toBeTypeOf("function"); expect(anthropicSpec.onPayload).toBeUndefined(); }); }); diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index 8b15c469..d3d9ddc8 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -3,6 +3,8 @@ import { CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, + createCuaBatchToolDefinition, + createCuaNavigationToolDefinition, type CuaActionType, anthropic, gemini, @@ -11,7 +13,7 @@ import { yutori, } from "../src/index"; -const providers = { openai, anthropic, gemini, tzafon, yutori }; +const providers = { openai, anthropic, gemini, tzafon }; function batchActionVariants(tool: { parameters: any }): any[] { const items = tool.parameters.properties.actions.items; @@ -20,70 +22,61 @@ function batchActionVariants(tool: { parameters: any }): any[] { describe("computer tool definitions", () => { for (const [provider, namespace] of Object.entries(providers)) { - it(`returns a default batch tool for ${provider}`, () => { - const tools = namespace.createComputerToolDefinitions(); - expect(tools.map((tool) => tool.name)).toEqual([CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME]); + it(`returns individual CUA action tools for ${provider}`, () => { + const tools = namespace.computerTools(); + expect(tools.map((tool) => tool.name)).toEqual([...CUA_ACTION_TYPES]); }); - it(`returns a narrowed batch tool for ${provider}`, () => { - const tools = namespace.createComputerToolDefinitions({ actions: ["click"] }); - expect(tools.map((tool) => tool.name)).toEqual([CUA_BATCH_TOOL_NAME]); + it(`returns narrowed individual tools for ${provider}`, () => { + const tools = namespace.computerTools({ actions: ["click"] }); + expect(tools.map((tool) => tool.name)).toEqual(["click"]); + expect(tools[0]!.parameters.properties.type).toBeUndefined(); + expect(tools[0]!.parameters.required).toEqual(["x", "y"]); }); - it(`default batch tool for ${provider} covers every CUA action type`, () => { - const tools = namespace.createComputerToolDefinitions(); - const variants = batchActionVariants(tools[0]!); - const seen = variants.map((variant) => variant.properties.type.const).sort(); - expect(seen).toEqual([...CUA_ACTION_TYPES].sort()); - }); - - it(`each action variant for ${provider} accepts only declared fields`, () => { - const tools = namespace.createComputerToolDefinitions(); - for (const variant of batchActionVariants(tools[0]!)) { - expect(variant.additionalProperties).toBe(false); - expect(variant.required).toContain("type"); + it(`each individual action schema for ${provider} accepts only declared fields`, () => { + const tools = namespace.computerTools(); + for (const tool of tools) { + expect(tool.parameters.additionalProperties).toBe(false); + expect(tool.parameters.properties.type).toBeUndefined(); } }); } - it("narrows the batch action schema when actions are provided", () => { - const tools = openai.createComputerToolDefinitions({ actions: ["click"] }); - expect(tools.map((tool) => tool.name)).toEqual([CUA_BATCH_TOOL_NAME]); - - const variants = batchActionVariants(tools[0]!); - expect(variants).toHaveLength(1); - expect(variants[0].properties.type.const).toBe("click"); - expect(variants[0].properties.x).toBeTruthy(); - expect(variants[0].properties.text).toBeUndefined(); - }); - - it("preserves action ordering in narrowed batch schemas", () => { + it("synthesizes a batch tool from an action subset", () => { const subset: CuaActionType[] = ["screenshot", "type", "click"]; - const tools = openai.createComputerToolDefinitions({ actions: subset }); - const variants = batchActionVariants(tools[0]!); - expect(variants.map((v) => v.properties.type.const)).toEqual(subset); + const tool = createCuaBatchToolDefinition(subset); + expect(tool.name).toBe(CUA_BATCH_TOOL_NAME); + expect(batchActionVariants(tool).map((v) => v.properties.type.const)).toEqual(subset); }); - it("emits a single-variant schema (not a union) when narrowed to one action", () => { - const tools = openai.createComputerToolDefinitions({ actions: ["click"] }); - const items = tools[0]!.parameters.properties.actions.items; + it("emits a single-variant batch schema when narrowed to one action", () => { + const tool = createCuaBatchToolDefinition(["click"]); + const items = tool.parameters.properties.actions.items; expect(items.anyOf).toBeUndefined(); expect(items.oneOf).toBeUndefined(); expect(items.properties.type.const).toBe("click"); }); - it("omits computer_use_extra navigation tool when actions are narrowed", () => { - const tools = openai.createComputerToolDefinitions({ actions: ["click", "goto"] }); - expect(tools).toHaveLength(1); - expect(tools[0]!.name).toBe(CUA_BATCH_TOOL_NAME); + it("synthesizes the navigation helper separately", () => { + const tool = createCuaNavigationToolDefinition(); + expect(tool.name).toBe(CUA_NAVIGATION_TOOL_NAME); + }); + + it("exposes local canonical executor definitions for Yutori", () => { + const tools = yutori.computerTools(); + expect(tools.map((tool) => tool.name)).toEqual([...yutori.YUTORI_CANONICAL_ACTION_TYPES]); + expect(tools.map((tool) => tool.name)).not.toContain(CUA_BATCH_TOOL_NAME); + expect(tools.map((tool) => tool.name)).not.toContain(CUA_NAVIGATION_TOOL_NAME); }); - it("exposes batch and navigation tool name constants identically across providers", () => { - for (const namespace of Object.values(providers)) { - const tools = namespace.createComputerToolDefinitions(); - expect(tools[0]!.name).toBe(CUA_BATCH_TOOL_NAME); - expect(tools[1]!.name).toBe(CUA_NAVIGATION_TOOL_NAME); - } + it("exports Yutori native action sets by model family", () => { + expect(yutori.yutoriNativeActionsForModel("n1-latest")).toEqual(yutori.YUTORI_N1_ACTION_TYPES); + expect(yutori.yutoriNativeActionsForModel("n1.5-latest")).toEqual(yutori.YUTORI_N15_CORE_ACTION_TYPES); + expect(yutori.YUTORI_N15_ACTION_TYPES).toEqual([ + ...yutori.YUTORI_N15_CORE_ACTION_TYPES, + ...yutori.YUTORI_N15_EXPANDED_ACTION_TYPES, + ]); }); it("exports provider coordinate systems", () => { diff --git a/packages/ai/test/tzafon-payload.test.ts b/packages/ai/test/tzafon-payload.test.ts new file mode 100644 index 00000000..5c26f4ed --- /dev/null +++ b/packages/ai/test/tzafon-payload.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { tzafon } from "../src/index"; + +describe("tzafonComputerUseOnPayload", () => { + it("replaces local CUA action tools with the native computer_use tool", () => { + const payload = { + tools: [ + { type: "function", name: "click" }, + { type: "function", name: "screenshot" }, + { type: "function", name: "custom_tool" }, + ], + }; + + const next = tzafon.tzafonComputerUseOnPayload(payload) as { tools?: Array<{ type?: string; name?: string }> }; + + expect(next.tools).toEqual([ + { + type: "computer_use", + display_width: 1920, + display_height: 1080, + environment: "browser", + }, + { type: "function", name: "custom_tool" }, + ]); + }); + + it("preserves caller-requested keep tools", () => { + const payload = { + tools: [ + { type: "function", name: "click" }, + { type: "function", name: "batch_computer_actions" }, + ], + }; + + const next = tzafon.tzafonComputerUseOnPayload(payload, undefined, { + keepToolNames: ["batch_computer_actions"], + }) as { tools?: Array<{ type?: string; name?: string }> }; + + expect(next.tools?.map((tool) => tool.type === "computer_use" ? "computer_use" : tool.name)).toEqual([ + "computer_use", + "batch_computer_actions", + ]); + }); + + it("returns undefined for non-object payloads", () => { + expect(tzafon.tzafonComputerUseOnPayload(undefined)).toBeUndefined(); + expect(tzafon.tzafonComputerUseOnPayload("x")).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/tzafon-provider.test.ts b/packages/ai/test/tzafon-provider.test.ts new file mode 100644 index 00000000..fd81451c --- /dev/null +++ b/packages/ai/test/tzafon-provider.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { tzafon } from "../src/index"; + +describe("streamTzafonResponses", () => { + it("derives unique ids when one computer_call expands to multiple actions", () => { + expect(tzafon.tzafonToolCallId("call_1", 0)).toBe("call_1"); + expect(tzafon.tzafonToolCallId("call_1", 1)).toBe("call_1:1"); + expect(tzafon.tzafonToolCallId("call_1", 2)).toBe("call_1:2"); + }); +}); diff --git a/packages/ai/test/yutori-actions.test.ts b/packages/ai/test/yutori-actions.test.ts new file mode 100644 index 00000000..5ec89566 --- /dev/null +++ b/packages/ai/test/yutori-actions.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { type CuaAction, yutori } from "../src/index"; + +const n15CoreActionArgs = { + left_click: { coordinates: [500, 250] }, + double_click: { coordinates: [500, 250] }, + triple_click: { coordinates: [500, 250] }, + middle_click: { coordinates: [500, 250] }, + right_click: { coordinates: [500, 250] }, + mouse_move: { coordinates: [100, 200] }, + mouse_down: { coordinates: [100, 200] }, + mouse_up: { coordinates: [100, 200] }, + drag: { start_coordinates: [100, 200], coordinates: [300, 400] }, + scroll: { coordinates: [500, 500], direction: "down", amount: 3 }, + type: { text: "hello" }, + key_press: { key: "ctrl+c" }, + hold_key: { key: "shift", duration: 1.5 }, + goto_url: { url: "https://example.com" }, + go_back: {}, + go_forward: {}, + refresh: {}, + wait: { duration: 1 }, +} satisfies Record<(typeof yutori.YUTORI_N15_CORE_ACTION_TYPES)[number], Record>; + +describe("Yutori native action normalization", () => { + it("has canonical mappings for every n1.5 core action", () => { + for (const action of yutori.YUTORI_N15_CORE_ACTION_TYPES) { + const canonical = yutori.toCanonicalActions(action, n15CoreActionArgs[action]); + expect(canonical, `${action} did not map to canonical CUA actions`).toBeDefined(); + expect(canonical!.length, `${action} mapped to an empty action list`).toBeGreaterThan(0); + for (const item of canonical as CuaAction[]) { + expect(typeof item.type).toBe("string"); + } + } + }); + + it("normalizes n1/n1.5 click actions to canonical individual actions", () => { + expect(yutori.toCanonicalActions("left_click", { coordinates: [500, 250] })).toEqual([ + { type: "click", x: 500, y: 250 }, + ]); + expect(yutori.toCanonicalActions("double_click", { coordinates: [500, 250] })).toEqual([ + { type: "double_click", x: 500, y: 250 }, + ]); + expect(yutori.toCanonicalActions("triple_click", { coordinates: [500, 250] })).toEqual([ + { type: "double_click", x: 500, y: 250 }, + { type: "click", x: 500, y: 250 }, + ]); + expect(yutori.toCanonicalActions("middle_click", { coordinates: [500, 250] })).toEqual([ + { type: "click", x: 500, y: 250, button: "middle" }, + ]); + }); + + it("normalizes mouse, drag, type, and keyboard actions", () => { + expect(yutori.toCanonicalActions("mouse_move", { coordinates: [100, 200] })).toEqual([ + { type: "move", x: 100, y: 200 }, + ]); + expect(yutori.toCanonicalActions("drag", { start_coordinates: [100, 200], coordinates: [300, 400] })).toEqual([ + { type: "drag", path: [{ x: 100, y: 200 }, { x: 300, y: 400 }], button: "left" }, + ]); + expect(yutori.toCanonicalActions("type", { text: "hello", clear_before_typing: true, press_enter_after: true })).toEqual([ + { type: "keypress", keys: ["ctrl", "a"] }, + { type: "keypress", keys: ["backspace"] }, + { type: "type", text: "hello" }, + { type: "keypress", keys: ["enter"] }, + ]); + expect(yutori.toCanonicalActions("hold_key", { key: "shift", duration: 1.5 })).toEqual([ + { type: "keypress", keys: ["shift"], duration: 1500 }, + ]); + expect(yutori.toCanonicalActions("key_press", { key: "ctrl+c" })).toEqual([ + { type: "keypress", keys: ["ctrl", "c"] }, + ]); + expect(yutori.toCanonicalActions("key_press", { key: "down down enter" })).toEqual([ + { type: "keypress", keys: ["down"] }, + { type: "keypress", keys: ["down"] }, + { type: "keypress", keys: ["enter"] }, + ]); + }); + + it("normalizes scroll and navigation actions", () => { + expect(yutori.toCanonicalActions("scroll", { coordinates: [500, 500], direction: "down", amount: 3 })).toEqual([ + { type: "scroll", x: 500, y: 500, scroll_x: 0, scroll_y: 360 }, + ]); + expect(yutori.toCanonicalActions("goto_url", { url: "https://example.com" })).toEqual([ + { type: "goto", url: "https://example.com" }, + { type: "wait", ms: 2000 }, + ]); + expect(yutori.toCanonicalActions("goto_url", { url: "example.com" })).toEqual([ + { type: "goto", url: "https://example.com" }, + { type: "wait", ms: 2000 }, + ]); + expect(yutori.toCanonicalActions("refresh", {})).toEqual([ + { type: "keypress", keys: ["f5"] }, + { type: "wait", ms: 2000 }, + ]); + }); +}); diff --git a/packages/ai/test/yutori-payload.test.ts b/packages/ai/test/yutori-payload.test.ts index dc27f85f..d3af6493 100644 --- a/packages/ai/test/yutori-payload.test.ts +++ b/packages/ai/test/yutori-payload.test.ts @@ -2,16 +2,41 @@ import { describe, expect, it } from "vitest"; import { yutori } from "../src/index"; describe("yutoriBuiltinToolsOnPayload", () => { - it("removes yutori built-in tool definitions from payload.tools", () => { + it("removes local Yutori action tool definitions from payload.tools", () => { const payload = { tools: [ { type: "function", function: { name: "click" } }, { type: "function", function: { name: "move" } }, { type: "function", function: { name: "batch_computer_actions" } }, + { type: "function", function: { name: "computer_use_extra" } }, + { type: "function", function: { name: "custom_tool" } }, ], }; const next = yutori.yutoriBuiltinToolsOnPayload(payload) as { tools?: Array<{ function?: { name?: string } }> }; - expect(next.tools).toEqual([{ type: "function", function: { name: "batch_computer_actions" } }]); + expect(next.tools?.map((tool) => tool.function?.name)).toEqual([ + "batch_computer_actions", + "computer_use_extra", + "custom_tool", + ]); + }); + + it("preserves caller-requested keep tools while adding the n1.5 core tool set", () => { + const payload = { + tools: [ + { type: "function", function: { name: "click" } }, + { type: "function", function: { name: "batch_computer_actions" } }, + ], + }; + const next = yutori.yutoriNativeToolSetOnPayload(payload, { id: "n1.5-latest" } as never, { + keepToolNames: ["batch_computer_actions"], + }) as { + tool_set?: string; + disable_tools?: string[]; + tools?: Array<{ function?: { name?: string } }>; + }; + expect(next.tool_set).toBe(yutori.YUTORI_N15_CORE_TOOL_SET); + expect(next.disable_tools).toEqual([...yutori.YUTORI_N15_EXPANDED_ACTION_TYPES]); + expect(next.tools?.map((tool) => tool.function?.name)).toEqual(["batch_computer_actions"]); }); it("returns undefined for non-object payloads", () => {