diff --git a/.gitignore b/.gitignore index aab323e7b..a6059c787 100644 --- a/.gitignore +++ b/.gitignore @@ -60,10 +60,11 @@ apps/desktop/resources/ apps/cloud/.dev-db/ apps/cloud/.e2e-db/ -# e2e suite: generated run artifacts + throwaway target state +# e2e suite: generated run artifacts + throwaway target state (the * also +# covers ad-hoc variants like .e2e-stub-db-manual from debugging boots) e2e/runs/ -apps/cloud/.e2e-stub-db/ -apps/host-selfhost/.e2e-data/ +apps/cloud/.e2e-stub-db*/ +apps/host-selfhost/.e2e-data*/ # playwright e2e artifacts test-results/ @@ -109,5 +110,3 @@ LEARNINGS.md # Throwaway UX prototype (not part of the app) ux-demo/ -# testkit run outputs (superseded by e2e/runs, also generated) -testkit/runs/ diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 79fa51c00..795e7bb76 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -6,7 +6,6 @@ ".turbo", "dist", "vendor", - "testkit", "e2e/runs", "integrationsdotsh", "node_modules", diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index e58d1b8d7..a117a09ec 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -152,7 +152,6 @@ "dist/", "vendor/", "emulators/", - "testkit/", "e2e/runs/", "integrationsdotsh/", "node_modules/", diff --git a/README.md b/README.md index be6ffd7e7..e848bd4af 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,17 @@ bun dev The dev server starts at `http://127.0.0.1:4788`. +### Tests + +```bash +bun run test # unit + integration suites +bun run test:e2e # full-stack e2e: boots the cloud and self-host apps and drives them +``` + +The browser e2e scenarios need Playwright's Chromium once per machine: +`bunx playwright install chromium`. The git submodules under `vendor/` are +optional — see [vendor/README.md](vendor/README.md). + ## Community Join the Discord: [https://discord.gg/eF29HBHwM6](https://discord.gg/eF29HBHwM6) diff --git a/e2e/src/surfaces/mcp.ts b/e2e/src/surfaces/mcp.ts index 75cf1e5a8..583e44701 100644 --- a/e2e/src/surfaces/mcp.ts +++ b/e2e/src/surfaces/mcp.ts @@ -1,7 +1,7 @@ -// MCP surface: the vendored mcporter fork as a programmatic MCP client, with -// headless OAuth via the target's consent strategy. Session methods are -// Effects; mcporter itself is promise-native underneath. Assertions are -// vitest's job. +// MCP surface: our mcporter fork (@executor-js/mcporter on npm; develop it in +// the vendor/mcporter submodule) as a programmatic MCP client, with headless +// OAuth via the target's consent strategy. Session methods are Effects; +// mcporter itself is promise-native underneath. Assertions are vitest's job. import { createHash, randomBytes, randomUUID } from "node:crypto"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; diff --git a/package.json b/package.json index 8f5c957e5..c6e0feb16 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "release:publish:packages:prepare": "bun run scripts/publish-packages.ts --prepare-only", "release:smoke:packages": "bun run scripts/smoke-test-packed.ts", "clean": "bun run scripts/clean.ts", - "prepare": "effect-language-service patch && effect-tsgo patch" + "prepare": "effect-language-service patch && effect-tsgo patch && bun run --cwd packages/core/vite-plugin build:bundle && bun run --cwd packages/react build" }, "dependencies": {}, "devDependencies": { diff --git a/packages/core/vite-plugin/package.json b/packages/core/vite-plugin/package.json index 4a7a7903b..af7bbeae1 100644 --- a/packages/core/vite-plugin/package.json +++ b/packages/core/vite-plugin/package.json @@ -35,6 +35,7 @@ }, "scripts": { "build": "tsup && (tsc --declaration --emitDeclarationOnly --outDir dist --rootDir src || true)", + "build:bundle": "tsup", "typecheck": "tsgo --noEmit", "test": "vitest run", "typecheck:slow": "tsc --noEmit" diff --git a/testkit/README.md b/testkit/README.md deleted file mode 100644 index b39b2465c..000000000 --- a/testkit/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# testkit (MCP surface) — MVP - -Black-box tests through the **MCP surface**, on the vendored mcporter fork -(`vendor/mcporter`, with its headless `cookieConsentStrategy`). Every run emits a -**chat-transcript recording** (`run.json`) — the MCP surface *is* an agent-in-a-chat, -so that's the natural shape. - -## Write a test - -```js -import { mcpTarget, mcpTest, cookieConsentStrategy } from "./src/testkit.mjs"; - -// the per-host seam: which MCP server + how to consent (headless, CI-safe) -const selfhost = mcpTarget({ - name: "selfhost", - server: "http://localhost:5173/mcp", - consent: cookieConsentStrategy({ appBaseUrl: "http://localhost:5173", email, password }), -}); - -mcpTest("execute runs code", selfhost, async (mcp) => { - mcp.say("Confirm the sandbox evaluates and returns a value"); // → reasoning turn - const r = await mcp.call("execute", { code: "return 6*7;" }); // → tool turn - mcp.expect(r.text).toBe("42"); // → assert turn -}); -``` - -## Run + watch - -```bash -node run.mjs # runs tests, writes runs/*.run.json, prints chat -node src/render.mjs runs/X.run.json # render a recording in the terminal -node src/render.mjs runs/X.run.json --html out.html # watchable HTML -node serve.mjs # serve runs/ (index + per-run HTML) on :8901 -``` - -## What a recording is - -`run.json` = a chat transcript (`turns`: user / assistant·reasoning / tool / assert / -error) + `asserts` + `ok`/`error`/`durationMs`. The HTML renders it as a chat: task -bubble → reasoning → collapsible tool cards (args + result) → inline asserts. Failures -record too — `ok:false` with the failing turn in place. - -## Status / next - -- **Now:** scripted brain (deterministic, no LLM). Proven against dev `/mcp`. -- **Next:** `agentTest` — LLM brain (real model drives via MCP, reasoning captured - verbatim); raw JSON-RPC `protocol[]` layer linked to tool turns; MCP-apps as `app` - turns. Productionize to TS + Effect Vitest later (see memory: effect-testing-stack-note). -- Self-contained ESM JS under `testkit/` (not in the bun workspace) — imports the fork's - built `dist`. Throwaway-grade until productionized; the proof is real. diff --git a/testkit/package-lock.json b/testkit/package-lock.json deleted file mode 100644 index 972e63a7f..000000000 --- a/testkit/package-lock.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "name": "@executor-js/testkit", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@executor-js/testkit", - "dependencies": { - "@kitlangton/terminal-control": "^0.3.0", - "playwright": "^1.60.0" - } - }, - "node_modules/@kitlangton/terminal-control": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@kitlangton/terminal-control/-/terminal-control-0.3.0.tgz", - "integrity": "sha512-YlOx9ztDInHBETgxWNRd6nq/YzpMlw1YEJ1OIrhdeWTqrQdJHIYThWGiWda9GxmQAThWNULkKR17as+QDgyKdQ==", - "license": "MIT", - "engines": { - "node": ">=20" - }, - "optionalDependencies": { - "@kitlangton/terminal-control-darwin-arm64": "0.3.0", - "@kitlangton/terminal-control-darwin-x64": "0.3.0", - "@kitlangton/terminal-control-linux-arm64-gnu": "0.3.0", - "@kitlangton/terminal-control-linux-x64-gnu": "0.3.0" - }, - "peerDependencies": { - "vitest": ">=3.0.0" - }, - "peerDependenciesMeta": { - "vitest": { - "optional": true - } - } - }, - "node_modules/@kitlangton/terminal-control-darwin-arm64": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@kitlangton/terminal-control-darwin-arm64/-/terminal-control-darwin-arm64-0.3.0.tgz", - "integrity": "sha512-HydPv5if5pAjwHNbA7TnZAJYKfq9AiZITaJuFyF1HsUMedzin3qzv9WRVBqIiGsQ/FiR1i1kaGhyLzaVpZR35w==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "bin": { - "termctrl": "bin/termctrl" - } - }, - "node_modules/@kitlangton/terminal-control-darwin-x64": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@kitlangton/terminal-control-darwin-x64/-/terminal-control-darwin-x64-0.3.0.tgz", - "integrity": "sha512-jrCSlJybh9NkpAXKdvP6p4HpMDQLtUki94V6dzKMoRh/PSTufgPQ66g8HphwjxUoiB8aTPcXgN2iBfPxOFpvqA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "bin": { - "termctrl": "bin/termctrl" - } - }, - "node_modules/@kitlangton/terminal-control-linux-arm64-gnu": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@kitlangton/terminal-control-linux-arm64-gnu/-/terminal-control-linux-arm64-gnu-0.3.0.tgz", - "integrity": "sha512-hwtLlpeRCBESveYl0Z8w6TphgAU7AQH9FSKCpwYKDL/n/hECW5kQ3zEuy7uULnPmtk7vwVlBXRTJKP553OUCXg==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "termctrl": "bin/termctrl" - } - }, - "node_modules/@kitlangton/terminal-control-linux-x64-gnu": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@kitlangton/terminal-control-linux-x64-gnu/-/terminal-control-linux-x64-gnu-0.3.0.tgz", - "integrity": "sha512-ge62a1I67X+aKX9VcMcEY92JHMQ3oxWS3ot3z85sKYEb3kse7q/hEtpgBYldYCBLxvu57U/Fa8sC6C3l5+xA5w==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "termctrl": "bin/termctrl" - } - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/playwright": { - "version": "1.60.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz", - "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==", - "license": "Apache-2.0", - "dependencies": { - "playwright-core": "1.60.0" - }, - "bin": { - "playwright": "cli.js" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" - } - }, - "node_modules/playwright-core": { - "version": "1.60.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz", - "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", - "license": "Apache-2.0", - "bin": { - "playwright-core": "cli.js" - }, - "engines": { - "node": ">=18" - } - } - } -} diff --git a/testkit/package.json b/testkit/package.json deleted file mode 100644 index 8cd1d870d..000000000 --- a/testkit/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@executor-js/testkit", - "private": true, - "type": "module", - "dependencies": { - "@kitlangton/terminal-control": "^0.3.0", - "playwright": "^1.60.0" - } -} diff --git a/testkit/record-flow.mjs b/testkit/record-flow.mjs deleted file mode 100644 index 18b7d4e1e..000000000 --- a/testkit/record-flow.mjs +++ /dev/null @@ -1,59 +0,0 @@ -// Records the full "OAuth → MCP → call" flow as one dark-themed mp4: -// 1) client title card (mcp add + pick OAuth) -// 2) REAL browser consent UI (login → connected), forced dark via colorScheme -// 3) chat replay of the real tool call -// Run from testkit/ (playwright resolves here): node record-flow.mjs -import { chromium } from "playwright"; -import { execSync } from "node:child_process"; -import { createRuntime, cookieConsentStrategy } from "../vendor/mcporter/dist/index.js"; -import { renderPlayer } from "./src/render.mjs"; -import { mkdtempSync, writeFileSync, readdirSync, renameSync, rmSync, mkdirSync } from "node:fs"; -import { tmpdir, homedir } from "node:os"; import { join } from "node:path"; - -// fresh OAuth so phase 2 records the real browser consent (not a cached token) -rmSync(join(homedir(), ".mcporter"), { recursive: true, force: true }); - -const W = 1000, H = 720, APP = process.env.MCP_APP ?? "http://localhost:5173"; -const EMAIL = process.env.MCP_EMAIL ?? "admin@demo.test", PW = process.env.MCP_PASSWORD ?? "demo-password-12345"; -const TMP = "/tmp/rec-flow"; rmSync(TMP, { recursive: true, force: true }); mkdirSync(TMP, { recursive: true }); -const newRt = () => { const d = mkdtempSync(join(tmpdir(), "rf-")); writeFileSync(join(d, "mcporter.json"), JSON.stringify({ mcpServers: { selfhost: { url: APP + "/mcp" } } })); return createRuntime({ configPath: join(d, "mcporter.json") }); }; -const recCtx = (b, dir) => b.newContext({ viewport: { width: W, height: H }, colorScheme: "dark", recordVideo: { dir, size: { width: W, height: H } } }); -const grab = (dir, name) => { const f = readdirSync(dir).find(x => x.endsWith(".webm")); renameSync(join(dir, f), `${TMP}/${name}.webm`); }; - -{ const html = `
`; - writeFileSync(`${TMP}/title.html`, html); - const b = await chromium.launch(); const ctx = await recCtx(b, `${TMP}/v1`); const p = await ctx.newPage(); - await p.goto(`file://${TMP}/title.html`); await p.waitForFunction("window.__done===true", { timeout: 15000 }).catch(()=>{}); await p.waitForTimeout(600); - await ctx.close(); await b.close(); grab(`${TMP}/v1`, "1"); console.log("phase1 ✓"); } - -{ let req; const rt = await newRt(); - try { await rt.listTools("selfhost", { autoAuthorize: true, oauthSessionOptions: { consentStrategy: async r => { req = r; throw new Error("x"); } } }); } catch {} - const b = await chromium.launch(); const ctx = await recCtx(b, `${TMP}/v2`); const p = await ctx.newPage(); - await p.goto(req.authorizationUrl, { waitUntil: "domcontentloaded" }); - await p.waitForSelector("input[type=email]", { timeout: 10000 }); await p.waitForTimeout(700); - await p.type("input[type=email]", EMAIL, { delay: 45 }); await p.waitForTimeout(250); - await p.type("input[type=password]", PW, { delay: 28 }); await p.waitForTimeout(450); - await p.click("button[type=submit]"); await p.waitForTimeout(2600); - await p.setContent('🔓 Connected to MCP server'); - await p.waitForTimeout(1700); await ctx.close(); await b.close(); grab(`${TMP}/v2`, "2"); console.log("phase2 ✓ (dark login)"); } - -{ const rt = await newRt(); - const opts = { autoAuthorize: true, oauthSessionOptions: { consentStrategy: cookieConsentStrategy({ appBaseUrl: APP, email: EMAIL, password: PW }) } }; - const tools = await rt.listTools("selfhost", opts); - const res = await rt.callTool("selfhost", "execute", { args: { code: "return 6 * 7;" }, ...opts }); - const text = res.content.filter(c=>c.type==="text").map(c=>c.text).join(""); - const run = { task: "Run 6 × 7 in the sandbox using the execute tool", brain: "scripted", meta: { server: APP + "/mcp" }, ok: text==="42", turns: [ - { role: "user", text: "Run 6 × 7 in the sandbox using the execute tool" }, - { role: "assistant", kind: "reasoning", text: "Connected to the MCP server. I'll call execute with a tiny program." }, - { role: "tool", call: { name: "tools/list", args: {} }, result: tools.map(t=>t.name), ok: true, text: tools.map(t=>t.name).join(", ") }, - { role: "tool", call: { name: "execute", args: { code: "return 6 * 7;" } }, result: res.content, ok: true, text }, - { role: "assert", kind: "toBe", actual: text, expected: "42", ok: text === "42" } ]}; - writeFileSync(`${TMP}/chat.html`, renderPlayer(run)); - const b = await chromium.launch(); const ctx = await recCtx(b, `${TMP}/v3`); const p = await ctx.newPage(); - await p.goto(`file://${TMP}/chat.html`); await p.waitForFunction("window.__done===true", { timeout: 30000 }).catch(()=>{}); await p.waitForTimeout(1000); - await ctx.close(); await b.close(); grab(`${TMP}/v3`, "3"); console.log("phase3 ✓ →", text); } - -for (const n of ["1", "2", "3"]) execSync(`ffmpeg -y -i ${TMP}/${n}.webm -vf "scale=${W}:${H}:force_original_aspect_ratio=decrease,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:color=0x0b0f17,fps=30" -pix_fmt yuv420p -c:v libx264 -preset veryfast ${TMP}/${n}.mp4 2>/dev/null`); -writeFileSync(`${TMP}/list.txt`, ["1","2","3"].map(n=>`file '${TMP}/${n}.mp4'`).join("\n")); -execSync(`ffmpeg -y -f concat -safe 0 -i ${TMP}/list.txt -c copy ${import.meta.dirname}/runs/oauth-flow.mp4 2>/dev/null`); -console.log("✓ runs/oauth-flow.mp4"); diff --git a/testkit/run.mjs b/testkit/run.mjs deleted file mode 100644 index d862833c9..000000000 --- a/testkit/run.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import { runTests } from "./src/testkit.mjs"; -import { renderTerminal } from "./src/render.mjs"; -import { readFileSync } from "node:fs"; -await import("./tests/execute.test.mjs"); -await import("./tests/x-oauth-basic.test.mjs"); -const { results } = await runTests(); -console.log("\n──────── recordings ────────"); -for (const r of results) console.log("\n" + renderTerminal(JSON.parse(readFileSync(r.file, "utf8")))); diff --git a/testkit/serve.mjs b/testkit/serve.mjs deleted file mode 100644 index bcb33644a..000000000 --- a/testkit/serve.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import http from "node:http"; import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; -const root = path.join(path.dirname(fileURLToPath(import.meta.url)), "runs"); -const TYPES = { ".html": "text/html", ".json": "application/json", ".mp4": "video/mp4", ".png": "image/png" }; -http.createServer((req, res) => { - let p = decodeURIComponent((req.url || "/").split("?")[0]); - if (p === "/" || p === "") p = "/index.html"; - const fp = path.normalize(path.join(root, p)); - if (!fp.startsWith(root)) { res.writeHead(403); return res.end(); } - let st; try { st = fs.statSync(fp); } catch { res.writeHead(404); return res.end("not found"); } - const type = TYPES[path.extname(fp)] || "application/octet-stream"; - const range = req.headers.range; - if (range) { // range support so