PR 1: wire cua-cli non-interactive paths onto CuaAgentHarness

.github/workflows/ci.yml

@@ -47,6 +47,21 @@ jobs:
       - name: Agent unit tests
         run: npm test --workspace @onkernel/cua-agent -- --exclude "**/*.live.test.ts"
 
+  cli-unit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v5
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - run: npm run build --workspace @onkernel/cua-ai
+      - run: npm run build --workspace @onkernel/cua-agent
+      - name: CLI unit tests
+        run: npm test --workspace @onkernel/cua-cli
+
   integration:
     runs-on: ubuntu-latest
     timeout-minutes: 15

packages/cua-cli/README.md

@@ -152,6 +152,12 @@ cua --print -o jsonl "open https://example.com" \
 Add `--jsonl-include-deltas` for assistant-token deltas and
 `--jsonl-include-images` for base64 screenshots in `tool_result` events.
 
+The first event of every `--print -o jsonl` run is
+`session_created` with a `schema_version` field. The current schema
+version is `1`. The `model` field carries a provider-qualified ref
+(e.g. `openai:gpt-5.5`); use `parseCuaModelRef` from `@onkernel/cua-ai`
+if you only need the bare model id.
+
 ## Sessions and transcripts
 
 `--print`, the interactive TUI, and any `-s <name>` invocation persist

packages/cua-cli/package.json

@@ -15,13 +15,16 @@
   "scripts": {
     "build": "tsc -b && chmod +x dist/cli.js",
     "clean": "tsc -b --clean",
-    "test": "node --test dist/tui/testing/*.test.js"
+    "test": "vitest --run"
   },
   "dependencies": {
+    "@earendil-works/pi-coding-agent": "0.79.1",
     "@mariozechner/pi-agent-core": "0.67.6",
     "@mariozechner/pi-ai": "0.67.6",
     "@mariozechner/pi-coding-agent": "0.67.6",
     "@mariozechner/pi-tui": "0.67.6",
+    "@onkernel/cua-agent": "0.3.3",
+    "@onkernel/cua-ai": "0.3.0",
     "@onkernel/cua-anthropic": "0.1.0",
     "@onkernel/cua-gemini": "0.1.0",
     "@onkernel/cua-openai": "0.1.0",
@@ -32,6 +35,7 @@
     "smol-toml": "1.5.1"
   },
   "devDependencies": {
-    "@onkernel/ptywright": "0.1.0"
+    "@onkernel/ptywright": "0.1.0",
+    "vitest": "^3.2.4"
   }
 }

packages/cua-cli/src/action/harness-runner.ts

@@ -0,0 +1,232 @@
+import type { AgentHarnessEvent, CuaAgentHarness, Session } from "@onkernel/cua-agent";
+import type { AssistantMessage, ImageContent } from "@onkernel/cua-ai";
+import { writeFile } from "node:fs/promises";
+import { stderr, stdout } from "node:process";
+import { captureScreenshot, type CuaBrowserHandle } from "../harness-browser";
+import { type ActionRequest, buildPrompt, DEFAULT_MAX_TURNS } from "./prompts";
+import { type ActionEventInfo, type ActionResult, exitCodeFor, formatCompact, parseResult } from "./result";
+
+export interface HarnessRunOptions {
+	harness: CuaAgentHarness;
+	browserHandle: CuaBrowserHandle;
+	session: Session;
+	/** Skip the auto-attached first-prompt screenshot (resume case). */
+	skipInitialScreenshot?: boolean;
+	maxTurns?: number;
+}
+
+export interface ScreenshotOutput {
+	out: string; // path or "-" for stdout
+}
+
+export interface RunActionResult {
+	result: ActionResult;
+	exitCode: number;
+}
+
+/**
+ * Run a single action subcommand against an existing harness + browser and
+ * return the parsed result plus exit code. The `screenshot` action is
+ * model-free — it captures directly through the SDK. All other actions
+ * drive the harness for at most `maxTurns` turns.
+ */
+export async function runAction(
+	req: ActionRequest,
+	opts: HarnessRunOptions,
+	screenshot?: ScreenshotOutput,
+): Promise<RunActionResult> {
+	const startedAt = Date.now();
+
+	if (req.action === "screenshot") {
+		const out = screenshot ?? { out: "screenshot.png" };
+		const png = await captureScreenshot(opts.browserHandle.client, opts.browserHandle.browser.session_id);
+		if (!png) {
+			const elapsed = Date.now() - startedAt;
+			const result: ActionResult = {
+				action: "screenshot",
+				status: "error",
+				text: "failed to capture screenshot",
+				elapsedMs: elapsed,
+				timestamp: Date.now(),
+			};
+			return { result, exitCode: exitCodeFor(result) };
+		}
+		if (out.out === "-") {
+			stdout.write(png);
+		} else {
+			await writeFile(out.out, png);
+		}
+		const elapsed = Date.now() - startedAt;
+		const result = parseResult("screenshot", "", [], elapsed);
+		result.text = out.out === "-" ? "(stdout)" : out.out;
+		return { result, exitCode: 0 };
+	}
+
+	const prompt = buildPrompt(req);
+	const maxTurns = req.maxTurns ?? opts.maxTurns ?? DEFAULT_MAX_TURNS;
+
+	const events: ActionEventInfo[] = [];
+	let assistantText = "";
+	let turns = 0;
+	let aborted = false;
+	let lastToolError: string | undefined;
+	let lastToolErrorDetail: string | undefined;
+
+	const unsubscribe = opts.harness.subscribe((event: AgentHarnessEvent) => {
+		switch (event.type) {
+			case "tool_execution_start":
+				collectActionEvent(event.toolName, event.args, events);
+				return;
+			case "tool_execution_end": {
+				if (event.isError) {
+					const { text, detail } = inspectToolError(event.result);
+					lastToolError = text ?? "tool execution failed";
+					lastToolErrorDetail = detail;
+				}
+				return;
+			}
+			case "message_update":
+				if (event.assistantMessageEvent.type === "text_delta") {
+					assistantText += event.assistantMessageEvent.delta;
+				}
+				return;
+			case "turn_end":
+				turns += 1;
+				if (turns >= maxTurns && !aborted) {
+					aborted = true;
+					void opts.harness.abort();
+				}
+				return;
+			default:
+				return;
+		}
+	});
+
+	let runError: Error | undefined;
+	let assistant: AssistantMessage | undefined;
+	try {
+		const images = await maybeInitialScreenshot(opts);
+		assistant = await opts.harness.prompt(prompt, images ? { images } : undefined);
+		if (assistant.stopReason === "error") {
+			runError = new Error(assistant.errorMessage ?? "agent stopped with error");
+		}
+	} catch (err) {
+		runError = err instanceof Error ? err : new Error(String(err));
+	} finally {
+		unsubscribe();
+	}
+
+	const elapsed = Date.now() - startedAt;
+
+	if (runError) {
+		const result: ActionResult = {
+			action: req.action,
+			status: "error",
+			text: runError.message,
+			elapsedMs: elapsed,
+			timestamp: Date.now(),
+		};
+		return { result, exitCode: exitCodeFor(result) };
+	}
+
+	if (!assistantText.trim() && assistant) {
+		assistantText = textFromAssistant(assistant);
+	}
+
+	const toolError = lastToolErrorDetail ?? lastToolError;
+	const result = parseResult(req.action, assistantText, events, elapsed, toolError);
+	return { result, exitCode: exitCodeFor(result) };
+}
+
+async function maybeInitialScreenshot(opts: HarnessRunOptions): Promise<ImageContent[] | undefined> {
+	if (opts.skipInitialScreenshot) return undefined;
+	const hasPriorTurn = await sessionHasPriorTurn(opts.session);
+	if (hasPriorTurn) return undefined;
+	const png = await captureScreenshot(opts.browserHandle.client, opts.browserHandle.browser.session_id);
+	if (!png) return undefined;
+	return [{ type: "image", data: png.toString("base64"), mimeType: "image/png" }];
+}
+
+async function sessionHasPriorTurn(session: Session): Promise<boolean> {
+	const entries = await session.getBranch();
+	for (const entry of entries) {
+		if (entry.type === "message" && (entry.message.role === "user" || entry.message.role === "assistant")) {
+			return true;
+		}
+	}
+	return false;
+}
+
+function textFromAssistant(message: AssistantMessage): string {
+	const parts: string[] = [];
+	for (const block of message.content) {
+		if (block && block.type === "text" && typeof block.text === "string") {
+			parts.push(block.text);
+		}
+	}
+	return parts.join("");
+}
+
+/**
+ * Collect click coordinates from canonical CUA tool calls. The harness
+ * dispatches batched calls via `computer_batch` (args: { actions: [...] })
+ * and single-action calls via per-action tools (args: cua action without
+ * the `type` field, which we recover from the tool name).
+ */
+function collectActionEvent(toolName: string, args: unknown, events: ActionEventInfo[]): void {
+	if (toolName === "computer_batch") {
+		const actions = (args as { actions?: unknown }).actions;
+		if (Array.isArray(actions)) {
+			for (const action of actions) {
+				if (action && typeof action === "object") {
+					addClickEvent(
+						(action as { type?: unknown }).type,
+						(action as { x?: unknown }).x,
+						(action as { y?: unknown }).y,
+						events,
+					);
+				}
+			}
+		}
+		return;
+	}
+	if (args && typeof args === "object") {
+		const x = (args as { x?: unknown }).x;
+		const y = (args as { y?: unknown }).y;
+		addClickEvent(toolName, x, y, events);
+	}
+}
+
+function addClickEvent(type: unknown, x: unknown, y: unknown, events: ActionEventInfo[]): void {
+	if (typeof type !== "string") return;
+	if (type !== "click" && type !== "double_click") return;
+	if (typeof x !== "number" || typeof y !== "number") return;
+	events.push({ actionType: type, x, y });
+}
+
+function inspectToolError(result: unknown): { text?: string; detail?: string } {
+	if (!result || typeof result !== "object") return {};
+	const detailsError = (result as { details?: { error?: unknown } }).details?.error;
+	const detail = typeof detailsError === "string" ? detailsError.trim() : undefined;
+	const content = (result as { content?: unknown }).content;
+	if (!Array.isArray(content)) return { detail };
+	const parts: string[] = [];
+	for (const block of content) {
+		if (block && typeof block === "object" && (block as { type?: unknown }).type === "text") {
+			const text = (block as { text?: unknown }).text;
+			if (typeof text === "string" && text.trim().length > 0) parts.push(text.trim());
+		}
+	}
+	const text = parts.length > 0 ? parts.join("\n") : undefined;
+	return { text, detail };
+}
+
+/** Print a compact result line and return its exit code. */
+export function emitCompact(res: RunActionResult): number {
+	const text = formatCompact(res.result);
+	if (text) stdout.write(`${text}\n`);
+	if (res.exitCode !== 0 && !text.startsWith("error") && res.result.status === "error") {
+		stderr.write(`error ${res.result.text ?? ""}\n`);
+	}
+	return res.exitCode;
+}