Add Grok 4.5 computer-use provider

.agents/skills/update-models/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: update-models
-description: Discover latest OpenAI, Anthropic, Google/Gemini, Meta, Tzafon, and Yutori models and verify computer-use support. Use when updating CUA model defaults, checking new model releases, auditing provider-native computer tool actions, or comparing provider metadata, official examples, and smoke-test results.
+description: Discover latest OpenAI, Anthropic, Google/Gemini, Meta, xAI, Tzafon, and Yutori models and verify computer-use support. Use when updating CUA model defaults, checking new model releases, auditing provider-native computer tool actions, or comparing provider metadata, official examples, and smoke-test results.
 ---
 
 # Update Models
@@ -9,14 +9,14 @@ Use this workflow to keep CUA current with provider model releases and computer-
 
 ## Quick Start
 
-1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `META_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`.
+1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `META_API_KEY`, `XAI_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`.
 2. If credentials live in `~/AGENTS.md`, load them into the current shell without printing them:
 
 ```bash
 eval "$(python3 - <<'PY'
 import pathlib, re, shlex
 text = pathlib.Path('~/AGENTS.md').expanduser().read_text()
-for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']:
+for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_API_KEY', 'XAI_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']:
     m = re.search(r'export\s+' + re.escape(key) + r'=(?:"([^"]+)"|([^\s\n]+))', text)
     if m:
         print(f'export {key}={shlex.quote(m.group(1) or m.group(2))}')
@@ -61,7 +61,7 @@ Treat example repos as strongest when they are provider-owned or linked from off
 
 There are two enumeration layers:
 
-- Live provider availability: `reference/discover-models.ts` uses provider APIs and docs (`OpenAI().models.list()`, `Anthropic().models.list({ limit: 1000 })`, `GoogleGenAI().models.list()` / documented Gemini computer-use IDs, Tzafon's `Lightcone().models.list()` with known-model fallback, and Yutori OpenAPI/docs model enums) to discover what the current API key can access.
+- Live provider availability: `reference/discover-models.ts` uses provider APIs and docs (`OpenAI().models.list()`, `Anthropic().models.list({ limit: 1000 })`, `GoogleGenAI().models.list()` / documented Gemini computer-use IDs, xAI's OpenAI-compatible `models.list()`, Tzafon's `Lightcone().models.list()` with known-model fallback, and Yutori OpenAPI/docs model enums) to discover what the current API key can access.
 - CUA-supported refs: `listCuaModels(provider?)` from `@onkernel/cua-ai` reads `packages/ai/src/models.ts` and returns the provider-qualified refs CUA accepts (e.g. `anthropic:claude-opus-4-7`). The `CUA_MODEL_ANNOTATIONS` table there is also what `getCuaModel()` and runtime provider routing use.
 
 When live discovery finds a new model with passing smoke tests, update `packages/ai/src/models.ts`; then verify it appears in `listCuaModels("<provider>")`.
@@ -104,6 +104,16 @@ Google/Gemini:
 - Pass condition: response contains provider-native `functionCall.name` values such as `open_web_browser`, `click_at`, or `type_text_at`.
 - Do not infer official computer-use support from CUA's custom Gemini `functionDeclarations`; those are a separate compatibility path.
 
+xAI:
+
+- Discover with the OpenAI SDK against `https://api.x.ai/v1` using `XAI_API_KEY`.
+- Record aliases, context length, standard and long-context token prices, and the 200k long-context threshold returned by `models.list()`.
+- Smoke-test the Responses API with screenshot input and explicit function tools matching CUA's canonical actions.
+- Pass condition: response output contains a `function_call` for one of the supplied browser actions.
+- Treat Grok computer use as custom-function-tool support, not a provider-native computer tool. xAI currently documents image understanding and function calling but no native coordinate protocol.
+- Use CUA's normalized 0-1000 coordinate instructions, `parallel_tool_calls: false`, `store: true`, and `previous_response_id` for browser loops. xAI accepts encrypted reasoning replay in these requests.
+- Use `reasoning: { effort: "low" }` for low-latency smoke tests; Grok 4.5 also supports `medium` and `high`, cannot disable reasoning, and defaults to `high`.
+
 Tzafon:
 
 - Discover with `new Lightcone({ apiKey }).models.list()` from `@tzafon/lightcone` when available.
@@ -126,9 +136,11 @@ Run action probes when updating adapters or when docs/examples show drift:
 
 ```bash
 npx tsx .agents/skills/update-models/reference/discover-models.ts --provider meta --models muse-spark-1.1
+npx tsx .agents/skills/update-models/reference/discover-models.ts --provider xai --models grok-4.5
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider openai --model gpt-5.5
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider anthropic --model claude-opus-4-7
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider gemini --model gemini-3-flash-preview
+npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider xai --model grok-4.5
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider tzafon --model tzafon.northstar-cua-fast
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider yutori --model n1.5-latest
 ```
@@ -147,7 +159,7 @@ Recommend a model as CUA-supported only if:
 
 - It appears in the provider metadata API for the available key.
 - Its model-specific docs do not rule out required CUA runtime features such as streaming.
-- Its provider-native computer-use smoke test passes.
+- Its provider-appropriate computer-use smoke test passes: native tools where offered, or supplied function tools for Meta and xAI.
 - Its local cua-ai smoke test emits a computer tool call: `CUA_MODEL=<provider>:<model> npm run example:quickstart --workspace @onkernel/cua-ai` returns a `toolCall` block.
 - Official docs or examples support the same tool mechanism, or the smoke result clearly supersedes stale docs.
 - The model is annotated in `CUA_MODEL_ANNOTATIONS` in `packages/ai/src/models.ts`, resolved from `pi-ai`'s registry or backed by a `CUA_MODEL_OVERRIDES` entry.
@@ -174,6 +186,7 @@ All CUA model and adapter support lives in `packages/ai` (`@onkernel/cua-ai`). W
   - OpenAI: update `packages/ai/src/providers/openai/index.ts` and its action vocabulary, plus the shared canonical types in `packages/ai/src/providers/common.ts` if the action set changes.
   - Anthropic: update the `ANTHROPIC_CUA_ACTION_TYPES` set in `packages/ai/src/providers/anthropic/actions.ts` and `index.ts`. The computer tool version and `computer-use-*` beta header are selected by `pi-ai` per model, so a new dated tool version usually means bumping `@earendil-works/pi-ai`, not editing this package.
   - Gemini: update `packages/ai/src/providers/gemini/index.ts`, including coordinate handling if needed.
+  - xAI: update `packages/ai/src/providers/xai/index.ts` and `provider.ts`, including normalized coordinate instructions, Responses threading, and reasoning compatibility.
   - Tzafon: update `packages/ai/src/providers/tzafon/index.ts` and `provider.ts`, including coordinate/action handling.
   - Yutori: update `packages/ai/src/providers/yutori/actions.ts`, `index.ts`, and `provider.ts`, including payload filtering and coordinate/action handling.
   - Shared canonical action semantics go in `packages/ai/src/providers/common.ts`.

.agents/skills/update-models/reference/README.md

@@ -12,10 +12,11 @@ These scripts support the `update-models` skill. Run them from the repository ro
   - `ANTHROPIC_API_KEY`
   - `GOOGLE_API_KEY` or `GEMINI_API_KEY`
   - `META_API_KEY`
+  - `XAI_API_KEY`
   - `TZAFON_API_KEY`
   - `YUTORI_API_KEY`
 
-The scripts never print API keys. Smoke tests are non-destructive: they ask each model to emit a computer-use tool call, then inspect the response without executing the action. Meta uses supplied function tools; other providers may use provider-native computer tools.
+The scripts never print API keys. Smoke tests are non-destructive: they ask each model to emit a computer-use tool call, then inspect the response without executing the action. Meta and xAI use supplied function tools; other providers may use provider-native computer tools.
 
 ## Common Commands
 
@@ -29,6 +30,7 @@ Probe native action vocabularies for a specific provider/model:
 
 ```bash
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider openai --model gpt-5.5 --out /tmp/openai-actions.json
+npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider xai --model grok-4.5 --out /tmp/xai-actions.json
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider tzafon --model tzafon.northstar-cua-fast --out /tmp/tzafon-actions.json
 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider yutori --model n1.5-latest --out /tmp/yutori-actions.json
 ```

.agents/skills/update-models/reference/audit-official-examples.ts

@@ -5,7 +5,7 @@ import { basename, join, resolve } from "node:path";
 import { spawnSync } from "node:child_process";
 import process from "node:process";
 
-type Provider = "openai" | "anthropic" | "gemini" | "meta" | "yutori";
+type Provider = "openai" | "anthropic" | "gemini" | "meta" | "xai" | "yutori";
 
 interface ExampleRepo {
 	provider: Provider;
@@ -59,6 +59,14 @@ const EXAMPLES: ExampleRepo[] = [
 		pathHint: "03_use_cases/13_macos_cua",
 		patterns: ["muse-spark-1.1", "function_call", "function_call_output", "previous_response_id", "reasoning.encrypted_content", "parallel_tool_calls"],
 	},
+	{
+		provider: "xai",
+		name: "xai-sdk-python",
+		repo: "https://github.com/xai-org/xai-sdk-python.git",
+		confidence: "provider-owned",
+		pathHint: "examples",
+		patterns: ["grok-4.5", "function_call", "tool_call", "previous_response_id", "reasoning_effort", "parallel_tool_calls"],
+	},
 	{
 		provider: "yutori",
 		name: "kernel-cli-yutori-template",
@@ -74,6 +82,7 @@ const ACTION_REGEXES: Record<Provider, RegExp[]> = {
 	anthropic: [/\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up|cursor_position|zoom)\b/g],
 	gemini: [/\b(open_web_browser|open_web|wait_5_seconds|go_back|go_forward|search|navigate|click_at|hover_at|type_text_at|key_combination|scroll_document|scroll_at|drag_and_drop)\b/g],
 	meta: [/\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up)\b/g],
+	xai: [/\b(screenshot|click|double_click|mouse_down|mouse_up|scroll|type|keypress|drag|move|wait)\b/g],
 	yutori: [/\b(left_click|double_click|triple_click|right_click|scroll|type|key_press|hover|drag|wait|refresh|go_back|goto_url|mouse_move|middle_click|mouse_down|mouse_up|go_forward|hold_key|extract_elements|find|set_element_value|execute_js)\b/g],
 };
 
@@ -156,7 +165,7 @@ async function auditRepo(example: ExampleRepo, cacheDir: string, args: Args): Pr
 		for (const regex of ACTION_REGEXES[example.provider] ?? []) {
 			extractAll(text, regex).forEach((v) => actionNames.add(v));
 		}
-		for (const field of ["computer_call", "actions", "action", "pending_safety_checks", "tool_use", "tool_result", "tool_calls", "function_call", "functionCall", "FunctionResponse", "safety_decision"]) {
+		for (const field of ["computer_call", "actions", "action", "pending_safety_checks", "tool_use", "tool_result", "tool_calls", "function_call", "functionCall", "FunctionResponse", "safety_decision", "previous_response_id", "parallel_tool_calls", "reasoning_effort"]) {
 			if (text.includes(field)) responseFields.add(field);
 		}
 

.agents/skills/update-models/reference/discover-models.ts

@@ -6,7 +6,7 @@ import { join } from "node:path";
 import process from "node:process";
 import { parse as parseToml } from "smol-toml";
 
-type Provider = "openai" | "anthropic" | "gemini" | "meta" | "tzafon" | "yutori";
+type Provider = "openai" | "anthropic" | "gemini" | "meta" | "xai" | "tzafon" | "yutori";
 
 interface Args {
 	provider: Provider | "all";
@@ -39,7 +39,7 @@ interface ModelResult {
 	cua?: Record<string, unknown>;
 }
 
-const PROVIDERS: Provider[] = ["openai", "anthropic", "gemini", "meta", "tzafon", "yutori"];
+const PROVIDERS: Provider[] = ["openai", "anthropic", "gemini", "meta", "xai", "tzafon", "yutori"];
 const TZAFON_KNOWN_MODELS = [
 	"tzafon.northstar-cua-fast",
 ];
@@ -110,7 +110,7 @@ function usage(): never {
   npx tsx .agents/skills/update-models/reference/discover-models.ts --provider openai --models gpt-5.5,gpt-5.4
 
 Options:
-  --provider <all|openai|anthropic|gemini|meta|tzafon|yutori>
+  --provider <all|openai|anthropic|gemini|meta|xai|tzafon|yutori>
   --models <comma-separated model ids>    Smoke-test explicit models instead of inferred candidates.
   --candidate-limit <n>                  Max inferred candidates per provider. Default: 20.
   --no-smoke                             Only list metadata.
@@ -139,6 +139,7 @@ async function runProvider(provider: Provider, args: Args): Promise<Record<strin
 		if (provider === "anthropic") return await discoverAnthropic(args);
 		if (provider === "gemini") return await discoverGemini(args);
 		if (provider === "meta") return await discoverMeta(args);
+		if (provider === "xai") return await discoverXai(args);
 		if (provider === "tzafon") return await discoverTzafon(args);
 		if (provider === "yutori") return await discoverYutori(args);
 		throw new Error(`unknown provider ${provider satisfies never}`);
@@ -203,12 +204,7 @@ async function discoverMeta(args: Args): Promise<Record<string, unknown>> {
 
 async function smokeMeta(client: any, model: string): Promise<SmokeResult> {
 	try {
-		const screenshotPath = [
-			join(process.cwd(), "examples", "screenshot.png"),
-			join(process.cwd(), "packages", "ai", "examples", "screenshot.png"),
-		].find(existsSync);
-		if (!screenshotPath) throw new Error("could not find packages/ai/examples/screenshot.png");
-		const screenshot = await readFile(screenshotPath);
+		const screenshot = await readFile(fixtureScreenshotPath());
 		const response = await client.responses.create({
 			model,
 			store: false,
@@ -250,6 +246,100 @@ async function smokeMeta(client: any, model: string): Promise<SmokeResult> {
 	}
 }
 
+async function discoverXai(args: Args): Promise<Record<string, unknown>> {
+	const OpenAI = await importDefault("openai", "OpenAI");
+	const client = new OpenAI({ apiKey: process.env.XAI_API_KEY, baseURL: "https://api.x.ai/v1" });
+	const rawModels = await collectAsync(client.models.list());
+	const models: ModelResult[] = rawModels.map((m) => ({
+		id: String(m.id),
+		display_name: String(m.id),
+		created_at: typeof m.created === "number" && m.created > 0 ? new Date(m.created * 1000).toISOString() : null,
+		raw: m,
+		supports_generation: likelyXaiGenerationModel(String(m.id)),
+		model_docs: {
+			url: String(m.id) === "grok-4.5" ? "https://docs.x.ai/developers/grok-4-5" : "https://docs.x.ai/developers/models",
+			responses_endpoint: "supported",
+			function_calling: "supported",
+			image_input: "verify-per-model",
+			coordinate_space: "CUA-defined 0-1000",
+		},
+	}));
+	const candidates = explicitOrCandidates(
+		args,
+		models
+			.filter((model) => model.supports_generation)
+			.sort(compareXaiCandidates)
+			.map((model) => model.id),
+	);
+	if (args.smoke) {
+		await Promise.all(candidates.map(async (id) => {
+			const model = models.find((candidate) => candidate.id === id) ?? { id, display_name: id, supports_generation: true };
+			model.computer_use = await smokeXai(client, id);
+			if (!models.find((candidate) => candidate.id === id)) models.unshift(model);
+		}));
+	}
+	await annotateCuaSupport("xai", models);
+	return { provider: "xai", metadata_source: "xAI models.list()", models, candidates };
+}
+
+async function smokeXai(client: any, model: string): Promise<SmokeResult> {
+	try {
+		const screenshot = await readFile(fixtureScreenshotPath());
+		const response = await client.responses.create({
+			model,
+			store: true,
+			parallel_tool_calls: false,
+			max_output_tokens: 768,
+			reasoning: { effort: "low" },
+			instructions: "Coordinates are normalized from 0 to 1000 relative to the screenshot.",
+			input: [{
+				role: "user",
+				content: [
+					{ type: "input_text", text: "Call the click tool for the sign in link. Do not answer only in text." },
+					{ type: "input_image", image_url: `data:image/png;base64,${screenshot.toString("base64")}`, detail: "high" },
+				],
+			}],
+			tools: [{
+				type: "function",
+				name: "click",
+				description: "Click at normalized 0-1000 screen coordinates.",
+				parameters: {
+					type: "object",
+					properties: { x: { type: "number" }, y: { type: "number" } },
+					required: ["x", "y"],
+					additionalProperties: false,
+				},
+			}],
+		});
+		const output: any[] = response.output ?? [];
+		const calls = output.filter((item) => item?.type === "function_call");
+		return {
+			status: calls.length > 0 ? "pass" : "inconclusive",
+			tool_name: "function_tools",
+			tool_version: null,
+			beta_header: null,
+			observed_actions: unique(calls.map((call) => call?.name).filter(Boolean)),
+			response_item_types: unique(output.map((item) => item?.type).filter(Boolean)),
+			reasoning_effort: "low",
+			coordinate_space: "0-1000",
+			error: null,
+		};
+	} catch (err) {
+		return smokeError(err, { tool_name: "function_tools" });
+	}
+}
+
+function likelyXaiGenerationModel(id: string): boolean {
+	const lower = id.toLowerCase();
+	return lower.startsWith("grok-") && !lower.includes("imagine");
+}
+
+function compareXaiCandidates(a: ModelResult, b: ModelResult): number {
+	if (a.id === "grok-4.5") return -1;
+	if (b.id === "grok-4.5") return 1;
+	return String(b.created_at ?? "").localeCompare(String(a.created_at ?? ""));
+}
+
 function likelyOpenAIGenerationModel(id: string): boolean {
 	const lower = id.toLowerCase();
 	if (OPENAI_EXCLUDE.some((needle) => lower.includes(needle))) return false;
@@ -750,9 +840,9 @@ const TZAFON_FUNCTION_TOOLS = [
 
 async function annotateCuaSupport(provider: Provider, models: ModelResult[]): Promise<void> {
 	const piProvider = provider === "gemini" ? "google" : provider;
-	const getModel = await import("@earendil-works/pi-ai").then((mod) => mod.getModel).catch(() => undefined);
+	const getBuiltinModel = await import("@earendil-works/pi-ai/providers/all").then((mod) => mod.getBuiltinModel).catch(() => undefined);
 	for (const model of models) {
-		const inRegistry = getModel ? !!getModel(piProvider as never, model.id as never) : false;
+		const inRegistry = getBuiltinModel ? !!getBuiltinModel(piProvider as never, model.id as never) : false;
 		const localAdapterSupport = localAdapterSupportStatus(provider, model);
 		model.cua = {
 			provider_inference: provider,
@@ -889,6 +979,15 @@ function unique<T>(values: T[]): T[] {
 	return [...new Set(values)];
 }
 
+function fixtureScreenshotPath(): string {
+	const path = [
+		join(process.cwd(), "examples", "screenshot.png"),
+		join(process.cwd(), "packages", "ai", "examples", "screenshot.png"),
+	].find(existsSync);
+	if (!path) throw new Error("could not find packages/ai/examples/screenshot.png");
+	return path;
+}
+
 async function emitJson(value: unknown, outPath: string): Promise<void> {
 	const text = `${JSON.stringify(value, null, 2)}\n`;
 	if (outPath) await writeFile(outPath, text);