From 51d13c155246f37f81b656528af23e2ada4cbf22 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 3 Aug 2026 12:37:59 +0100 Subject: [PATCH 1/6] (MOT-4001) feat(computer): add the computer worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full-desktop computer use on the bus, the sibling of the browser worker: `computer::*` starts a desktop session, screenshots it, and clicks, types, scrolls and drags by coordinate, so a model that can see an image can operate a GUI with no glue in between. One `Driver` trait, three ways to reach a desktop, picked per session: this machine (capture and input built in), a desktop booted inside an iii-sandbox microVM and driven through `sandbox::exec` alone, or a remote desktop reached through the executor inside it. Sessions are durable — mirrored into `state` and reconnected best-effort on boot — and a screencast pump keeps `computer:frames` fed so watchers follow the screen without polling. Sibling workers bind `computer::session-started` / `session-stopped` instead of asking. The worker also ships its own console page (`#/ext/computer`): a session rail, a live viewport, and click/type/scroll forwarding, plus how every `computer::*` call renders in chat. Injected at registration, so no console rebuild. macOS degrades screen capture and synthetic input silently rather than failing, so the native driver checks both permissions and fails loud instead of handing back a wallpaper-only screenshot or a click that never landed. --- computer/Cargo.lock | 3908 +++++++++++++++++ computer/Cargo.toml | 46 + computer/README.md | 169 + computer/architecture/README.md | 48 + computer/architecture/integration.md | 68 + computer/architecture/internals.md | 82 + computer/build.rs | 179 + computer/iii.worker.yaml | 8 + computer/images/desktop/Dockerfile | 57 + computer/images/desktop/README.md | 61 + computer/skills/SKILL.md | 92 + computer/src/config.rs | 184 + computer/src/configuration.rs | 143 + computer/src/driver/mod.rs | 133 + computer/src/driver/native.rs | 533 +++ computer/src/driver/remote.rs | 323 ++ computer/src/driver/sandbox.rs | 498 +++ computer/src/events.rs | 404 ++ computer/src/functions/act.rs | 51 + computer/src/functions/displays.rs | 15 + computer/src/functions/frame.rs | 51 + computer/src/functions/mod.rs | 505 +++ computer/src/functions/observe.rs | 36 + computer/src/functions/screenshot.rs | 61 + computer/src/functions/sessions.rs | 81 + computer/src/lib.rs | 12 + computer/src/main.rs | 168 + computer/src/manifest.rs | 47 + computer/src/session.rs | 621 +++ computer/src/ui.rs | 71 + .../tests/golden/schemas/computer.act.json | 121 + .../golden/schemas/computer.displays.json | 68 + .../tests/golden/schemas/computer.frame.json | 76 + .../golden/schemas/computer.observe.json | 119 + .../schemas/computer.screencast.start.json | 30 + .../schemas/computer.screencast.stop.json | 31 + .../golden/schemas/computer.screenshot.json | 97 + .../schemas/computer.sessions.list.json | 85 + .../schemas/computer.sessions.start.json | 99 + .../schemas/computer.sessions.stop.json | 36 + computer/tests/manifest.rs | 28 + computer/tests/schemas.rs | 92 + computer/tests/support/mod.rs | 117 + computer/ui/build.mjs | 37 + computer/ui/package.json | 19 + computer/ui/page.tsx | 33 + .../ComputerViews.tsx | 155 + .../ui/src/function-trigger-message/index.tsx | 137 + computer/ui/src/lib/cn.ts | 8 + computer/ui/src/lib/computer.ts | 311 ++ computer/ui/src/lib/errors.ts | 10 + computer/ui/src/lib/events.ts | 137 + computer/ui/src/lib/format.ts | 24 + computer/ui/src/page/SessionRail.tsx | 81 + computer/ui/src/page/StartSessionForm.tsx | 107 + computer/ui/src/page/Viewport.tsx | 234 + computer/ui/src/page/index.tsx | 201 + computer/ui/src/page/useLiveFrames.ts | 130 + computer/ui/src/page/useSessionsLive.ts | 77 + computer/ui/styles.css | 310 ++ computer/ui/tsconfig.json | 14 + pnpm-lock.yaml | 19 + pnpm-workspace.yaml | 1 + 63 files changed, 11699 insertions(+) create mode 100644 computer/Cargo.lock create mode 100644 computer/Cargo.toml create mode 100644 computer/README.md create mode 100644 computer/architecture/README.md create mode 100644 computer/architecture/integration.md create mode 100644 computer/architecture/internals.md create mode 100644 computer/build.rs create mode 100644 computer/iii.worker.yaml create mode 100644 computer/images/desktop/Dockerfile create mode 100644 computer/images/desktop/README.md create mode 100644 computer/skills/SKILL.md create mode 100644 computer/src/config.rs create mode 100644 computer/src/configuration.rs create mode 100644 computer/src/driver/mod.rs create mode 100644 computer/src/driver/native.rs create mode 100644 computer/src/driver/remote.rs create mode 100644 computer/src/driver/sandbox.rs create mode 100644 computer/src/events.rs create mode 100644 computer/src/functions/act.rs create mode 100644 computer/src/functions/displays.rs create mode 100644 computer/src/functions/frame.rs create mode 100644 computer/src/functions/mod.rs create mode 100644 computer/src/functions/observe.rs create mode 100644 computer/src/functions/screenshot.rs create mode 100644 computer/src/functions/sessions.rs create mode 100644 computer/src/lib.rs create mode 100644 computer/src/main.rs create mode 100644 computer/src/manifest.rs create mode 100644 computer/src/session.rs create mode 100644 computer/src/ui.rs create mode 100644 computer/tests/golden/schemas/computer.act.json create mode 100644 computer/tests/golden/schemas/computer.displays.json create mode 100644 computer/tests/golden/schemas/computer.frame.json create mode 100644 computer/tests/golden/schemas/computer.observe.json create mode 100644 computer/tests/golden/schemas/computer.screencast.start.json create mode 100644 computer/tests/golden/schemas/computer.screencast.stop.json create mode 100644 computer/tests/golden/schemas/computer.screenshot.json create mode 100644 computer/tests/golden/schemas/computer.sessions.list.json create mode 100644 computer/tests/golden/schemas/computer.sessions.start.json create mode 100644 computer/tests/golden/schemas/computer.sessions.stop.json create mode 100644 computer/tests/manifest.rs create mode 100644 computer/tests/schemas.rs create mode 100644 computer/tests/support/mod.rs create mode 100644 computer/ui/build.mjs create mode 100644 computer/ui/package.json create mode 100644 computer/ui/page.tsx create mode 100644 computer/ui/src/function-trigger-message/ComputerViews.tsx create mode 100644 computer/ui/src/function-trigger-message/index.tsx create mode 100644 computer/ui/src/lib/cn.ts create mode 100644 computer/ui/src/lib/computer.ts create mode 100644 computer/ui/src/lib/errors.ts create mode 100644 computer/ui/src/lib/events.ts create mode 100644 computer/ui/src/lib/format.ts create mode 100644 computer/ui/src/page/SessionRail.tsx create mode 100644 computer/ui/src/page/StartSessionForm.tsx create mode 100644 computer/ui/src/page/Viewport.tsx create mode 100644 computer/ui/src/page/index.tsx create mode 100644 computer/ui/src/page/useLiveFrames.ts create mode 100644 computer/ui/src/page/useSessionsLive.ts create mode 100644 computer/ui/styles.css create mode 100644 computer/ui/tsconfig.json diff --git a/computer/Cargo.lock b/computer/Cargo.lock new file mode 100644 index 000000000..0f8ab06e7 --- /dev/null +++ b/computer/Cargo.lock @@ -0,0 +1,3908 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey", + "rayon", + "thiserror 2.0.18", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2 0.5.2", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2 0.6.4", +] + +[[package]] +name = "built" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "computer" +version = "0.1.0" +dependencies = [ + "anyhow", + "arc-swap", + "async-trait", + "base64", + "clap", + "enigo", + "futures-util", + "iii-console-ui", + "iii-sdk", + "image 0.25.10", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "tokio-tungstenite 0.24.0", + "tracing", + "tracing-subscriber", + "xcap", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" +dependencies = [ + "bitflags 2.13.0", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.13.0", + "core-foundation", + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dispatch2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a0d569e003ff27784e0e14e4a594048698e0c0f0b66cabcb51511be55a7caa0" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "libc", + "objc2 0.6.4", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "objc2 0.6.4", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "enigo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cf6f550bbbdd5fe66f39d429cb2604bcdacbf00dca0f5bbe2e9306a0009b7c6" +dependencies = [ + "core-foundation", + "core-graphics", + "foreign-types-shared", + "libc", + "log", + "objc2 0.5.2", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", + "windows 0.58.0", + "xkbcommon", + "xkeysym", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "exr" +version = "1.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link 0.2.1", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "iii-console-ui" +version = "0.1.0" +dependencies = [ + "iii-sdk", + "schemars", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "iii-helpers" +version = "0.21.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bdc7bbc3abfde934a62cdc5d3045adf52914dfc1ed6c20f8af691fc561dc55" +dependencies = [ + "futures-util", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "reqwest", + "schemars", + "serde", + "serde_json", + "sysinfo", + "tokio", + "tokio-tungstenite 0.28.0", + "tracing", + "uuid", +] + +[[package]] +name = "iii-sdk" +version = "0.21.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07dd060fddcc9153b0dd07c038a14cf172ce15ce1d4edb98155563ed55b2caba" +dependencies = [ + "async-trait", + "futures-util", + "hostname", + "iii-helpers", + "reqwest", + "schemars", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite 0.28.0", + "tracing", + "uuid", +] + +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "num-traits", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40fac9d56ed6437b198fddba683305e8e2d651aa42647f00f5ae542e7f5c94a2" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "pkg-config", +] + +[[package]] +name = "libfuzzer-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libwayshot" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2efa01ecfd021b1e7db27f21f4e79b35b048081c9cae9d2f898eddc98444d69" +dependencies = [ + "image 0.24.9", + "log", + "memmap2", + "nix", + "thiserror 1.0.69", + "wayland-client", + "wayland-protocols", + "wayland-protocols-wlr", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "libc", +] + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.13.0", + "block2 0.5.1", + "libc", + "objc2 0.5.2", + "objc2-core-data 0.2.2", + "objc2-core-image 0.2.2", + "objc2-foundation 0.2.2", + "objc2-quartz-core 0.2.2", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "libc", + "objc2 0.6.4", + "objc2-cloud-kit", + "objc2-core-data 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image 0.3.2", + "objc2-core-text", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "dispatch2 0.3.1", + "objc2 0.6.4", + "objc2-avf-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image 0.3.2", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-image-io", + "objc2-media-toolbox", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2 0.3.1", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.13.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-data" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "dispatch2 0.3.1", + "libc", + "objc2 0.6.4", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "dispatch2 0.3.1", + "libc", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-io-surface", + "objc2-metal 0.3.2", +] + +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal 0.2.2", +] + +[[package]] +name = "objc2-core-image" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "dispatch2 0.3.1", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + +[[package]] +name = "objc2-core-text" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", + "objc2-metal 0.3.2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.13.0", + "block2 0.5.1", + "libc", + "objc2 0.5.2", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "libc", + "objc2 0.6.4", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-image-io" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b0446e98cf4a784cc7a0177715ff317eeaa8463841c616cfc78aa4f953c4ea" +dependencies = [ + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-media-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd9fdde720df3da7046bb9097811000c1e7ab5cd579fa89d96b27d56781fb30" +dependencies = [ + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-media", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.13.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.13.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal 0.2.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand 0.9.5", + "thiserror 2.0.18", + "tokio", + "tokio-stream", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.0", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.5", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror 2.0.18", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex-automata" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.62.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.24.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite 0.28.0", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.0", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.7", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wayland-backend" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" +dependencies = [ + "bitflags 2.13.0", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f81f365b8b4a97f422ac0e8737c438024b5951734506b0e1d775c73030561f4" +dependencies = [ + "bitflags 2.13.0", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1f61b76b6c2d8742e10f9ba5c3737f6530b4c243132c2a2ccc8aa96fe25cd6" +dependencies = [ + "bitflags 2.13.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" +dependencies = [ + "proc-macro2", + "quick-xml 0.39.4", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets", +] + +[[package]] +name = "windows" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf874e74c7a99773e62b1c671427abf01a425e77c3d3fb9fb1e4883ea934529" +dependencies = [ + "windows-collections 0.1.1", + "windows-core 0.60.1", + "windows-future 0.1.1", + "windows-link 0.1.3", + "windows-numerics 0.1.1", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", +] + +[[package]] +name = "windows-collections" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5467f79cc1ba3f52ebb2ed41dbb459b8e7db636cc3429458d9a852e15bc24dec" +dependencies = [ + "windows-core 0.60.1", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.60.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" +dependencies = [ + "windows-implement 0.59.0", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.3.1", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" +dependencies = [ + "windows-core 0.60.1", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" +dependencies = [ + "windows-core 0.60.1", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xcap" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd25cdb442bb7f63f13fdee2f59d991b04668d37c69aee00dc2a1cc9d0e9a1" +dependencies = [ + "dbus", + "dispatch2 0.2.0", + "image 0.25.10", + "lazy_static", + "libwayshot", + "log", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", + "percent-encoding", + "scopeguard", + "thiserror 2.0.18", + "widestring", + "windows 0.60.0", + "xcb", +] + +[[package]] +name = "xcb" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4c580d8205abb0a5cf4eb7e927bd664e425b6c3263f9c5310583da96970cf6" +dependencies = [ + "bitflags 1.3.2", + "libc", + "quick-xml 0.30.0", +] + +[[package]] +name = "xkbcommon" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d66ca9352cbd4eecbbc40871d8a11b4ac8107cfc528a6e14d7c19c69d0e1ac9" +dependencies = [ + "libc", + "memmap2", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/computer/Cargo.toml b/computer/Cargo.toml new file mode 100644 index 000000000..f4a64ea02 --- /dev/null +++ b/computer/Cargo.toml @@ -0,0 +1,46 @@ +[workspace] + +[package] +name = "computer" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "computer" +path = "src/main.rs" + +[lib] +name = "computer" +path = "src/lib.rs" + +[dependencies] +iii-sdk = "=0.21.6" +iii-console-ui = { path = "../crates/console-ui" } +arc-swap = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } +tokio-tungstenite = "0.24" +futures-util = "0.3" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +clap = { version = "4", features = ["derive"] } +schemars = "0.8" +async-trait = "0.1" +base64 = "0.22" + +# Native host backend (screen capture + input injection). Only meaningful on a +# desktop OS, and these pull system libraries (dbus/xcb/xdo) that headless Linux +# servers and CI do not have, so they are gated to macOS/Windows. Other targets +# build with the computer-server backend only. +[target.'cfg(any(target_os = "macos", target_os = "windows"))'.dependencies] +enigo = "0.3" +xcap = "0.4" +image = "0.25" + +[dev-dependencies] +serde_json = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } diff --git a/computer/README.md b/computer/README.md new file mode 100644 index 000000000..45f01c7ab --- /dev/null +++ b/computer/README.md @@ -0,0 +1,169 @@ +# computer + +Drive a full desktop on the iii bus. Where the +[browser](https://github.com/iii-hq/workers/tree/main/browser) worker gives an +agent a Chromium tab, `computer` gives it a whole screen: it hands the model a +screenshot and clicks, types, and scrolls by coordinate. The harness discovers +`computer::*` as functions automatically, so a model sees the screen and acts on +it with no glue, and the console gets a live viewport of whatever the desktop is +doing. + +The worker does the one thing no other worker can: capture the screen and move +the cursor. Three drivers sit behind one surface, picked per session: + +- **native** (no `endpoint`, no `image`): drive the local machine this worker + runs on, capture and input injection built in. +- **sandbox** (an `image`): boot a fresh desktop inside an + [iii-sandbox](https://github.com/iii-hq/workers/tree/main/iii-sandbox) microVM + and drive it through iii primitives alone (`sandbox::exec` / `sandbox::fs`), + with no socket into the guest. A fixed virtual display means 1:1 coordinates, + no HiDPI or multi-monitor ambiguity, and nothing to grant on the host. +- **remote** (an `endpoint`): drive an already-running desktop through the + executor inside it, over a WebSocket. `computer` connects; that desktop boots + out of band. + +Everything else composes with workers that already exist: run commands and touch +files with the [shell](https://github.com/iii-hq/workers/tree/main/shell) +worker, persist with `state`, schedule with `cron`. + +## Install + +```bash +iii worker add computer +``` + +To drive a throwaway desktop instead of your own machine, add the sandbox worker +too — it boots the microVM the `image` driver runs in: + +```bash +iii worker add iii-sandbox +``` + +**macOS, native driver only.** Capture and input are permission-gated and macOS +degrades both silently, so the worker checks and fails loud. Grant the app that +runs the worker **Screen Recording** (without it every screenshot is the +wallpaper with all windows stripped) and **Accessibility** (without it clicks and +typing are dropped) in System Settings > Privacy & Security, then restart it. The +sandbox driver needs neither. + +## Quickstart + +Start a session, look at the screen, click what you see: + +```bash +# drive this machine (native driver) +iii trigger computer::sessions::start --json '{}' +# -> { "session_id": "c1", "endpoint": "native", "os": "macos", "screen": { "width": 1512, "height": 945 } } + +# see the screen: an image block the vision model renders inline +iii trigger computer::screenshot --json '{"session_id":"c1"}' + +# click a pixel read off that screenshot (top-left origin) +iii trigger computer::act --json '{"session_id":"c1","action":"click","x":640,"y":360}' + +iii trigger computer::sessions::stop --json '{"session_id":"c1"}' +``` + +Swap the first call for `{"image":"desktop"}` to boot a sandboxed Linux desktop +instead (see [Sandbox desktop image](#sandbox-desktop-image)), or +`{"endpoint":"ws://host:8000"}` to drive a remote one. Everything after that call +is identical: the session id is the only handle. + +The same from a sibling worker: + +```rust +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::{register_worker, InitOptions}; +use serde_json::json; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let iii = register_worker("ws://localhost:49134", InitOptions::default()); + + let started = iii + .trigger(TriggerRequest { + function_id: "computer::sessions::start".into(), + payload: json!({}), + action: None, + timeout_ms: Some(30_000), + }) + .await?; + let session_id = started["session_id"].as_str().unwrap(); + + iii.trigger(TriggerRequest { + function_id: "computer::act".into(), + payload: json!({ "session_id": session_id, "action": "click", "x": 640, "y": 360 }), + action: None, + timeout_ms: Some(10_000), + }) + .await?; + + Ok(()) +} +``` + +The rest of the surface: `computer::act` (click, right_click, double_click, +move, drag, scroll, type, press, hotkey — all by coordinate), +`computer::observe` (screenshot plus the accessibility tree on macOS guests), +`computer::displays`, `computer::sessions::list` and `computer::sessions::stop`. +For running commands or reading and writing files on the desktop, use the +[shell](https://github.com/iii-hq/workers/tree/main/shell) worker. + +## Console + +The worker ships its own console page (`#/ext/computer`): a session rail, a live +viewport fed by the screencast stream, and click / type / scroll forwarding +straight into the desktop. It is injected into any running console at +registration time — nothing to install, nothing to rebuild. Every `computer::*` +call in chat and traces renders through the same asset. + +## Configuration + +Stored in the `configuration` worker under the `computer` key; every field is +editable live from the console. Timeouts and the screencast rate hot-reload; +`default_endpoint` and `os` apply to sessions started after the change. + +```yaml +computer: + default_endpoint: '' # guest-executor endpoint for a remote desktop; empty = drive the local machine + os: linux # guest OS label recorded on sessions + max_sessions: 2 # concurrent desktop connections + idle_stop_ms: 300000 # stop sessions idle this long; 0 disables + screencast_fps: 15 # live-view frame rate cap + max_screenshot_dimension: 1280 # downscale screenshots/frames to this longest edge (native) + screenshot_quality: 70 # JPEG quality 1-100 (native) + command_timeout_ms: 120000 # timeout for each driver action (fixed at connect) + connect_timeout_ms: 15000 # driver connect timeout at session start + sandbox_image: '' # iii-sandbox image for a sandbox session; empty = name it per call + sandbox_width: 1280 # sandbox virtual display width + sandbox_height: 800 # sandbox virtual display height + sandbox_idle_timeout_secs: 86400 # idle_timeout_secs for sandbox::create; kept high, the worker owns teardown + screen_capture_preflight: true # macOS: ask for Screen Recording at native start, fail loud if denied +``` + +## Sandbox desktop image + +The sandbox driver needs a desktop inside the guest (Xvfb, xdotool, imagemagick, +openbox). A prebaked image lives in [`images/desktop`](images/desktop): build it, +push it, register it as a `custom_images` entry on the iii-sandbox worker, then +start a session with `{ "image": "desktop" }`. iii-sandbox boots on macOS Apple +Silicon (libkrun) and Linux (`/dev/kvm`). + +## Custom trigger types + +Sibling workers (and the console page) subscribe to session activity instead of +polling. Both bindings accept an optional `{ "session_id": "..." }` filter. + +| Trigger type | Fires when | Payload to subscribers | +|---|---|---| +| `computer::session-started` | A session connected and is ready | `{ session_id, endpoint, os, screen, timestamp }` | +| `computer::session-stopped` | A session ended | `{ session_id, reason: "stopped" \| "idle", timestamp }` | + +## Live screen + +`computer::screencast::start` / `stop` and `computer::frame` drive the live +viewport: the worker captures at `screencast_fps` and pushes the newest frame +onto the `computer:frames` stream (one item per session), so the console and any +number of watchers follow the desktop without polling. These three are internal +console plumbing rather than agent surface, and stay out of agent function +lists. diff --git a/computer/architecture/README.md b/computer/architecture/README.md new file mode 100644 index 000000000..4e2a046c2 --- /dev/null +++ b/computer/architecture/README.md @@ -0,0 +1,48 @@ +# computer — architecture + +One desktop, three ways to reach it, behind one function surface. The worker is +a capability primitive — it captures pixels and moves a cursor. Everything an +agentic computer-use product needs around that (the model loop, approval, +transcripts, tracing) belongs to the engine and the harness, not here. + +```text + agent / console worker the desktop + ──────────────── ────── ─────────── + computer::act ──────▶ functions/ ─▶ session ─▶ Driver ─┬─▶ native (this machine) + computer::screenshot │ ├─▶ sandbox (microVM) + │ └─▶ remote (guest executor) + console viewport ◀── computer:frames ◀──┘ screencast pump + sibling workers ◀── session-started / session-stopped +``` + +## Module map + +| Module | Role | +|---|---| +| `config.rs` | `WorkerConfig` schema (endpoint, session cap, timeouts, capture limits, sandbox display) and its shared hot-reloadable handle | +| `configuration.rs` | `configuration::register` + the `computer::on-config-change` trigger | +| `driver/mod.rs` | The `Driver` trait: the whole desktop semantic (capture, click, type, keypress, a11y tree, close) | +| `driver/native.rs` | This machine: `xcap` capture + `enigo` input, display pinning, downscale, macOS permission gates | +| `driver/sandbox.rs` | A desktop in an iii-sandbox microVM, driven through `sandbox::exec` / `sandbox::fs` | +| `driver/remote.rs` | A desktop reached through its guest executor over a WebSocket | +| `session.rs` | Session registry, driver selection, durable records in `state`, the screencast pump | +| `events.rs` | `computer::session-started` / `session-stopped` trigger types and their subscriber fan-out | +| `functions/` | The `computer::*` wire surface, one module per function plus the golden-tested catalog | +| `ui.rs` + `ui/` | The injected console page and chat renderer (see the injectable-console-UI SOP) | + +## Vocabulary + +| Term | Means | +|---|---| +| driver | One implementation of `Driver` — how this session reaches its desktop | +| session | A live desktop plus its id, screen size, and screencast state | +| screen | Desktop pixel dimensions; the coordinate space `act` and screenshots share | +| frame | One screencast capture, pushed onto `computer:frames` under the session id | + +## Doc map + +| Doc | For | +|---|---| +| [`internals.md`](internals.md) | Changing the worker: driver selection, capture pipeline, durability, permission gates | +| [`integration.md`](integration.md) | Calling the worker: function ids, trigger types, the guardrail split | +| [`../README.md`](../README.md) | Operating the worker: install, quickstart, configuration | diff --git a/computer/architecture/integration.md b/computer/architecture/integration.md new file mode 100644 index 000000000..25be75a5c --- /dev/null +++ b/computer/architecture/integration.md @@ -0,0 +1,68 @@ +# computer — integration + +For calling the worker from an agent, a sibling worker, or the console. + +## Functions + +Every call after `sessions::start` takes the `session_id` it returned. + +| Function | In | Out | +|---|---|---| +| `computer::sessions::start` | `image?`, `endpoint?`, `os?`, `monitor?` | `session_id`, `endpoint`, `os`, `screen` | +| `computer::sessions::list` | — | `sessions[]` with endpoint, os, screen, timestamps, screencast state | +| `computer::sessions::stop` | `session_id` | `ok`, `was_running` (idempotent) | +| `computer::displays` | — | local `displays[]`; empty off a desktop host | +| `computer::screenshot` | `session_id` | image + text content blocks, `details.width/height/mime` | +| `computer::observe` | `session_id`, `include_a11y?` | screenshot plus the accessibility tree where the guest has one | +| `computer::act` | `session_id`, `action`, coordinates / `text` / `keys` | `ok`, `detail` | + +`act` actions: `click` (`left_click`), `right_click`, `double_click`, `move`, +`drag` (`to_x`/`to_y`), `scroll` (`scroll_x`/`scroll_y`), `type` (`text`), +`press` / `hotkey` (`keys`, e.g. `["cmd","c"]`). + +**Coordinates are the screenshot's pixels**, top-left origin. That is the whole +contract: screenshot, read a position off the image, act on it. After anything +that changes the screen, screenshot again before acting — a stale coordinate is +a click somewhere else. + +`computer::screencast::start` / `stop` and `computer::frame` are console +plumbing (flagged internal, denied in `iii-permissions.yaml`). Agents use +`screenshot`. + +## Trigger types + +| Trigger type | Fires when | Payload | +|---|---|---| +| `computer::session-started` | A session connected and probed | `session_id`, `endpoint`, `os`, `screen`, `timestamp` | +| `computer::session-stopped` | A session ended | `session_id`, `reason` (`stopped` \| `idle`), `timestamp` | + +Both accept an optional `{ "session_id": "..." }` equality filter. Bind them +instead of polling `sessions::list`. + +## The guardrail split (read this before wiring an agent) + +`computer::act` has **no** confirm step, denylist, or read-only mode. That is +deliberate, and it is the same split the browser worker uses: device-level +limits live in the worker, action-level policy lives above it. + +| Layer | Lives in | Examples | +|---|---|---| +| Device limits | this worker | permission preflight, `max_sessions`, capture downscale, command timeouts | +| Action policy | engine + harness | dispatch policy (`options.functions`), the approval gate on `computer::act`, permission profiles | + +A raw `act` does what it is told, on a real machine. Anything that should ask a +human first belongs in an approval-gate `pre-trigger` hook or a dispatch +allow-list, where the human-facing surface (the console renders every capture +inline) already is. + +## What not to do + +- Do not use this worker to open a URL. That is the + [browser](https://github.com/iii-hq/workers/tree/main/browser) worker, which + gives you a DOM instead of pixels. +- Do not use it to run commands or move files on the desktop — that is `shell`. + This worker sees the screen and drives the cursor, nothing else. +- Do not screenshot after every action. Captures are large and land in the + transcript; `act` returns a `detail` line that says what happened. +- Do not hold a session open across unrelated work. The cap is small by design; + stop it and start another. diff --git a/computer/architecture/internals.md b/computer/architecture/internals.md new file mode 100644 index 000000000..3fc6cf2a5 --- /dev/null +++ b/computer/architecture/internals.md @@ -0,0 +1,82 @@ +# computer — internals + +For changing the worker. Consumers want [`integration.md`](integration.md). + +## Driver selection + +`sessions::start` resolves exactly one driver, in this order (`session.rs`): + +1. **`image`** (argument, else the configured `sandbox_image`) — boot a desktop + in an iii-sandbox microVM. Endpoint label: `sandbox:`, guest OS + `linux`. +2. **`endpoint`** (argument, else the configured `default_endpoint`) — connect + to the desktop's guest executor. Endpoint label: the normalized url. +3. **neither** — the native driver, this machine. Endpoint label: `native`. + +An image beats an endpoint deliberately: a caller who names an image wants a +fresh desktop, not whatever the operator configured as a default target. + +Selection ends with a `screen_size` probe. A driver that connects but cannot +report a screen is not a session — the start fails there instead of handing +back an id that breaks on first use. + +## Capture pipeline + +`Shot` carries encoded bytes plus the mime **detected from the magic bytes**, +never assumed: a driver returns whatever its guest encodes, and the content +block advertises the truth to the model. + +The native driver does the work no guest does for it: + +- One display per session, chosen at first capture (the display under the + cursor unless `monitor` names one) and then pinned by id, so coordinates stay + stable for the life of the session. +- Downscale to `max_screenshot_dimension` and JPEG-encode at + `screenshot_quality`. A full Retina frame is tens of megabytes of PNG; that + floods both the model context and the frame stream. +- Input maps back through the display's **logical point** size and global + origin — the space `enigo` absolute coordinates use — so a click on a scaled + display lands where the model saw it. + +The sandbox driver sidesteps all of it with one fixed virtual display: 1:1 +coordinates, no HiDPI, no multi-monitor ambiguity. + +## Durability and the screencast + +Sessions are mirrored into `state` (scope `computer_sessions`) on start and +deleted on stop; `Sessions::restore` reconnects them best-effort at boot, so a +worker restart does not lose live desktops. + +The screencast pump is one task per session. It captures at +`screencast_fps`, pushes each frame onto the `computer:frames` stream +(`stream::set`, group = session id, one item), and keeps the newest frame in +memory for `computer::frame`. Both writes matter: the stream is how the console +follows without polling, the in-memory copy is how a late subscriber paints +immediately. A capture failure stops the pump rather than looping on a broken +driver, and `stop_screencast` clears both the stream item and the buffer so a +stopped session never leaves a multi-megabyte image resident. + +## macOS permission gates + +macOS degrades both capabilities silently, which is worse than failing: + +- **Screen Recording** missing → capture returns wallpaper and menu bar with + every window stripped. Looks like a screenshot, shows nothing. +- **Accessibility** missing → synthetic input is dropped while the input + library still reports success, so `act` would claim a click that never landed. + +So the worker checks. Input is preflighted with `AXIsProcessTrusted`. Capture +calls `CGRequestScreenCaptureAccess` — the *request* API, which surfaces the +system prompt — and fails loud, gated by `screen_capture_preflight` for setups +where the grant is already in place. Do **not** switch that to +`CGPreflightScreenCaptureAccess`: it reports per-process state a child of a +granted terminal does not inherit, so it false-negatives on capture that works. +The grant is per binary, so a rebuild loses it. + +## Tests + +`tests/schemas.rs` is the wire contract: a golden snapshot per function, plus +an assertion that no function publishes an untyped schema. Regenerate +deliberately with `UPDATE_GOLDENS=1 cargo test` — a diff there is a change to +what every agent sees. `src/ui.rs` tests assert the embedded console assets are +present and scoped. diff --git a/computer/build.rs b/computer/build.rs new file mode 100644 index 000000000..c7d67619b --- /dev/null +++ b/computer/build.rs @@ -0,0 +1,179 @@ +//! Build script for the `computer` worker. +//! +//! 1. Forwards the build-time target triple to the binary as `env!("TARGET")` +//! (used by `manifest.rs` for the registry `supported_targets` field). +//! 2. Ensures the injected console UI assets exist: `src/ui.rs` embeds +//! `ui/dist/page.js` and `ui/dist/styles.css` via `include_str!`, so if +//! either is missing or stale we run `pnpm install && pnpm build` inside +//! `ui/` first (the state worker's precedent). Set `SKIP_UI_BUILD=1` to use +//! the existing `ui/dist/` outputs as-is. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::SystemTime; + +fn main() { + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); + + // `dist/` itself is not listed: include_str! reads it directly, and + // listing it would rebuild-loop on our own output. + println!("cargo:rerun-if-changed=ui/page.tsx"); + println!("cargo:rerun-if-changed=ui/styles.css"); + println!("cargo:rerun-if-changed=ui/src"); + println!("cargo:rerun-if-changed=ui/build.mjs"); + println!("cargo:rerun-if-changed=ui/package.json"); + // The lockfile lives at the workers-repo root (pnpm workspace: the ui + // project links @iii-dev/console-ui from packages/console-ui). + println!("cargo:rerun-if-changed=../pnpm-lock.yaml"); + println!("cargo:rerun-if-changed=ui/tsconfig.json"); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let ui_dir = manifest_dir.join("ui"); + let dist_assets = [ + ui_dir.join("dist").join("page.js"), + ui_dir.join("dist").join("styles.css"), + ]; + + if dist_assets + .iter() + .all(|a| a.exists() && dist_is_fresh(a, &ui_dir)) + { + return; + } + + if std::env::var_os("SKIP_UI_BUILD").is_some() { + for asset in &dist_assets { + if !asset.exists() { + panic!( + "SKIP_UI_BUILD set but {} is missing — build the UI manually \ + (cd ui && pnpm install && pnpm build) or unset the env var", + asset.display() + ); + } + } + return; + } + + let pnpm = locate_pnpm(); + + let status = Command::new(&pnpm) + .args(["install"]) + .current_dir(&ui_dir) + .status() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `pnpm install` in {}: {e}", + ui_dir.display() + ) + }); + if !status.success() { + panic!("`pnpm install` exited with {status} — see logs above"); + } + + let status = Command::new(&pnpm) + .args(["build"]) + .current_dir(&ui_dir) + .status() + .unwrap_or_else(|e| panic!("failed to spawn `pnpm build` in {}: {e}", ui_dir.display())); + if !status.success() { + panic!("`pnpm build` exited with {status} — see logs above"); + } + + for asset in &dist_assets { + if !asset.exists() { + panic!( + "`pnpm build` finished but {} is still missing — check the esbuild \ + output above", + asset.display() + ); + } + } +} + +/// `true` when the built asset is at least as new as every source that +/// contributes to it. Conservative: any I/O failure forces a rebuild. +fn dist_is_fresh(dist_asset: &Path, ui_dir: &Path) -> bool { + let Ok(dist_mtime) = dist_asset.metadata().and_then(|m| m.modified()) else { + return false; + }; + + let watched_files = [ + ui_dir.join("page.tsx"), + ui_dir.join("styles.css"), + ui_dir.join("build.mjs"), + ui_dir.join("package.json"), + ui_dir.join("../../pnpm-lock.yaml"), + ui_dir.join("tsconfig.json"), + ]; + for f in watched_files.iter() { + if !f.exists() { + continue; + } + let Ok(m) = f.metadata().and_then(|m| m.modified()) else { + return false; + }; + if m > dist_mtime { + return false; + } + } + + for dir in [ui_dir.join("src")] { + if dir.exists() && !subtree_older_than(&dir, dist_mtime) { + return false; + } + } + + true +} + +fn subtree_older_than(root: &Path, ceiling: SystemTime) -> bool { + let Ok(read) = std::fs::read_dir(root) else { + return false; + }; + for entry in read.flatten() { + let path = entry.path(); + let Ok(meta) = entry.metadata() else { + return false; + }; + if meta.is_dir() { + if !subtree_older_than(&path, ceiling) { + return false; + } + } else { + let Ok(m) = meta.modified() else { + return false; + }; + if m > ceiling { + return false; + } + } + } + true +} + +fn locate_pnpm() -> PathBuf { + if let Ok(explicit) = std::env::var("PNPM") { + return PathBuf::from(explicit); + } + let candidates = if cfg!(windows) { + ["pnpm.cmd", "pnpm.exe", "pnpm"].as_slice() + } else { + ["pnpm"].as_slice() + }; + let path = std::env::var_os("PATH").unwrap_or_default(); + for dir in std::env::split_paths(&path) { + for name in candidates { + let candidate = dir.join(name); + if candidate.is_file() { + return candidate; + } + } + } + panic!( + "pnpm not found on PATH — install Node + pnpm, or set SKIP_UI_BUILD=1 \ + after building the UI manually with `cd ui && pnpm install && pnpm build`" + ); +} diff --git a/computer/iii.worker.yaml b/computer/iii.worker.yaml new file mode 100644 index 000000000..ece67cdd5 --- /dev/null +++ b/computer/iii.worker.yaml @@ -0,0 +1,8 @@ +iii: v1 +name: computer +language: rust +deploy: binary +manifest: Cargo.toml +bin: computer +tags: [computer-use, desktop, screenshot, gui, automation] +description: Drive a full desktop on the iii bus. Start a session on the local machine, a sandboxed desktop, or a remote one, screenshot it, click and type by coordinate, and stream the live screen. diff --git a/computer/images/desktop/Dockerfile b/computer/images/desktop/Dockerfile new file mode 100644 index 000000000..76d0fc500 --- /dev/null +++ b/computer/images/desktop/Dockerfile @@ -0,0 +1,57 @@ +# Desktop image for iii-sandbox-backed computer-use sessions. +# +# iii-sandbox boots this image's filesystem as a libkrun microVM rootfs. It does +# NOT run the ENTRYPOINT/CMD (PID 1 is the engine's supervisor), so nothing here +# autostarts: the computer worker brings up Xvfb + openbox via `sandbox::exec` +# on session start. This image only needs the tools present on the filesystem. +# +# The worker drives the guest with: +# - Xvfb the virtual display (fixed resolution, e.g. 1280x800) +# - openbox a minimal window manager +# - xdotool pointer + keyboard injection (DISPLAY=:0) +# - imagemagick `import` grabs the X root to JPEG for screenshots +# - x11-utils `xdpyinfo` for the display-readiness probe +# - procps `pgrep` used by the bootstrap +# - util-linux `setsid` to detach Xvfb into its own session +# +# Build and publish, then register it with the sandbox worker as a custom image: +# +# docker build -t ghcr.io//iii-desktop:latest computer/images/desktop +# docker push ghcr.io//iii-desktop:latest +# +# # engine config.yaml, under the iii-sandbox worker: +# # custom_images: +# # desktop: ghcr.io//iii-desktop:latest +# # image_allowlist: +# # - desktop +# +# Then: computer::sessions::start { "image": "desktop" } + +FROM debian:bookworm-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + DISPLAY=:0 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + xvfb \ + x11-utils \ + xdotool \ + imagemagick \ + openbox \ + procps \ + util-linux \ + dbus-x11 \ + fonts-dejavu-core \ + ca-certificates \ + xterm \ + && rm -rf /var/lib/apt/lists/* + +# ImageMagick's default policy blocks some coders; the X-root grab used for +# screenshots is unaffected, but relax the width/height/memory limits so a +# full-screen capture is never rejected on resource policy. +RUN if [ -f /etc/ImageMagick-6/policy.xml ]; then \ + sed -i 's/name="memory" value="[^"]*"/name="memory" value="512MiB"/' /etc/ImageMagick-6/policy.xml; \ + sed -i 's/name="width" value="[^"]*"/name="width" value="32KP"/' /etc/ImageMagick-6/policy.xml; \ + sed -i 's/name="height" value="[^"]*"/name="height" value="32KP"/' /etc/ImageMagick-6/policy.xml; \ + fi diff --git a/computer/images/desktop/README.md b/computer/images/desktop/README.md new file mode 100644 index 000000000..03bf43b32 --- /dev/null +++ b/computer/images/desktop/README.md @@ -0,0 +1,61 @@ +# iii-desktop image + +A minimal Linux desktop for sandboxed computer-use sessions. The `computer` +worker boots this inside an [`iii-sandbox`](https://workers.iii.dev/workers/iii-sandbox) +microVM and drives it entirely through iii primitives (`sandbox::exec` + +`sandbox::fs`), with no executor and no socket into the guest. + +## Why a sandboxed desktop + +Driving a real host desktop means fighting multi-monitor layout, HiDPI/Retina +point-vs-pixel scaling, and OS permission prompts. A sandbox sidesteps all of +it: one fixed-resolution virtual display means coordinates are 1:1 with the +screenshot, and there is nothing to grant. It is also reproducible and works +headless (CI included). + +## How it works + +iii-sandbox boots the image filesystem as a libkrun microVM rootfs but does +**not** run its ENTRYPOINT/CMD (PID 1 is the engine supervisor). So the worker +starts the display itself on `sessions::start`: + +1. `sandbox::create { image, network: true, idle_timeout_secs }` +2. `sandbox::exec` runs `setsid Xvfb :0 -screen 0 xx24 &` + `openbox`, + then waits for `xdpyinfo` (detached with `setsid` so it survives between + exec calls). +3. Per action: `import -window root jpg:- | base64` for a screenshot, `xdotool` + for pointer/keyboard, all under `DISPLAY=:0`. + +## Build and register + +```bash +docker build -t ghcr.io//iii-desktop:latest computer/images/desktop +docker push ghcr.io//iii-desktop:latest +``` + +Register it with the sandbox worker in the engine `config.yaml`: + +```yaml +- name: iii-sandbox + config: + auto_install: true + image_allowlist: + - desktop + custom_images: + desktop: ghcr.io//iii-desktop:latest +``` + +Then start a session: + +```json +{ "trigger": "computer::sessions::start", "payload": { "image": "desktop" } } +``` + +Or set `sandbox_image: desktop` in the `computer` worker config so a bare +`sessions::start` uses it by default. Resolution comes from the worker config +(`sandbox_width` / `sandbox_height`, default 1280x800). + +## Extending + +Add your own apps to the image (a browser, an editor, whatever the task needs). +Nothing else changes: the worker launches and drives them via `xdotool`. diff --git a/computer/skills/SKILL.md b/computer/skills/SKILL.md new file mode 100644 index 000000000..0e56b57c4 --- /dev/null +++ b/computer/skills/SKILL.md @@ -0,0 +1,92 @@ +--- +name: computer +description: >- + Drive a full desktop computer over the iii bus: start a session, see the + screen as an image, and click, type, and scroll by coordinate. Reach for it + when a task needs to operate a real GUI app or whole desktop, not just a web + page. +--- + +# computer + +The computer worker turns a live desktop into iii functions. Start a session +with no endpoint to drive the local machine this worker runs on, pass an `image` +to boot a sandboxed desktop, or pass an `endpoint` to drive a remote one. Take a +screenshot to see the screen, then act on it by pixel coordinate: the screenshot +is the source of truth for where things are, and `computer::act` clicks and +types at those coordinates. The session stays alive, so you can act, screenshot +the result, and act again. + +Sessions are durable (a worker restart reconnects them) and cost one driver +connection each; the configured session cap is small. Stop sessions when a task +is done. Screenshots are large; take one when you need to see the screen, not +after every action. + +## When to Use + +- Operating a desktop GUI application that is not a web page (an installer, a + native app, a settings panel). +- End-to-end tasks that span the whole screen: move a window, drag between + apps, use a system dialog. +- Watching the effect of a change on screen: run the command with the `shell` + worker, then `computer::screenshot` to see the result. + +## Boundaries + +- Web-only tasks belong to the + [browser](https://github.com/iii-hq/workers/tree/main/browser) worker (a real + Chromium tab with an accessibility outline and page console) or `web::fetch` + for a one-shot page. Do not start a desktop session just to open a URL. +- Shell and files on the desktop belong to the `shell` worker (`shell::exec`, + `shell::fs::*`), not this worker. `computer` only sees the screen and drives + the cursor. +- With no `endpoint` and no `image` it drives the local machine; an `image` + boots a sandboxed desktop through the iii-sandbox worker; an `endpoint` + connects to a desktop somebody else booted. +- `computer::screencast::*` and `computer::frame` are console-UI plumbing, not + agent surface. +- Coordinates are integer pixels, top-left origin, in the space of the most + recent screenshot. Re-screenshot after the screen changes before acting. + +## Functions + +- `computer::sessions::start` — connect a desktop session; returns the + session_id every other function needs, plus the screen size. +- `computer::sessions::list` — live sessions with endpoint, OS, and screen. +- `computer::sessions::stop` — stop a session; idempotent. +- `computer::screenshot` — the desktop as a viewable image; how you see the + screen before acting. +- `computer::observe` — screenshot plus, on macOS guests, the accessibility + tree (`include_a11y: true`). +- `computer::act` — click, right_click, double_click, move, drag, scroll, type, + press, or hotkey, addressed by pixel coordinates. + +## Keeping context small + +Desktop screenshots are large and land in the transcript, so a few careless +captures fill the context window. Screenshot when you need to see the screen, +not reflexively after each action; a `computer::act` returns a short confirming +`detail` on its own. Reuse one session across steps and stop it when done. + +## Reactive triggers + +Bind a `computer::*` trigger when another function should react to session +activity instead of polling. The types: `computer::session-started` (payload +carries `endpoint`, `os`, `screen`) and `computer::session-stopped` (payload +carries `reason`). Both accept an optional `session_id` equality filter. + +### How to bind + +1. Register a handler: `registerFunction('mywatcher::on-desktop', handler)`. +2. Register the trigger: + +```typescript +iii.registerTrigger({ + type: 'computer::session-stopped', + function_id: 'mywatcher::on-desktop', + config: { session_id: 'c1' }, +}) +``` + +Omit `session_id` to receive events for all sessions. For event payload shapes, +call `get function info` on the trigger type. diff --git a/computer/src/config.rs b/computer/src/config.rs new file mode 100644 index 000000000..c7a40c2f8 --- /dev/null +++ b/computer/src/config.rs @@ -0,0 +1,184 @@ +//! Config for the computer worker. `max_sessions` is a hard cap. The screencast +//! rate is read per pump tick, so it hot-reloads; the action and connect +//! timeouts are read when a session connects, and `default_endpoint` and `os` +//! are read at session start, so a change to any of those applies to sessions +//! started after it. Running sessions keep the driver they were launched with. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub type SharedConfig = Arc>; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct WorkerConfig { + /// Guest-executor endpoint new sessions connect to when + /// `sessions::start` omits one. Accepts `ws://host:port`, + /// `http://host:port`, or a bare `host:port` (defaults to ws, `/ws` + /// appended). Empty means every `sessions::start` must pass its own + /// `endpoint`. + pub default_endpoint: String, + /// Informational OS label recorded on started sessions and surfaced in + /// `session-started` events (`linux`, `macos`, `windows`, `android`). + /// Does not change the driver; it labels which guest you connected to. + pub os: String, + /// Maximum concurrently running sessions; `sessions::start` beyond this + /// fails until one stops. + pub max_sessions: u64, + /// Stop sessions idle longer than this (ms). 0 disables the sweep. + pub idle_stop_ms: u64, + /// Live-view frame rate cap for the screencast (frames/sec). Clamped to + /// at least 1. The pump polls the driver screenshot at most this often. + pub screencast_fps: u64, + /// Longest edge (px) a native screenshot/frame is downscaled to before + /// JPEG encoding. Full-resolution Retina captures are huge; this caps the + /// image the model sees and the coordinate space it acts in. + pub max_screenshot_dimension: u64, + /// JPEG quality (1-100) for native screenshots and frames. + pub screenshot_quality: u64, + /// Timeout for each driver action (ms). Fixed when a session connects. + pub command_timeout_ms: u64, + /// Timeout for establishing the driver connection at session start (ms). + pub connect_timeout_ms: u64, + /// OCI image (an iii-sandbox preset name or `custom_images` key) booted for + /// a sandbox-backed session when `sessions::start` omits `image`. Empty + /// means a sandbox session must name its own image. The image must ship + /// Xvfb, xdotool, imagemagick, and openbox (see images/desktop). + pub sandbox_image: String, + /// Virtual display width (px) for a sandbox-backed session. Fixed + /// resolution keeps coordinates 1:1 with the screenshot. + pub sandbox_width: u64, + /// Virtual display height (px) for a sandbox-backed session. + pub sandbox_height: u64, + /// Idle timeout (seconds) passed to `sandbox::create`. Set well above the + /// worker's own `idle_stop_ms` so the sandbox reaper never kills a live + /// desktop out from under a session; the worker owns teardown. + pub sandbox_idle_timeout_secs: u64, + /// Ask macOS for Screen Recording at native session start and fail loudly if + /// it is not granted (macOS otherwise returns a wallpaper-only capture with + /// every window stripped, silently). Set false if capture works in your + /// setup but the TCC API under-reports (e.g. the worker runs as a child of + /// an already-granted app). Ignored off macOS and for non-native drivers. + pub screen_capture_preflight: bool, +} + +impl Default for WorkerConfig { + fn default() -> Self { + Self { + default_endpoint: String::new(), + os: "linux".to_string(), + max_sessions: 2, + idle_stop_ms: 300_000, + screencast_fps: 15, + max_screenshot_dimension: 1280, + screenshot_quality: 70, + command_timeout_ms: 120_000, + connect_timeout_ms: 15_000, + sandbox_image: String::new(), + sandbox_width: 1280, + sandbox_height: 800, + sandbox_idle_timeout_secs: 86_400, + screen_capture_preflight: true, + } + } +} + +impl WorkerConfig { + pub fn json_schema() -> serde_json::Value { + serde_json::to_value(schemars::schema_for!(WorkerConfig)) + .expect("WorkerConfig schema serializes") + } + + /// Parse from a JSON object; missing keys fall back to defaults + /// (`#[serde(default)]`). The configuration worker may store the value + /// under a `computer` wrapper or flat: accept both. + pub fn from_json(v: &serde_json::Value) -> Result { + let inner = v.get("computer").unwrap_or(v); + serde_json::from_value(inner.clone()).map_err(|e| format!("invalid computer config: {e}")) + } + + pub fn to_json(&self) -> serde_json::Value { + serde_json::to_value(self).expect("WorkerConfig serializes") + } + + pub fn into_shared(self) -> SharedConfig { + Arc::new(ArcSwap::from_pointee(self)) + } + + /// Minimum wall-clock interval between screencast frames (ms), derived + /// from `screencast_fps` (floored at 1 fps so the divisor is never zero). + pub fn screencast_interval_ms(&self) -> u64 { + 1_000 / self.screencast_fps.max(1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_match_spec() { + let c = WorkerConfig::default(); + assert_eq!(c.default_endpoint, ""); + assert_eq!(c.os, "linux"); + assert_eq!(c.max_sessions, 2); + assert_eq!(c.idle_stop_ms, 300_000); + assert_eq!(c.screencast_fps, 15); + assert_eq!(c.max_screenshot_dimension, 1280); + assert_eq!(c.screenshot_quality, 70); + assert_eq!(c.command_timeout_ms, 120_000); + assert_eq!(c.connect_timeout_ms, 15_000); + assert_eq!(c.sandbox_image, ""); + assert_eq!(c.sandbox_width, 1280); + assert_eq!(c.sandbox_height, 800); + assert_eq!(c.sandbox_idle_timeout_secs, 86_400); + assert!(c.screen_capture_preflight); + } + + #[test] + fn json_roundtrip() { + let c = WorkerConfig { + os: "macos".to_string(), + max_sessions: 1, + ..WorkerConfig::default() + }; + let back = WorkerConfig::from_json(&c.to_json()).unwrap(); + assert_eq!(back.os, "macos"); + assert_eq!(back.max_sessions, 1); + } + + #[test] + fn from_json_fills_missing_with_defaults() { + let v = serde_json::json!({ "max_sessions": 4 }); + let c = WorkerConfig::from_json(&v).unwrap(); + assert_eq!(c.max_sessions, 4); + assert_eq!(c.screencast_fps, 15); + } + + #[test] + fn from_json_accepts_wrapped_value() { + let v = serde_json::json!({ "computer": { "os": "windows" } }); + let c = WorkerConfig::from_json(&v).unwrap(); + assert_eq!(c.os, "windows"); + } + + #[test] + fn screencast_interval_floors_fps_at_one() { + let mut c = WorkerConfig::default(); + assert_eq!(c.screencast_interval_ms(), 66); + c.screencast_fps = 0; + assert_eq!(c.screencast_interval_ms(), 1_000); + } + + #[test] + fn schema_lists_fields() { + let s = WorkerConfig::json_schema(); + let props = &s["properties"]; + assert!(props.get("default_endpoint").is_some()); + assert!(props.get("os").is_some()); + assert!(props.get("screencast_fps").is_some()); + } +} diff --git a/computer/src/configuration.rs b/computer/src/configuration.rs new file mode 100644 index 000000000..7be70da12 --- /dev/null +++ b/computer/src/configuration.rs @@ -0,0 +1,143 @@ +//! Integration with the `configuration` worker: register a JSON Schema + seed +//! at boot, read the authoritative (env-expanded) value, and bind a +//! `configuration` trigger so `configuration:updated` re-fetches and applies +//! the change. Timeouts and the screencast rate hot-reload; `default_endpoint` +//! and `os` apply to sessions started after the change. + +use std::time::Duration; + +use iii_sdk::errors::Error; +use iii_sdk::protocol::{RegisterTriggerInput, TriggerRequest}; +use iii_sdk::{IIIClient, RegisterFunction}; +use serde_json::{json, Value}; + +use crate::config::{SharedConfig, WorkerConfig}; + +pub const CONFIG_ID: &str = "computer"; +const CONFIG_FN_ID: &str = "computer::on-config-change"; +const CONFIG_TIMEOUT_MS: u64 = 5_000; +const CONFIG_RETRIES: u32 = 3; + +pub async fn register_config(iii: &IIIClient, seed: Option<&WorkerConfig>) -> Result<(), String> { + let mut payload = json!({ + "id": CONFIG_ID, + "name": "computer", + "description": "Endpoint, OS label, session cap, timeouts, and screencast rate for the computer worker.", + "schema": WorkerConfig::json_schema(), + }); + if let Some(seed) = seed { + payload["initial_value"] = seed.to_json(); + } else if should_seed_default(iii).await? { + payload["initial_value"] = WorkerConfig::default().to_json(); + } + trigger_with_retry(iii, "configuration::register", payload).await?; + Ok(()) +} + +pub async fn fetch_config(iii: &IIIClient) -> Result { + match try_get_value(iii).await? { + Some(v) if !v.is_null() => WorkerConfig::from_json(&v), + _ => { + tracing::info!("no configuration value found; using built-in defaults"); + Ok(WorkerConfig::default()) + } + } +} + +async fn should_seed_default(iii: &IIIClient) -> Result { + match try_get_value(iii).await? { + None => Ok(true), + Some(v) if v.is_null() => Ok(true), + Some(_) => Ok(false), + } +} + +async fn try_get_value(iii: &IIIClient) -> Result, String> { + match trigger_with_retry(iii, "configuration::get", json!({ "id": CONFIG_ID })).await { + Ok(resp) => Ok(resp.get("value").cloned()), + Err(e) if e.contains("NOT_FOUND") => Ok(None), + Err(e) => Err(e), + } +} + +#[derive(Debug, Default, serde::Deserialize, schemars::JsonSchema)] +struct OnConfigChangeRequest {} + +#[derive(Debug, serde::Serialize, schemars::JsonSchema)] +struct OnConfigChangeResponse { + ok: bool, +} + +pub fn register_config_trigger(iii: &IIIClient, config: SharedConfig) -> Result<(), Error> { + let cfg = config.clone(); + let engine = iii.clone(); + iii.register_function( + CONFIG_FN_ID, + RegisterFunction::new_async(move |_req: OnConfigChangeRequest| { + let cfg = cfg.clone(); + let engine = engine.clone(); + async move { + on_config_change(&engine, &cfg).await; + Ok::(OnConfigChangeResponse { ok: true }) + } + }) + .description( + "Internal: reload computer settings from the authoritative configuration on change.", + ) + .metadata(json!({ "internal": true })), + ); + + iii.register_trigger(RegisterTriggerInput { + trigger_type: "configuration".to_string(), + function_id: CONFIG_FN_ID.to_string(), + config: json!({ "configuration_id": CONFIG_ID, "event_types": ["configuration:updated"] }), + metadata: None, + })?; + Ok(()) +} + +async fn on_config_change(iii: &IIIClient, config: &SharedConfig) { + match fetch_config(iii).await { + Ok(cfg) => { + config.store(std::sync::Arc::new(cfg)); + tracing::info!("computer configuration reloaded"); + } + Err(e) => tracing::error!(error = %e, "config-change: keeping previous config"), + } +} + +async fn trigger_with_retry( + iii: &IIIClient, + function_id: &str, + payload: Value, +) -> Result { + let mut last_err = String::new(); + for attempt in 1..=CONFIG_RETRIES { + match iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload: payload.clone(), + action: None, + timeout_ms: Some(CONFIG_TIMEOUT_MS), + }) + .await + { + Ok(v) => return Ok(v), + Err(e) => { + last_err = e.to_string(); + if attempt < CONFIG_RETRIES { + tracing::warn!( + function_id, + attempt, + error = %last_err, + "configuration RPC failed; retrying" + ); + tokio::time::sleep(Duration::from_millis(250 * u64::from(attempt))).await; + } + } + } + } + Err(format!( + "{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}" + )) +} diff --git a/computer/src/driver/mod.rs b/computer/src/driver/mod.rs new file mode 100644 index 000000000..08f989285 --- /dev/null +++ b/computer/src/driver/mod.rs @@ -0,0 +1,133 @@ +//! The environment driver: one desktop the worker drives. The [`Driver`] +//! trait keeps [`crate::session::Session`] driver-agnostic; three +//! implementations slot in behind it: +//! +//! - [`RemoteClient`] speaks the guest-executor wire (`{command, params}` +//! over a WebSocket) to a remote or sandboxed desktop. +//! - [`NativeHost`] drives the local machine this worker runs on (macOS / +//! Windows), capturing and injecting input directly. +//! - [`IiiSandboxHost`] boots a desktop inside an iii-sandbox microVM and +//! drives it through iii primitives (`sandbox::exec` / `sandbox::fs`), with +//! no executor and no socket into the guest. + +#[cfg(any(target_os = "macos", target_os = "windows"))] +pub mod native; +pub mod remote; +pub mod sandbox; + +use async_trait::async_trait; +use base64::engine::general_purpose::STANDARD; +use base64::Engine; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[cfg(any(target_os = "macos", target_os = "windows"))] +pub use native::NativeHost; +pub use remote::RemoteClient; +pub use sandbox::IiiSandboxHost; + +/// Desktop pixel dimensions. Coordinates handed to pointer actions are in this +/// space: integer pixels, top-left origin, 1:1 with the screenshot. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct Screen { + pub width: u32, + pub height: u32, +} + +/// One display available to the native host (from `computer::displays`). +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct DisplayInfo { + /// Index to pass as `monitor` to `computer::sessions::start`. + pub index: u32, + pub name: String, + pub primary: bool, + pub builtin: bool, + /// Logical width in points. + pub width: u32, + /// Logical height in points. + pub height: u32, +} + +/// An encoded screenshot plus the detected mime of its bytes. The driver +/// returns whatever its guest encodes (png by default); the mime is read from +/// the magic bytes rather than assumed, so the content block advertises the +/// truth to the model. +#[derive(Debug, Clone)] +pub struct Shot { + pub bytes: Vec, + pub mime: String, +} + +impl Shot { + /// Detect `image/png` / `image/jpeg` from the leading magic bytes, falling + /// back to `application/octet-stream` for anything unrecognized. + pub fn new(bytes: Vec) -> Self { + let mime = detect_mime(&bytes).to_string(); + Self { bytes, mime } + } + + /// Base64 of the encoded image bytes. + pub fn to_base64(&self) -> String { + STANDARD.encode(&self.bytes) + } + + pub fn byte_len(&self) -> usize { + self.bytes.len() + } +} + +pub fn detect_mime(bytes: &[u8]) -> &'static str { + if bytes.starts_with(&[0x89, b'P', b'N', b'G']) { + "image/png" + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + "image/jpeg" + } else { + "application/octet-stream" + } +} + +/// A live computer-use environment. Every method maps a desktop semantic onto +/// the concrete driver wire. Errors are human-readable strings surfaced back +/// through the function envelope. +#[async_trait] +pub trait Driver: Send + Sync { + /// Current desktop dimensions. + async fn screen_size(&self) -> Result; + /// Capture the desktop; bytes carry their own encoding (see [`Shot`]). + async fn screenshot(&self) -> Result; + async fn left_click(&self, x: i64, y: i64) -> Result<(), String>; + async fn right_click(&self, x: i64, y: i64) -> Result<(), String>; + async fn double_click(&self, x: i64, y: i64) -> Result<(), String>; + async fn move_cursor(&self, x: i64, y: i64) -> Result<(), String>; + async fn scroll(&self, x: i64, y: i64, scroll_x: i64, scroll_y: i64) -> Result<(), String>; + async fn drag(&self, from: (i64, i64), to: (i64, i64), button: &str) -> Result<(), String>; + async fn type_text(&self, text: &str) -> Result<(), String>; + /// Press a chord: a single key (`["enter"]`) or a combination + /// (`["ctrl", "c"]`). + async fn keypress(&self, keys: &[String]) -> Result<(), String>; + /// Best-effort accessibility tree; `null` when the guest does not expose a + /// real one (Linux/Windows guests return a stub or nothing). + async fn accessibility_tree(&self) -> Result; + /// Close the underlying connection. Idempotent: closing an + /// already-closed driver succeeds. + async fn close(&self) -> Result<(), String>; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_png_and_jpeg() { + assert_eq!(detect_mime(&[0x89, b'P', b'N', b'G', 0x0d]), "image/png"); + assert_eq!(detect_mime(&[0xFF, 0xD8, 0xFF, 0xE0]), "image/jpeg"); + assert_eq!(detect_mime(&[0x00, 0x01]), "application/octet-stream"); + assert_eq!(detect_mime(&[]), "application/octet-stream"); + } + + #[test] + fn shot_carries_detected_mime() { + let shot = Shot::new(vec![0x89, b'P', b'N', b'G', 0x0d, 0x0a]); + assert_eq!(shot.mime, "image/png"); + } +} diff --git a/computer/src/driver/native.rs b/computer/src/driver/native.rs new file mode 100644 index 000000000..9f01bd243 --- /dev/null +++ b/computer/src/driver/native.rs @@ -0,0 +1,533 @@ +//! Native host driver: drive the machine this worker runs on directly, with +//! no executor process in between. Screen capture via `xcap`, mouse/keyboard +//! via `enigo`. +//! +//! Multi-monitor and Retina aware. A session captures ONE display, chosen at +//! its first capture: by default the display under the cursor (so it follows +//! where you are working), pinned by id thereafter for coordinate stability. +//! The captured frame is downscaled to `max_dimension` and JPEG-encoded (a full +//! Retina frame is tens of megabytes of PNG, which floods the model context and +//! the frame stream). The model sees the downscaled image and its coordinate +//! space; pointer actions map those coordinates through the display's LOGICAL +//! point size and its global origin, which is exactly what `enigo`'s absolute +//! coordinates use, so clicks land on the right display at the right point +//! regardless of scale factor. +//! +//! Capture and input are synchronous, blocking OS calls, so each runs on a +//! blocking thread. `enigo` state is not `Send`, so an `Enigo` is created, used, +//! and dropped inside each blocking closure. On macOS the worker process needs +//! Screen Recording (capture) and Accessibility (input) permission. Input is +//! preflighted with `AXIsProcessTrusted` so a dropped click fails loudly instead +//! of a false success. Screen Recording is NOT preflighted: +//! `CGPreflightScreenCaptureAccess` reports per-process state that a child of a +//! granted terminal does not inherit, so it false-negatives on a capture that +//! actually works; a missing grant instead surfaces as a wallpaper-only image. + +use std::io::Cursor; +use std::sync::OnceLock; + +use async_trait::async_trait; +use enigo::{Axis, Button, Coordinate, Direction, Enigo, Key, Keyboard, Mouse, Settings}; +use image::codecs::jpeg::JpegEncoder; +use image::imageops::{resize, FilterType}; +use image::DynamicImage; +use tokio::task::spawn_blocking; +use xcap::Monitor; + +use super::{DisplayInfo, Driver, Screen, Shot}; + +/// macOS permission gates. Two things macOS silently degrades instead of +/// erroring, so the worker must check them or it lies to the model: +/// +/// - Accessibility: without it, macOS drops synthetic input while `enigo` still +/// reports success, so `act` would claim a click landed when it did not. +/// Checked with `AXIsProcessTrusted`. +/// - Screen Recording: without it, capture returns the desktop wallpaper and +/// menu bar with every window stripped, so a screenshot looks fine but shows +/// nothing. `request_screen_capture` calls the *request* API +/// (`CGRequestScreenCaptureAccess`), which surfaces the macOS prompt rather +/// than the *preflight* API, which shows no dialog and under-reports for a +/// process launched as a child of a granted terminal. +/// +/// Windows has no equivalent gates, so both checks are always true there. +#[cfg(target_os = "macos")] +mod perms { + #[link(name = "ApplicationServices", kind = "framework")] + extern "C" { + fn AXIsProcessTrusted() -> u8; + } + #[link(name = "CoreGraphics", kind = "framework")] + extern "C" { + fn CGRequestScreenCaptureAccess() -> u8; + } + pub fn accessibility_granted() -> bool { + unsafe { AXIsProcessTrusted() != 0 } + } + pub fn request_screen_capture() -> bool { + unsafe { CGRequestScreenCaptureAccess() != 0 } + } +} + +#[cfg(not(target_os = "macos"))] +mod perms { + pub fn accessibility_granted() -> bool { + true + } + pub fn request_screen_capture() -> bool { + true + } +} + +/// Ask macOS for Screen Recording (surfacing the prompt) and fail loudly if the +/// worker is not authorized, so a native session never hands back a +/// wallpaper-only screenshot with no explanation. Gated by config +/// (`screen_capture_preflight`), because the TCC APIs under-report for a process +/// launched as a child of an already-granted app, and a user whose capture +/// works must be able to skip the check. +pub fn preflight_screen_capture() -> Result<(), String> { + if perms::request_screen_capture() { + Ok(()) + } else { + Err( + "Screen Recording is not granted to this worker, so a screenshot \ + would capture only the desktop wallpaper (every window stripped). \ + macOS has been asked for it: open System Settings > Privacy & \ + Security > Screen Recording, enable this worker (or the app that \ + launched it), then start the session again. If capture already \ + works in your setup, set screen_capture_preflight: false in the \ + computer config to skip this check." + .to_string(), + ) + } +} + +/// Current cursor position in the global point space (origin = main display +/// top-left) — the same space `xcap::Monitor::from_point` and `enigo` absolute +/// coordinates use. Returns `None` if unavailable; callers fall back to the +/// primary display. +#[cfg(target_os = "macos")] +fn cursor_point() -> Option<(i32, i32)> { + #[repr(C)] + struct CGPoint { + x: f64, + y: f64, + } + #[link(name = "CoreGraphics", kind = "framework")] + extern "C" { + fn CGEventCreate(source: *const core::ffi::c_void) -> *mut core::ffi::c_void; + fn CGEventGetLocation(event: *mut core::ffi::c_void) -> CGPoint; + } + #[link(name = "CoreFoundation", kind = "framework")] + extern "C" { + fn CFRelease(cf: *const core::ffi::c_void); + } + unsafe { + let ev = CGEventCreate(core::ptr::null()); + if ev.is_null() { + return None; + } + let p = CGEventGetLocation(ev); + CFRelease(ev as *const _); + Some((p.x.round() as i32, p.y.round() as i32)) + } +} + +#[cfg(not(target_os = "macos"))] +fn cursor_point() -> Option<(i32, i32)> { + None +} + +/// Everything needed to map a downscaled-image coordinate to an absolute +/// `enigo` coordinate: the captured display's global origin (points) and the +/// scale from downscaled pixels to the display's logical points. +#[derive(Clone, Copy)] +struct Geom { + origin_x: i32, + origin_y: i32, + scale_x: f64, + scale_y: f64, +} + +pub struct NativeHost { + max_dimension: u32, + jpeg_quality: u8, + /// Optional display index override (from `sessions::start`); `None` picks + /// the display under the cursor at first capture. + monitor: Option, + /// The pinned display (id) and its coordinate geometry, learned on the + /// first capture so the session stays on one display with stable input + /// mapping. + pinned: OnceLock<(u32, Geom)>, +} + +impl NativeHost { + pub fn new(max_dimension: u32, jpeg_quality: u8, monitor: Option) -> Self { + Self { + max_dimension: max_dimension.max(320), + jpeg_quality: jpeg_quality.clamp(1, 100), + monitor, + pinned: OnceLock::new(), + } + } + + fn pinned_id(&self) -> Option { + self.pinned.get().map(|(id, _)| *id) + } + + fn geom(&self) -> Option { + self.pinned.get().map(|(_, g)| *g) + } +} + +/// List the local displays for `computer::displays`. +pub fn list_displays() -> Result, String> { + let monitors = Monitor::all().map_err(|e| format!("enumerate monitors: {e}"))?; + Ok(monitors + .iter() + .enumerate() + .map(|(i, m)| DisplayInfo { + index: i as u32, + name: m.name().unwrap_or_default(), + primary: m.is_primary().unwrap_or(false), + builtin: m.is_builtin().unwrap_or(false), + width: m.width().unwrap_or(0), + height: m.height().unwrap_or(0), + }) + .collect()) +} + +/// Resolve which display to capture: a pinned id if the session already chose +/// one, else the index override, else the display under the cursor, else the +/// primary, else the first. +fn resolve_monitor(pinned_id: Option, override_idx: Option) -> Result { + let monitors = Monitor::all().map_err(|e| format!("enumerate monitors: {e}"))?; + if monitors.is_empty() { + return Err("no display found".to_string()); + } + if let Some(id) = pinned_id { + if let Some(m) = monitors + .into_iter() + .find(|m| m.id().map(|mid| mid == id).unwrap_or(false)) + { + return Ok(m); + } + // Pinned display went away (unplugged); fall through to a fresh pick. + return resolve_monitor(None, override_idx); + } + if let Some(i) = override_idx { + return monitors + .into_iter() + .nth(i as usize) + .ok_or_else(|| format!("display index {i} is out of range")); + } + if let Some((cx, cy)) = cursor_point() { + if let Ok(m) = Monitor::from_point(cx, cy) { + return Ok(m); + } + } + let mut first: Option = None; + for m in monitors { + if m.is_primary().unwrap_or(false) { + return Ok(m); + } + if first.is_none() { + first = Some(m); + } + } + first.ok_or_else(|| "no display found".to_string()) +} + +/// Downscale target: longest edge capped at `max_dim`, aspect preserved. +fn target_dims(w: u32, h: u32, max_dim: u32) -> (u32, u32) { + let longest = w.max(h); + if longest <= max_dim || longest == 0 { + return (w, h); + } + let s = max_dim as f64 / longest as f64; + ( + ((w as f64 * s).round() as u32).max(1), + ((h as f64 * s).round() as u32).max(1), + ) +} + +/// Capture the resolved display, downscale + JPEG-encode. Returns +/// (jpeg bytes, target w, target h, display id, geometry). +fn capture( + pinned_id: Option, + override_idx: Option, + max_dim: u32, + quality: u8, +) -> Result<(Vec, u32, u32, u32, Geom), String> { + let monitor = resolve_monitor(pinned_id, override_idx)?; + let id = monitor.id().unwrap_or(0); + let img = monitor + .capture_image() + .map_err(|e| format!("screen capture failed: {e}"))?; + let (rw, rh) = (img.width(), img.height()); + let (tw, th) = target_dims(rw, rh, max_dim); + + // The display's LOGICAL size + global origin (points). `enigo` absolute + // coordinates and `Monitor::x/y/width/height` share this space, so mapping + // downscaled pixels -> logical points -> +origin lands the click correctly + // on this display at any scale factor. Fall back to the backing pixels at + // origin (0,0) if the geometry query fails. + let lx = monitor.x().unwrap_or(0); + let ly = monitor.y().unwrap_or(0); + let lw = monitor.width().unwrap_or(rw); + let lh = monitor.height().unwrap_or(rh); + let geom = Geom { + origin_x: lx, + origin_y: ly, + scale_x: lw as f64 / tw.max(1) as f64, + scale_y: lh as f64 / th.max(1) as f64, + }; + + let rgb = if (tw, th) == (rw, rh) { + DynamicImage::ImageRgba8(img).to_rgb8() + } else { + DynamicImage::ImageRgba8(resize(&img, tw, th, FilterType::Triangle)).to_rgb8() + }; + let mut out = Vec::new(); + JpegEncoder::new_with_quality(&mut Cursor::new(&mut out), quality) + .encode_image(&rgb) + .map_err(|e| format!("jpeg encode failed: {e}"))?; + Ok((out, tw, th, id, geom)) +} + +/// Map a downscaled-image coordinate to an absolute `enigo` coordinate through +/// the pinned display geometry (identity if not yet known). +fn to_global(x: i64, y: i64, geom: Option) -> (i32, i32) { + match geom { + Some(g) => ( + g.origin_x + (x as f64 * g.scale_x).round() as i32, + g.origin_y + (y as f64 * g.scale_y).round() as i32, + ), + None => (x as i32, y as i32), + } +} + +/// Run a closure with a fresh `Enigo` and the pinned geometry, after preflighting +/// Accessibility. +async fn with_enigo(geom: Option, f: F) -> Result<(), String> +where + F: FnOnce(&mut Enigo, Option) -> Result<(), String> + Send + 'static, +{ + spawn_blocking(move || { + // Without Accessibility, macOS drops the event and enigo reports ok, so + // the agent thinks a click landed when it did not. Fail clearly instead. + if !perms::accessibility_granted() { + return Err( + "Accessibility is not granted to this process, so macOS is silently \ + dropping synthetic input. Enable it in System Settings > Privacy & \ + Security > Accessibility for the app running this worker, then retry." + .to_string(), + ); + } + let mut enigo = + Enigo::new(&Settings::default()).map_err(|e| format!("enigo init failed: {e}"))?; + f(&mut enigo, geom) + }) + .await + .map_err(|e| format!("input task failed: {e}"))? +} + +fn input_err(e: enigo::InputError) -> String { + format!("input failed: {e}") +} + +/// Map a key name (from `press`/`hotkey`) onto an enigo key. +fn key_for(name: &str) -> Key { + match name.to_ascii_lowercase().as_str() { + "enter" | "return" => Key::Return, + "tab" => Key::Tab, + "escape" | "esc" => Key::Escape, + "backspace" => Key::Backspace, + "delete" | "del" => Key::Delete, + "space" => Key::Space, + "up" | "arrowup" => Key::UpArrow, + "down" | "arrowdown" => Key::DownArrow, + "left" | "arrowleft" => Key::LeftArrow, + "right" | "arrowright" => Key::RightArrow, + "home" => Key::Home, + "end" => Key::End, + "pageup" => Key::PageUp, + "pagedown" => Key::PageDown, + "cmd" | "command" | "meta" | "super" | "win" => Key::Meta, + "ctrl" | "control" => Key::Control, + "alt" | "option" => Key::Alt, + "shift" => Key::Shift, + other => other.chars().next().map(Key::Unicode).unwrap_or(Key::Space), + } +} + +impl NativeHost { + async fn capture_shot(&self) -> Result<(Vec, u32, u32), String> { + let (pinned, over, max_dim, q) = ( + self.pinned_id(), + self.monitor, + self.max_dimension, + self.jpeg_quality, + ); + let (bytes, tw, th, id, geom) = spawn_blocking(move || capture(pinned, over, max_dim, q)) + .await + .map_err(|e| format!("capture task failed: {e}"))??; + let _ = self.pinned.set((id, geom)); + Ok((bytes, tw, th)) + } +} + +#[async_trait] +impl Driver for NativeHost { + async fn screen_size(&self) -> Result { + let (_, tw, th) = self.capture_shot().await?; + Ok(Screen { + width: tw, + height: th, + }) + } + + async fn screenshot(&self) -> Result { + let (bytes, _, _) = self.capture_shot().await?; + if bytes.is_empty() { + return Err("screenshot: capture produced an empty image".to_string()); + } + Ok(Shot::new(bytes)) + } + + async fn left_click(&self, x: i64, y: i64) -> Result<(), String> { + with_enigo(self.geom(), move |e, geom| { + let (rx, ry) = to_global(x, y, geom); + e.move_mouse(rx, ry, Coordinate::Abs).map_err(input_err)?; + e.button(Button::Left, Direction::Click).map_err(input_err) + }) + .await + } + + async fn right_click(&self, x: i64, y: i64) -> Result<(), String> { + with_enigo(self.geom(), move |e, geom| { + let (rx, ry) = to_global(x, y, geom); + e.move_mouse(rx, ry, Coordinate::Abs).map_err(input_err)?; + e.button(Button::Right, Direction::Click).map_err(input_err) + }) + .await + } + + async fn double_click(&self, x: i64, y: i64) -> Result<(), String> { + with_enigo(self.geom(), move |e, geom| { + let (rx, ry) = to_global(x, y, geom); + e.move_mouse(rx, ry, Coordinate::Abs).map_err(input_err)?; + e.button(Button::Left, Direction::Click) + .map_err(input_err)?; + e.button(Button::Left, Direction::Click).map_err(input_err) + }) + .await + } + + async fn move_cursor(&self, x: i64, y: i64) -> Result<(), String> { + with_enigo(self.geom(), move |e, geom| { + let (rx, ry) = to_global(x, y, geom); + e.move_mouse(rx, ry, Coordinate::Abs).map_err(input_err) + }) + .await + } + + async fn scroll(&self, x: i64, y: i64, scroll_x: i64, scroll_y: i64) -> Result<(), String> { + with_enigo(self.geom(), move |e, geom| { + let (rx, ry) = to_global(x, y, geom); + e.move_mouse(rx, ry, Coordinate::Abs).map_err(input_err)?; + if scroll_x != 0 { + e.scroll(scroll_x as i32, Axis::Horizontal) + .map_err(input_err)?; + } + if scroll_y != 0 { + e.scroll(scroll_y as i32, Axis::Vertical) + .map_err(input_err)?; + } + Ok(()) + }) + .await + } + + async fn drag(&self, from: (i64, i64), to: (i64, i64), _button: &str) -> Result<(), String> { + with_enigo(self.geom(), move |e, geom| { + let (fx, fy) = to_global(from.0, from.1, geom); + let (tx, ty) = to_global(to.0, to.1, geom); + e.move_mouse(fx, fy, Coordinate::Abs).map_err(input_err)?; + e.button(Button::Left, Direction::Press) + .map_err(input_err)?; + e.move_mouse(tx, ty, Coordinate::Abs).map_err(input_err)?; + e.button(Button::Left, Direction::Release) + .map_err(input_err) + }) + .await + } + + async fn type_text(&self, text: &str) -> Result<(), String> { + let text = text.to_string(); + with_enigo(None, move |e, _| e.text(&text).map_err(input_err)).await + } + + async fn keypress(&self, keys: &[String]) -> Result<(), String> { + let keys: Vec = keys.to_vec(); + with_enigo(None, move |e, _| { + // A single key is a tap; a chord holds all but the last as + // modifiers, taps the last, then releases the modifiers. + let Some((last, mods)) = keys.split_last() else { + return Ok(()); + }; + for m in mods { + e.key(key_for(m), Direction::Press).map_err(input_err)?; + } + let tapped = e.key(key_for(last), Direction::Click).map_err(input_err); + for m in mods.iter().rev() { + let _ = e.key(key_for(m), Direction::Release); + } + tapped + }) + .await + } + + async fn accessibility_tree(&self) -> Result { + // No native a11y tree yet; observe falls back to the screenshot. + Ok(serde_json::Value::Null) + } + + async fn close(&self) -> Result<(), String> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn target_dims_caps_longest_edge() { + assert_eq!(target_dims(3024, 1964, 1280), (1280, 831)); + assert_eq!(target_dims(800, 600, 1280), (800, 600)); + assert_eq!(target_dims(0, 0, 1280), (0, 0)); + } + + #[test] + fn to_global_maps_downscaled_to_display_points() { + // Built-in Retina: 1280-wide downscale of a 1512-point display -> ~1.18x, + // origin (0,0). + let g = Geom { + origin_x: 0, + origin_y: 0, + scale_x: 1512.0 / 1280.0, + scale_y: 982.0 / 831.0, + }; + assert_eq!(to_global(640, 415, Some(g)), (756, 490)); + // Secondary display to the right of a 1512-pt main, 1:1 scale. + let g2 = Geom { + origin_x: 1512, + origin_y: 0, + scale_x: 1.0, + scale_y: 1.0, + }; + assert_eq!(to_global(100, 200, Some(g2)), (1612, 200)); + // Identity when the display is not yet known. + assert_eq!(to_global(50, 60, None), (50, 60)); + } +} diff --git a/computer/src/driver/remote.rs b/computer/src/driver/remote.rs new file mode 100644 index 000000000..482f25b07 --- /dev/null +++ b/computer/src/driver/remote.rs @@ -0,0 +1,323 @@ +//! The remote driver: an executor running inside the desktop guest, reached +//! over a socket. It speaks a tiny request/response protocol: each call is a +//! JSON frame `{"command": , "params": {...}}` and the reply is +//! `{"success": , ...}`. We use the WebSocket channel (`/ws`), which is +//! strictly one-reply-per-command, so the socket is held behind a mutex and +//! each command serializes send-then-recv under it. +//! +//! Transport failures (send error, server close, transport error, timeout) +//! drop the socket so every later call fails fast with a clear message rather +//! than hanging; logical failures (`success: false`) surface as errors but +//! keep the connection. + +use std::time::Duration; + +use async_trait::async_trait; +use base64::engine::general_purpose::STANDARD; +use base64::Engine; +use futures_util::{SinkExt, StreamExt}; +use serde_json::{json, Value}; +use tokio::net::TcpStream; +use tokio::sync::Mutex; +use tokio::time::timeout; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; + +use super::{Driver, Screen, Shot}; + +type Socket = WebSocketStream>; + +pub struct RemoteClient { + endpoint: String, + socket: Mutex>, + command_timeout: Duration, +} + +impl RemoteClient { + /// Connect to a guest executor endpoint. `endpoint` may be a full ws/wss + /// url, an http/https url (mapped to ws/wss), or a bare `host:port` (ws + /// assumed). A `/ws` path is appended unless already present. + pub async fn connect( + endpoint: &str, + connect_timeout_ms: u64, + command_timeout_ms: u64, + ) -> Result { + let url = ws_url(endpoint)?; + let (socket, _resp) = timeout( + Duration::from_millis(connect_timeout_ms), + connect_async(&url), + ) + .await + .map_err(|_| format!("timed out connecting to {url} after {connect_timeout_ms}ms"))? + .map_err(|e| format!("failed to connect to {url}: {e}"))?; + Ok(Self { + endpoint: url, + socket: Mutex::new(Some(socket)), + command_timeout: Duration::from_millis(command_timeout_ms), + }) + } + + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + /// Send one command, read its reply. Serializes on the socket mutex; skips + /// server-initiated ping/pong; distinguishes transport failures (drop the + /// socket) from logical `success:false` failures (keep it). + async fn command(&self, command: &str, params: Value) -> Result { + let mut guard = self.socket.lock().await; + let socket = guard + .as_mut() + .ok_or_else(|| format!("{command}: driver connection is closed"))?; + let frame = json!({ "command": command, "params": params }).to_string(); + + let outcome: Result = timeout(self.command_timeout, async { + socket + .send(Message::Text(frame)) + .await + .map_err(|e| CommandError::Transport(format!("{command}: send failed: {e}")))?; + loop { + match socket.next().await { + Some(Ok(Message::Text(t))) => { + return parse_reply(command, &t).map_err(CommandError::Logical) + } + Some(Ok(Message::Binary(b))) => { + return parse_reply(command, &String::from_utf8_lossy(&b)) + .map_err(CommandError::Logical) + } + Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => continue, + Some(Ok(Message::Close(_))) => { + return Err(CommandError::Transport(format!( + "{command}: server closed the connection" + ))) + } + Some(Ok(_)) => continue, + Some(Err(e)) => { + return Err(CommandError::Transport(format!( + "{command}: transport error: {e}" + ))) + } + None => { + return Err(CommandError::Transport(format!( + "{command}: connection ended" + ))) + } + } + } + }) + .await + .unwrap_or_else(|_| { + Err(CommandError::Transport(format!( + "{command}: timed out after {}ms", + self.command_timeout.as_millis() + ))) + }); + + match outcome { + Ok(v) => Ok(v), + // A transport failure may leave the socket desynced; drop it so the + // next call fails fast instead of hanging on a stale connection. + Err(CommandError::Transport(msg)) => { + *guard = None; + Err(msg) + } + Err(CommandError::Logical(msg)) => Err(msg), + } + } + + /// A command whose reply carries nothing beyond success (pointer, keyboard, + /// and file-write actions). Collapses the `.await?; Ok(())` boilerplate. + async fn command_ok(&self, command: &str, params: Value) -> Result<(), String> { + self.command(command, params).await.map(|_| ()) + } +} + +/// Whether a failed command left the socket usable. Transport failures drop the +/// connection; logical `success:false` failures keep it. +enum CommandError { + Transport(String), + Logical(String), +} + +/// Parse an executor reply. `success:true` returns the whole object (so +/// callers can read `image_data`, `stdout`, etc.); `success:false` maps to a +/// readable error, preferring an `error` string and falling back to the +/// `return_code`/`stderr` composite the wire uses for shell failures. +fn parse_reply(command: &str, text: &str) -> Result { + let v: Value = + serde_json::from_str(text).map_err(|e| format!("{command}: invalid reply json: {e}"))?; + if v.get("success").and_then(Value::as_bool).unwrap_or(false) { + return Ok(v); + } + if let Some(err) = v.get("error").and_then(Value::as_str) { + return Err(format!("{command}: {err}")); + } + if let Some(rc) = v.get("return_code").or_else(|| v.get("returncode")) { + let stderr = v.get("stderr").and_then(Value::as_str).unwrap_or(""); + return Err(format!("{command}: failed (return_code={rc}) {stderr}")); + } + Err(format!("{command}: driver returned success=false")) +} + +#[async_trait] +impl Driver for RemoteClient { + async fn screen_size(&self) -> Result { + let v = self.command("get_screen_size", json!({})).await?; + // Some builds nest under `size`, others return width/height flat. + let obj = v.get("size").unwrap_or(&v); + let width = obj + .get("width") + .and_then(Value::as_u64) + .ok_or("get_screen_size: reply missing width")?; + let height = obj + .get("height") + .and_then(Value::as_u64) + .ok_or("get_screen_size: reply missing height")?; + Ok(Screen { + width: width as u32, + height: height as u32, + }) + } + + async fn screenshot(&self) -> Result { + let v = self.command("screenshot", json!({})).await?; + let b64 = v + .get("image_data") + .and_then(Value::as_str) + .ok_or("screenshot: reply missing image_data")?; + let bytes = STANDARD + .decode(b64) + .map_err(|e| format!("screenshot: invalid base64: {e}"))?; + if bytes.is_empty() { + return Err("screenshot: driver returned an empty image".to_string()); + } + Ok(Shot::new(bytes)) + } + + async fn left_click(&self, x: i64, y: i64) -> Result<(), String> { + self.command_ok("left_click", json!({ "x": x, "y": y })) + .await + } + + async fn right_click(&self, x: i64, y: i64) -> Result<(), String> { + self.command_ok("right_click", json!({ "x": x, "y": y })) + .await + } + + async fn double_click(&self, x: i64, y: i64) -> Result<(), String> { + self.command_ok("double_click", json!({ "x": x, "y": y })) + .await + } + + async fn move_cursor(&self, x: i64, y: i64) -> Result<(), String> { + self.command_ok("move_cursor", json!({ "x": x, "y": y })) + .await + } + + async fn scroll(&self, x: i64, y: i64, scroll_x: i64, scroll_y: i64) -> Result<(), String> { + self.command_ok( + "scroll", + json!({ "x": x, "y": y, "scroll_x": scroll_x, "scroll_y": scroll_y }), + ) + .await + } + + async fn drag(&self, from: (i64, i64), to: (i64, i64), button: &str) -> Result<(), String> { + self.command_ok( + "drag", + json!({ "path": [[from.0, from.1], [to.0, to.1]], "button": button }), + ) + .await + } + + async fn type_text(&self, text: &str) -> Result<(), String> { + self.command_ok("type_text", json!({ "text": text })).await + } + + async fn keypress(&self, keys: &[String]) -> Result<(), String> { + self.command_ok("hotkey", json!({ "keys": keys })).await + } + + async fn accessibility_tree(&self) -> Result { + let v = self.command("get_accessibility_tree", json!({})).await?; + Ok(v.get("tree").cloned().unwrap_or(v)) + } + + async fn close(&self) -> Result<(), String> { + let mut guard = self.socket.lock().await; + if let Some(mut socket) = guard.take() { + let _ = socket.close(None).await; + } + Ok(()) + } +} + +/// Normalize a caller endpoint into a `ws://.../ws` (or `wss`) url. +fn ws_url(endpoint: &str) -> Result { + let e = endpoint.trim(); + if e.is_empty() { + return Err( + "endpoint is empty: pass one to sessions::start or set default_endpoint".to_string(), + ); + } + let base = if let Some(rest) = e.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = e.strip_prefix("http://") { + format!("ws://{rest}") + } else if e.starts_with("ws://") || e.starts_with("wss://") { + e.to_string() + } else { + format!("ws://{e}") + }; + let trimmed = base.trim_end_matches('/'); + if trimmed.ends_with("/ws") { + Ok(trimmed.to_string()) + } else { + Ok(format!("{trimmed}/ws")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ws_url_normalizes_schemes_and_path() { + assert_eq!(ws_url("127.0.0.1:8000").unwrap(), "ws://127.0.0.1:8000/ws"); + assert_eq!(ws_url("http://host:8000").unwrap(), "ws://host:8000/ws"); + assert_eq!(ws_url("https://host:8443").unwrap(), "wss://host:8443/ws"); + assert_eq!(ws_url("ws://host:8000/ws").unwrap(), "ws://host:8000/ws"); + assert_eq!(ws_url("ws://host:8000/ws/").unwrap(), "ws://host:8000/ws"); + assert_eq!(ws_url("wss://host/ws").unwrap(), "wss://host/ws"); + assert!(ws_url(" ").is_err()); + } + + #[test] + fn parse_reply_ok_returns_object() { + let v = parse_reply("screenshot", r#"{"success":true,"image_data":"abc"}"#).unwrap(); + assert_eq!(v["image_data"], "abc"); + } + + #[test] + fn parse_reply_surfaces_error_string() { + let err = + parse_reply("left_click", r#"{"success":false,"error":"no display"}"#).unwrap_err(); + assert!(err.contains("no display"), "{err}"); + } + + #[test] + fn parse_reply_surfaces_return_code_composite() { + let err = parse_reply( + "run_command", + r#"{"success":false,"return_code":2,"stderr":"boom"}"#, + ) + .unwrap_err(); + assert!(err.contains("return_code=2"), "{err}"); + assert!(err.contains("boom"), "{err}"); + } + + #[test] + fn parse_reply_rejects_invalid_json() { + assert!(parse_reply("x", "not json").is_err()); + } +} diff --git a/computer/src/driver/sandbox.rs b/computer/src/driver/sandbox.rs new file mode 100644 index 000000000..a425fd66e --- /dev/null +++ b/computer/src/driver/sandbox.rs @@ -0,0 +1,498 @@ +//! The iii-sandbox driver: a desktop running inside an iii-sandbox microVM, +//! driven entirely through iii primitives. No executor, no socket into the +//! guest. +//! +//! iii-sandbox boots a libkrun microVM from an OCI image and exposes it only +//! through `sandbox::exec` / `sandbox::fs` over the engine bus; there is no +//! inbound TCP into the guest. So this driver does not connect to anything. +//! It boots a sandbox, brings up a virtual display (Xvfb) with a fixed +//! resolution, and maps every desktop semantic onto a `sandbox::exec` call: +//! +//! - screenshot: `import` grabs the X root and pipes JPEG to stdout, base64'd. +//! - pointer/keyboard: `xdotool` against `DISPLAY=:0`. +//! +//! A fixed virtual resolution means 1:1 coordinates and no HiDPI or +//! multi-monitor ambiguity: the pixel the model reads is the pixel it clicks. +//! Pointer/keyboard calls run as a raw argv (no shell), so caller text and +//! coordinates are literal arguments and never shell-interpreted. + +use async_trait::async_trait; +use base64::engine::general_purpose::STANDARD; +use base64::Engine; +use serde_json::{json, Value}; +use std::sync::Arc; + +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::IIIClient; + +use super::{Driver, Screen, Shot}; + +/// Longest a `sandbox::create` (incl. a cold OCI pull) may take before we give +/// up. The daemon recommends 300s for a cold pull. +const BOOT_TIMEOUT_MS: u64 = 300_000; +/// The virtual display the desktop renders to. +const DISPLAY: &str = ":0"; + +/// One live desktop inside an iii-sandbox microVM. +pub struct IiiSandboxHost { + iii: Arc, + sandbox_id: String, + screen: Screen, + jpeg_quality: u8, + command_timeout_ms: u64, +} + +impl IiiSandboxHost { + /// Boot a sandbox from `image`, bring up Xvfb at `width`x`height`, and + /// return a ready host. On any bootstrap failure the sandbox is stopped so + /// a failed start never leaks a live VM. + #[allow(clippy::too_many_arguments)] + pub async fn create( + iii: Arc, + image: &str, + width: u32, + height: u32, + jpeg_quality: u8, + idle_timeout_secs: u64, + command_timeout_ms: u64, + ) -> Result { + let created = iii + .trigger(TriggerRequest { + function_id: "sandbox::create".to_string(), + payload: json!({ + "image": image, + "network": true, + "idle_timeout_secs": idle_timeout_secs, + }), + action: None, + timeout_ms: Some(BOOT_TIMEOUT_MS), + }) + .await + .map_err(|e| format!("sandbox::create({image}) failed: {e}"))?; + let sandbox_id = created + .get("sandbox_id") + .and_then(Value::as_str) + .ok_or("sandbox::create reply missing sandbox_id")? + .to_string(); + + let host = Self { + iii, + sandbox_id, + screen: Screen { width, height }, + jpeg_quality, + command_timeout_ms, + }; + if let Err(e) = host.bootstrap().await { + // Best-effort teardown: a half-booted sandbox must not linger. + let _ = host.stop_sandbox().await; + return Err(e); + } + Ok(host) + } + + /// Re-attach to a sandbox that outlived a worker restart (its id was + /// persisted). Re-runs the idempotent bootstrap; if the sandbox is gone the + /// bootstrap exec fails and the caller drops the record. + pub async fn attach( + iii: Arc, + sandbox_id: String, + width: u32, + height: u32, + jpeg_quality: u8, + command_timeout_ms: u64, + ) -> Result { + let host = Self { + iii, + sandbox_id, + screen: Screen { width, height }, + jpeg_quality, + command_timeout_ms, + }; + host.bootstrap().await?; + Ok(host) + } + + pub fn sandbox_id(&self) -> &str { + &self.sandbox_id + } + + /// Start Xvfb + a window manager and block until the display answers. + /// Detach with `setsid` (a new session, not just `nohup`): the daemon + /// SIGKILLs the exec child's process group on host disconnect, and only a + /// fresh session escapes that, so the display keeps running between exec + /// calls. Idempotent: re-running against a live display is a no-op, so + /// `attach` reuses it. + async fn bootstrap(&self) -> Result<(), String> { + let script = format!( + "if ! xdpyinfo >/dev/null 2>&1; then \ + setsid Xvfb {DISPLAY} -screen 0 {w}x{h}x24 -nolisten tcp >/tmp/xvfb.log 2>&1 & \ + fi; \ + n=0; \ + while ! xdpyinfo >/dev/null 2>&1; do \ + n=$((n+1)); \ + if [ $n -gt 100 ]; then echo xvfb-timeout >&2; tail -n 20 /tmp/xvfb.log >&2; exit 1; fi; \ + sleep 0.1; \ + done; \ + if ! pgrep -x openbox >/dev/null 2>&1; then setsid openbox >/tmp/openbox.log 2>&1 & fi; \ + echo ready", + w = self.screen.width, + h = self.screen.height, + ); + self.exec_sh(&script) + .await + .map_err(|e| format!("sandbox {} display bootstrap failed: {e}", self.sandbox_id))?; + Ok(()) + } + + /// Run a command as a raw argv (no shell): caller text and coordinates are + /// literal arguments, never shell-parsed. `DISPLAY` is injected so X tools + /// target the virtual display. + async fn exec_argv(&self, cmd: &str, args: Vec) -> Result { + self.exec(json!({ + "sandbox_id": self.sandbox_id, + "cmd": cmd, + "args": args, + "env": { "DISPLAY": DISPLAY }, + "timeout_ms": self.command_timeout_ms, + })) + .await + } + + /// Run a `sh -lc` line, for the few cases that need a pipe (screenshot) or + /// shell control flow (bootstrap). Never interpolates caller-supplied text. + async fn exec_sh(&self, script: &str) -> Result { + self.exec(json!({ + "sandbox_id": self.sandbox_id, + "cmd": "sh", + "args": ["-lc", script], + "env": { "DISPLAY": DISPLAY }, + "timeout_ms": self.command_timeout_ms, + })) + .await + } + + async fn exec(&self, payload: Value) -> Result { + let v = self + .iii + .trigger(TriggerRequest { + function_id: "sandbox::exec".to_string(), + payload, + action: None, + // Give the bus a margin over the daemon's own exec deadline. + timeout_ms: Some(self.command_timeout_ms + 5_000), + }) + .await + .map_err(|e| format!("sandbox::exec failed: {e}"))?; + ExecOut::from_reply(&v) + } + + async fn stop_sandbox(&self) -> Result<(), String> { + self.iii + .trigger(TriggerRequest { + function_id: "sandbox::stop".to_string(), + payload: json!({ "sandbox_id": self.sandbox_id }), + action: None, + timeout_ms: Some(self.command_timeout_ms), + }) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) + } + + /// Move the pointer, then click `button` `count` times. + async fn click_at(&self, x: i64, y: i64, button: &str, count: u32) -> Result<(), String> { + let mut args = vec![ + "mousemove".to_string(), + clamp(x), + clamp(y), + "click".to_string(), + ]; + if count > 1 { + args.push("--repeat".to_string()); + args.push(count.to_string()); + args.push("--delay".to_string()); + args.push("40".to_string()); + } + args.push(button.to_string()); + self.exec_argv("xdotool", args).await.map(|_| ()) + } + + /// Spin the wheel `button` `count` times at the current pointer position. + async fn wheel(&self, x: i64, y: i64, button: &str, count: i64) -> Result<(), String> { + if count <= 0 { + return Ok(()); + } + self.exec_argv( + "xdotool", + vec![ + "mousemove".to_string(), + clamp(x), + clamp(y), + "click".to_string(), + "--repeat".to_string(), + count.to_string(), + button.to_string(), + ], + ) + .await + .map(|_| ()) + } +} + +#[async_trait] +impl Driver for IiiSandboxHost { + async fn screen_size(&self) -> Result { + // Fixed virtual resolution set at boot; no round-trip needed. + Ok(self.screen) + } + + async fn screenshot(&self) -> Result { + let script = format!( + "import -window root -quality {q} jpg:- | base64 -w0", + q = self.jpeg_quality + ); + let out = self.exec_sh(&script).await?; + let b64 = out.stdout.trim(); + if b64.is_empty() { + return Err(format!( + "screenshot: empty capture (stderr: {})", + out.stderr.trim() + )); + } + let bytes = STANDARD + .decode(b64) + .map_err(|e| format!("screenshot: invalid base64 from guest: {e}"))?; + if bytes.is_empty() { + return Err("screenshot: guest returned an empty image".to_string()); + } + Ok(Shot::new(bytes)) + } + + async fn left_click(&self, x: i64, y: i64) -> Result<(), String> { + self.click_at(x, y, "1", 1).await + } + + async fn right_click(&self, x: i64, y: i64) -> Result<(), String> { + self.click_at(x, y, "3", 1).await + } + + async fn double_click(&self, x: i64, y: i64) -> Result<(), String> { + self.click_at(x, y, "1", 2).await + } + + async fn move_cursor(&self, x: i64, y: i64) -> Result<(), String> { + self.exec_argv("xdotool", vec!["mousemove".to_string(), clamp(x), clamp(y)]) + .await + .map(|_| ()) + } + + async fn scroll(&self, x: i64, y: i64, scroll_x: i64, scroll_y: i64) -> Result<(), String> { + // X wheel buttons: 4 up, 5 down, 6 left, 7 right. Positive scroll_y + // scrolls down (matches the act default), positive scroll_x right. + if scroll_y > 0 { + self.wheel(x, y, "5", scroll_y).await?; + } else if scroll_y < 0 { + self.wheel(x, y, "4", -scroll_y).await?; + } + if scroll_x > 0 { + self.wheel(x, y, "7", scroll_x).await?; + } else if scroll_x < 0 { + self.wheel(x, y, "6", -scroll_x).await?; + } + Ok(()) + } + + async fn drag(&self, from: (i64, i64), to: (i64, i64), button: &str) -> Result<(), String> { + let b = button_number(button); + self.exec_argv( + "xdotool", + vec![ + "mousemove".to_string(), + clamp(from.0), + clamp(from.1), + "mousedown".to_string(), + b.to_string(), + "mousemove".to_string(), + clamp(to.0), + clamp(to.1), + "mouseup".to_string(), + b.to_string(), + ], + ) + .await + .map(|_| ()) + } + + async fn type_text(&self, text: &str) -> Result<(), String> { + self.exec_argv( + "xdotool", + vec![ + "type".to_string(), + "--clearmodifiers".to_string(), + "--".to_string(), + text.to_string(), + ], + ) + .await + .map(|_| ()) + } + + async fn keypress(&self, keys: &[String]) -> Result<(), String> { + let combo = keys + .iter() + .map(|k| map_key(k)) + .collect::>() + .join("+"); + self.exec_argv( + "xdotool", + vec!["key".to_string(), "--clearmodifiers".to_string(), combo], + ) + .await + .map(|_| ()) + } + + async fn accessibility_tree(&self) -> Result { + // A Linux X guest exposes no macOS-style AX tree. + Ok(serde_json::Value::Null) + } + + async fn close(&self) -> Result<(), String> { + // Idempotent: a stop against an already-reaped sandbox is a success. + match self.stop_sandbox().await { + Ok(()) => Ok(()), + Err(e) => { + tracing::debug!(sandbox = %self.sandbox_id, error = %e, "sandbox stop on close (treated as gone)"); + Ok(()) + } + } + } +} + +/// The subset of a `sandbox::exec` reply we act on. +#[derive(Debug)] +struct ExecOut { + stdout: String, + stderr: String, +} + +impl ExecOut { + fn from_reply(v: &Value) -> Result { + let stdout = v + .get("stdout") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let stderr = v + .get("stderr") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let exit_code = v.get("exit_code").and_then(Value::as_i64); + let success = v + .get("success") + .and_then(Value::as_bool) + .unwrap_or(exit_code == Some(0)); + if !success { + let code = exit_code.unwrap_or(-1); + let detail = if !stderr.trim().is_empty() { + stderr.trim() + } else { + stdout.trim() + }; + return Err(format!("guest command exited {code}: {detail}")); + } + Ok(Self { stdout, stderr }) + } +} + +/// Clamp a coordinate to a non-negative integer string for xdotool. +fn clamp(v: i64) -> String { + v.max(0).to_string() +} + +/// X button number for a named mouse button. +fn button_number(button: &str) -> u8 { + match button { + "right" => 3, + "middle" => 2, + _ => 1, + } +} + +/// Map a caller key name onto the X keysym `xdotool key` expects. Unknown +/// single names pass through (a literal char or an already-valid keysym). +fn map_key(key: &str) -> String { + match key.to_ascii_lowercase().as_str() { + "enter" | "return" => "Return", + "esc" | "escape" => "Escape", + "tab" => "Tab", + "space" | "spacebar" => "space", + "backspace" => "BackSpace", + "delete" | "del" => "Delete", + "up" | "arrowup" => "Up", + "down" | "arrowdown" => "Down", + "left" | "arrowleft" => "Left", + "right" | "arrowright" => "Right", + "home" => "Home", + "end" => "End", + "pageup" | "pgup" => "Prior", + "pagedown" | "pgdn" => "Next", + "ctrl" | "control" => "ctrl", + "alt" | "option" => "alt", + "shift" => "shift", + "cmd" | "command" | "super" | "meta" | "win" => "super", + _ => return key.to_string(), + } + .to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clamp_floors_negatives() { + assert_eq!(clamp(-5), "0"); + assert_eq!(clamp(0), "0"); + assert_eq!(clamp(1280), "1280"); + } + + #[test] + fn button_number_maps_names() { + assert_eq!(button_number("left"), 1); + assert_eq!(button_number("middle"), 2); + assert_eq!(button_number("right"), 3); + assert_eq!(button_number("weird"), 1); + } + + #[test] + fn map_key_normalizes_common_names() { + assert_eq!(map_key("enter"), "Return"); + assert_eq!(map_key("Escape"), "Escape"); + assert_eq!(map_key("cmd"), "super"); + assert_eq!(map_key("ctrl"), "ctrl"); + assert_eq!(map_key("a"), "a"); + assert_eq!(map_key("F5"), "F5"); + } + + #[test] + fn exec_out_flags_nonzero_exit() { + let ok = ExecOut::from_reply(&json!({ "stdout": "hi", "exit_code": 0, "success": true })); + assert!(ok.is_ok()); + let bad = ExecOut::from_reply( + &json!({ "stdout": "", "stderr": "boom", "exit_code": 2, "success": false }), + ); + let err = bad.unwrap_err(); + assert!(err.contains("exited 2"), "{err}"); + assert!(err.contains("boom"), "{err}"); + } + + #[test] + fn exec_out_infers_success_from_exit_code() { + // No explicit success flag: fall back to exit_code == 0. + let ok = ExecOut::from_reply(&json!({ "stdout": "x", "exit_code": 0 })); + assert!(ok.is_ok()); + let bad = ExecOut::from_reply(&json!({ "stderr": "no", "exit_code": 1 })); + assert!(bad.is_err()); + } +} diff --git a/computer/src/events.rs b/computer/src/events.rs new file mode 100644 index 000000000..a4478bd8a --- /dev/null +++ b/computer/src/events.rs @@ -0,0 +1,404 @@ +//! The two custom trigger types this worker emits, and the fan-out behind +//! them. Consumers bind handlers with the standard two-step pattern; the +//! engine routes each registration to our [`TriggerHandler`]. Delivery is +//! fire-and-forget (`TriggerAction::Void`) and at-least-once; the optional +//! per-binding `session_id` filter is evaluated here by the emitting worker, +//! and malformed configs are rejected at registration time. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use iii_sdk::errors::Error; +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::trigger::{TriggerConfig, TriggerHandler}; +use iii_sdk::{IIIClient, RegisterTriggerType, TriggerAction}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::driver::Screen; + +pub const SESSION_STARTED: &str = "computer::session-started"; +pub const SESSION_STOPPED: &str = "computer::session-stopped"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EventKind { + SessionStarted, + SessionStopped, +} + +impl EventKind { + pub fn trigger_type(&self) -> &'static str { + match self { + EventKind::SessionStarted => SESSION_STARTED, + EventKind::SessionStopped => SESSION_STOPPED, + } + } + + pub fn all() -> [EventKind; 2] { + [EventKind::SessionStarted, EventKind::SessionStopped] + } +} + +/// Config accepted by every `computer::*` trigger binding. The only filter is +/// an optional session-id equality match; unknown fields fail at registration +/// so a misspelled filter key fails loudly instead of silently receiving +/// nothing. +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BindingConfig { + /// Only deliver events for this computer session. + #[serde(skip_serializing_if = "Option::is_none")] + pub session_id: Option, +} + +pub fn binding_matches(filter: &BindingConfig, session_id: &str) -> bool { + match &filter.session_id { + Some(want) => want == session_id, + None => true, + } +} + +// --------------------------------------------------------------------------- +// Event payloads (what subscribers receive) +// --------------------------------------------------------------------------- + +/// `computer::session-started` — a desktop session is connected and ready. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct SessionStartedEvent { + pub session_id: String, + /// The endpoint the session is driving (`native` for the local machine). + pub endpoint: String, + /// Informational guest OS label (`linux`, `macos`, ...). + pub os: String, + pub screen: Screen, + pub timestamp: i64, +} + +/// `computer::session-stopped` — a session ended; see `reason`. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct SessionStoppedEvent { + pub session_id: String, + /// One of `stopped`, `idle`. + pub reason: String, + pub timestamp: i64, +} + +// --------------------------------------------------------------------------- +// Subscriber registry + trigger type registration +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct Binding { + pub id: String, + pub function_id: String, + pub filter: BindingConfig, +} + +/// Thread-safe subscriber registry for one trigger type. +#[derive(Clone)] +pub struct SubscriberSet { + kind: EventKind, + inner: Arc>>, +} + +impl SubscriberSet { + pub fn new(kind: EventKind) -> Self { + Self { + kind, + inner: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Parse + insert a binding. Rejects malformed configs at registration. + pub fn add(&self, config: TriggerConfig) -> Result<(), String> { + let raw = if config.config.is_null() { + Value::Object(serde_json::Map::new()) + } else { + config.config.clone() + }; + let filter: BindingConfig = serde_json::from_value(raw) + .map_err(|e| format!("invalid {} config: {e}", self.kind.trigger_type()))?; + let binding = Binding { + id: config.id.clone(), + function_id: config.function_id, + filter, + }; + self.lock().insert(config.id, binding); + Ok(()) + } + + pub fn remove(&self, id: &str) { + self.lock().remove(id); + } + + /// Snapshot so the mutex is never held across awaits. + pub fn snapshot(&self) -> Vec { + self.lock().values().cloned().collect() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.inner.lock().unwrap_or_else(|p| p.into_inner()) + } +} + +/// The two subscriber sets, one per trigger type. +#[derive(Clone)] +pub struct TriggerSets { + inner: HashMap<&'static str, SubscriberSet>, +} + +impl TriggerSets { + pub fn new() -> Self { + let mut inner = HashMap::new(); + for kind in EventKind::all() { + inner.insert(kind.trigger_type(), SubscriberSet::new(kind)); + } + Self { inner } + } + + pub fn for_kind(&self, kind: EventKind) -> &SubscriberSet { + self.inner + .get(kind.trigger_type()) + .expect("all kinds are populated at construction") + } +} + +impl Default for TriggerSets { + fn default() -> Self { + Self::new() + } +} + +struct ComputerTriggerHandler { + set: SubscriberSet, +} + +#[async_trait] +impl TriggerHandler for ComputerTriggerHandler { + async fn register_trigger(&self, config: TriggerConfig) -> Result<(), Error> { + let id = config.id.clone(); + let function_id = config.function_id.clone(); + self.set.add(config).map_err(Error::Handler)?; + tracing::info!(id = %id, function_id = %function_id, "trigger subscription registered"); + Ok(()) + } + + async fn unregister_trigger(&self, config: TriggerConfig) -> Result<(), Error> { + tracing::info!(id = %config.id, "trigger subscription unregistered"); + self.set.remove(&config.id); + Ok(()) + } +} + +/// Register the custom trigger types with the engine. Must run before +/// `functions::register_all` so handlers can capture the subscriber sets. +pub fn register_trigger_types(iii: &Arc) -> TriggerSets { + let sets = TriggerSets::new(); + let descriptions: [(EventKind, &str); 2] = [ + ( + EventKind::SessionStarted, + "A desktop session connected and is ready to drive.", + ), + ( + EventKind::SessionStopped, + "A desktop session ended (stopped or idle).", + ), + ]; + for (kind, description) in descriptions { + let _ = iii.register_trigger_type( + RegisterTriggerType::new( + kind.trigger_type(), + description, + ComputerTriggerHandler { + set: sets.for_kind(kind).clone(), + }, + ) + .trigger_request_format::(), + ); + tracing::info!( + trigger_type = kind.trigger_type(), + "registered trigger type" + ); + } + sets +} + +// --------------------------------------------------------------------------- +// Emission +// --------------------------------------------------------------------------- + +/// How a matched event reaches a subscriber. Production delivers over the bus; +/// tests record. +#[async_trait] +pub trait EventDeliverer: Send + Sync { + async fn deliver(&self, trigger_type: &str, function_id: &str, payload: Value); +} + +/// Fire-and-forget bus delivery so the caller that produced the event is never +/// blocked on subscriber latency. Failures are logged and swallowed. +pub struct IiiDeliverer { + iii: Arc, +} + +impl IiiDeliverer { + pub fn new(iii: Arc) -> Self { + Self { iii } + } +} + +#[async_trait] +impl EventDeliverer for IiiDeliverer { + async fn deliver(&self, trigger_type: &str, function_id: &str, payload: Value) { + let iii = self.iii.clone(); + let trigger_type = trigger_type.to_string(); + let function_id = function_id.to_string(); + tokio::spawn(async move { + let res = iii + .trigger(TriggerRequest { + function_id: function_id.clone(), + payload, + action: Some(TriggerAction::Void), + timeout_ms: None, + }) + .await; + if let Err(e) = res { + tracing::warn!(trigger_type, function_id, error = %e, "event fan-out failed"); + } + }); + } +} + +/// Evaluates each binding's session filter against each event and delivers the +/// payload to every match. +pub struct Emitter { + sets: TriggerSets, + deliverer: Arc, +} + +impl Emitter { + pub fn new(sets: TriggerSets, deliverer: Arc) -> Self { + Self { sets, deliverer } + } + + pub async fn emit(&self, kind: EventKind, session_id: &str, payload: &T) { + let bindings = self.sets.for_kind(kind).snapshot(); + let mut matched = bindings + .into_iter() + .filter(|b| binding_matches(&b.filter, session_id)) + .peekable(); + if matched.peek().is_none() { + return; + } + let payload = match serde_json::to_value(payload) { + Ok(v) => v, + Err(e) => { + tracing::warn!(error = %e, "event payload failed to serialize"); + return; + } + }; + for binding in matched { + self.deliverer + .deliver(kind.trigger_type(), &binding.function_id, payload.clone()) + .await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn trigger_config(config: Value) -> TriggerConfig { + TriggerConfig { + id: "t1".to_string(), + function_id: "console::on-event".to_string(), + config, + metadata: None, + } + } + + #[test] + fn empty_filter_matches_everything() { + assert!(binding_matches(&BindingConfig::default(), "c1")); + } + + #[test] + fn session_filter_is_equality() { + let filter = BindingConfig { + session_id: Some("c1".to_string()), + }; + assert!(binding_matches(&filter, "c1")); + assert!(!binding_matches(&filter, "c2")); + } + + #[test] + fn add_rejects_unknown_filter_keys() { + let set = SubscriberSet::new(EventKind::SessionStarted); + let err = set + .add(trigger_config(json!({ "sesion_id": "c1" }))) + .unwrap_err(); + assert!(err.contains("computer::session-started"), "{err}"); + } + + #[test] + fn add_accepts_null_config() { + let set = SubscriberSet::new(EventKind::SessionStopped); + set.add(trigger_config(Value::Null)).unwrap(); + assert_eq!(set.snapshot().len(), 1); + } + + #[tokio::test] + async fn emit_delivers_only_to_matching_bindings() { + struct Recorder(Mutex>); + + #[async_trait] + impl EventDeliverer for Recorder { + async fn deliver(&self, trigger_type: &str, function_id: &str, _payload: Value) { + self.0 + .lock() + .unwrap() + .push((trigger_type.to_string(), function_id.to_string())); + } + } + + let sets = TriggerSets::new(); + let set = sets.for_kind(EventKind::SessionStopped); + set.add(TriggerConfig { + id: "match".to_string(), + function_id: "fn::match".to_string(), + config: json!({ "session_id": "c1" }), + metadata: None, + }) + .unwrap(); + set.add(TriggerConfig { + id: "other".to_string(), + function_id: "fn::other".to_string(), + config: json!({ "session_id": "c2" }), + metadata: None, + }) + .unwrap(); + + let recorder = Arc::new(Recorder(Mutex::new(Vec::new()))); + let emitter = Emitter::new(sets, recorder.clone()); + emitter + .emit( + EventKind::SessionStopped, + "c1", + &SessionStoppedEvent { + session_id: "c1".to_string(), + reason: "stopped".to_string(), + timestamp: 0, + }, + ) + .await; + + let seen = recorder.0.lock().unwrap().clone(); + assert_eq!( + seen, + vec![(SESSION_STOPPED.to_string(), "fn::match".to_string())] + ); + } +} diff --git a/computer/src/functions/act.rs b/computer/src/functions/act.rs new file mode 100644 index 000000000..c234a9804 --- /dev/null +++ b/computer/src/functions/act.rs @@ -0,0 +1,51 @@ +//! `computer::act` — pointer and keyboard input against the desktop, by +//! coordinate. Coordinates are integer pixels, top-left origin, in the space +//! of the screenshot (`computer::screenshot` / the live frame). + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ActInput { + pub session_id: String, + /// What to do: `click`, `right_click`, `double_click`, `move`, `drag`, + /// `scroll`, `type`, `press`, or `hotkey`. + pub action: String, + /// Target x (pixels). Required for click/right_click/double_click/move and + /// the drag start; the scroll anchor defaults to screen center. + #[serde(default)] + pub x: Option, + /// Target y (pixels). + #[serde(default)] + pub y: Option, + /// Drag end x (with `to_y`), for `action=drag`. + #[serde(default)] + pub to_x: Option, + /// Drag end y. + #[serde(default)] + pub to_y: Option, + /// Mouse button for `drag` (`left`, `right`, `middle`). Default `left`. + #[serde(default)] + pub button: Option, + /// Text to type, for `action=type`. + #[serde(default)] + pub text: Option, + /// Keys for `press`/`hotkey`: a single key (`["enter"]`) or a chord + /// (`["ctrl", "c"]`). + #[serde(default)] + pub keys: Option>, + /// Horizontal scroll amount for `action=scroll`. Default 0. + #[serde(default)] + pub scroll_x: Option, + /// Vertical scroll amount for `action=scroll`; positive scrolls down. + /// Default 3. + #[serde(default)] + pub scroll_y: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ActOutput { + pub ok: bool, + /// What was done, for the transcript. + pub detail: String, +} diff --git a/computer/src/functions/displays.rs b/computer/src/functions/displays.rs new file mode 100644 index 000000000..a35bf768f --- /dev/null +++ b/computer/src/functions/displays.rs @@ -0,0 +1,15 @@ +//! `computer::displays` — list the local displays so a caller can pick which +//! one a native session drives. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::driver::DisplayInfo; + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct DisplaysInput {} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct DisplaysOutput { + pub displays: Vec, +} diff --git a/computer/src/functions/frame.rs b/computer/src/functions/frame.rs new file mode 100644 index 000000000..d5d296378 --- /dev/null +++ b/computer/src/functions/frame.rs @@ -0,0 +1,51 @@ +//! `computer::screencast::start` / `stop` / `computer::frame` — the live-view +//! pipeline behind the console viewport. The worker polls the driver +//! screenshot at the configured fps and pushes each frame onto the +//! `computer:frames` stream; `computer::frame` hands out the newest frame +//! without a capture round-trip so the UI can poll fast. All three are +//! internal console-UI plumbing, not agent surface — agents read the desktop +//! with `computer::screenshot` and `computer::observe`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ScreencastStartInput { + pub session_id: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ScreencastStopInput { + /// Stopping the screencast on an unknown session succeeds. + pub session_id: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct AckOutput { + pub ok: bool, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct FrameInput { + pub session_id: String, + /// Frame cursor from the previous read; when the newest frame still has + /// this seq the response omits `frame` (nothing changed). + #[serde(default)] + pub since_frame: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct FrameOutput { + /// Base64 image of the newest frame; absent when `since_frame` is still + /// current or no frame has arrived yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub frame: Option, + /// Image mime of the frame bytes. + pub mime: String, + pub width: u32, + pub height: u32, + pub frame_seq: u64, + pub timestamp: i64, + /// False when no screencast is running (call screencast::start first). + pub active: bool, +} diff --git a/computer/src/functions/mod.rs b/computer/src/functions/mod.rs new file mode 100644 index 000000000..c50cbd335 --- /dev/null +++ b/computer/src/functions/mod.rs @@ -0,0 +1,505 @@ +//! Function registration: the `computer::*` wire surface. One module per +//! function (or small family) holds the typed request/response structs; +//! registration lives here so `register_all` reads as the product surface. + +pub mod act; +pub mod displays; +pub mod frame; +pub mod observe; +pub mod screenshot; +pub mod sessions; + +use std::sync::Arc; + +use iii_sdk::errors::Error; +use iii_sdk::{IIIClient, RegisterFunction}; + +use crate::driver::Screen; +use crate::session::{Session, Sessions}; + +pub const SESSIONS_START_ID: &str = "computer::sessions::start"; +pub const SESSIONS_START_DESC: &str = + "Start a computer-use session and return its session_id. Pass `image` to boot a fresh desktop \ + in an iii-sandbox microVM (fixed virtual display, no host setup), an `endpoint` to drive a \ + desktop through its guest executor, or omit both to drive the local machine. Sessions are \ + durable; stop them with computer::sessions::stop when done."; +pub const SESSIONS_LIST_ID: &str = "computer::sessions::list"; +pub const SESSIONS_LIST_DESC: &str = + "List live computer sessions with their endpoint, guest OS, and screen size."; +pub const SESSIONS_STOP_ID: &str = "computer::sessions::stop"; +pub const SESSIONS_STOP_DESC: &str = + "Stop a computer session and close its driver connection. Idempotent: stopping an unknown \ + or already-stopped session succeeds with was_running=false."; +pub const DISPLAYS_ID: &str = "computer::displays"; +pub const DISPLAYS_DESC: &str = + "List the local displays (index, name, primary, size). Pass the chosen index as `monitor` \ + to computer::sessions::start to drive that display; omit it to use the display under the \ + cursor. Native host only."; +pub const SCREENSHOT_ID: &str = "computer::screenshot"; +pub const SCREENSHOT_DESC: &str = + "Capture the desktop as a viewable image. This is how you see the screen before acting; the \ + coordinate space of computer::act is this image's pixels (top-left origin)."; +pub const OBSERVE_ID: &str = "computer::observe"; +pub const OBSERVE_DESC: &str = + "Capture the desktop plus, optionally, the accessibility tree. Use include_a11y on macOS \ + guests for a machine-readable element tree; elsewhere prefer computer::screenshot."; +pub const ACT_ID: &str = "computer::act"; +pub const ACT_DESC: &str = + "Drive the desktop: click, right_click, double_click, move, drag, scroll, type, press, or \ + hotkey. Address by pixel coordinates read off the screenshot (top-left origin)."; +pub const SCREENCAST_START_ID: &str = "computer::screencast::start"; +pub const SCREENCAST_START_DESC: &str = + "Internal: start pushing live desktop frames onto the computer:frames stream for the console \ + viewport. Console-UI plumbing; agents use computer::screenshot. Not an agent function."; +pub const SCREENCAST_STOP_ID: &str = "computer::screencast::stop"; +pub const SCREENCAST_STOP_DESC: &str = + "Internal: stop the live frame push. Idempotent. Not an agent function."; +pub const FRAME_ID: &str = "computer::frame"; +pub const FRAME_DESC: &str = + "Internal: newest screencast frame, or nothing when since_frame is still current. No capture \ + round-trip; poll fast. Not an agent function."; + +/// One wire-surface entry: everything the golden schema test pins. +pub struct FunctionSpec { + pub function_id: &'static str, + pub description: &'static str, + pub request_schema: schemars::schema::RootSchema, + pub response_schema: schemars::schema::RootSchema, +} + +/// Build a schema exactly the way iii-sdk does at registration, so the catalog +/// snapshot pins what registration emits. +fn spec(function_id: &'static str, description: &'static str) -> FunctionSpec +where + Req: schemars::JsonSchema, + Resp: schemars::JsonSchema, +{ + let generator = || schemars::r#gen::SchemaSettings::draft07().into_generator(); + FunctionSpec { + function_id, + description, + request_schema: generator().into_root_schema_for::(), + response_schema: generator().into_root_schema_for::(), + } +} + +/// The full wire-surface catalog, in registration order. Golden-tested in +/// `tests/schemas.rs`; keep in lockstep with `register_all`. +pub fn catalog() -> Vec { + vec![ + spec::(SESSIONS_START_ID, SESSIONS_START_DESC), + spec::(SESSIONS_LIST_ID, SESSIONS_LIST_DESC), + spec::(SESSIONS_STOP_ID, SESSIONS_STOP_DESC), + spec::(DISPLAYS_ID, DISPLAYS_DESC), + spec::( + SCREENSHOT_ID, + SCREENSHOT_DESC, + ), + spec::(OBSERVE_ID, OBSERVE_DESC), + spec::(ACT_ID, ACT_DESC), + spec::( + SCREENCAST_START_ID, + SCREENCAST_START_DESC, + ), + spec::( + SCREENCAST_STOP_ID, + SCREENCAST_STOP_DESC, + ), + spec::(FRAME_ID, FRAME_DESC), + ] +} + +pub fn register_all(iii: &Arc, sessions: &Arc) { + register_sessions_start(iii, sessions); + register_sessions_list(iii, sessions); + register_sessions_stop(iii, sessions); + register_displays(iii); + register_screenshot(iii, sessions); + register_observe(iii, sessions); + register_act(iii, sessions); + register_screencast_start(iii, sessions); + register_screencast_stop(iii, sessions); + register_frame(iii, sessions); +} + +// --------------------------------------------------------------------------- +// Handler helpers +// --------------------------------------------------------------------------- + +async fn get_session(sessions: &Arc, id: &str) -> Result, Error> { + sessions.get(id).await.ok_or_else(|| { + Error::Handler(format!( + "unknown session '{id}' (start one with computer::sessions::start)" + )) + }) +} + +fn require_xy(req: &act::ActInput) -> Result<(i64, i64), Error> { + match (req.x, req.y) { + (Some(x), Some(y)) => Ok((x, y)), + _ => Err(Error::Handler(format!( + "action '{}' needs x and y", + req.action + ))), + } +} + +fn xy_or_center(req: &act::ActInput, screen: Screen) -> (i64, i64) { + ( + req.x.unwrap_or(screen.width as i64 / 2), + req.y.unwrap_or(screen.height as i64 / 2), + ) +} + +// --------------------------------------------------------------------------- +// Registration +// --------------------------------------------------------------------------- + +/// Local displays, enumerated live from the OS. Empty on non-desktop targets. +#[cfg(any(target_os = "macos", target_os = "windows"))] +fn host_displays() -> Vec { + crate::driver::native::list_displays().unwrap_or_default() +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn host_displays() -> Vec { + Vec::new() +} + +fn register_displays(iii: &Arc) { + iii.register_function( + DISPLAYS_ID, + RegisterFunction::new_async(|_req: displays::DisplaysInput| async move { + Ok::<_, Error>(displays::DisplaysOutput { + displays: host_displays(), + }) + }) + .description(DISPLAYS_DESC), + ); +} + +fn register_sessions_start(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + SESSIONS_START_ID, + RegisterFunction::new_async(move |req: sessions::StartInput| { + let sessions = sessions.clone(); + async move { + let session = sessions + .start(req.image, req.endpoint, req.os, req.monitor) + .await + .map_err(Error::Handler)?; + Ok::<_, Error>(sessions::StartOutput { + session_id: session.id.clone(), + endpoint: session.endpoint.clone(), + os: session.os.clone(), + screen: session.screen, + }) + } + }) + .description(SESSIONS_START_DESC), + ); +} + +fn register_sessions_list(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + SESSIONS_LIST_ID, + RegisterFunction::new_async(move |_req: sessions::ListInput| { + let sessions = sessions.clone(); + async move { + let infos = sessions + .list() + .await + .iter() + .map(|s| sessions::SessionInfo { + session_id: s.id.clone(), + endpoint: s.endpoint.clone(), + os: s.os.clone(), + screen: s.screen, + created_ms: s.created_ms, + last_used_ms: s.last_used_ms(), + screencast_active: s.screencast_active(), + }) + .collect(); + Ok::<_, Error>(sessions::ListOutput { sessions: infos }) + } + }) + .description(SESSIONS_LIST_DESC), + ); +} + +fn register_sessions_stop(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + SESSIONS_STOP_ID, + RegisterFunction::new_async(move |req: sessions::StopInput| { + let sessions = sessions.clone(); + async move { + let was_running = sessions.stop(&req.session_id, "stopped").await; + Ok::<_, Error>(sessions::StopOutput { + ok: true, + was_running, + }) + } + }) + .description(SESSIONS_STOP_DESC), + ); +} + +fn register_screenshot(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + SCREENSHOT_ID, + RegisterFunction::new_async(move |req: screenshot::ScreenshotInput| { + let sessions = sessions.clone(); + async move { + let session = get_session(&sessions, &req.session_id).await?; + session.touch(); + let shot = session + .driver() + .screenshot() + .await + .map_err(Error::Handler)?; + let bytes = shot.byte_len(); + Ok::<_, Error>(screenshot::ScreenshotOutput { + content: vec![ + screenshot::ContentBlock::image(shot.mime.clone(), shot.to_base64()), + screenshot::ContentBlock::text(format!( + "Desktop of session {} ({}x{}, {bytes} bytes)", + session.id, session.screen.width, session.screen.height + )), + ], + details: screenshot::ScreenshotDetails { + session_id: session.id.clone(), + width: session.screen.width, + height: session.screen.height, + mime: shot.mime, + }, + }) + } + }) + .description(SCREENSHOT_DESC), + ); +} + +fn register_observe(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + OBSERVE_ID, + RegisterFunction::new_async(move |req: observe::ObserveInput| { + let sessions = sessions.clone(); + async move { + let session = get_session(&sessions, &req.session_id).await?; + session.touch(); + let driver = session.driver(); + let shot = driver.screenshot().await.map_err(Error::Handler)?; + let accessibility = if req.include_a11y.unwrap_or(false) { + match driver.accessibility_tree().await { + Ok(v) if !v.is_null() => Some(v), + Ok(_) => None, + Err(e) => { + tracing::debug!(session = %session.id, error = %e, "observe: a11y tree unavailable"); + None + } + } + } else { + None + }; + let bytes = shot.byte_len(); + let has_a11y = accessibility.is_some(); + Ok::<_, Error>(observe::ObserveOutput { + content: vec![ + screenshot::ContentBlock::image(shot.mime.clone(), shot.to_base64()), + screenshot::ContentBlock::text(format!( + "Desktop of session {} ({}x{}, {bytes} bytes){}", + session.id, + session.screen.width, + session.screen.height, + if has_a11y { + ", accessibility tree attached" + } else { + "" + } + )), + ], + details: observe::ObserveDetails { + session_id: session.id.clone(), + screen: session.screen, + mime: shot.mime, + }, + accessibility, + }) + } + }) + .description(OBSERVE_DESC), + ); +} + +fn register_act(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + ACT_ID, + RegisterFunction::new_async(move |req: act::ActInput| { + let sessions = sessions.clone(); + async move { + let session = get_session(&sessions, &req.session_id).await?; + session.touch(); + let driver = session.driver(); + let detail = match req.action.as_str() { + "click" | "left_click" => { + let (x, y) = require_xy(&req)?; + driver.left_click(x, y).await.map_err(Error::Handler)?; + format!("left click at ({x}, {y})") + } + "right_click" => { + let (x, y) = require_xy(&req)?; + driver.right_click(x, y).await.map_err(Error::Handler)?; + format!("right click at ({x}, {y})") + } + "double_click" => { + let (x, y) = require_xy(&req)?; + driver.double_click(x, y).await.map_err(Error::Handler)?; + format!("double click at ({x}, {y})") + } + "move" | "move_cursor" => { + let (x, y) = require_xy(&req)?; + driver.move_cursor(x, y).await.map_err(Error::Handler)?; + format!("move to ({x}, {y})") + } + "scroll" => { + let (x, y) = xy_or_center(&req, session.screen); + let sx = req.scroll_x.unwrap_or(0); + let sy = req.scroll_y.unwrap_or(3); + driver.scroll(x, y, sx, sy).await.map_err(Error::Handler)?; + format!("scroll ({sx}, {sy}) at ({x}, {y})") + } + "drag" => { + let (x, y) = require_xy(&req)?; + let (tx, ty) = match (req.to_x, req.to_y) { + (Some(tx), Some(ty)) => (tx, ty), + _ => { + return Err(Error::Handler( + "action 'drag' needs to_x and to_y".to_string(), + )) + } + }; + let button = req.button.as_deref().unwrap_or("left"); + driver + .drag((x, y), (tx, ty), button) + .await + .map_err(Error::Handler)?; + format!("drag ({x}, {y}) -> ({tx}, {ty})") + } + "type" => { + let text = req.text.as_deref().ok_or_else(|| { + Error::Handler("action 'type' needs text".to_string()) + })?; + driver.type_text(text).await.map_err(Error::Handler)?; + format!("type {} chars", text.chars().count()) + } + "press" | "hotkey" | "key" => { + let keys = + req.keys.as_ref().filter(|k| !k.is_empty()).ok_or_else(|| { + Error::Handler(format!( + "action '{}' needs a non-empty keys array", + req.action + )) + })?; + driver.keypress(keys).await.map_err(Error::Handler)?; + format!("press {}", keys.join("+")) + } + other => { + return Err(Error::Handler(format!( + "unknown action '{other}' (click, right_click, double_click, move, \ + drag, scroll, type, press, hotkey)" + ))) + } + }; + Ok::<_, Error>(act::ActOutput { ok: true, detail }) + } + }) + .description(ACT_DESC), + ); +} + +fn register_screencast_start(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + SCREENCAST_START_ID, + RegisterFunction::new_async(move |req: frame::ScreencastStartInput| { + let sessions = sessions.clone(); + async move { + let session = get_session(&sessions, &req.session_id).await?; + session.start_screencast().await; + Ok::<_, Error>(frame::AckOutput { ok: true }) + } + }) + .description(SCREENCAST_START_DESC) + .metadata(serde_json::json!({ "internal": true })), + ); +} + +fn register_screencast_stop(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + SCREENCAST_STOP_ID, + RegisterFunction::new_async(move |req: frame::ScreencastStopInput| { + let sessions = sessions.clone(); + async move { + if let Some(session) = sessions.get(&req.session_id).await { + session.stop_screencast().await; + } + Ok::<_, Error>(frame::AckOutput { ok: true }) + } + }) + .description(SCREENCAST_STOP_DESC) + .metadata(serde_json::json!({ "internal": true })), + ); +} + +fn register_frame(iii: &Arc, sessions: &Arc) { + let sessions = sessions.clone(); + iii.register_function( + FRAME_ID, + RegisterFunction::new_async(move |req: frame::FrameInput| { + let sessions = sessions.clone(); + async move { + let session = get_session(&sessions, &req.session_id).await?; + let active = session.screencast_active(); + let out = match session.latest_frame() { + Some(f) => { + // Fast no-change poll copies only metadata; the base64 + // is cloned once, only when a new frame is delivered. + let unchanged = req.since_frame == Some(f.frame_seq); + frame::FrameOutput { + frame: if unchanged { + None + } else { + Some(f.data_b64.clone()) + }, + mime: f.mime.clone(), + width: f.width, + height: f.height, + frame_seq: f.frame_seq, + timestamp: f.timestamp, + active, + } + } + None => frame::FrameOutput { + frame: None, + mime: "image/png".to_string(), + width: session.screen.width, + height: session.screen.height, + frame_seq: 0, + timestamp: 0, + active, + }, + }; + Ok::<_, Error>(out) + } + }) + .description(FRAME_DESC) + .metadata(serde_json::json!({ "internal": true })), + ); +} diff --git a/computer/src/functions/observe.rs b/computer/src/functions/observe.rs new file mode 100644 index 000000000..8d2eb2efa --- /dev/null +++ b/computer/src/functions/observe.rs @@ -0,0 +1,36 @@ +//! `computer::observe` — a screenshot plus, optionally, the accessibility +//! tree. The image is the reliable signal on every guest; the a11y tree is +//! real on macOS and a stub or absent elsewhere, so it is opt-in and its +//! absence never fails the call. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::screenshot::ContentBlock; +use crate::driver::Screen; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ObserveInput { + pub session_id: String, + /// Also fetch the accessibility tree. macOS returns a real tree; other + /// guests may return a stub or nothing (the field is then omitted). + #[serde(default)] + pub include_a11y: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ObserveDetails { + pub session_id: String, + pub screen: Screen, + pub mime: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ObserveOutput { + pub content: Vec, + pub details: ObserveDetails, + /// Accessibility tree, present only when requested and the guest exposes + /// one. + #[serde(skip_serializing_if = "Option::is_none")] + pub accessibility: Option, +} diff --git a/computer/src/functions/screenshot.rs b/computer/src/functions/screenshot.rs new file mode 100644 index 000000000..3f3f3ef89 --- /dev/null +++ b/computer/src/functions/screenshot.rs @@ -0,0 +1,61 @@ +//! `computer::screenshot` — desktop capture returned as viewable content +//! blocks (an image block plus a text line), the same envelope the browser +//! worker and `web::fetch` use, so the harness renders it inline for the +//! model. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ScreenshotInput { + pub session_id: String, +} + +/// One block of a viewable response: an image block or a text line. +#[derive(Debug, Serialize, JsonSchema)] +pub struct ContentBlock { + pub r#type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub mime: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, +} + +impl ContentBlock { + /// An image block: base64 `data` with its `mime`. + pub fn image(mime: String, data: String) -> Self { + Self { + r#type: "image".to_string(), + mime: Some(mime), + data: Some(data), + text: None, + } + } + + /// A text block. + pub fn text(text: String) -> Self { + Self { + r#type: "text".to_string(), + mime: None, + data: None, + text: Some(text), + } + } +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ScreenshotDetails { + pub session_id: String, + pub width: u32, + pub height: u32, + /// Detected image mime (`image/png` or `image/jpeg`). + pub mime: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ScreenshotOutput { + pub content: Vec, + pub details: ScreenshotDetails, +} diff --git a/computer/src/functions/sessions.rs b/computer/src/functions/sessions.rs new file mode 100644 index 000000000..425fbbf1e --- /dev/null +++ b/computer/src/functions/sessions.rs @@ -0,0 +1,81 @@ +//! `computer::sessions::start` / `list` / `stop` — session lifecycle. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::driver::Screen; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct StartInput { + /// Boot a fresh desktop inside an iii-sandbox microVM from this OCI image + /// (a sandbox preset name or `custom_images` key) and drive it through iii + /// primitives alone. A fixed virtual display means 1:1 + /// coordinates, no HiDPI or multi-monitor ambiguity. Falls back to the + /// configured `sandbox_image` when omitted. Takes precedence over + /// `endpoint`. + #[serde(default)] + pub image: Option, + /// Desktop to drive when not using a sandbox `image`. Omit (and leave + /// `image` unset) to drive the local machine this worker runs on (native + /// driver, nothing else to run). Pass the endpoint of a desktop guest's + /// executor (a `ws`/`wss`/`http`/`https` url or a bare `host:port`) to + /// drive a remote desktop; falls back to the configured + /// `default_endpoint` when omitted. + #[serde(default)] + pub endpoint: Option, + /// Guest OS label recorded on the session and surfaced in + /// `session-started` (`linux`, `macos`, `windows`, `android`). Omit to + /// use the configured `os`. + #[serde(default)] + pub os: Option, + /// Display index (from `computer::displays`) for a native session. Omit to + /// drive the display under the cursor. Ignored for a remote `endpoint` + /// or a sandbox `image`. + #[serde(default)] + pub monitor: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct StartOutput { + /// Pass this to every other computer function. + pub session_id: String, + /// What the session drives: `native` for the local machine, or the + /// normalized remote endpoint. + pub endpoint: String, + pub os: String, + /// Desktop pixel dimensions; the coordinate space for `computer::act`. + pub screen: Screen, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct ListInput {} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct SessionInfo { + pub session_id: String, + pub endpoint: String, + pub os: String, + pub screen: Screen, + pub created_ms: i64, + pub last_used_ms: i64, + /// True while a live screen stream is running for this session. + pub screencast_active: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ListOutput { + pub sessions: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct StopInput { + /// Session to stop. Stopping an unknown or already-stopped id succeeds. + pub session_id: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct StopOutput { + pub ok: bool, + /// False when the session was already gone. + pub was_running: bool, +} diff --git a/computer/src/lib.rs b/computer/src/lib.rs new file mode 100644 index 000000000..df3d01bc9 --- /dev/null +++ b/computer/src/lib.rs @@ -0,0 +1,12 @@ +//! Library surface for the `computer` worker: drive a full desktop on the iii +//! bus through a computer-use driver. The binary (`src/main.rs`) is a thin +//! boot sequence; everything testable lives here. + +pub mod config; +pub mod configuration; +pub mod driver; +pub mod events; +pub mod functions; +pub mod manifest; +pub mod session; +pub mod ui; diff --git a/computer/src/main.rs b/computer/src/main.rs new file mode 100644 index 000000000..a5ea59649 --- /dev/null +++ b/computer/src/main.rs @@ -0,0 +1,168 @@ +//! `computer` binary entry: connect, register configuration + fetch the +//! authoritative value, register the `computer::*` trigger types and +//! functions, restore any durable sessions, start the idle sweep, then sleep +//! until Ctrl+C / SIGTERM. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use clap::Parser; +use iii_sdk::runtime::WorkerMetadata; +use iii_sdk::{register_worker, InitOptions}; + +use computer::config::WorkerConfig; +use computer::events::{self, IiiDeliverer}; +use computer::session::Sessions; +use computer::{configuration, functions, manifest}; + +#[derive(Parser, Debug)] +#[command( + name = "computer", + about = "Drive a full desktop on the iii bus (computer::*)." +)] +struct Cli { + /// Optional YAML seed used to populate `initial_value` on first + /// configuration registration. + #[arg(long)] + config: Option, + #[arg(long, default_value = "ws://127.0.0.1:49134")] + url: String, + #[arg(long)] + manifest: bool, +} + +/// SIGINT and SIGTERM both shut down cleanly: sessions own live driver +/// connections, and `kill`/`docker stop`/the worker manager all deliver +/// SIGTERM, so ctrl_c alone would leak them. +async fn wait_for_shutdown_signal() -> Result<()> { + #[cfg(unix)] + { + let mut sigterm = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + r = tokio::signal::ctrl_c() => r?, + _ = sigterm.recv() => {} + } + Ok(()) + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await?; + Ok(()) + } +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let cli = Cli::parse(); + if cli.manifest { + println!( + "{}", + serde_json::to_string_pretty(&manifest::build_manifest()).unwrap() + ); + return Ok(()); + } + + let iii = Arc::new(register_worker( + &cli.url, + InitOptions { + metadata: Some(WorkerMetadata { + runtime: "rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + name: "computer".to_string(), + os: std::env::consts::OS.to_string(), + description: Some("Drive a full desktop on the iii bus (computer::*).".to_string()), + pid: Some(std::process::id()), + telemetry: None, + ..WorkerMetadata::default() + }), + ..InitOptions::default() + }, + )); + + // Optional YAML seed -> initial_value on first registration. + let seed = cli.config.as_deref().and_then(|path| { + let contents = match std::fs::read_to_string(path) { + Ok(s) => s, + Err(e) => { + tracing::warn!(path, error = %e, "failed to read seed config; ignoring"); + return None; + } + }; + match serde_yaml::from_str::(&contents) { + Ok(v) => match WorkerConfig::from_json(&v) { + Ok(cfg) => Some(cfg), + Err(e) => { + tracing::warn!(path, error = %e, "failed to parse seed config; ignoring"); + None + } + }, + Err(e) => { + tracing::warn!(path, error = %e, "failed to parse YAML seed config; ignoring"); + None + } + } + }); + + configuration::register_config(&iii, seed.as_ref()) + .await + .map_err(anyhow::Error::msg) + .context("registering computer configuration schema")?; + let cfg = configuration::fetch_config(&iii) + .await + .map_err(anyhow::Error::msg) + .context("loading computer configuration")?; + tracing::info!( + os = %cfg.os, + max_sessions = cfg.max_sessions, + default_endpoint = %cfg.default_endpoint, + "loaded computer configuration" + ); + let shared = cfg.into_shared(); + + // Trigger types before functions, so handlers capture live subscriber sets. + let sets = events::register_trigger_types(&iii); + let emitter = Arc::new(events::Emitter::new( + sets, + Arc::new(IiiDeliverer::new(iii.clone())), + )); + + let sessions = Sessions::new(shared.clone(), emitter, iii.clone()); + functions::register_all(&iii, &sessions); + + configuration::register_config_trigger(&iii, shared.clone()) + .context("registering configuration change trigger")?; + + // Injectable console UI — after the computer::* functions so the console + // can attribute the assets. + computer::ui::register(&iii); + + // Durable sessions: reconnect anything persisted on a previous run. + sessions.restore().await; + + // Idle sweep: stop sessions nobody has touched for idle_stop_ms. + let sweep_sessions = sessions.clone(); + let sweep = tokio::spawn(async move { + let mut tick = tokio::time::interval(Duration::from_secs(60)); + loop { + tick.tick().await; + sweep_sessions.sweep_idle().await; + } + }); + + tracing::info!("computer ready: computer::* sessions + act + screencast"); + wait_for_shutdown_signal().await?; + tracing::info!("computer shutting down"); + sweep.abort(); + sessions.stop_all().await; + iii.shutdown_async().await; + Ok(()) +} diff --git a/computer/src/manifest.rs b/computer/src/manifest.rs new file mode 100644 index 000000000..6b50ab50f --- /dev/null +++ b/computer/src/manifest.rs @@ -0,0 +1,47 @@ +use serde::Serialize; + +use crate::config::WorkerConfig; + +#[derive(Serialize)] +pub struct ModuleManifest { + pub name: String, + pub version: String, + pub description: String, + pub default_config: serde_json::Value, + pub supported_targets: Vec, +} + +pub fn build_manifest() -> ModuleManifest { + ModuleManifest { + name: env!("CARGO_PKG_NAME").to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + description: "Drive a full desktop on the iii bus. Screenshot, click and type by \ + coordinate, run shell, and stream the live screen from a computer-use \ + session." + .to_string(), + default_config: WorkerConfig::default().to_json(), + supported_targets: vec![env!("TARGET").to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_roundtrip_has_required_fields() { + let m = build_manifest(); + let json = serde_json::to_string_pretty(&m).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["name"], env!("CARGO_PKG_NAME")); + assert_eq!(parsed["version"], env!("CARGO_PKG_VERSION")); + assert!(parsed["default_config"].is_object()); + assert!(!parsed["supported_targets"].as_array().unwrap().is_empty()); + } + + #[test] + fn default_config_mirrors_worker_config() { + let m = build_manifest(); + assert_eq!(m.default_config, WorkerConfig::default().to_json()); + } +} diff --git a/computer/src/session.rs b/computer/src/session.rs new file mode 100644 index 000000000..6d7b5f1de --- /dev/null +++ b/computer/src/session.rs @@ -0,0 +1,621 @@ +//! Session registry and lifecycle. A `Session` owns one live driver +//! connection to a desktop plus its screencast pump; `Sessions` is the live +//! table keyed by session id (`c1`, `c2`, ...). +//! +//! Two things here go beyond an ephemeral computer-use client, because iii +//! gives us the primitives for free: +//! +//! - **Durable sessions.** Every session is mirrored into `state` (scope +//! `computer_sessions`). On boot, [`Sessions::restore`] reconnects them +//! best-effort, so a worker restart does not lose live desktops. +//! - **A live screen stream.** The screencast pump pushes frames onto the +//! `computer:frames` stream (`stream::set`, one item per session), so the +//! console and any number of watchers follow the desktop without polling. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::IIIClient; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; + +use crate::config::SharedConfig; +use crate::driver::{Driver, RemoteClient, Screen}; +use crate::events::{Emitter, EventKind, SessionStartedEvent, SessionStoppedEvent}; + +/// Last-value stream carrying the newest screencast frame per session. +pub const FRAMES_STREAM: &str = "computer:frames"; +/// Constant item id: the stream keeps only the newest frame per session group. +const FRAME_ITEM_ID: &str = "frame"; +/// State scope for persisted session records. +const STATE_SCOPE: &str = "computer_sessions"; +/// Bus RPC timeout for the state/stream side-writes. +const SIDE_WRITE_TIMEOUT_MS: u64 = 5_000; + +pub fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +/// Build the native (drive-this-machine) driver. Only available on desktop +/// OSes; elsewhere the worker requires a guest-executor endpoint. +#[cfg(any(target_os = "macos", target_os = "windows"))] +fn native_driver( + cfg: &crate::config::WorkerConfig, + monitor: Option, +) -> Result, String> { + // Surface the macOS Screen Recording prompt and fail loudly if capture would + // be wallpaper-only, so the model is never handed a blank desktop silently. + if cfg.screen_capture_preflight { + crate::driver::native::preflight_screen_capture()?; + } + Ok(Arc::new(crate::driver::NativeHost::new( + cfg.max_screenshot_dimension as u32, + cfg.screenshot_quality as u8, + monitor, + ))) +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn native_driver( + _cfg: &crate::config::WorkerConfig, + _monitor: Option, +) -> Result, String> { + Err( + "native driver (drive this machine) is only available on macOS and Windows; \ + pass an `endpoint` to drive a remote or sandboxed desktop" + .to_string(), + ) +} + +/// The newest screencast frame, handed to `computer::frame` without a capture +/// round-trip. +#[derive(Clone)] +pub struct LatestFrame { + pub data_b64: String, + pub mime: String, + pub width: u32, + pub height: u32, + pub frame_seq: u64, + pub timestamp: i64, +} + +/// Persisted session record (state value). Enough to reconnect on boot. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SessionRecord { + session_id: String, + endpoint: String, + os: String, + width: u32, + height: u32, + created_ms: i64, +} + +pub struct Session { + pub id: String, + pub endpoint: String, + pub os: String, + pub screen: Screen, + pub created_ms: i64, + last_used_ms: AtomicI64, + driver: Arc, + screencast_active: AtomicBool, + frame_seq: AtomicU64, + latest_frame: StdMutex>>, + screencast_task: Mutex>>, + config: SharedConfig, + iii: Arc, +} + +impl Session { + #[allow(clippy::too_many_arguments)] + fn new( + id: String, + endpoint: String, + os: String, + screen: Screen, + created_ms: i64, + driver: Arc, + config: SharedConfig, + iii: Arc, + ) -> Arc { + Arc::new(Self { + id, + endpoint, + os, + screen, + created_ms, + last_used_ms: AtomicI64::new(now_ms()), + driver, + screencast_active: AtomicBool::new(false), + frame_seq: AtomicU64::new(0), + latest_frame: StdMutex::new(None), + screencast_task: Mutex::new(None), + config, + iii, + }) + } + + pub fn driver(&self) -> &Arc { + &self.driver + } + + pub fn touch(&self) { + self.last_used_ms.store(now_ms(), Ordering::Relaxed); + } + + pub fn last_used_ms(&self) -> i64 { + self.last_used_ms.load(Ordering::Relaxed) + } + + pub fn screencast_active(&self) -> bool { + self.screencast_active.load(Ordering::Relaxed) + } + + pub fn latest_frame(&self) -> Option> { + self.latest_frame + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() + } + + fn record(&self) -> SessionRecord { + SessionRecord { + session_id: self.id.clone(), + endpoint: self.endpoint.clone(), + os: self.os.clone(), + width: self.screen.width, + height: self.screen.height, + created_ms: self.created_ms, + } + } + + /// Start (or confirm) the screencast pump. Idempotent: a second call while + /// active is a no-op. The pump polls the driver screenshot at the + /// configured fps and pushes each frame onto `computer:frames`. + pub async fn start_screencast(self: &Arc) { + if self.screencast_active.swap(true, Ordering::SeqCst) { + return; + } + let session = self.clone(); + let handle = tokio::spawn(async move { session.screencast_pump().await }); + if let Some(old) = self.screencast_task.lock().await.replace(handle) { + old.abort(); + } + } + + /// Stop the screencast pump and clear the stream group so a later + /// subscriber does not see a stale frame. Idempotent. + pub async fn stop_screencast(&self) { + self.screencast_active.store(false, Ordering::SeqCst); + if let Some(handle) = self.screencast_task.lock().await.take() { + handle.abort(); + } + self.delete_frame_stream().await; + // Release the last frame's buffer immediately; a stopped screencast + // must not keep a multi-MB image resident. + *self.latest_frame.lock().unwrap_or_else(|p| p.into_inner()) = None; + } + + async fn screencast_pump(self: Arc) { + loop { + if !self.screencast_active.load(Ordering::Relaxed) { + break; + } + let interval = self.config.load().screencast_interval_ms().max(1); + tokio::time::sleep(Duration::from_millis(interval)).await; + if !self.screencast_active.load(Ordering::Relaxed) { + break; + } + match self.driver.screenshot().await { + Ok(shot) => { + let seq = self.frame_seq.fetch_add(1, Ordering::Relaxed) + 1; + let frame = Arc::new(LatestFrame { + data_b64: shot.to_base64(), + mime: shot.mime, + width: self.screen.width, + height: self.screen.height, + frame_seq: seq, + timestamp: now_ms(), + }); + // Push first (borrows), then store the Arc (moves): the + // base64 is copied once into the RPC payload and never + // deep-cloned into the slot. + self.push_frame_stream(&frame).await; + *self.latest_frame.lock().unwrap_or_else(|p| p.into_inner()) = Some(frame); + } + Err(e) => { + tracing::warn!(session = %self.id, error = %e, "screencast capture failed; stopping pump"); + self.screencast_active.store(false, Ordering::Relaxed); + break; + } + } + } + } + + async fn push_frame_stream(&self, frame: &LatestFrame) { + let payload = json!({ + "stream_name": FRAMES_STREAM, + "group_id": self.id, + "item_id": FRAME_ITEM_ID, + "data": { + "data": frame.data_b64, + "mime": frame.mime, + "width": frame.width, + "height": frame.height, + "frame_seq": frame.frame_seq, + "timestamp": frame.timestamp, + } + }); + if let Err(e) = side_write(&self.iii, "stream::set", payload).await { + tracing::debug!(session = %self.id, error = %e, "frame stream write failed"); + } + } + + async fn delete_frame_stream(&self) { + let payload = + json!({ "stream_name": FRAMES_STREAM, "group_id": self.id, "item_id": FRAME_ITEM_ID }); + if let Err(e) = side_write(&self.iii, "stream::delete", payload).await { + tracing::debug!(session = %self.id, error = %e, "frame stream delete failed"); + } + } + + /// Tear down: stop the screencast pump (clearing its stream and buffer), + /// then close the driver. Idempotent. + async fn shutdown(&self) { + self.stop_screencast().await; + if let Err(e) = self.driver.close().await { + tracing::warn!(session = %self.id, error = %e, "driver close failed"); + } + } +} + +pub struct Sessions { + map: Mutex>>, + counter: AtomicU64, + config: SharedConfig, + emitter: Arc, + iii: Arc, +} + +impl Sessions { + pub fn new(config: SharedConfig, emitter: Arc, iii: Arc) -> Arc { + Arc::new(Self { + map: Mutex::new(HashMap::new()), + counter: AtomicU64::new(0), + config, + emitter, + iii, + }) + } + + /// Connect a new desktop session. Enforces the concurrency cap, resolves + /// the endpoint (arg overrides `default_endpoint`), verifies the + /// connection with a `screen_size` probe, persists the record, and emits + /// `computer::session-started`. + pub async fn start( + &self, + image: Option, + endpoint: Option, + os: Option, + monitor: Option, + ) -> Result, String> { + let cfg = self.config.load_full(); + if (self.map.lock().await.len() as u64) >= cfg.max_sessions { + return Err(format!( + "session cap reached ({}); stop a session before starting another", + cfg.max_sessions + )); + } + // A sandbox image (arg or configured default) boots a fresh desktop in + // an iii-sandbox; it takes precedence over an endpoint. + let image = image + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .or_else(|| { + let d = cfg.sandbox_image.trim(); + (!d.is_empty()).then(|| d.to_string()) + }); + let endpoint = endpoint + .map(|e| e.trim().to_string()) + .filter(|e| !e.is_empty()) + .or_else(|| { + let d = cfg.default_endpoint.trim(); + (!d.is_empty()).then(|| d.to_string()) + }); + + // Driver selection: a sandbox image wins, else a remote endpoint, + // else the local machine (native driver). + let (driver, endpoint_used, default_os): (Arc, String, String) = + if let Some(img) = &image { + let host = crate::driver::IiiSandboxHost::create( + self.iii.clone(), + img, + cfg.sandbox_width as u32, + cfg.sandbox_height as u32, + cfg.screenshot_quality as u8, + cfg.sandbox_idle_timeout_secs, + cfg.command_timeout_ms, + ) + .await?; + let label = format!("sandbox:{}", host.sandbox_id()); + (Arc::new(host), label, "linux".to_string()) + } else if let Some(ep) = &endpoint { + let client = + RemoteClient::connect(ep, cfg.connect_timeout_ms, cfg.command_timeout_ms) + .await?; + let label = client.endpoint().to_string(); + (Arc::new(client), label, cfg.os.clone()) + } else { + ( + native_driver(&cfg, monitor)?, + "native".to_string(), + std::env::consts::OS.to_string(), + ) + }; + let os = os.filter(|s| !s.is_empty()).unwrap_or(default_os); + let screen = driver + .screen_size() + .await + .map_err(|e| format!("driver '{endpoint_used}' screen_size failed: {e}"))?; + + let slot = self.counter.fetch_add(1, Ordering::Relaxed) + 1; + let id = format!("c{slot}"); + let session = Session::new( + id.clone(), + endpoint_used, + os, + screen, + now_ms(), + driver, + self.config.clone(), + self.iii.clone(), + ); + self.map.lock().await.insert(id.clone(), session.clone()); + self.persist(&session).await; + self.emitter + .emit( + EventKind::SessionStarted, + &id, + &SessionStartedEvent { + session_id: id.clone(), + endpoint: session.endpoint.clone(), + os: session.os.clone(), + screen, + timestamp: now_ms(), + }, + ) + .await; + Ok(session) + } + + pub async fn get(&self, id: &str) -> Option> { + self.map.lock().await.get(id).cloned() + } + + pub async fn list(&self) -> Vec> { + let mut sessions: Vec> = self.map.lock().await.values().cloned().collect(); + sessions.sort_by_key(|s| s.created_ms); + sessions + } + + /// Stop a session and delete its persisted record. Idempotent: stopping an + /// unknown or already-stopped id returns `false`. + pub async fn stop(&self, id: &str, reason: &str) -> bool { + let removed = self.map.lock().await.remove(id); + match removed { + Some(session) => { + session.shutdown().await; + self.forget(id).await; + self.emitter + .emit( + EventKind::SessionStopped, + id, + &SessionStoppedEvent { + session_id: id.to_string(), + reason: reason.to_string(), + timestamp: now_ms(), + }, + ) + .await; + true + } + None => false, + } + } + + /// Tear down local resources for every session on worker shutdown. Keeps + /// the persisted records so [`restore`](Self::restore) can reconnect on + /// the next boot; does not emit stopped events (subscribers are gone). + pub async fn stop_all(&self) { + let sessions: Vec> = { + let mut map = self.map.lock().await; + map.drain().map(|(_, s)| s).collect() + }; + for session in sessions { + session.shutdown().await; + } + } + + /// Stop sessions idle longer than `idle_stop_ms` (0 disables). + pub async fn sweep_idle(&self) { + let idle_ms = self.config.load().idle_stop_ms; + if idle_ms == 0 { + return; + } + let cutoff = now_ms() - idle_ms as i64; + let stale: Vec = { + let map = self.map.lock().await; + map.values() + .filter(|s| s.last_used_ms() < cutoff) + .map(|s| s.id.clone()) + .collect() + }; + for id in stale { + tracing::info!(session = %id, "stopping idle session"); + self.stop(&id, "idle").await; + } + } + + /// Reconnect sessions persisted on a previous run. Best-effort: a record + /// whose driver no longer answers is dropped. Run once at boot. + pub async fn restore(&self) { + let records = match self.list_records().await { + Ok(r) => r, + Err(e) => { + tracing::warn!(error = %e, "restore: state::list failed; starting empty"); + return; + } + }; + if records.is_empty() { + return; + } + let cfg = self.config.load_full(); + let mut restored = 0u64; + for rec in records { + if (self.map.lock().await.len() as u64) >= cfg.max_sessions { + tracing::warn!("restore: cap reached; leaving remaining records for a later start"); + break; + } + match self.reconnect(&rec, &cfg).await { + Ok(session) => { + self.advance_counter_past(&rec.session_id); + self.map + .lock() + .await + .insert(rec.session_id.clone(), session); + restored += 1; + } + Err(e) => { + tracing::warn!(session = %rec.session_id, endpoint = %rec.endpoint, error = %e, "restore: reconnect failed; dropping record"); + self.forget(&rec.session_id).await; + } + } + } + if restored > 0 { + tracing::info!(restored, "restored persisted computer sessions"); + } + } + + async fn reconnect( + &self, + rec: &SessionRecord, + cfg: &crate::config::WorkerConfig, + ) -> Result, String> { + let (driver, endpoint_used): (Arc, String) = if rec.endpoint == "native" { + (native_driver(cfg, None)?, "native".to_string()) + } else if let Some(sid) = rec.endpoint.strip_prefix("sandbox:") { + // Re-attach to a sandbox that outlived the restart; attach re-runs + // the idempotent display bootstrap and fails if the VM is gone. + let host = crate::driver::IiiSandboxHost::attach( + self.iii.clone(), + sid.to_string(), + rec.width, + rec.height, + cfg.screenshot_quality as u8, + cfg.command_timeout_ms, + ) + .await?; + let label = format!("sandbox:{}", host.sandbox_id()); + (Arc::new(host), label) + } else { + let client = RemoteClient::connect( + &rec.endpoint, + cfg.connect_timeout_ms, + cfg.command_timeout_ms, + ) + .await?; + let label = client.endpoint().to_string(); + (Arc::new(client), label) + }; + let screen = driver.screen_size().await?; + Ok(Session::new( + rec.session_id.clone(), + endpoint_used, + rec.os.clone(), + screen, + rec.created_ms, + driver, + self.config.clone(), + self.iii.clone(), + )) + } + + /// Keep the id counter ahead of a restored `cN` so a new session never + /// reuses a live id. + fn advance_counter_past(&self, session_id: &str) { + if let Some(n) = session_id + .strip_prefix('c') + .and_then(|n| n.parse::().ok()) + { + let mut cur = self.counter.load(Ordering::Relaxed); + while n > cur { + match self.counter.compare_exchange_weak( + cur, + n, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(observed) => cur = observed, + } + } + } + } + + async fn persist(&self, session: &Session) { + let payload = json!({ "scope": STATE_SCOPE, "key": session.id, "value": session.record() }); + if let Err(e) = side_write(&self.iii, "state::set", payload).await { + tracing::warn!(session = %session.id, error = %e, "failed to persist session record; it may be lost on restart"); + } + } + + async fn forget(&self, id: &str) { + let payload = json!({ "scope": STATE_SCOPE, "key": id }); + if let Err(e) = side_write(&self.iii, "state::delete", payload).await { + tracing::warn!(session = %id, error = %e, "failed to delete session record"); + } + } + + async fn list_records(&self) -> Result, String> { + let listed = self + .iii + .trigger(TriggerRequest { + function_id: "state::list".to_string(), + payload: json!({ "scope": STATE_SCOPE }), + action: None, + timeout_ms: Some(10_000), + }) + .await + .map_err(|e| e.to_string())?; + match listed { + Value::Array(items) => Ok(items + .into_iter() + .filter_map(|v| serde_json::from_value(v).ok()) + .collect()), + _ => Ok(Vec::new()), + } + } +} + +/// Best-effort side-write onto the bus (state/stream mutation). Returns the +/// error string for the caller to log at the level that fits: durability +/// writes (persist/forget) at warn, high-volume stream writes at debug. +async fn side_write(iii: &IIIClient, function_id: &str, payload: Value) -> Result<(), String> { + iii.trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: Some(SIDE_WRITE_TIMEOUT_MS), + }) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) +} diff --git a/computer/src/ui.rs b/computer/src/ui.rs new file mode 100644 index 000000000..bac1fca7a --- /dev/null +++ b/computer/src/ui.rs @@ -0,0 +1,71 @@ +//! Injectable console UI for the computer worker (authoring SOP: +//! workers/docs/sops/injectable-console-ui.md). +//! +//! Ships two assets into any running console: +//! +//! - `computer/page.js` (`console:script`) — the `#/ext/computer` page +//! (session rail, screencast-fed live desktop that forwards clicks, scroll +//! and typing back as `computer::act`) AND the function-trigger renderer for +//! every `computer::*` call in chat and the traces span tab. +//! - `computer/styles.css` (`console:style`) — the stylesheet, every rule +//! scoped under `[data-iii-ui="computer"]`; the console mounts it as a +//! `` and link-swaps it on change, styles-before-scripts on boot. +//! +//! The registration machinery (content function `computer::ui-content`, one +//! Message-path trigger per asset, `III_COMPUTER_UI_WATCH` hot-reload watcher) +//! lives in the shared `iii-console-ui` crate; this module only names the +//! assets and embeds their bytes. +//! +//! The assets are compiled from `ui/` by esbuild (react + @iii-dev/console-ui +//! external — they resolve through the console's import map at runtime) and +//! embedded at compile time so the worker stays one self-contained binary. + +use std::sync::Arc; + +use iii_console_ui::ConsoleUi; +use iii_sdk::IIIClient; + +pub const PAGE_PATH: &str = "computer/page.js"; +pub const STYLES_PATH: &str = "computer/styles.css"; + +/// Built by `build.rs` (esbuild over `ui/`). +const PAGE_JS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/page.js")); +const STYLES_CSS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/ui/dist/styles.css")); + +fn console_ui() -> ConsoleUi { + ConsoleUi::new("computer") + .script(PAGE_PATH, PAGE_JS) + .style(STYLES_PATH, STYLES_CSS) +} + +/// Register the computer worker's console UI. Call after the `computer::*` +/// functions are registered. +pub fn register(iii: &Arc) { + console_ui().register(iii); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ui_builder_accepts_the_assets() { + // The builder panics on any path/kind the console would reject. + let _ = console_ui(); + } + + #[test] + fn embedded_page_is_nonempty_esm() { + assert!(PAGE_JS.contains("export"), "built page.js looks wrong"); + } + + #[test] + fn embedded_styles_are_scoped() { + // esbuild prints the attribute selector unquoted ([data-iii-ui=computer]). + assert!( + STYLES_CSS.contains(r#"[data-iii-ui="computer"]"#) + || STYLES_CSS.contains("[data-iii-ui=computer]"), + "built styles.css must be scoped under the worker's data-iii-ui attribute" + ); + } +} diff --git a/computer/tests/golden/schemas/computer.act.json b/computer/tests/golden/schemas/computer.act.json new file mode 100644 index 000000000..224c3cedf --- /dev/null +++ b/computer/tests/golden/schemas/computer.act.json @@ -0,0 +1,121 @@ +{ + "description": "Drive the desktop: click, right_click, double_click, move, drag, scroll, type, press, or hotkey. Address by pixel coordinates read off the screenshot (top-left origin).", + "function_id": "computer::act", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "action": { + "description": "What to do: `click`, `right_click`, `double_click`, `move`, `drag`, `scroll`, `type`, `press`, or `hotkey`.", + "type": "string" + }, + "button": { + "default": null, + "description": "Mouse button for `drag` (`left`, `right`, `middle`). Default `left`.", + "type": [ + "string", + "null" + ] + }, + "keys": { + "default": null, + "description": "Keys for `press`/`hotkey`: a single key (`[\"enter\"]`) or a chord (`[\"ctrl\", \"c\"]`).", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "scroll_x": { + "default": null, + "description": "Horizontal scroll amount for `action=scroll`. Default 0.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "scroll_y": { + "default": null, + "description": "Vertical scroll amount for `action=scroll`; positive scrolls down. Default 3.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "text": { + "default": null, + "description": "Text to type, for `action=type`.", + "type": [ + "string", + "null" + ] + }, + "to_x": { + "default": null, + "description": "Drag end x (with `to_y`), for `action=drag`.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "to_y": { + "default": null, + "description": "Drag end y.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "x": { + "default": null, + "description": "Target x (pixels). Required for click/right_click/double_click/move and the drag start; the scroll anchor defaults to screen center.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "y": { + "default": null, + "description": "Target y (pixels).", + "format": "int64", + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "action", + "session_id" + ], + "title": "ActInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "detail": { + "description": "What was done, for the transcript.", + "type": "string" + }, + "ok": { + "type": "boolean" + } + }, + "required": [ + "detail", + "ok" + ], + "title": "ActOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.displays.json b/computer/tests/golden/schemas/computer.displays.json new file mode 100644 index 000000000..a8c4467fa --- /dev/null +++ b/computer/tests/golden/schemas/computer.displays.json @@ -0,0 +1,68 @@ +{ + "description": "List the local displays (index, name, primary, size). Pass the chosen index as `monitor` to computer::sessions::start to drive that display; omit it to use the display under the cursor. Native host only.", + "function_id": "computer::displays", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DisplaysInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "DisplayInfo": { + "description": "One display available to the native host (from `computer::displays`).", + "properties": { + "builtin": { + "type": "boolean" + }, + "height": { + "description": "Logical height in points.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "index": { + "description": "Index to pass as `monitor` to `computer::sessions::start`.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "name": { + "type": "string" + }, + "primary": { + "type": "boolean" + }, + "width": { + "description": "Logical width in points.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "builtin", + "height", + "index", + "name", + "primary", + "width" + ], + "type": "object" + } + }, + "properties": { + "displays": { + "items": { + "$ref": "#/definitions/DisplayInfo" + }, + "type": "array" + } + }, + "required": [ + "displays" + ], + "title": "DisplaysOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.frame.json b/computer/tests/golden/schemas/computer.frame.json new file mode 100644 index 000000000..4efd2c9b1 --- /dev/null +++ b/computer/tests/golden/schemas/computer.frame.json @@ -0,0 +1,76 @@ +{ + "description": "Internal: newest screencast frame, or nothing when since_frame is still current. No capture round-trip; poll fast. Not an agent function.", + "function_id": "computer::frame", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "session_id": { + "type": "string" + }, + "since_frame": { + "default": null, + "description": "Frame cursor from the previous read; when the newest frame still has this seq the response omits `frame` (nothing changed).", + "format": "uint64", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "session_id" + ], + "title": "FrameInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "active": { + "description": "False when no screencast is running (call screencast::start first).", + "type": "boolean" + }, + "frame": { + "description": "Base64 image of the newest frame; absent when `since_frame` is still current or no frame has arrived yet.", + "type": [ + "string", + "null" + ] + }, + "frame_seq": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "mime": { + "description": "Image mime of the frame bytes.", + "type": "string" + }, + "timestamp": { + "format": "int64", + "type": "integer" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "active", + "frame_seq", + "height", + "mime", + "timestamp", + "width" + ], + "title": "FrameOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.observe.json b/computer/tests/golden/schemas/computer.observe.json new file mode 100644 index 000000000..e4bf908db --- /dev/null +++ b/computer/tests/golden/schemas/computer.observe.json @@ -0,0 +1,119 @@ +{ + "description": "Capture the desktop plus, optionally, the accessibility tree. Use include_a11y on macOS guests for a machine-readable element tree; elsewhere prefer computer::screenshot.", + "function_id": "computer::observe", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "include_a11y": { + "default": null, + "description": "Also fetch the accessibility tree. macOS returns a real tree; other guests may return a stub or nothing (the field is then omitted).", + "type": [ + "boolean", + "null" + ] + }, + "session_id": { + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "ObserveInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "ContentBlock": { + "description": "One block of a viewable response: an image block or a text line.", + "properties": { + "data": { + "type": [ + "string", + "null" + ] + }, + "mime": { + "type": [ + "string", + "null" + ] + }, + "text": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "ObserveDetails": { + "properties": { + "mime": { + "type": "string" + }, + "screen": { + "$ref": "#/definitions/Screen" + }, + "session_id": { + "type": "string" + } + }, + "required": [ + "mime", + "screen", + "session_id" + ], + "type": "object" + }, + "Screen": { + "description": "Desktop pixel dimensions. Coordinates handed to pointer actions are in this space: integer pixels, top-left origin, 1:1 with the screenshot.", + "properties": { + "height": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "height", + "width" + ], + "type": "object" + } + }, + "properties": { + "accessibility": { + "description": "Accessibility tree, present only when requested and the guest exposes one." + }, + "content": { + "items": { + "$ref": "#/definitions/ContentBlock" + }, + "type": "array" + }, + "details": { + "$ref": "#/definitions/ObserveDetails" + } + }, + "required": [ + "content", + "details" + ], + "title": "ObserveOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.screencast.start.json b/computer/tests/golden/schemas/computer.screencast.start.json new file mode 100644 index 000000000..5d3270ef3 --- /dev/null +++ b/computer/tests/golden/schemas/computer.screencast.start.json @@ -0,0 +1,30 @@ +{ + "description": "Internal: start pushing live desktop frames onto the computer:frames stream for the console viewport. Console-UI plumbing; agents use computer::screenshot. Not an agent function.", + "function_id": "computer::screencast::start", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "session_id": { + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "ScreencastStartInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "ok": { + "type": "boolean" + } + }, + "required": [ + "ok" + ], + "title": "AckOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.screencast.stop.json b/computer/tests/golden/schemas/computer.screencast.stop.json new file mode 100644 index 000000000..4ae82a444 --- /dev/null +++ b/computer/tests/golden/schemas/computer.screencast.stop.json @@ -0,0 +1,31 @@ +{ + "description": "Internal: stop the live frame push. Idempotent. Not an agent function.", + "function_id": "computer::screencast::stop", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "session_id": { + "description": "Stopping the screencast on an unknown session succeeds.", + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "ScreencastStopInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "ok": { + "type": "boolean" + } + }, + "required": [ + "ok" + ], + "title": "AckOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.screenshot.json b/computer/tests/golden/schemas/computer.screenshot.json new file mode 100644 index 000000000..3e84efc57 --- /dev/null +++ b/computer/tests/golden/schemas/computer.screenshot.json @@ -0,0 +1,97 @@ +{ + "description": "Capture the desktop as a viewable image. This is how you see the screen before acting; the coordinate space of computer::act is this image's pixels (top-left origin).", + "function_id": "computer::screenshot", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "session_id": { + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "ScreenshotInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "ContentBlock": { + "description": "One block of a viewable response: an image block or a text line.", + "properties": { + "data": { + "type": [ + "string", + "null" + ] + }, + "mime": { + "type": [ + "string", + "null" + ] + }, + "text": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "ScreenshotDetails": { + "properties": { + "height": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "mime": { + "description": "Detected image mime (`image/png` or `image/jpeg`).", + "type": "string" + }, + "session_id": { + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "height", + "mime", + "session_id", + "width" + ], + "type": "object" + } + }, + "properties": { + "content": { + "items": { + "$ref": "#/definitions/ContentBlock" + }, + "type": "array" + }, + "details": { + "$ref": "#/definitions/ScreenshotDetails" + } + }, + "required": [ + "content", + "details" + ], + "title": "ScreenshotOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.sessions.list.json b/computer/tests/golden/schemas/computer.sessions.list.json new file mode 100644 index 000000000..4a1e9b466 --- /dev/null +++ b/computer/tests/golden/schemas/computer.sessions.list.json @@ -0,0 +1,85 @@ +{ + "description": "List live computer sessions with their endpoint, guest OS, and screen size.", + "function_id": "computer::sessions::list", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ListInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Screen": { + "description": "Desktop pixel dimensions. Coordinates handed to pointer actions are in this space: integer pixels, top-left origin, 1:1 with the screenshot.", + "properties": { + "height": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "height", + "width" + ], + "type": "object" + }, + "SessionInfo": { + "properties": { + "created_ms": { + "format": "int64", + "type": "integer" + }, + "endpoint": { + "type": "string" + }, + "last_used_ms": { + "format": "int64", + "type": "integer" + }, + "os": { + "type": "string" + }, + "screen": { + "$ref": "#/definitions/Screen" + }, + "screencast_active": { + "description": "True while a live screen stream is running for this session.", + "type": "boolean" + }, + "session_id": { + "type": "string" + } + }, + "required": [ + "created_ms", + "endpoint", + "last_used_ms", + "os", + "screen", + "screencast_active", + "session_id" + ], + "type": "object" + } + }, + "properties": { + "sessions": { + "items": { + "$ref": "#/definitions/SessionInfo" + }, + "type": "array" + } + }, + "required": [ + "sessions" + ], + "title": "ListOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.sessions.start.json b/computer/tests/golden/schemas/computer.sessions.start.json new file mode 100644 index 000000000..95f565d02 --- /dev/null +++ b/computer/tests/golden/schemas/computer.sessions.start.json @@ -0,0 +1,99 @@ +{ + "description": "Start a computer-use session and return its session_id. Pass `image` to boot a fresh desktop in an iii-sandbox microVM (fixed virtual display, no host setup), an `endpoint` to drive a desktop through its guest executor, or omit both to drive the local machine. Sessions are durable; stop them with computer::sessions::stop when done.", + "function_id": "computer::sessions::start", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "endpoint": { + "default": null, + "description": "Desktop to drive when not using a sandbox `image`. Omit (and leave `image` unset) to drive the local machine this worker runs on (native driver, nothing else to run). Pass the endpoint of a desktop guest's executor (a `ws`/`wss`/`http`/`https` url or a bare `host:port`) to drive a remote desktop; falls back to the configured `default_endpoint` when omitted.", + "type": [ + "string", + "null" + ] + }, + "image": { + "default": null, + "description": "Boot a fresh desktop inside an iii-sandbox microVM from this OCI image (a sandbox preset name or `custom_images` key) and drive it through iii primitives alone. A fixed virtual display means 1:1 coordinates, no HiDPI or multi-monitor ambiguity. Falls back to the configured `sandbox_image` when omitted. Takes precedence over `endpoint`.", + "type": [ + "string", + "null" + ] + }, + "monitor": { + "default": null, + "description": "Display index (from `computer::displays`) for a native session. Omit to drive the display under the cursor. Ignored for a remote `endpoint` or a sandbox `image`.", + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "os": { + "default": null, + "description": "Guest OS label recorded on the session and surfaced in `session-started` (`linux`, `macos`, `windows`, `android`). Omit to use the configured `os`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "StartInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Screen": { + "description": "Desktop pixel dimensions. Coordinates handed to pointer actions are in this space: integer pixels, top-left origin, 1:1 with the screenshot.", + "properties": { + "height": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "height", + "width" + ], + "type": "object" + } + }, + "properties": { + "endpoint": { + "description": "What the session drives: `native` for the local machine, or the normalized remote endpoint.", + "type": "string" + }, + "os": { + "type": "string" + }, + "screen": { + "allOf": [ + { + "$ref": "#/definitions/Screen" + } + ], + "description": "Desktop pixel dimensions; the coordinate space for `computer::act`." + }, + "session_id": { + "description": "Pass this to every other computer function.", + "type": "string" + } + }, + "required": [ + "endpoint", + "os", + "screen", + "session_id" + ], + "title": "StartOutput", + "type": "object" + } +} diff --git a/computer/tests/golden/schemas/computer.sessions.stop.json b/computer/tests/golden/schemas/computer.sessions.stop.json new file mode 100644 index 000000000..29b52b004 --- /dev/null +++ b/computer/tests/golden/schemas/computer.sessions.stop.json @@ -0,0 +1,36 @@ +{ + "description": "Stop a computer session and close its driver connection. Idempotent: stopping an unknown or already-stopped session succeeds with was_running=false.", + "function_id": "computer::sessions::stop", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "session_id": { + "description": "Session to stop. Stopping an unknown or already-stopped id succeeds.", + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "StopInput", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "ok": { + "type": "boolean" + }, + "was_running": { + "description": "False when the session was already gone.", + "type": "boolean" + } + }, + "required": [ + "ok", + "was_running" + ], + "title": "StopOutput", + "type": "object" + } +} diff --git a/computer/tests/manifest.rs b/computer/tests/manifest.rs new file mode 100644 index 000000000..7b21b4331 --- /dev/null +++ b/computer/tests/manifest.rs @@ -0,0 +1,28 @@ +//! The `--manifest` subcommand must emit valid module-manifest JSON: cargo +//! builds the binary for this test and hands us its path via +//! `CARGO_BIN_EXE_computer`. + +use std::process::Command; + +#[test] +fn manifest_subcommand_emits_valid_json() { + let output = Command::new(env!("CARGO_BIN_EXE_computer")) + .arg("--manifest") + .output() + .expect("run computer --manifest"); + assert!( + output.status.success(), + "--manifest exited with {:?}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); + let parsed: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("manifest stdout is valid JSON"); + assert_eq!(parsed["name"], "computer"); + assert!(parsed["version"].is_string()); + assert!(parsed["default_config"].is_object()); + assert!(!parsed["supported_targets"] + .as_array() + .expect("supported_targets is an array") + .is_empty()); +} diff --git a/computer/tests/schemas.rs b/computer/tests/schemas.rs new file mode 100644 index 000000000..10dc826ca --- /dev/null +++ b/computer/tests/schemas.rs @@ -0,0 +1,92 @@ +//! Wire-schema snapshots for the ten `computer::*` functions. +//! +//! `computer::functions::catalog()` is the single source of truth for each +//! function's id, registration description, and schemars-derived +//! request/response schemas (generated with the same `SchemaSettings::draft07()` +//! construction iii-sdk uses at registration, from the same input/output +//! structs). Each entry is serialized to pretty JSON and compared against +//! `tests/golden/schemas/.json` (`::` maps to `.` in filenames). +//! +//! These snapshots ARE the product surface consumed by callers and agents; any +//! schema or description change must land as an explicit golden diff. +//! Regenerate with `UPDATE_GOLDENS=1 cargo test`. + +mod support; + +use computer::functions::{catalog, FunctionSpec}; + +fn golden_file_name(function_id: &str) -> String { + format!("schemas/{}.json", function_id.replace("::", ".")) +} + +fn spec_to_pretty_json(spec: &FunctionSpec) -> String { + let value = serde_json::json!({ + "function_id": spec.function_id, + "description": spec.description, + "request_schema": spec.request_schema, + "response_schema": spec.response_schema, + }); + let mut pretty = serde_json::to_string_pretty(&value).expect("spec serializes"); + pretty.push('\n'); + pretty +} + +/// The catalog must cover exactly the ten registered functions, in +/// registration order (kept in lockstep with `register_all`). +#[test] +fn catalog_lists_all_functions_in_registration_order() { + let ids: Vec<&str> = catalog().iter().map(|s| s.function_id).collect(); + assert_eq!( + ids, + vec![ + "computer::sessions::start", + "computer::sessions::list", + "computer::sessions::stop", + "computer::displays", + "computer::screenshot", + "computer::observe", + "computer::act", + "computer::screencast::start", + "computer::screencast::stop", + "computer::frame", + ] + ); +} + +/// Every catalog entry matches its committed golden. Mismatches are collected +/// across ALL functions before failing so one run shows the full drift, not +/// just the first file. +#[test] +fn wire_schema_snapshots_match_goldens() { + let mut failures = Vec::new(); + for spec in catalog() { + let rel = golden_file_name(spec.function_id); + let actual = spec_to_pretty_json(&spec); + if let Err(msg) = support::check_golden(&rel, &actual) { + failures.push(msg); + } + } + assert!( + failures.is_empty(), + "{} wire-schema golden(s) drifted:\n\n{}", + failures.len(), + failures.join("\n") + ); +} + +/// No function may ship the permissive `AnyValue` schema (the deploy-time +/// "unknown" request/response schema this convention exists to prevent). Every +/// request and response schema must be a typed struct. +#[test] +fn every_function_has_typed_request_and_response_schemas() { + for spec in catalog() { + support::assert_typed_schema( + &format!("{} request_schema", spec.function_id), + &spec.request_schema, + ); + support::assert_typed_schema( + &format!("{} response_schema", spec.function_id), + &spec.response_schema, + ); + } +} diff --git a/computer/tests/support/mod.rs b/computer/tests/support/mod.rs new file mode 100644 index 000000000..da8a00df8 --- /dev/null +++ b/computer/tests/support/mod.rs @@ -0,0 +1,117 @@ +//! Hand-rolled golden-file harness (deliberately no `insta`/snapshot +//! dependency). Goldens live under `tests/golden/` and are committed; any +//! wire-surface change must show up as an explicit, reviewed diff. +//! +//! Workflow: +//! - `cargo test` compares actual output against the committed goldens. +//! - `UPDATE_GOLDENS=1 cargo test` regenerates the files; review the git +//! diff, then commit the new goldens alongside the change that caused them. + +#![allow(dead_code)] + +use std::fs; +use std::path::PathBuf; + +/// Root of the committed golden files. +pub fn golden_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/golden") +} + +fn update_mode() -> bool { + std::env::var("UPDATE_GOLDENS") + .map(|v| v == "1") + .unwrap_or(false) +} + +/// Compare `actual` against the golden file at `tests/golden/`. Returns +/// `Err(readable diff hint)` on mismatch or missing golden; with +/// `UPDATE_GOLDENS=1` the file is (re)written and the check passes. +pub fn check_golden(rel: &str, actual: &str) -> Result<(), String> { + let path = golden_root().join(rel); + if update_mode() { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; + } + fs::write(&path, actual).map_err(|e| format!("write {}: {e}", path.display()))?; + return Ok(()); + } + let expected = fs::read_to_string(&path).map_err(|e| { + format!( + "golden file {} unreadable ({e}).\n\ + Run `UPDATE_GOLDENS=1 cargo test` to (re)generate, then review \ + and commit the diff.", + path.display() + ) + })?; + if expected == actual { + return Ok(()); + } + Err(diff_hint(rel, &expected, actual)) +} + +/// Readable first-divergence diff hint: line number, expected vs actual around +/// the mismatch, and the regeneration instructions. +fn diff_hint(rel: &str, expected: &str, actual: &str) -> String { + let exp_lines: Vec<&str> = expected.lines().collect(); + let act_lines: Vec<&str> = actual.lines().collect(); + let first_diff = exp_lines + .iter() + .zip(act_lines.iter()) + .position(|(e, a)| e != a) + .unwrap_or_else(|| exp_lines.len().min(act_lines.len())); + + const CONTEXT: usize = 3; + let lo = first_diff.saturating_sub(CONTEXT); + let hi = (first_diff + CONTEXT + 1).max(first_diff + 1); + + let mut out = format!( + "golden mismatch: tests/golden/{rel}\n\ + first divergence at line {} (expected {} lines, actual {} lines)\n", + first_diff + 1, + exp_lines.len(), + act_lines.len() + ); + out.push_str("--- expected (golden) ---\n"); + for (i, line) in exp_lines.iter().enumerate().skip(lo).take(hi - lo) { + let marker = if i == first_diff { ">" } else { " " }; + out.push_str(&format!("{marker} {:>4} | {line}\n", i + 1)); + } + out.push_str("--- actual ---\n"); + for (i, line) in act_lines.iter().enumerate().skip(lo).take(hi - lo) { + let marker = if i == first_diff { ">" } else { " " }; + out.push_str(&format!("{marker} {:>4} | {line}\n", i + 1)); + } + out.push_str( + "If this change is intentional, run `UPDATE_GOLDENS=1 cargo test`, \ + review the git diff, and commit the updated goldens.\n", + ); + out +} + +/// Assert a schemars-derived request/response schema is a *real* schema and +/// not the permissive `AnyValue` schema a `Value` handler emits (the "unknown" +/// schema this whole convention exists to prevent). A real schema carries at +/// least one schema-defining keyword. +pub fn assert_typed_schema(label: &str, schema: &schemars::schema::RootSchema) { + let value = serde_json::to_value(schema).expect("schema serializes"); + let obj = value + .as_object() + .unwrap_or_else(|| panic!("{label}: schema is not a JSON object")); + const DEFINING: [&str; 8] = [ + "type", + "properties", + "$ref", + "allOf", + "anyOf", + "oneOf", + "enum", + "items", + ]; + let has_defining = DEFINING.iter().any(|k| obj.contains_key(*k)); + assert!( + has_defining, + "{label}: schema is the permissive AnyValue/empty schema (no type/properties/$ref/...). \ + The handler is registered with `Value` — give it a typed struct deriving JsonSchema. \ + Got: {value}" + ); +} diff --git a/computer/ui/build.mjs b/computer/ui/build.mjs new file mode 100644 index 000000000..e6759a633 --- /dev/null +++ b/computer/ui/build.mjs @@ -0,0 +1,37 @@ +/** + * Build the worker's two console assets: + * + * page.tsx → dist/page.js (injected over `console:script`) + * styles.css → dist/styles.css (injected over `console:style`) + * + * The five shared specifiers stay EXTERNAL — they resolve at runtime + * through the console's import map (a bundled React copy would surface as + * a cryptic "Invalid hook call"). Everything else the page needs gets + * bundled in. `--watch` pairs with the worker's III_COMPUTER_UI_WATCH + * poller for the hot-reload dev loop. + */ + +import esbuild from 'esbuild' + +const options = { + entryPoints: ['page.tsx', 'styles.css'], + bundle: true, + format: 'esm', + jsx: 'automatic', + outdir: 'dist', + external: [ + 'react', + 'react-dom', + 'react-dom/client', + 'react/jsx-runtime', + '@iii-dev/console-ui', + ], + logLevel: 'info', +} + +if (process.argv.includes('--watch')) { + const ctx = await esbuild.context(options) + await ctx.watch() +} else { + await esbuild.build(options) +} diff --git a/computer/ui/package.json b/computer/ui/package.json new file mode 100644 index 000000000..46ddac1b3 --- /dev/null +++ b/computer/ui/package.json @@ -0,0 +1,19 @@ +{ + "name": "@iii-workers/computer-ui", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "build": "tsc --noEmit && node build.mjs", + "watch": "node build.mjs --watch" + }, + "dependencies": { + "@iii-dev/console-ui": "workspace:*", + "zod": "^4.0.0" + }, + "devDependencies": { + "@types/react": "^19.2.14", + "esbuild": "^0.25.0", + "typescript": "^5.9.2" + } +} diff --git a/computer/ui/page.tsx b/computer/ui/page.tsx new file mode 100644 index 000000000..cabed58b4 --- /dev/null +++ b/computer/ui/page.tsx @@ -0,0 +1,33 @@ +/** + * Entry for the computer worker's injected console UI — compiled by esbuild + * (react + @iii-dev/console-ui external) into dist/page.js and served over the + * `console:script` trigger (see src/ui.rs). The stylesheet is its own asset: + * styles.css ships over `console:style` as computer/styles.css — the console + * mounts and link-swaps it, styles-before-scripts on boot. + * + * `setup(host)` registers two contributions: + * - src/page/ — the `#/ext/computer` page: session rail plus a screencast-fed + * live desktop that forwards clicks, scroll and typing as `computer::act`. + * - src/function-trigger-message/ — how every `computer::*` call renders in + * chat and the traces span tab. + * + * No config form is registered: the computer worker's configuration is plain + * scalar fields, so the console's schema-generated form is sufficient. + * + * Registrations go through `host` so the loader disposes them on hot reload / + * worker disconnect. + */ + +import type { Host } from '@iii-dev/console-ui' +import { createComputerRenderer } from './src/function-trigger-message' +import { ComputerPage } from './src/page' + +export default function setup(host: Host) { + host.pages.register({ + id: 'computer', + title: 'computer', + render: () => , + }) + + host.functionTriggers.register(createComputerRenderer(host)) +} diff --git a/computer/ui/src/function-trigger-message/ComputerViews.tsx b/computer/ui/src/function-trigger-message/ComputerViews.tsx new file mode 100644 index 000000000..1de8439b4 --- /dev/null +++ b/computer/ui/src/function-trigger-message/ComputerViews.tsx @@ -0,0 +1,155 @@ +import { Badge } from '@iii-dev/console-ui' +import { z } from 'zod' +import { + actResultSchema, + type ComputerSessionInfo, + decodeComputerResult, + displayInfoSchema, + parseCapture, + sessionInfoSchema, + sessionStartSchema, + sessionStopSchema, +} from '../lib/computer' +import { shortEndpoint } from '../lib/format' + +/** + * Per-function bodies for `computer::*` chat cards. Each view parses the + * worker's own result shape and returns `null` when it does not match, so the + * caller falls back to the decoded JSON rather than rendering a half-card. + */ + +export function CaptureView({ output }: { output: unknown }) { + const capture = parseCapture(output) + if (!capture?.dataUrl) return null + return ( +
+ {`desktop + {capture.width > 0 ? ( +
+ {capture.width}x{capture.height} +
+ ) : null} +
+ ) +} + +export function SessionStartView({ output }: { output: unknown }) { + const parsed = sessionStartSchema.safeParse(decodeComputerResult(output)) + if (!parsed.success) return null + const session = parsed.data + return ( +
+ + + + +
+ ) +} + +export function SessionStopView({ output }: { output: unknown }) { + const parsed = sessionStopSchema.safeParse(decodeComputerResult(output)) + if (!parsed.success) return null + return ( +

+ {parsed.data.was_running ? 'session stopped' : 'already stopped'} +

+ ) +} + +export function SessionListView({ output }: { output: unknown }) { + const parsed = z + .object({ sessions: z.array(sessionInfoSchema) }) + .safeParse(decodeComputerResult(output)) + if (!parsed.success) return null + const sessions: ComputerSessionInfo[] = parsed.data.sessions + if (sessions.length === 0) { + return

no live sessions

+ } + return ( +
    + {sessions.map((session) => ( +
  • + {session.session_id} + + {shortEndpoint(session.endpoint)} · {session.os} ·{' '} + {session.screen.width}x{session.screen.height} + + {session.screencast_active ? ( + + streaming + + ) : null} +
    • + ))} +
+ ) +} + +export function ActView({ + input, + output, +}: { + input: unknown + output: unknown +}) { + const parsed = actResultSchema.safeParse(decodeComputerResult(output)) + if (!parsed.success) return null + const action = + input && typeof input === 'object' + ? (input as Record).action + : undefined + return ( +

+ {typeof action === 'string' ? ( + + {action} + + ) : null} + {parsed.data.detail} +

+ ) +} + +export function DisplaysView({ output }: { output: unknown }) { + const parsed = z + .object({ displays: z.array(displayInfoSchema) }) + .safeParse(decodeComputerResult(output)) + if (!parsed.success) return null + if (parsed.data.displays.length === 0) { + return

no local displays (not a desktop host)

+ } + return ( +
    + {parsed.data.displays.map((display) => ( +
  • + {display.index} + + {display.name || 'display'} · {display.width}x{display.height} + + {display.primary ? ( + + primary + + ) : null} +
    • + ))} +
+ ) +} + +function Kv({ label, value }: { label: string; value: string }) { + return ( +
+ {label} + {value} +
+ ) +} diff --git a/computer/ui/src/function-trigger-message/index.tsx b/computer/ui/src/function-trigger-message/index.tsx new file mode 100644 index 000000000..6ea9def5a --- /dev/null +++ b/computer/ui/src/function-trigger-message/index.tsx @@ -0,0 +1,137 @@ +/** + * Injected function-trigger renderer for every `computer::*` call — + * registered through `host.functionTriggers`, so it dispatches before the + * console's built-in families and owns how computer calls render in chat and + * in the traces span tab. Screenshots render inline (that image IS the + * result), actions collapse to one line, and anything unrecognised falls back + * to the decoded JSON. Errors are left to the console's own cards. + */ + +import { + type FunctionTriggerMessage, + type FunctionTriggerRenderer, + type Host, + JsonHighlight, +} from '@iii-dev/console-ui' +import { + decodeComputerResult, + isComputerFunction, + sessionIdFromCall, +} from '../lib/computer' +import { + ActView, + CaptureView, + DisplaysView, + SessionListView, + SessionStartView, + SessionStopView, +} from './ComputerViews' + +/** The injected page's route — where "open the desktop" navigates. */ +const COMPUTER_PAGE_HASH = '#/ext/computer' + +/** + * Header label for `computer::*` ids: dims the namespace prefix so the op + * (`act`, `screenshot`, …) reads clearly. + */ +function FunctionIdLabel({ functionId }: { functionId: string }) { + if (!functionId.startsWith('computer::')) { + return {functionId} + } + const tail = functionId.slice('computer::'.length) + return ( + <> + computer:: + {tail} + + ) +} + +function formatJson(value: unknown): string { + try { + return JSON.stringify(value, null, 2) + } catch { + return String(value) + } +} + +function renderBody(message: FunctionTriggerMessage): React.ReactNode | null { + const { input, output } = message + switch (message.functionId) { + case 'computer::sessions::start': + return + case 'computer::sessions::list': + return + case 'computer::sessions::stop': + return + case 'computer::displays': + return + case 'computer::screenshot': + case 'computer::observe': + return + case 'computer::act': + return + default: + return null + } +} + +function ComputerCallView({ message }: { message: FunctionTriggerMessage }) { + const sessionId = sessionIdFromCall(message.input, message.output) + const running = !!message.running + + const body = !running && message.output != null ? renderBody(message) : null + const fallback = + !body && message.output != null + ? decodeComputerResult(message.output) + : null + + return ( +
+
+ + {sessionId ? ( + <> + session {sessionId} + + ) : ( + 'computer' + )} + + {sessionId ? ( + + open the desktop + + ) : null} +
+ {running && message.output == null ? ( +

running...

+ ) : body ? ( + body + ) : fallback != null ? ( +
+ +
+ ) : ( +

no result

+ )} +
+ ) +} + +function renderCall(message: FunctionTriggerMessage): React.ReactNode | null { + if (!isComputerFunction(message.functionId)) return null + if (message.pendingApproval) return null + return +} + +export function createComputerRenderer(_host: Host): FunctionTriggerRenderer { + return { + id: 'computer/page.js#calls', + isMatch: isComputerFunction, + tryRender: (message) => renderCall(message), + tryRenderRunning: (message) => renderCall(message), + tryRenderPreview: () => null, + FunctionIdLabel, + } +} diff --git a/computer/ui/src/lib/cn.ts b/computer/ui/src/lib/cn.ts new file mode 100644 index 000000000..34f4b5942 --- /dev/null +++ b/computer/ui/src/lib/cn.ts @@ -0,0 +1,8 @@ +/** + * Minimal class-name joiner for the injected UI. The console's Tailwind `cn` + * is not available here — injected UI ships its own scoped stylesheet with + * semantic classes, so there are no utility classes to merge. + */ +export function cn(...parts: Array): string { + return parts.filter(Boolean).join(' ') +} diff --git a/computer/ui/src/lib/computer.ts b/computer/ui/src/lib/computer.ts new file mode 100644 index 000000000..250df03e8 --- /dev/null +++ b/computer/ui/src/lib/computer.ts @@ -0,0 +1,311 @@ +import type { ExtensionIii } from '@iii-dev/console-ui' +import { z } from 'zod' + +/** + * Control plane for the `computer` worker's own injected UI: typed wrappers + * over its session surface (start / list / stop / screenshot / act) plus the + * screencast plumbing the live viewport rides on, and parsers for the chat + * function-trigger views. + * + * Every call goes through the tab's `host.iii` client, passed in by the page — + * there is no module-level singleton in injected UI. Wire source: + * `computer/src/functions/*.rs` (verbatim ids and payloads). + */ + +export const SESSIONS_START_FUNCTION_ID = 'computer::sessions::start' +export const SESSIONS_LIST_FUNCTION_ID = 'computer::sessions::list' +export const SESSIONS_STOP_FUNCTION_ID = 'computer::sessions::stop' +export const DISPLAYS_FUNCTION_ID = 'computer::displays' +export const SCREENSHOT_FUNCTION_ID = 'computer::screenshot' +export const OBSERVE_FUNCTION_ID = 'computer::observe' +export const ACT_FUNCTION_ID = 'computer::act' +export const SCREENCAST_START_FUNCTION_ID = 'computer::screencast::start' +export const SCREENCAST_STOP_FUNCTION_ID = 'computer::screencast::stop' +export const FRAME_FUNCTION_ID = 'computer::frame' + +export const SESSION_STARTED_TRIGGER = 'computer::session-started' +export const SESSION_STOPPED_TRIGGER = 'computer::session-stopped' + +/** Session lifecycle trigger types the rail re-reads on. */ +export const LIFECYCLE_TRIGGERS = [ + SESSION_STARTED_TRIGGER, + SESSION_STOPPED_TRIGGER, +] as const + +/** + * Stream the worker pushes live desktop frames onto (group = session id). The + * page subscribes with a `type:'stream'` trigger instead of polling. + */ +export const FRAMES_STREAM = 'computer:frames' + +/** Every `computer::*` bus function belongs to this family. */ +export function isComputerFunction(functionId: string): boolean { + return functionId.startsWith('computer::') +} + +const screenSchema = z.object({ width: z.number(), height: z.number() }) +export type Screen = z.infer + +export const sessionInfoSchema = z.object({ + session_id: z.string(), + endpoint: z.string(), + os: z.string(), + screen: screenSchema, + created_ms: z.number(), + last_used_ms: z.number(), + screencast_active: z.boolean(), +}) +export type ComputerSessionInfo = z.infer + +export const sessionStartSchema = z.object({ + session_id: z.string(), + endpoint: z.string(), + os: z.string(), + screen: screenSchema, +}) +export type ComputerSessionStart = z.infer + +export const sessionStopSchema = z.object({ + ok: z.boolean(), + was_running: z.boolean(), +}) + +export const actResultSchema = z.object({ + ok: z.boolean(), + detail: z.string(), +}) + +export const displayInfoSchema = z.object({ + index: z.number(), + name: z.string(), + primary: z.boolean(), + builtin: z.boolean(), + width: z.number(), + height: z.number(), +}) +export type ComputerDisplay = z.infer + +const frameSchema = z.object({ + frame: z.string().nullish(), + mime: z.string(), + width: z.number(), + height: z.number(), + frame_seq: z.number(), + timestamp: z.number(), + active: z.boolean(), +}) +export type ComputerFrame = z.infer + +/** One pushed screencast frame, as it arrives on the stream. */ +const streamFrameSchema = z.object({ + data: z.string(), + mime: z.string(), + width: z.number(), + height: z.number(), + frame_seq: z.number(), + timestamp: z.number(), +}) +export type ComputerStreamFrame = z.infer + +const contentBlockSchema = z.object({ + type: z.string(), + mime: z.string().nullish(), + data: z.string().nullish(), + text: z.string().nullish(), +}) + +const captureSchema = z.object({ + content: z.array(contentBlockSchema), + details: z.object({ + session_id: z.string(), + mime: z.string(), + width: z.number().optional(), + height: z.number().optional(), + screen: screenSchema.optional(), + }), +}) + +export interface ComputerCapture { + sessionId: string + dataUrl: string + width: number + height: number + note?: string +} + +/** + * Unwrap a transcript output into the worker's plain result. Through the + * harness, results arrive as `{content:[{type:'text', text:}, ...], details}`; the text block is authoritative, `details` the + * fallback. Direct bus results pass through untouched. + */ +export function decodeComputerResult(output: unknown): unknown { + if (!output || typeof output !== 'object' || Array.isArray(output)) { + return output + } + const obj = output as Record + if (!Array.isArray(obj.content)) return output + for (const block of obj.content) { + if (!block || typeof block !== 'object') continue + const b = block as Record + if (b.type !== 'text' || typeof b.text !== 'string') continue + try { + return JSON.parse(b.text) + } catch { + // Not a stringified result; keep looking. + } + } + return 'details' in obj ? obj.details : output +} + +/** `screenshot` / `observe` output → a renderable image, or null. */ +export function parseCapture(output: unknown): ComputerCapture | null { + const parsed = captureSchema.safeParse(output) + if (!parsed.success) return null + const image = parsed.data.content.find( + (b) => b.type === 'image' && typeof b.data === 'string', + ) + if (!image?.data) return null + const note = parsed.data.content.find( + (b) => b.type === 'text' && typeof b.text === 'string', + )?.text + const details = parsed.data.details + return { + sessionId: details.session_id, + dataUrl: `data:${image.mime ?? details.mime};base64,${image.data}`, + width: details.width ?? details.screen?.width ?? 0, + height: details.height ?? details.screen?.height ?? 0, + note: note ?? undefined, + } +} + +/** Session id carried by a `computer::*` call, from its input or its result. */ +export function sessionIdFromCall( + input: unknown, + output: unknown, +): string | null { + if (input && typeof input === 'object') { + const id = (input as Record).session_id + if (typeof id === 'string' && id) return id + } + const decoded = decodeComputerResult(output) + if (decoded && typeof decoded === 'object') { + const id = (decoded as Record).session_id + if (typeof id === 'string' && id) return id + } + return null +} + +/** A stream push (`{event:{data}}` or `{data}`) → the frame it carries. */ +export function extractStreamFrame(raw: unknown): ComputerStreamFrame | null { + if (!raw || typeof raw !== 'object') return null + const obj = raw as Record + const outer = + obj.event && typeof obj.event === 'object' + ? (obj.event as Record) + : obj + const data = 'data' in outer ? outer.data : obj.data + const parsed = streamFrameSchema.safeParse(data) + return parsed.success ? parsed.data : null +} + +export interface StartSessionInput { + image?: string + endpoint?: string + os?: string + monitor?: number +} + +export async function startSession( + iii: ExtensionIii, + input: StartSessionInput, +): Promise { + const payload: Record = {} + if (input.image) payload.image = input.image + if (input.endpoint) payload.endpoint = input.endpoint + if (input.os) payload.os = input.os + if (input.monitor != null) payload.monitor = input.monitor + const res = await iii.trigger(SESSIONS_START_FUNCTION_ID, payload, { + timeoutMs: 120_000, + }) + return sessionStartSchema.parse(decodeComputerResult(res)) +} + +export async function listSessions( + iii: ExtensionIii, +): Promise { + const res = await iii.trigger(SESSIONS_LIST_FUNCTION_ID, {}) + const decoded = decodeComputerResult(res) + const parsed = z + .object({ sessions: z.array(sessionInfoSchema).optional() }) + .safeParse(decoded) + return parsed.success ? (parsed.data.sessions ?? []) : [] +} + +export async function stopSession( + iii: ExtensionIii, + sessionId: string, +): Promise { + await iii.trigger(SESSIONS_STOP_FUNCTION_ID, { session_id: sessionId }) +} + +export async function listDisplays( + iii: ExtensionIii, +): Promise { + const res = await iii.trigger(DISPLAYS_FUNCTION_ID, {}) + const parsed = z + .object({ displays: z.array(displayInfoSchema).optional() }) + .safeParse(decodeComputerResult(res)) + return parsed.success ? (parsed.data.displays ?? []) : [] +} + +export async function takeScreenshot( + iii: ExtensionIii, + sessionId: string, +): Promise { + const res = await iii.trigger(SCREENSHOT_FUNCTION_ID, { + session_id: sessionId, + }) + return parseCapture(res) +} + +export type ActPayload = Record & { action: string } + +export async function act( + iii: ExtensionIii, + sessionId: string, + payload: ActPayload, +): Promise { + await iii.trigger(ACT_FUNCTION_ID, { session_id: sessionId, ...payload }) +} + +export async function startScreencast( + iii: ExtensionIii, + sessionId: string, +): Promise { + await iii.trigger(SCREENCAST_START_FUNCTION_ID, { session_id: sessionId }) +} + +export async function stopScreencast( + iii: ExtensionIii, + sessionId: string, +): Promise { + await iii.trigger(SCREENCAST_STOP_FUNCTION_ID, { session_id: sessionId }) +} + +/** + * Newest pushed screencast frame; a memory read on the worker, cheap to poll. + * `frame` is absent while `sinceFrame` is still the newest seq. + */ +export async function readFrame( + iii: ExtensionIii, + sessionId: string, + sinceFrame?: number, +): Promise { + const res = await iii.trigger(FRAME_FUNCTION_ID, { + session_id: sessionId, + ...(sinceFrame != null ? { since_frame: sinceFrame } : {}), + }) + const parsed = frameSchema.safeParse(res) + return parsed.success ? parsed.data : null +} diff --git a/computer/ui/src/lib/errors.ts b/computer/ui/src/lib/errors.ts new file mode 100644 index 000000000..540bc2855 --- /dev/null +++ b/computer/ui/src/lib/errors.ts @@ -0,0 +1,10 @@ +/** Any thrown value → a message worth showing in the page. */ +export function errorMessage(err: unknown): string { + if (err instanceof Error) return err.message + if (typeof err === 'string') return err + try { + return JSON.stringify(err) + } catch { + return String(err) + } +} diff --git a/computer/ui/src/lib/events.ts b/computer/ui/src/lib/events.ts new file mode 100644 index 000000000..40ecaec41 --- /dev/null +++ b/computer/ui/src/lib/events.ts @@ -0,0 +1,137 @@ +import type { Host } from '@iii-dev/console-ui' +import { useEffect, useId, useRef, useState } from 'react' +import { LIFECYCLE_TRIGGERS } from './computer' + +/** + * Page-local bindings to the computer worker's custom trigger types and its + * screencast stream. Each binding is `host.iii.on(fnId)` plus + * `host.iii.registerTrigger` targeting `::` (the SDK + * registers the handler under the same namespaced id, so they match). The + * handler base ids carry the `iii::` prefix so per-event invocations stay + * span-suppressed and out of the trace feed; the per-mount `instanceId` keeps + * two hook instances from colliding. + * + * Every binding is GC'd with the tab and unregistered on unmount, so the + * injected UI's subscriptions die and revive with the page script. + */ + +const LIFECYCLE_FN = 'iii::computer-ui::lifecycle' + +export interface UseLifecycleEventsOptions { + host: Host + /** Only subscribe while the page is live. */ + enabled: boolean + onEvent: () => void +} + +export interface LifecycleSubscription { + /** + * True once both trigger bindings registered. While false (worker absent, + * SDK failure) callers fall back to polling. + */ + bound: boolean +} + +/** + * Feed of the session lifecycle trigger types, for surfaces that re-read the + * session list on any change. + */ +export function useComputerLifecycleEvents( + opts: UseLifecycleEventsOptions, +): LifecycleSubscription { + const { host, enabled } = opts + const onEventRef = useRef(opts.onEvent) + onEventRef.current = opts.onEvent + + const instanceId = useId().replace(/[^a-zA-Z0-9]/g, '') + const [bound, setBound] = useState(false) + + useEffect(() => { + if (!enabled) { + setBound(false) + return + } + const offs: Array<() => void> = [] + let registered = 0 + for (const triggerType of LIFECYCLE_TRIGGERS) { + const suffix = triggerType.replace(/[^a-zA-Z0-9]/g, '-') + const localFnId = `${LIFECYCLE_FN}::${suffix}::${instanceId}` + try { + offs.push( + host.iii.on(localFnId, () => { + onEventRef.current() + }), + ) + offs.push( + host.iii.registerTrigger({ + type: triggerType, + function_id: `${localFnId}::${host.iii.browserId}`, + config: {}, + }), + ) + registered += 1 + } catch { + // Worker absent or trigger type unregistered; drop the binding. + } + } + setBound(registered === LIFECYCLE_TRIGGERS.length) + + return () => { + setBound(false) + for (const off of offs) off() + } + }, [host, enabled, instanceId]) + + return { bound } +} + +export interface UseComputerStreamOptions { + host: Host + enabled: boolean + /** iii stream name to subscribe to. */ + streamName: string + /** Stream group (the session id for per-session streams). */ + groupId: string | null + /** Base id for this binding's browser-local handler. */ + fnId: string + onFrame: (payload: unknown) => void +} + +/** + * Subscribe to an iii stream (`type:'stream'`) for a session: the engine + * pushes, the client appends. Rebinds when the group (session) changes and + * unregisters on unmount. + */ +export function useComputerStream(opts: UseComputerStreamOptions): void { + const { host, enabled, streamName, groupId, fnId } = opts + const onFrameRef = useRef(opts.onFrame) + onFrameRef.current = opts.onFrame + + const instanceId = useId().replace(/[^a-zA-Z0-9]/g, '') + + useEffect(() => { + if (!enabled || !groupId) return + const offs: Array<() => void> = [] + const localFnId = `${fnId}::${instanceId}` + try { + offs.push( + host.iii.on(localFnId, (payload: unknown) => { + onFrameRef.current(payload) + }), + ) + offs.push( + host.iii.registerTrigger({ + type: 'stream', + function_id: `${localFnId}::${host.iii.browserId}`, + config: { stream_name: streamName, group_id: groupId }, + }), + ) + } catch { + // Stream not available; the seed read is the fallback. + } + + return () => { + for (const off of offs) off() + } + }, [host, enabled, streamName, groupId, fnId, instanceId]) +} diff --git a/computer/ui/src/lib/format.ts b/computer/ui/src/lib/format.ts new file mode 100644 index 000000000..615858d15 --- /dev/null +++ b/computer/ui/src/lib/format.ts @@ -0,0 +1,24 @@ +/** Pure formatting helpers for the computer page. */ + +/** Epoch millis → short relative time ("3m ago"). */ +export function formatAge(unixMs: number, now = Date.now()): string { + if (!unixMs || unixMs <= 0) return '—' + const secs = Math.max(0, Math.floor((now - unixMs) / 1000)) + if (secs < 60) return secs <= 1 ? 'just now' : `${secs}s ago` + const mins = Math.floor(secs / 60) + if (mins < 60) return `${mins}m ago` + const hours = Math.floor(mins / 60) + if (hours < 24) return `${hours}h ago` + return `${Math.floor(hours / 24)}d ago` +} + +/** `native` stays as-is; a url is trimmed to host:port for the rail. */ +export function shortEndpoint(endpoint: string): string { + if (!endpoint || endpoint === 'native') return 'native' + try { + const url = new URL(endpoint) + return url.host || endpoint + } catch { + return endpoint + } +} diff --git a/computer/ui/src/page/SessionRail.tsx b/computer/ui/src/page/SessionRail.tsx new file mode 100644 index 000000000..c5bbee788 --- /dev/null +++ b/computer/ui/src/page/SessionRail.tsx @@ -0,0 +1,81 @@ +import { Badge, Button, StatusDot } from '@iii-dev/console-ui' +import { cn } from '../lib/cn' +import type { ComputerSessionInfo } from '../lib/computer' +import { formatAge, shortEndpoint } from '../lib/format' + +/** + * The left rail: every live session, newest first. Each row carries what + * decides which desktop you are looking at — where it runs, the guest OS, the + * coordinate space, and whether the live view is streaming. + */ + +interface SessionRailProps { + sessions: ComputerSessionInfo[] + selectedId: string | null + loading: boolean + busyId: string | null + onSelect: (sessionId: string) => void + onStop: (sessionId: string) => void +} + +export function SessionRail({ + sessions, + selectedId, + loading, + busyId, + onSelect, + onStop, +}: SessionRailProps) { + if (sessions.length === 0) { + return ( +

+ {loading ? 'loading sessions...' : 'no sessions yet'} +

+ ) + } + + return ( +
    + {sessions.map((session) => { + const selected = session.session_id === selectedId + return ( +
  • +
    + + +
    +
    • + ) + })} +
+ ) +} diff --git a/computer/ui/src/page/StartSessionForm.tsx b/computer/ui/src/page/StartSessionForm.tsx new file mode 100644 index 000000000..53d4066f3 --- /dev/null +++ b/computer/ui/src/page/StartSessionForm.tsx @@ -0,0 +1,107 @@ +import { Button, Input, Select } from '@iii-dev/console-ui' +import { useState } from 'react' +import type { ComputerDisplay, StartSessionInput } from '../lib/computer' + +/** + * Start control: the three ways to get a desktop, as one choice rather than + * three optional fields. `native` drives this machine (with a display picker + * when the host reports more than one), `sandbox` boots a desktop image in a + * microVM, `remote` connects to a desktop somebody else booted. + */ + +type Mode = 'native' | 'sandbox' | 'remote' + +const MODE_OPTIONS = [ + { value: 'native' as const, label: 'this machine' }, + { value: 'sandbox' as const, label: 'sandbox image' }, + { value: 'remote' as const, label: 'remote endpoint' }, +] + +interface StartSessionFormProps { + displays: ComputerDisplay[] + starting: boolean + onStart: (input: StartSessionInput) => void +} + +export function StartSessionForm({ + displays, + starting, + onStart, +}: StartSessionFormProps) { + const [mode, setMode] = useState('native') + const [image, setImage] = useState('') + const [endpoint, setEndpoint] = useState('') + const [monitor, setMonitor] = useState(undefined) + + const submit = () => { + if (starting) return + if (mode === 'sandbox') { + onStart({ image: image.trim() || 'desktop' }) + return + } + if (mode === 'remote') { + const trimmed = endpoint.trim() + if (!trimmed) return + onStart({ endpoint: trimmed }) + return + } + onStart(monitor != null ? { monitor: Number(monitor) } : {}) + } + + return ( +
{ + e.preventDefault() + submit() + }} + > + + value={mode} + options={MODE_OPTIONS} + onChange={setMode} + aria-label="what to drive" + className="cp-ui-start-mode" + /> + {mode === 'sandbox' ? ( + + ) : null} + {mode === 'remote' ? ( + + ) : null} + {mode === 'native' && displays.length > 1 ? ( +