From b03635685338054dfce92904d34e3b2c12767ac9 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan Date: Sat, 13 Jun 2026 16:56:01 -0700 Subject: [PATCH 1/6] e2e: VM substrate for cross-OS supervised-daemon targets + build --target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for codifying the supervised-daemon reboot-survival proof as e2e tests (today it's only a by-hand bash harness). Adds a VM substrate that can provision a guest, drive it over SSH, REBOOT the OS for real, and tear it down: - e2e/src/vm/{types,tart,build-binary}.ts — the tart provider (macOS + Linux guests on an Apple-Silicon host) with a reconnecting SSH forward that survives a guest reboot, plus a helper to compile the guest's `executor` binary. - apps/cli/src/build.ts: a first-class `--target ` flag so the harness can build a specific platform's binary (replaces a throwaway env hack). Validated standalone: provision macOS guest → ssh → real reboot (boottime changes) → clean discard. Next: the `cli` target wires this into the scenario suite so restart-persistence runs against a real reboot. --- apps/cli/src/build.ts | 23 ++++- e2e/src/vm/build-binary.ts | 35 +++++++ e2e/src/vm/tart.ts | 204 +++++++++++++++++++++++++++++++++++++ e2e/src/vm/types.ts | 43 ++++++++ 4 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 e2e/src/vm/build-binary.ts create mode 100644 e2e/src/vm/tart.ts create mode 100644 e2e/src/vm/types.ts diff --git a/apps/cli/src/build.ts b/apps/cli/src/build.ts index 74ec74394..162951fbd 100644 --- a/apps/cli/src/build.ts +++ b/apps/cli/src/build.ts @@ -501,18 +501,24 @@ const buildPreviewTarballs = async (binaries: Record) => { } }; +// Resolve a comma-separated list of target package names (e.g. +// "executor-windows-x64") to Targets. Shared by `--target` and the +// EXECUTOR_PREVIEW_TARGETS env used by the preview-wrapper CI job. const resolveTargetsFromEnv = (env: string | undefined): Target[] => { - if (!env) throw new Error("EXECUTOR_PREVIEW_TARGETS must be set (comma-separated package names)"); + if (!env) throw new Error("No build targets given (comma-separated package names)"); const names = env .split(",") .map((s) => s.trim()) .filter(Boolean); const resolved = names.map((name) => { const match = ALL_TARGETS.find((t) => targetPackageName(t) === name); - if (!match) throw new Error(`Unknown preview target: ${name}`); + if (!match) { + const valid = ALL_TARGETS.map(targetPackageName).join(", "); + throw new Error(`Unknown build target: ${name}. Expected one of: ${valid}`); + } return match; }); - if (resolved.length === 0) throw new Error("EXECUTOR_PREVIEW_TARGETS resolved to an empty list"); + if (resolved.length === 0) throw new Error("Build target list resolved to empty"); return resolved; }; @@ -919,6 +925,11 @@ const { values, positionals } = parseArgs({ args: process.argv.slice(2), options: { single: { type: "boolean", default: false }, + // Build a specific target (or comma-separated set) by package name, e.g. + // `--target executor-windows-x64`. Used by the e2e VM harness to compile + // the guest's binary; without it `binary` builds the current platform + // (`--single`) or all targets. + target: { type: "string" }, mode: { type: "string", default: "production" }, }, allowPositionals: true, @@ -932,7 +943,11 @@ if (mode !== "production" && mode !== "development") { } if (command === "binary") { - const targets = values.single ? ALL_TARGETS.filter(isCurrentPlatform) : ALL_TARGETS; + const targets = values.target + ? resolveTargetsFromEnv(values.target) + : values.single + ? ALL_TARGETS.filter(isCurrentPlatform) + : ALL_TARGETS; const binaries = await buildBinaries(targets, mode); await buildWrapperPackage(binaries); } else if (command === "preview") { diff --git a/e2e/src/vm/build-binary.ts b/e2e/src/vm/build-binary.ts new file mode 100644 index 000000000..b5d32c26d --- /dev/null +++ b/e2e/src/vm/build-binary.ts @@ -0,0 +1,35 @@ +// Build the compiled `executor` for a guest os/arch. `service install` refuses +// to run from a dev (.ts) entrypoint, so the VM targets need a real binary — +// produced via the `--target` flag on apps/cli/src/build.ts. + +import { execFile } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { promisify } from "node:util"; + +import type { VmArch, VmOs } from "./types"; + +const execFileP = promisify(execFile); + +const PLATFORM_TAG: Record = { macos: "darwin", linux: "linux", windows: "windows" }; + +// e2e/src/vm/build-binary.ts → repo root. +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_DIR = path.join(REPO_ROOT, "apps", "cli"); + +/** + * Build the `executor` binary for a guest and return its `bin` directory + * (executor[.exe] + the native libsql/keyring modules). The build cleans + * `dist/` each run, so callers should push the result before building another. + */ +export const buildGuestBinary = async (os: VmOs, arch: VmArch): Promise => { + const target = `executor-${PLATFORM_TAG[os]}-${arch}`; + await execFileP("bun", ["run", "src/build.ts", "binary", "--target", target], { + cwd: CLI_DIR, + maxBuffer: 256 * 1024 * 1024, + }); + const binDir = path.join(CLI_DIR, "dist", target, "bin"); + const exe = path.join(binDir, os === "windows" ? "executor.exe" : "executor"); + if (!existsSync(exe)) throw new Error(`buildGuestBinary: ${exe} not produced`); + return binDir; +}; diff --git a/e2e/src/vm/tart.ts b/e2e/src/vm/tart.ts new file mode 100644 index 000000000..fad7a7324 --- /dev/null +++ b/e2e/src/vm/tart.ts @@ -0,0 +1,204 @@ +// tart provider: macOS + Linux guests on an Apple-Silicon host (the Mini). +// Mirrors the by-hand reboot harness — clone a base image, boot headless, drive +// over sshpass, reboot the guest OS for real, tear down the clone. + +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import net from "node:net"; +import { promisify } from "node:util"; + +import { + type SshResult, + sleep, + type Tunnel, + type VmArch, + type VmHandle, + type VmProvider, +} from "./types"; + +const execFileP = promisify(execFile); + +const TART = process.env.E2E_TART_BIN ?? "/opt/homebrew/bin/tart"; +const SSHPASS = process.env.E2E_SSHPASS_BIN ?? "/opt/homebrew/bin/sshpass"; +const SSH_OPTS = [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "ConnectTimeout=8", + "-o", + "ServerAliveInterval=5", + "-o", + "LogLevel=ERROR", +]; +const GUEST_USER = "admin"; +const GUEST_PASS = "admin"; + +const baseImage = (os: "macos" | "linux"): string => + os === "macos" + ? (process.env.E2E_TART_MACOS_BASE ?? "executor-macos-base") + : (process.env.E2E_TART_LINUX_BASE ?? "executor-linux-base"); + +/** Ask the OS for a free localhost port (for SSH tunnels). */ +const freePort = (): Promise => + new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on("error", reject); + srv.listen(0, "127.0.0.1", () => { + const port = (srv.address() as net.AddressInfo).port; + srv.close(() => resolve(port)); + }); + }); + +/** Resolve once a TCP connect to localhost:port succeeds (SSH bound the forward). */ +const waitLocalPort = async (port: number, attempts = 40): Promise => { + for (let i = 0; i < attempts; i++) { + const ok = await new Promise((resolve) => { + const sock = net.connect({ host: "127.0.0.1", port }, () => { + sock.destroy(); + resolve(true); + }); + sock.on("error", () => resolve(false)); + sock.setTimeout(1000, () => { + sock.destroy(); + resolve(false); + }); + }); + if (ok) return; + await sleep(500); + } + throw new Error(`tunnel local port ${port} never came up`); +}; + +export const tartVm = (os: "macos" | "linux", arch: VmArch = "arm64"): VmProvider => ({ + os, + provision: async () => { + const name = `executor-e2e-${os}-${process.pid}-${Math.floor(performance.now())}`; + await execFileP(TART, ["clone", baseImage(os), name]); + const runProc = spawn(TART, ["run", name, "--no-graphics"], { stdio: "ignore" }); + + const tunnelClosers: Array<() => void> = []; + let ip = ""; + + const fetchIp = async (): Promise => { + for (let i = 0; i < 90; i++) { + try { + const { stdout } = await execFileP(TART, ["ip", name]); + if (stdout.trim()) { + ip = stdout.trim(); + return true; + } + } catch { + /* not booted yet */ + } + await sleep(2000); + } + return false; + }; + + // Linux systemctl --user calls need XDG_RUNTIME_DIR; harmless elsewhere. + const wrap = (command: string): string => + os === "linux" ? `export XDG_RUNTIME_DIR=/run/user/$(id -u); ${command}` : command; + + const ssh = async (command: string): Promise => { + try { + const { stdout, stderr } = await execFileP( + SSHPASS, + ["-p", GUEST_PASS, "ssh", ...SSH_OPTS, `${GUEST_USER}@${ip}`, wrap(command)], + { maxBuffer: 32 * 1024 * 1024 }, + ); + return { stdout, stderr, code: 0 }; + } catch (err) { + const e = err as { stdout?: string; stderr?: string; code?: number }; + return { + stdout: e.stdout ?? "", + stderr: e.stderr ?? "", + code: typeof e.code === "number" ? e.code : 1, + }; + } + }; + + const waitSsh = async (attempts: number): Promise => { + for (let i = 0; i < attempts; i++) { + if ((await ssh("true")).code === 0) return true; + await sleep(2000); + } + return false; + }; + + const handle: VmHandle = { + os, + arch, + ssh, + push: async (localPath, remotePath) => { + await execFileP(SSHPASS, [ + "-p", + GUEST_PASS, + "scp", + "-r", + ...SSH_OPTS, + localPath, + `${GUEST_USER}@${ip}:${remotePath}`, + ]); + }, + reboot: async () => { + await ssh("sudo reboot").catch(() => undefined); // connection drops mid-call + await sleep(5000); + if (!(await fetchIp())) throw new Error(`tart ${os}: no IP after reboot`); + if (!(await waitSsh(120))) throw new Error(`tart ${os}: SSH did not return after reboot`); + }, + tunnel: async (guestPort) => { + const localPort = await freePort(); + // Reconnecting forward: when the guest reboots the ssh exits, so respawn + // it until closed. `restart()` health-polls through this local port, so + // it only goes green once the daemon AND the forward are back. + let closed = false; + let child: ChildProcess | undefined; + const spawnOnce = (): void => { + child = spawn( + SSHPASS, + [ + "-p", + GUEST_PASS, + "ssh", + ...SSH_OPTS, + "-N", + "-L", + `${localPort}:127.0.0.1:${guestPort}`, + `${GUEST_USER}@${ip}`, + ], + { stdio: "ignore" }, + ); + child.on("exit", () => { + if (!closed) setTimeout(spawnOnce, 2000); + }); + }; + spawnOnce(); + const close = (): void => { + closed = true; + child?.kill(); + }; + tunnelClosers.push(close); + await waitLocalPort(localPort); + const tunnel: Tunnel = { localPort, close }; + return tunnel; + }, + discard: async () => { + for (const close of tunnelClosers) close(); + runProc.kill(); + await sleep(1500); + await execFileP(TART, ["delete", name]).catch(() => undefined); + }, + }; + + if (!(await fetchIp())) { + await handle.discard(); + throw new Error(`tart ${os}: no IP within 180s`); + } + if (!(await waitSsh(90))) { + await handle.discard(); + throw new Error(`tart ${os}: SSH never came up`); + } + return handle; + }, +}); diff --git a/e2e/src/vm/types.ts b/e2e/src/vm/types.ts new file mode 100644 index 000000000..f0edaf513 --- /dev/null +++ b/e2e/src/vm/types.ts @@ -0,0 +1,43 @@ +// VM substrate for the cross-OS supervised-daemon e2e targets. +// +// A VmHandle is a booted guest we can drive over SSH, REBOOT for real, and tear +// down. Providers: tart (macOS + Linux, local on an Apple-Silicon host) and ec2 +// (Windows, ephemeral). This is the codified form of the by-hand reboot harness. + +export type VmOs = "macos" | "linux" | "windows"; +export type VmArch = "arm64" | "x64"; + +export interface SshResult { + readonly stdout: string; + readonly stderr: string; + readonly code: number; +} + +/** An open SSH local-forward (`localhost:localPort` → guest:guestPort). */ +export interface Tunnel { + readonly localPort: number; + close(): void; +} + +export interface VmHandle { + readonly os: VmOs; + readonly arch: VmArch; + /** Run a command in the guest over SSH (shell on Unix, PowerShell on Windows). */ + ssh(command: string): Promise; + /** Copy a local file or directory into the guest (recursive for directories). */ + push(localPath: string, remotePath: string): Promise; + /** Reboot the guest OS; resolves only once SSH is reachable again. */ + reboot(): Promise; + /** Forward `localhost:` → `guest:` over SSH. */ + tunnel(guestPort: number): Promise; + /** Discard the VM (delete the tart clone / terminate the EC2 instance). */ + discard(): Promise; +} + +export interface VmProvider { + readonly os: VmOs; + /** Boot a fresh guest and wait until SSH answers. */ + provision(): Promise; +} + +export const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); From e2bff6ecb67bfdd773f9dcd1631cc7893681acec Mon Sep 17 00:00:00 2001 From: Rhys Sullivan Date: Sat, 13 Jun 2026 17:03:59 -0700 Subject: [PATCH 2/6] =?UTF-8?q?e2e:=20cli=20target=20=E2=80=94=20supervise?= =?UTF-8?q?d=20daemon=20in=20a=20VM,=20restart()=20is=20a=20real=20reboot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the VM substrate into the scenario suite as a `cli` target (one project per guest OS: cli-macos / cli-linux / cli-windows). The globalsetup builds the guest's executor binary, provisions a VM, `executor service install`s the supervised daemon, and forwards its loopback port over a reconnecting SSH tunnel — so target.baseUrl works unchanged for the api surface. restart() reboots the guest OS for REAL and waits for the daemon to auto-start, so the existing restart-persistence scenario now proves the boot-time auto-start path (launchd RunAtLoad / systemd linger / Task Scheduler AtStartup), not a process restart. Because globalsetup (main process) and restart() (test worker) are different processes, restart() re-derives the guest address from env and reboots statelessly, mirroring how selfhost-docker's restart re-derives container ops. Verified: `vitest run --project cli-macos` boots a macOS guest, installs the daemon, addSpec's an integration, reboots the guest, and asserts the data survived — green, with a run-record in the matrix. cli-linux reuses the tart provider; cli-windows awaits the ec2 provider. --- e2e/setup/cli-linux.globalsetup.ts | 3 ++ e2e/setup/cli-macos.globalsetup.ts | 3 ++ e2e/setup/cli.globalsetup.ts | 65 ++++++++++++++++++++++++++++++ e2e/src/vm/tart.ts | 20 +++++++++ e2e/src/vm/types.ts | 2 + e2e/targets/cli.ts | 47 +++++++++++++++++++++ e2e/targets/registry.ts | 6 +++ e2e/vitest.config.ts | 15 +++++++ 8 files changed, 161 insertions(+) create mode 100644 e2e/setup/cli-linux.globalsetup.ts create mode 100644 e2e/setup/cli-macos.globalsetup.ts create mode 100644 e2e/setup/cli.globalsetup.ts create mode 100644 e2e/targets/cli.ts diff --git a/e2e/setup/cli-linux.globalsetup.ts b/e2e/setup/cli-linux.globalsetup.ts new file mode 100644 index 000000000..bf04cd6af --- /dev/null +++ b/e2e/setup/cli-linux.globalsetup.ts @@ -0,0 +1,3 @@ +import { setupCliTarget } from "./cli.globalsetup"; + +export default (): Promise<(() => Promise) | void> => setupCliTarget("linux"); diff --git a/e2e/setup/cli-macos.globalsetup.ts b/e2e/setup/cli-macos.globalsetup.ts new file mode 100644 index 000000000..efa8f091e --- /dev/null +++ b/e2e/setup/cli-macos.globalsetup.ts @@ -0,0 +1,3 @@ +import { setupCliTarget } from "./cli.globalsetup"; + +export default (): Promise<(() => Promise) | void> => setupCliTarget("macos"); diff --git a/e2e/setup/cli.globalsetup.ts b/e2e/setup/cli.globalsetup.ts new file mode 100644 index 000000000..fc7e2b9ff --- /dev/null +++ b/e2e/setup/cli.globalsetup.ts @@ -0,0 +1,65 @@ +// Boot the `cli` target: build the guest's `executor` binary, provision a VM, +// install it as a supervised OS service, and forward its loopback HTTP port to +// the host over a reconnecting SSH tunnel. Publishes connection + reboot info +// via env (inherited by the test workers spawned afterward). Per-OS entrypoints +// (cli-macos.globalsetup.ts, …) call setupCliTarget with their OS. +import { buildGuestBinary } from "../src/vm/build-binary"; +import { tartVm } from "../src/vm/tart"; +import type { VmArch, VmOs } from "../src/vm/types"; +import { waitForHttp } from "./boot"; + +const PORT = 4789; +const GUEST_DIR = "~/ed"; + +export async function setupCliTarget(os: VmOs): Promise<(() => Promise) | void> { + process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS + if (os === "windows") { + throw new Error("cli-windows is pending the ec2 provider; run cli-macos / cli-linux for now"); + } + + const arch: VmArch = "arm64"; // tart guests on an Apple-Silicon host + const binDir = await buildGuestBinary(os, arch); + const vm = await tartVm(os, arch).provision(); + + let tunnelClose: (() => void) | undefined; + try { + await vm.ssh(`rm -rf ${GUEST_DIR} && mkdir -p ${GUEST_DIR}`); + await vm.push(`${binDir}/.`, `${GUEST_DIR}/`); + // macOS quarantines scp'd executables; clear it so the binary can run. + await vm.ssh( + os === "macos" + ? `chmod +x ${GUEST_DIR}/executor; xattr -dr com.apple.quarantine ${GUEST_DIR} 2>/dev/null || true` + : `chmod +x ${GUEST_DIR}/executor`, + ); + + const install = await vm.ssh(`${GUEST_DIR}/executor service install --port ${PORT}`); + if (install.code !== 0) { + throw new Error(`service install failed: ${install.stderr.trim() || install.stdout.trim()}`); + } + + const keyRaw = (await vm.ssh("cat ~/.executor/server-control/service.key")).stdout.trim(); + const password = (JSON.parse(keyRaw) as { password: string }).password; + + const tunnel = await vm.tunnel(PORT); + tunnelClose = tunnel.close; + const baseUrl = `http://127.0.0.1:${tunnel.localPort}`; + await waitForHttp(`${baseUrl}/`, { timeoutMs: 60_000 }); + + process.env.E2E_CLI_BASE_URL = baseUrl; + process.env.E2E_CLI_AUTH_PASSWORD = password; + process.env.E2E_CLI_VM_HOST = vm.host; + process.env.E2E_CLI_TUNNEL_PORT = String(tunnel.localPort); + process.env.E2E_CLI_BIN_DIR = GUEST_DIR; + } catch (error) { + tunnelClose?.(); + await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined); + await vm.discard(); + throw error; + } + + return async () => { + tunnelClose?.(); + await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined); + await vm.discard(); + }; +} diff --git a/e2e/src/vm/tart.ts b/e2e/src/vm/tart.ts index fad7a7324..5ca696420 100644 --- a/e2e/src/vm/tart.ts +++ b/e2e/src/vm/tart.ts @@ -34,6 +34,23 @@ const SSH_OPTS = [ const GUEST_USER = "admin"; const GUEST_PASS = "admin"; +/** + * Reboot a tart guest by address, with no live handle. `restart()` runs in a + * vitest worker (separate process from the globalsetup that owns the VM), so it + * re-derives the guest address from env and triggers the reboot statelessly — + * the reconnecting tunnel and a health poll confirm recovery. + */ +export const sshRebootGuest = async (ip: string): Promise => { + await execFileP(SSHPASS, [ + "-p", + GUEST_PASS, + "ssh", + ...SSH_OPTS, + `${GUEST_USER}@${ip}`, + "sudo reboot", + ]).catch(() => undefined); // the connection drops mid-call +}; + const baseImage = (os: "macos" | "linux"): string => os === "macos" ? (process.env.E2E_TART_MACOS_BASE ?? "executor-macos-base") @@ -129,6 +146,9 @@ export const tartVm = (os: "macos" | "linux", arch: VmArch = "arm64"): VmProvide const handle: VmHandle = { os, arch, + get host() { + return ip; + }, ssh, push: async (localPath, remotePath) => { await execFileP(SSHPASS, [ diff --git a/e2e/src/vm/types.ts b/e2e/src/vm/types.ts index f0edaf513..28b07961c 100644 --- a/e2e/src/vm/types.ts +++ b/e2e/src/vm/types.ts @@ -22,6 +22,8 @@ export interface Tunnel { export interface VmHandle { readonly os: VmOs; readonly arch: VmArch; + /** Current reachable address of the guest (re-resolved across reboots). */ + readonly host: string; /** Run a command in the guest over SSH (shell on Unix, PowerShell on Windows). */ ssh(command: string): Promise; /** Copy a local file or directory into the guest (recursive for directories). */ diff --git a/e2e/targets/cli.ts b/e2e/targets/cli.ts new file mode 100644 index 000000000..00385252b --- /dev/null +++ b/e2e/targets/cli.ts @@ -0,0 +1,47 @@ +// The supervised local CLI daemon as a target. `executor service install` boots +// an OS-managed daemon (launchd / systemd / Task Scheduler) inside a guest VM; +// `restart()` reboots that guest for REAL, so restart-persistence proves the +// boot-time auto-start path (RunAtLoad / linger / AtStartup), not just a process +// restart. The guest daemon binds loopback, so globalsetup forwards it to a +// local port over a reconnecting SSH tunnel — making target.baseUrl work +// unchanged for the api surface. Boot + tunnel live in setup/cli.globalsetup.ts; +// this target only reads what that published via env. +import { Effect } from "effect"; + +import { waitForHttp } from "../setup/boot"; +import { sshRebootGuest } from "../src/vm/tart"; +import type { VmOs } from "../src/vm/types"; +import type { Capability, Identity, Target } from "../src/target"; + +const env = (key: string): string => { + const value = process.env[key]; + if (!value) throw new Error(`cli target: ${key} not set — did cli.globalsetup run?`); + return value; +}; + +export const cliTarget = (): Target => { + const baseUrl = env("E2E_CLI_BASE_URL"); + const os = (process.env.E2E_VM_OS ?? "macos") as VmOs; + const username = process.env.E2E_CLI_AUTH_USER ?? "executor"; + + return { + name: process.env.E2E_TARGET ?? "cli", + baseUrl, + mcpUrl: `${baseUrl}/mcp`, + capabilities: new Set(["api"]), + newIdentity: () => + Effect.sync((): Identity => { + const basic = Buffer.from(`${username}:${env("E2E_CLI_AUTH_PASSWORD")}`).toString("base64"); + return { label: "cli-daemon", headers: { Authorization: `Basic ${basic}` } }; + }), + // A genuine machine reboot, not a service kick: reboot the guest OS and wait + // for the supervised daemon to auto-start and serve again (401 = up). The + // reconnecting tunnel re-establishes the forward, so the same baseUrl works. + restart: () => + Effect.promise(async () => { + if (os === "windows") throw new Error("cli-windows restart pending the ec2 provider"); + await sshRebootGuest(env("E2E_CLI_VM_HOST")); + await waitForHttp(`${baseUrl}/`, { timeoutMs: 240_000 }); + }), + }; +}; diff --git a/e2e/targets/registry.ts b/e2e/targets/registry.ts index c1449bec2..93e4edc66 100644 --- a/e2e/targets/registry.ts +++ b/e2e/targets/registry.ts @@ -2,6 +2,7 @@ // once per worker. Adding a target = one factory entry here + a project in // vitest.config.ts + a globalsetup that boots (or attaches to) the instance. import type { Target } from "../src/target"; +import { cliTarget } from "./cli"; import { cloudTarget } from "./cloud"; import { cloudflareTarget } from "./cloudflare"; import { desktopTarget } from "./desktop"; @@ -16,6 +17,11 @@ const factories: Record Target> = { cloudflare: cloudflareTarget, desktop: desktopTarget, local: localTarget, + // The supervised CLI daemon inside a VM, one project per guest OS — restart() + // is a real reboot. See setup/cli.globalsetup.ts. + "cli-macos": cliTarget, + "cli-linux": cliTarget, + "cli-windows": cliTarget, }; let current: Target | undefined; diff --git a/e2e/vitest.config.ts b/e2e/vitest.config.ts index 3c3d98a55..449de4b32 100644 --- a/e2e/vitest.config.ts +++ b/e2e/vitest.config.ts @@ -68,6 +68,21 @@ export default defineConfig({ fileParallelism: true, testTimeout: 180_000, }), + // The supervised CLI daemon inside a guest VM, one project per OS. The + // globalsetup provisions a VM, `executor service install`s the daemon, and + // tunnels it; restart() reboots the guest for REAL, so restart-persistence + // proves the boot-time auto-start path. Needs tart (macOS/Linux) or an EC2 + // credential (Windows); not part of the default `npm run test` chain — run + // with `vitest run --project cli-macos` (etc.) on the Mini. + ...(["macos", "linux", "windows"] as const).map((os) => + project(`cli-${os}`, { + include: ["scenarios/restart-persistence.test.ts", "cli/**/*.test.ts"], + env: { E2E_TARGET: `cli-${os}`, E2E_VM_OS: os }, + fileParallelism: false, + testTimeout: 300_000, + hookTimeout: 900_000, + }), + ), ], }, }); From 5bacd3c7044175cf24bdc1eb7b9967dd772b1792 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan Date: Sat, 13 Jun 2026 17:17:16 -0700 Subject: [PATCH 3/6] =?UTF-8?q?e2e:=20recorded=20film=20=E2=80=94=20integr?= =?UTF-8?q?ation=20survives=20a=20real=20reboot=20(terminal=20cast)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A watchable companion to restart-persistence: drives the same register → reboot → survived flow through the chat theater so the on-screen tool spinner runs for the ACTUAL guest reboot. Produces a terminal.cast the viewer plays in the matrix — press-play evidence instead of trusting a green check. Runs against the cli-* VM targets; verified on cli-macos (357-event cast). --- e2e/cli/service-lifecycle.test.ts | 128 ++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 e2e/cli/service-lifecycle.test.ts diff --git a/e2e/cli/service-lifecycle.test.ts b/e2e/cli/service-lifecycle.test.ts new file mode 100644 index 000000000..e38505ba9 --- /dev/null +++ b/e2e/cli/service-lifecycle.test.ts @@ -0,0 +1,128 @@ +// The supervised daemon's durability as a WATCHABLE terminal recording: register +// a REAL integration, REBOOT the daemon's machine for real (the on-screen +// spinner runs for the actual reboot), then show the integration still there. +// Same assertions as restart-persistence — but filmed, so you can press play +// instead of trusting a green check. Runs against the cli-* VM targets. +import { randomBytes } from "node:crypto"; +import { join } from "node:path"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; + +import { composePluginApi } from "@executor-js/api/server"; +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; + +import { withChatTheater } from "../src/clients/chat-theater"; +import { scenario } from "../src/scenario"; +import { Api, Cli, Restart, RunDir, Target } from "../src/services"; + +const api = composePluginApi([openApiHttpPlugin()] as const); + +/** Inline OpenAPI 3 spec with a single GET /ping (its server is never called). */ +const pingSpec = JSON.stringify({ + openapi: "3.0.3", + info: { title: "Reboot Lifecycle API", version: "1.0.0" }, + servers: [{ url: "http://127.0.0.1:59998" }], + paths: { + "/ping": { + get: { + operationId: "getPing", + summary: "Liveness ping", + responses: { "200": { description: "pong" } }, + }, + }, + }, +}); + +scenario( + "Supervised daemon · an integration survives a real machine reboot (recorded)", + {}, + Effect.gen(function* () { + const target = yield* Target; + const restart = yield* Restart; + const { client } = yield* Api; + const cli = yield* Cli; + const runDir = yield* RunDir; + + const slug = `reboot-film-${randomBytes(4).toString("hex")}`; + + yield* withChatTheater( + cli, + { title: "executor — supervised daemon", record: join(runDir, "terminal.cast") }, + (chat) => + Effect.gen(function* () { + yield* chat.user("If this machine reboots, do my connected integrations survive?"); + yield* chat.assistant( + "Let's prove it — register a real integration, reboot the daemon's machine for real, then check it's still there.", + ); + + const before = yield* client(api, yield* target.newIdentity()); + + const added = yield* chat.tool( + { + name: "executor call executor.openapi.addSpec", + input: `slug: ${slug}\nspec: inline OpenAPI (GET /ping)`, + result: (a) => `registered — ${a.toolCount} tool(s)`, + }, + before.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: pingSpec }, + slug, + authenticationTemplate: [], + }, + }), + ); + expect(added.toolCount, "the spec registered with tools").toBeGreaterThan(0); + + const listed = yield* chat.tool( + { + name: "executor tools sources", + result: (rows) => + rows.map((r) => String(r.slug)).includes(slug) ? `${slug} is listed` : "NOT listed", + }, + before.integrations.list(), + ); + expect( + listed.map((i) => String(i.slug)), + "listed before the reboot", + ).toContain(slug); + + // The spinner here runs for the ENTIRE real reboot — the supervised + // service must auto-start at boot for this to ever return. + yield* chat.tool( + { + name: "reboot the daemon's machine", + input: "guest OS reboot — the OS service manager must auto-start the daemon at boot", + result: () => "back online; daemon auto-started", + }, + restart(), + ); + + const after = yield* client(api, yield* target.newIdentity()); + yield* Effect.ensuring( + Effect.gen(function* () { + const survived = yield* chat.tool( + { + name: "executor tools sources", + result: (rows) => + rows.map((r) => String(r.slug)).includes(slug) + ? `${slug} SURVIVED the reboot` + : "VANISHED", + }, + after.integrations.list(), + ); + expect( + survived.map((i) => String(i.slug)), + "survived the reboot", + ).toContain(slug); + yield* chat.assistant( + "It survived — the OS restarted the daemon at boot and its data was intact.", + ); + }), + // Shared guest, but ephemeral; still, never leave the spec behind. + after.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore), + ); + }), + ); + }), +); From cfd22840bb9140c0715a75d45a2fd0eea3b183ac Mon Sep 17 00:00:00 2001 From: Rhys Sullivan Date: Sat, 13 Jun 2026 17:40:15 -0700 Subject: [PATCH 4/6] =?UTF-8?q?e2e:=20desktop=20attach=20film=20=E2=80=94?= =?UTF-8?q?=20app=20connects=20to=20the=20supervised=20daemon=20(not=20a?= =?UTF-8?q?=20sidecar)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Films the desktop ATTACHING to an already-running OS-supervised daemon instead of spawning its own sidecar: starts a real cli-daemon (the desktop sidecar server in EXECUTOR_SUPERVISED mode → kind "cli-daemon") against a throwaway HOME, launches Electron pointed at the same HOME, and proves the attach — the server manifest still names OUR daemon's pid (a spawned sidecar would be a new pid + kind "desktop-sidecar"). session.mp4 is the artifact. The daemon half is verified (EXECUTOR_READY + kind cli-daemon + 401). Like the existing desktop scenario it drives Electron via Playwright, so it needs a GUI display session to run (the desktop project is already display-gated, not in the default test chain) — run on a real desktop / CI-with-display to capture the video. --- e2e/desktop/supervised-attach.test.ts | 182 ++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 e2e/desktop/supervised-attach.test.ts diff --git a/e2e/desktop/supervised-attach.test.ts b/e2e/desktop/supervised-attach.test.ts new file mode 100644 index 000000000..0ef2459af --- /dev/null +++ b/e2e/desktop/supervised-attach.test.ts @@ -0,0 +1,182 @@ +// Desktop-only, on camera: the app ATTACHES to an already-running OS-supervised +// daemon instead of spawning its own sidecar. We start a real supervised gateway +// (the desktop sidecar server in EXECUTOR_SUPERVISED mode → it self-publishes a +// manifest of kind "cli-daemon"), launch the Electron app pointed at the same +// HOME, and prove it attached: the manifest still names OUR daemon's pid (a +// spawned sidecar would be a fresh pid + kind "desktop-sidecar"). The recording +// (session.mp4 + screenshots) is the artifact; the waits are the assertions. No +// launchd — only a throwaway home and one short-lived daemon process. +import { type ChildProcess, execFile, spawn } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { _electron } from "playwright"; + +import { scenario } from "../src/scenario"; +import { RunDir } from "../src/services"; +import { waitForHttp } from "../setup/boot"; + +const appDir = fileURLToPath(new URL("../../apps/desktop/", import.meta.url)); +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); +const sidecarServer = join(appDir, "src/sidecar/server.ts"); +const clientDir = join(repoRoot, "apps/local/dist"); +const electronBinary = createRequire(join(appDir, "package.json"))("electron") as string; + +const freePort = (): Promise => + new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on("error", reject); + srv.listen(0, "127.0.0.1", () => { + const port = (srv.address() as net.AddressInfo).port; + srv.close(() => resolve(port)); + }); + }); + +interface Manifest { + readonly kind: string; + readonly pid: number; +} + +interface DaemonStart { + readonly child: ChildProcess; + readonly ready: boolean; + readonly stderr: string; +} + +/** Spawn the supervised gateway; resolves once it announces EXECUTOR_READY (or + * times out / exits early, with `ready: false`). The caller asserts readiness, + * so the executor only ever resolves. */ +const startSupervisedDaemon = (env: NodeJS.ProcessEnv): Promise => + new Promise((resolve) => { + const child = spawn("bun", ["run", sidecarServer], { + cwd: repoRoot, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stderr = ""; + const settle = (ready: boolean) => resolve({ child, ready, stderr }); + const timer = setTimeout(() => settle(false), 60_000); + child.stdout.on("data", (chunk: Buffer) => { + if (chunk.toString().includes("EXECUTOR_READY:")) { + clearTimeout(timer); + settle(true); + } + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.on("exit", () => { + clearTimeout(timer); + settle(false); + }); + }); + +scenario( + "Desktop · attaches to the OS-supervised daemon instead of spawning a sidecar", + { timeout: 240_000 }, + Effect.gen(function* () { + const runDir = yield* RunDir; + yield* Effect.promise(() => run(runDir)); + }), +); + +const run = async (runDir: string) => { + const home = mkdtempSync(join(tmpdir(), "executor-attach-e2e-")); + const dataDir = join(home, ".executor"); + const manifestPath = join(dataDir, "server-control", "server.json"); + const videoTmp = join(runDir, ".video-tmp"); + const port = await freePort(); + + let daemon: ChildProcess | undefined; + let app: Awaited> | undefined; + let stepIndex = 0; + + try { + const started = await startSupervisedDaemon({ + ...process.env, + HOME: home, + EXECUTOR_SUPERVISED: "1", + EXECUTOR_DATA_DIR: dataDir, + EXECUTOR_PORT: String(port), + EXECUTOR_HOST: "127.0.0.1", + EXECUTOR_AUTH_PASSWORD: "supervised-attach-film", + EXECUTOR_CLIENT_DIR: clientDir, + }); + daemon = started.child; + expect(started.ready, `supervised daemon became ready; stderr:\n${started.stderr}`).toBe(true); + await waitForHttp(`http://127.0.0.1:${port}/`, { timeoutMs: 30_000 }); + + const daemonManifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Manifest; + expect(daemonManifest.kind, "the running daemon advertises itself as cli-daemon").toBe( + "cli-daemon", + ); + const daemonPid = daemonManifest.pid; + + app = await _electron.launch({ + executablePath: electronBinary, + args: [appDir], + cwd: appDir, + env: { ...process.env, HOME: home }, + recordVideo: { dir: videoTmp, size: { width: 1280, height: 800 } }, + timeout: 120_000, + }); + + const page = await app.firstWindow({ timeout: 120_000 }); + const step = async (label: string, body: () => Promise) => { + await body(); + stepIndex += 1; + const slug = label.toLowerCase().replace(/[^a-z0-9]+/g, "-"); + await page.screenshot({ + path: join(runDir, `${String(stepIndex).padStart(2, "0")}-${slug}.png`), + }); + }; + + // The window only loads the console once the app has a connection — and it + // attaches to the supervised daemon before it would ever spawn a sidecar. + await step("desktop boots into the console", async () => { + await page.getByText("Settings").first().waitFor({ timeout: 120_000 }); + }); + + // The proof it ATTACHED rather than spawned: the manifest is untouched — + // same pid, still cli-daemon. A managed sidecar would have rewritten it to + // kind "desktop-sidecar" with a fresh child pid. + await step("server manifest still names the supervised daemon", async () => { + const after = JSON.parse(readFileSync(manifestPath, "utf8")) as Manifest; + expect(after.kind, "still the supervised daemon (not a desktop sidecar)").toBe("cli-daemon"); + expect(after.pid, "the desktop attached to our daemon, not a new sidecar").toBe(daemonPid); + }); + } finally { + const page = app?.windows()[0]; + const video = page?.video(); + await app?.close().catch(() => {}); + const recordedPath = await video?.path().catch(() => undefined); + if (recordedPath && existsSync(recordedPath)) { + await promisify(execFile)("ffmpeg", [ + "-y", + "-i", + recordedPath, + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "26", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + join(runDir, "session.mp4"), + ]).catch(() => {}); + } + daemon?.kill("SIGTERM"); + rmSync(videoTmp, { recursive: true, force: true }); + rmSync(home, { recursive: true, force: true }); + } +}; From d5ae77d904afef5528489a7df511551ac6b061a9 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan Date: Sat, 13 Jun 2026 23:57:16 -0700 Subject: [PATCH 5/6] e2e/cli: bearer-gated daemon target, reboot down-gate, EC2 provider for Windows - cli target authenticates with the daemon's auth.json bearer (was Basic from service.key); globalsetup reads the token from the guest and publishes it. - restart() gates on the daemon going DOWN before polling it back up, so an orderly shutdown / reconnecting tunnel can't false-pass a reboot that never happened (waitForHttpDown). - EC2 VmProvider (e2e/src/vm/ec2.ts) + cli-windows globalsetup unblock Windows: launch, key-based SSH/PowerShell drive, real boot-time-gated reboot, terminate. --- e2e/setup/boot.ts | 24 ++ e2e/setup/cli-windows.globalsetup.ts | 3 + e2e/setup/cli.globalsetup.ts | 76 +++-- e2e/src/vm/ec2.ts | 422 +++++++++++++++++++++++++++ e2e/src/vm/types.ts | 6 + e2e/targets/cli.ts | 34 ++- 6 files changed, 532 insertions(+), 33 deletions(-) create mode 100644 e2e/setup/cli-windows.globalsetup.ts create mode 100644 e2e/src/vm/ec2.ts diff --git a/e2e/setup/boot.ts b/e2e/setup/boot.ts index 7e6bd3b1b..2b6b13731 100644 --- a/e2e/setup/boot.ts +++ b/e2e/setup/boot.ts @@ -100,3 +100,27 @@ export const waitForHttp = async ( } throw new Error(`timed out waiting for ${url}: ${String(lastError)}`); }; + +/** + * Reboot DOWN-GATE: wait until `url` stops answering (fetch rejects — connection + * refused/reset). An orderly OS shutdown keeps the daemon serving for several + * seconds, and a reconnecting tunnel re-establishes the forward, so polling for + * "up" right after a reboot command can false-pass a reboot that never happened. + * Gating on the server actually going DOWN first makes restart-persistence prove + * a real reboot. Throws if it never goes down within the deadline. + */ +export const waitForHttpDown = async ( + url: string, + options: { readonly timeoutMs?: number } = {}, +): Promise => { + const deadline = Date.now() + (options.timeoutMs ?? 120_000); + while (Date.now() < deadline) { + try { + await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(2000) }); + } catch { + return; // the server (or its tunnel) is gone — the reboot took + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`${url} never became unreachable — the reboot may not have taken`); +}; diff --git a/e2e/setup/cli-windows.globalsetup.ts b/e2e/setup/cli-windows.globalsetup.ts new file mode 100644 index 000000000..89e1d3fe2 --- /dev/null +++ b/e2e/setup/cli-windows.globalsetup.ts @@ -0,0 +1,3 @@ +import { setupCliTarget } from "./cli.globalsetup"; + +export default (): Promise<(() => Promise) | void> => setupCliTarget("windows"); diff --git a/e2e/setup/cli.globalsetup.ts b/e2e/setup/cli.globalsetup.ts index fc7e2b9ff..0c45d4225 100644 --- a/e2e/setup/cli.globalsetup.ts +++ b/e2e/setup/cli.globalsetup.ts @@ -3,42 +3,73 @@ // the host over a reconnecting SSH tunnel. Publishes connection + reboot info // via env (inherited by the test workers spawned afterward). Per-OS entrypoints // (cli-macos.globalsetup.ts, …) call setupCliTarget with their OS. +// +// macOS + Linux run on local tart guests (Apple-Silicon host); Windows runs on +// an ephemeral EC2 instance. The supervised daemon is bearer-gated: it mints/ +// loads its token into auth.json, which we read from the guest and publish so +// the api surface authenticates with `Authorization: Bearer`. import { buildGuestBinary } from "../src/vm/build-binary"; +import { ec2Vm } from "../src/vm/ec2"; import { tartVm } from "../src/vm/tart"; -import type { VmArch, VmOs } from "../src/vm/types"; +import type { VmArch, VmHandle, VmOs } from "../src/vm/types"; import { waitForHttp } from "./boot"; const PORT = 4789; -const GUEST_DIR = "~/ed"; -export async function setupCliTarget(os: VmOs): Promise<(() => Promise) | void> { - process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS +// Per-OS guest specifics: working dir, binary name, and the shell idioms for +// prep/cleanup — Unix `sh` on tart (macOS/Linux), PowerShell on EC2 Windows. +const guestPlan = (os: VmOs) => { if (os === "windows") { - throw new Error("cli-windows is pending the ec2 provider; run cli-macos / cli-linux for now"); + const dir = "C:/ed"; + const exe = `${dir}/executor.exe`; + return { + dir, + prep: `Remove-Item -Recurse -Force '${dir}' -ErrorAction SilentlyContinue; New-Item -ItemType Directory -Force -Path '${dir}' | Out-Null`, + postPush: "Write-Output ok", // no chmod/quarantine on Windows + install: `& '${exe}' service install --port ${PORT}`, + readToken: 'Get-Content "$env:USERPROFILE\\.executor\\server-control\\auth.json" -Raw', + uninstall: `& '${exe}' service uninstall`, + }; } + const dir = "~/ed"; + const exe = `${dir}/executor`; + return { + dir, + prep: `rm -rf ${dir} && mkdir -p ${dir}`, + // macOS quarantines scp'd executables; clear it so the binary can run. + postPush: + os === "macos" + ? `chmod +x ${exe}; xattr -dr com.apple.quarantine ${dir} 2>/dev/null || true` + : `chmod +x ${exe}`, + install: `${exe} service install --port ${PORT}`, + readToken: "cat ~/.executor/server-control/auth.json", + uninstall: `${exe} service uninstall`, + }; +}; - const arch: VmArch = "arm64"; // tart guests on an Apple-Silicon host +export async function setupCliTarget(os: VmOs): Promise<(() => Promise) | void> { + process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS + + const arch: VmArch = os === "windows" ? "x64" : "arm64"; const binDir = await buildGuestBinary(os, arch); - const vm = await tartVm(os, arch).provision(); + const vm: VmHandle = + os === "windows" ? await ec2Vm(os, arch).provision() : await tartVm(os, arch).provision(); + const plan = guestPlan(os); let tunnelClose: (() => void) | undefined; try { - await vm.ssh(`rm -rf ${GUEST_DIR} && mkdir -p ${GUEST_DIR}`); - await vm.push(`${binDir}/.`, `${GUEST_DIR}/`); - // macOS quarantines scp'd executables; clear it so the binary can run. - await vm.ssh( - os === "macos" - ? `chmod +x ${GUEST_DIR}/executor; xattr -dr com.apple.quarantine ${GUEST_DIR} 2>/dev/null || true` - : `chmod +x ${GUEST_DIR}/executor`, - ); + await vm.ssh(plan.prep); + await vm.push(`${binDir}/.`, os === "windows" ? plan.dir : `${plan.dir}/`); + await vm.ssh(plan.postPush); - const install = await vm.ssh(`${GUEST_DIR}/executor service install --port ${PORT}`); + const install = await vm.ssh(plan.install); if (install.code !== 0) { throw new Error(`service install failed: ${install.stderr.trim() || install.stdout.trim()}`); } - const keyRaw = (await vm.ssh("cat ~/.executor/server-control/service.key")).stdout.trim(); - const password = (JSON.parse(keyRaw) as { password: string }).password; + // The supervised daemon mints/loads its bearer into auth.json on first boot. + const tokenRaw = (await vm.ssh(plan.readToken)).stdout.trim(); + const token = (JSON.parse(tokenRaw) as { token: string }).token; const tunnel = await vm.tunnel(PORT); tunnelClose = tunnel.close; @@ -46,20 +77,21 @@ export async function setupCliTarget(os: VmOs): Promise<(() => Promise) | await waitForHttp(`${baseUrl}/`, { timeoutMs: 60_000 }); process.env.E2E_CLI_BASE_URL = baseUrl; - process.env.E2E_CLI_AUTH_PASSWORD = password; + process.env.E2E_CLI_AUTH_TOKEN = token; process.env.E2E_CLI_VM_HOST = vm.host; + if (vm.sshKeyPath) process.env.E2E_CLI_SSH_KEY = vm.sshKeyPath; process.env.E2E_CLI_TUNNEL_PORT = String(tunnel.localPort); - process.env.E2E_CLI_BIN_DIR = GUEST_DIR; + process.env.E2E_CLI_BIN_DIR = plan.dir; } catch (error) { tunnelClose?.(); - await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined); + await vm.ssh(plan.uninstall).catch(() => undefined); await vm.discard(); throw error; } return async () => { tunnelClose?.(); - await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined); + await vm.ssh(plan.uninstall).catch(() => undefined); await vm.discard(); }; } diff --git a/e2e/src/vm/ec2.ts b/e2e/src/vm/ec2.ts new file mode 100644 index 000000000..fdbe7dce8 --- /dev/null +++ b/e2e/src/vm/ec2.ts @@ -0,0 +1,422 @@ +// ec2 provider: ephemeral guests on AWS EC2 for the cross-OS supervised-daemon +// e2e where tart can't run (Windows; optionally Linux). Mirrors tart.ts — launch +// a fresh instance, drive over SSH (key-based; PowerShell on Windows), REBOOT for +// real, tear down. +// +// Credentials are NEVER embedded here: the `aws` CLI uses the ambient sign-in +// (`aws configure` / env). Every instance is tagged `executor-e2e` and always +// terminated on discard; the security group is scoped to this host's egress IP. +// +// Reboot is gated on a real boot-time change (Windows `LastBootUpTime`), not mere +// SSH reachability — an orderly shutdown keeps the daemon serving for several +// seconds, so "SSH answered" alone can false-pass a reboot that never happened. + +import { execFile } from "node:child_process"; +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { + type SshResult, + sleep, + type VmArch, + type VmHandle, + type VmOs, + type VmProvider, +} from "./types"; + +const execFileP = promisify(execFile); + +const REGION = process.env.E2E_EC2_REGION ?? "us-west-2"; +const INSTANCE_TYPE = process.env.E2E_EC2_INSTANCE_TYPE ?? "t3.medium"; +const TAG = "executor-e2e"; + +const SSH_OPTS = [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "LogLevel=ERROR", +]; + +const guestUser = (os: VmOs): string => + os === "windows" ? "Administrator" : (process.env.E2E_EC2_LINUX_USER ?? "ubuntu"); + +/** + * Reboot an EC2 guest by address, statelessly (no live handle) — the mirror of + * tart's sshRebootGuest, for the worker-side `restart()`. The connection drops + * mid-call, so errors are swallowed; the caller's down-gate + up-poll confirm + * the real reboot. + */ +export const ec2RebootGuest = async ( + host: string, + keyPath: string, + os: VmOs = "windows", +): Promise => { + const cmd = os === "windows" ? "Restart-Computer -Force" : "sudo reboot"; + await execFileP("ssh", ["-i", keyPath, ...SSH_OPTS, `${guestUser(os)}@${host}`, cmd]).catch( + () => undefined, + ); +}; + +const aws = async (args: ReadonlyArray): Promise => { + const { stdout } = await execFileP("aws", ["--region", REGION, "--output", "text", ...args], { + maxBuffer: 64 * 1024 * 1024, + }); + return stdout.trim(); +}; + +/** This host's public egress IP, for the inbound-SSH security-group rule. */ +const egressIp = async (): Promise => { + const { stdout } = await execFileP("curl", [ + "-s", + "--max-time", + "10", + "https://checkip.amazonaws.com", + ]); + return stdout.trim(); +}; + +/** Latest AWS-published base AMI for the guest OS (resolve dynamically — ids rotate). */ +const latestAmi = async (os: VmOs): Promise => { + if (os === "windows") { + const viaSsm = await aws([ + "ssm", + "get-parameters", + "--names", + "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base", + "--query", + "Parameters[0].Value", + ]).catch(() => ""); + if (viaSsm && viaSsm !== "None") return viaSsm; + return aws([ + "ec2", + "describe-images", + "--owners", + "amazon", + "--filters", + "Name=name,Values=Windows_Server-2022-English-Full-Base-*", + "Name=state,Values=available", + "--query", + "reverse(sort_by(Images,&CreationDate))[0].ImageId", + ]); + } + return aws([ + "ssm", + "get-parameters", + "--names", + "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id", + "--query", + "Parameters[0].Value", + ]); +}; + +const defaultSubnet = async (): Promise => { + const vpc = await aws([ + "ec2", + "describe-vpcs", + "--filters", + "Name=isDefault,Values=true", + "--query", + "Vpcs[0].VpcId", + ]); + const subnet = await aws([ + "ec2", + "describe-subnets", + "--filters", + `Name=vpc-id,Values=${vpc}`, + "Name=default-for-az,Values=true", + "--query", + "Subnets[0].SubnetId", + ]); + return subnet && subnet !== "None" + ? subnet + : aws([ + "ec2", + "describe-subnets", + "--filters", + `Name=vpc-id,Values=${vpc}`, + "--query", + "Subnets[0].SubnetId", + ]); +}; + +/** Create (idempotently) a security group allowing inbound SSH from this host. */ +const ensureSecurityGroup = async (myIp: string): Promise => { + const name = `${TAG}-sg`; + let sg = await aws([ + "ec2", + "describe-security-groups", + "--filters", + `Name=group-name,Values=${name}`, + "--query", + "SecurityGroups[0].GroupId", + ]).catch(() => ""); + if (!sg || sg === "None") { + sg = await aws([ + "ec2", + "create-security-group", + "--group-name", + name, + "--description", + "executor e2e ephemeral guests (SSH from CI host)", + "--query", + "GroupId", + ]); + } + // Authorize this host's IP for SSH; ignore "already exists". + await aws([ + "ec2", + "authorize-security-group-ingress", + "--group-id", + sg, + "--protocol", + "tcp", + "--port", + "22", + "--cidr", + `${myIp}/32`, + ]).catch(() => undefined); + return sg; +}; + +/** PowerShell user-data: enable OpenSSH, default the shell to PowerShell, and + * authorize our public key for the Administrator account. */ +const windowsUserData = (publicKey: string): string => + [ + "", + "Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0", + "Set-Service -Name sshd -StartupType Automatic", + "Start-Service sshd", + "New-ItemProperty -Path 'HKLM:\\SOFTWARE\\OpenSSH' -Name DefaultShell -Value 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' -PropertyType String -Force", + "New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -ErrorAction SilentlyContinue", + "$akf = 'C:\\ProgramData\\ssh\\administrators_authorized_keys'", + `Set-Content -Path $akf -Value '${publicKey}'`, + "icacls $akf /inheritance:r /grant 'Administrators:F' /grant 'SYSTEM:F'", + "", + ].join("\n"); + +const linuxUserData = (publicKey: string): string => + ["#cloud-config", "ssh_authorized_keys:", ` - ${publicKey}`].join("\n"); + +const freePort = (): Promise => + new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on("error", reject); + srv.listen(0, "127.0.0.1", () => { + const port = (srv.address() as net.AddressInfo).port; + srv.close(() => resolve(port)); + }); + }); + +const waitLocalPort = async (port: number, attempts = 40): Promise => { + for (let i = 0; i < attempts; i++) { + const ok = await new Promise((resolve) => { + const sock = net.connect({ host: "127.0.0.1", port }, () => { + sock.destroy(); + resolve(true); + }); + sock.on("error", () => resolve(false)); + sock.setTimeout(1000, () => { + sock.destroy(); + resolve(false); + }); + }); + if (ok) return; + await sleep(500); + } + throw new Error(`tunnel local port ${port} never came up`); +}; + +export const ec2Vm = (os: VmOs, arch: VmArch = "x64"): VmProvider => ({ + os, + provision: async () => { + const user = guestUser(os); + // A throwaway SSH keypair, authorized via user-data (no EC2 key pair needed — + // we drive over OpenSSH key auth, not the Windows password). + const keyDir = mkdtempSync(join(tmpdir(), "executor-ec2-")); + const keyPath = join(keyDir, "id"); + await execFileP("ssh-keygen", ["-t", "ed25519", "-N", "", "-q", "-f", keyPath]); + chmodSync(keyPath, 0o600); + const publicKey = (await execFileP("ssh-keygen", ["-y", "-f", keyPath])).stdout.trim(); + + const [myIp, ami, subnet] = await Promise.all([egressIp(), latestAmi(os), defaultSubnet()]); + const sg = await ensureSecurityGroup(myIp); + const userData = os === "windows" ? windowsUserData(publicKey) : linuxUserData(publicKey); + const userDataFile = join(keyDir, "user-data.txt"); + writeFileSync(userDataFile, userData); + + const instanceId = await aws([ + "ec2", + "run-instances", + "--image-id", + ami, + "--instance-type", + INSTANCE_TYPE, + "--count", + "1", + "--security-group-ids", + sg, + "--subnet-id", + subnet, + "--associate-public-ip-address", + "--instance-initiated-shutdown-behavior", + "terminate", + "--user-data", + `file://${userDataFile}`, + "--tag-specifications", + `ResourceType=instance,Tags=[{Key=Name,Value=${TAG}-${os}},{Key=purpose,Value=e2e}]`, + "--query", + "Instances[0].InstanceId", + ]); + + let ip = ""; + const tunnelClosers: Array<() => void> = []; + + const ssh = async (command: string): Promise => { + try { + const { stdout, stderr } = await execFileP( + "ssh", + ["-i", keyPath, ...SSH_OPTS, `${user}@${ip}`, command], + { maxBuffer: 64 * 1024 * 1024 }, + ); + return { stdout, stderr, code: 0 }; + } catch (err) { + const e = err as { stdout?: string; stderr?: string; code?: number }; + return { + stdout: e.stdout ?? "", + stderr: e.stderr ?? "", + code: typeof e.code === "number" ? e.code : 1, + }; + } + }; + + const waitSshUp = async (attempts: number): Promise => { + for (let i = 0; i < attempts; i++) { + if ((await ssh(os === "windows" ? "echo ok" : "true")).code === 0) return true; + await sleep(5000); + } + return false; + }; + + const waitSshDown = async (attempts = 40): Promise => { + for (let i = 0; i < attempts; i++) { + if ((await ssh("echo up").catch(() => ({ code: 1 }) as SshResult)).code !== 0) return; + await sleep(3000); + } + // never observed down — caller's boot-time check is the backstop. + }; + + const bootTime = async (): Promise => + os === "windows" + ? ( + await ssh("(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o')") + ).stdout.trim() + : (await ssh("cat /proc/sys/kernel/random/boot_id")).stdout.trim(); + + const handle: VmHandle = { + os, + arch, + sshKeyPath: keyPath, + get host() { + return ip; + }, + ssh, + push: async (localPath, remotePath) => { + await execFileP("scp", [ + "-i", + keyPath, + "-r", + ...SSH_OPTS, + localPath, + `${user}@${ip}:${remotePath}`, + ]); + }, + reboot: async () => { + const before = await bootTime(); + await ssh(os === "windows" ? "Restart-Computer -Force" : "sudo reboot").catch( + () => undefined, + ); + await waitSshDown(); + if (!(await waitSshUp(60))) throw new Error(`ec2 ${os}: SSH did not return after reboot`); + const after = await bootTime(); + if (before && after && before === after) { + throw new Error( + `ec2 ${os}: boot time unchanged after reboot — the guest never actually rebooted`, + ); + } + }, + tunnel: async (guestPort) => { + const localPort = await freePort(); + let closed = false; + let child: ReturnType | undefined; + const { spawn } = await import("node:child_process"); + const spawnOnce = (): void => { + child = spawn( + "ssh", + [ + "-i", + keyPath, + ...SSH_OPTS, + "-N", + "-L", + `${localPort}:127.0.0.1:${guestPort}`, + `${user}@${ip}`, + ], + { stdio: "ignore" }, + ); + child.on("exit", () => { + if (!closed) setTimeout(spawnOnce, 2000); + }); + }; + spawnOnce(); + const close = (): void => { + closed = true; + child?.kill(); + }; + tunnelClosers.push(close); + await waitLocalPort(localPort); + return { localPort, close }; + }, + discard: async () => { + for (const close of tunnelClosers) close(); + await aws(["ec2", "terminate-instances", "--instance-ids", instanceId]).catch( + () => undefined, + ); + }, + }; + + // Wait for a public IP, then for OpenSSH (Windows boot + FoD install ≈ 2-4 min). + for (let i = 0; i < 60; i++) { + const got = await aws([ + "ec2", + "describe-instances", + "--instance-ids", + instanceId, + "--query", + "Reservations[0].Instances[0].PublicIpAddress", + ]).catch(() => ""); + if (got && got !== "None") { + ip = got; + break; + } + await sleep(5000); + } + if (!ip) { + await handle.discard(); + throw new Error(`ec2 ${os}: no public IP within 300s`); + } + if (!(await waitSshUp(60))) { + await handle.discard(); + throw new Error(`ec2 ${os}: SSH never came up`); + } + return handle; + }, +}); diff --git a/e2e/src/vm/types.ts b/e2e/src/vm/types.ts index 28b07961c..20a259f1d 100644 --- a/e2e/src/vm/types.ts +++ b/e2e/src/vm/types.ts @@ -24,6 +24,12 @@ export interface VmHandle { readonly arch: VmArch; /** Current reachable address of the guest (re-resolved across reboots). */ readonly host: string; + /** + * Path to the SSH private key for key-based providers (EC2). Undefined for + * password-based providers (tart/sshpass). Published by globalsetup so the + * stateless worker-side `restart()` can reboot the guest. + */ + readonly sshKeyPath?: string; /** Run a command in the guest over SSH (shell on Unix, PowerShell on Windows). */ ssh(command: string): Promise; /** Copy a local file or directory into the guest (recursive for directories). */ diff --git a/e2e/targets/cli.ts b/e2e/targets/cli.ts index 00385252b..3b6a45e83 100644 --- a/e2e/targets/cli.ts +++ b/e2e/targets/cli.ts @@ -8,7 +8,8 @@ // this target only reads what that published via env. import { Effect } from "effect"; -import { waitForHttp } from "../setup/boot"; +import { waitForHttp, waitForHttpDown } from "../setup/boot"; +import { ec2RebootGuest } from "../src/vm/ec2"; import { sshRebootGuest } from "../src/vm/tart"; import type { VmOs } from "../src/vm/types"; import type { Capability, Identity, Target } from "../src/target"; @@ -22,25 +23,36 @@ const env = (key: string): string => { export const cliTarget = (): Target => { const baseUrl = env("E2E_CLI_BASE_URL"); const os = (process.env.E2E_VM_OS ?? "macos") as VmOs; - const username = process.env.E2E_CLI_AUTH_USER ?? "executor"; return { name: process.env.E2E_TARGET ?? "cli", baseUrl, mcpUrl: `${baseUrl}/mcp`, capabilities: new Set(["api"]), + // The supervised daemon is bearer-gated (auth.json); globalsetup reads the + // token from the guest and publishes it. A wrong/absent token still gets a + // clean 401, which the api surface treats as "up". newIdentity: () => - Effect.sync((): Identity => { - const basic = Buffer.from(`${username}:${env("E2E_CLI_AUTH_PASSWORD")}`).toString("base64"); - return { label: "cli-daemon", headers: { Authorization: `Basic ${basic}` } }; - }), - // A genuine machine reboot, not a service kick: reboot the guest OS and wait - // for the supervised daemon to auto-start and serve again (401 = up). The - // reconnecting tunnel re-establishes the forward, so the same baseUrl works. + Effect.sync( + (): Identity => ({ + label: "cli-daemon", + headers: { Authorization: `Bearer ${env("E2E_CLI_AUTH_TOKEN")}` }, + }), + ), + // A genuine machine reboot, not a service kick: reboot the guest OS, GATE on + // the daemon actually going down (an orderly shutdown serves for several + // seconds + the reconnecting tunnel re-forwards, so "reachable" right after + // the reboot command would false-pass), then wait for the supervised daemon + // to auto-start and serve again — proving the boot-time auto-start path. restart: () => Effect.promise(async () => { - if (os === "windows") throw new Error("cli-windows restart pending the ec2 provider"); - await sshRebootGuest(env("E2E_CLI_VM_HOST")); + const host = env("E2E_CLI_VM_HOST"); + if (os === "windows") { + await ec2RebootGuest(host, env("E2E_CLI_SSH_KEY"), os); + } else { + await sshRebootGuest(host); + } + await waitForHttpDown(`${baseUrl}/`, { timeoutMs: 120_000 }); await waitForHttp(`${baseUrl}/`, { timeoutMs: 240_000 }); }), }; From 6f4f99100e4606ea3c2dd5be3f7f8b97154a6165 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan Date: Sun, 14 Jun 2026 00:11:48 -0700 Subject: [PATCH 6/6] e2e/desktop + cli tests: bearer auth token, not a password The supervised-attach film passed EXECUTOR_AUTH_PASSWORD, a dead env the sidecar no longer reads (it reads EXECUTOR_AUTH_TOKEN and otherwise mints/loads the bearer from auth.json). Pin a real bearer instead. Fix two stale launchd/wrapper comments that still pointed the secret at service.key. --- apps/cli/src/service.test.ts | 4 ++-- e2e/desktop/supervised-attach.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/service.test.ts b/apps/cli/src/service.test.ts index 0510b4749..4bf5979c6 100644 --- a/apps/cli/src/service.test.ts +++ b/apps/cli/src/service.test.ts @@ -50,7 +50,7 @@ describe("service unit generation", () => { it("never leaks the auth password into the unit", () => { const plist = generateLaunchdPlist(launchdInput); - // The secret lives in the 0600 service.key, never in the plist env. + // No secret in the unit — the daemon reads the bearer from auth.json at boot. expect(plist).not.toContain("EXECUTOR_AUTH_PASSWORD"); }); @@ -95,7 +95,7 @@ describe("service unit generation", () => { '"C:\\Program Files\\Executor\\executor.exe" daemon run --foreground --port 4789', ); expect(wrapper).toContain('1>> "C:\\Users\\x\\.executor\\logs\\daemon.log"'); - // The secret is never baked into the wrapper — the daemon reads service.key. + // No secret in the wrapper — the daemon reads the bearer from auth.json at boot. expect(wrapper).not.toContain("EXECUTOR_AUTH_PASSWORD"); }); diff --git a/e2e/desktop/supervised-attach.test.ts b/e2e/desktop/supervised-attach.test.ts index 0ef2459af..ce8eef41f 100644 --- a/e2e/desktop/supervised-attach.test.ts +++ b/e2e/desktop/supervised-attach.test.ts @@ -106,7 +106,7 @@ const run = async (runDir: string) => { EXECUTOR_DATA_DIR: dataDir, EXECUTOR_PORT: String(port), EXECUTOR_HOST: "127.0.0.1", - EXECUTOR_AUTH_PASSWORD: "supervised-attach-film", + EXECUTOR_AUTH_TOKEN: "supervised-attach-film", EXECUTOR_CLIENT_DIR: clientDir, }); daemon = started.child;