diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 886daecd3..78eafed0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: # scripts/validate-docs.test.mjs covers the docs validator that the # validate-docs job runs. - name: Test release runtime archive, packaging, and docs-validator guards - run: node --test apps/ade-cli/scripts/native-archive-verification.test.mjs apps/ade-cli/scripts/native-deps-entry-filter.test.mjs apps/ade-cli/scripts/notarize-static-runtime.test.mjs apps/desktop/scripts/mac-runtime-archive-mode.test.mjs apps/desktop/scripts/packaged-ade-cli-resources.test.mjs apps/desktop/scripts/runtime-fetched-tool-packages.test.mjs apps/desktop/scripts/runtime-resource-targets.test.mjs scripts/validate-docs.test.mjs scripts/validate-platform-gates.test.mjs + run: node --test apps/ade-cli/scripts/install-runtime-rollback.test.mjs apps/ade-cli/scripts/native-archive-verification.test.mjs apps/ade-cli/scripts/native-deps-entry-filter.test.mjs apps/ade-cli/scripts/notarize-static-runtime.test.mjs apps/desktop/scripts/mac-runtime-archive-mode.test.mjs apps/desktop/scripts/packaged-ade-cli-resources.test.mjs apps/desktop/scripts/runtime-fetched-tool-packages.test.mjs apps/desktop/scripts/runtime-resource-targets.test.mjs apps/desktop/scripts/windows-release-contract.test.mjs scripts/validate-docs.test.mjs scripts/validate-platform-gates.test.mjs typecheck-web: needs: install @@ -474,7 +474,6 @@ jobs: src/bootstrap.test.ts src/serviceManager/common.test.ts src/serviceManager/installWindows.test.ts - src/serviceManager/windowsSupervisor.test.ts src/services/builtInBrowser/desktopBridgeClient.test.ts src/services/modelPickerStore.test.ts src/services/projects/machineLayout.test.ts @@ -518,6 +517,17 @@ jobs: src/lib/trustedWindowsTools.test.ts src/services/credentials/credentialStore.test.ts + # The standalone PowerShell installer (apps/ade-cli/scripts/install-runtime.ps1) + # has no runnable harness off a Windows host: its POSIX twin is covered by + # `node --test apps/ade-cli/scripts/install-runtime-rollback.test.mjs` on + # the ubuntu job, but nothing there can parse or execute PowerShell. This + # suite does both on a real runner -- it parses the installer through + # `[Parser]::ParseFile` and exercises the path normalizer it shares with + # the uninstall cleanup -- so a syntax error or a broken staging path in + # the installer fails CI instead of a user's machine. + - name: Test Windows standalone installer and uninstall cleanup scripts + run: cd apps/desktop && node --test ./scripts/windows-uninstall-cleanup.test.mjs + - name: Test Windows desktop, SQLite, and capability contracts run: cd apps/desktop && npx vitest run src/main/packagedRuntimeSmoke.test.ts src/main/services/computerUse/localComputerUse.test.ts src/renderer/lib/platform.test.ts diff --git a/.gitleaksignore b/.gitleaksignore index 08b604fca..6d17ea827 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -30,3 +30,10 @@ # Public Clerk OAuth client id (PKCE public client) documented in the dev recipe. baa7a0bb4b5f4c3112680c37ab7572663bb87551:apps/desktop/vite.webclient.config.ts:generic-api-key:77 + +# Synthetic JWT used as a redaction fixture: the test proves `token=` is +# stripped from diagnostic reports, so it must contain a JWT-shaped string. The +# working tree now assembles it from three segments at runtime, but the commit +# that introduced the literal still carries it in its own patch, and gitleaks +# fingerprints are commit-scoped. Scoped to that one commit and finding. +359ae24fd3fdf638b1e4b28cbeea34c068285aad:apps/ade-cli/src/services/diagnostics/diagnosticReport.test.ts:generic-api-key:54 diff --git a/apps/ade-cli/README.md b/apps/ade-cli/README.md index 2a3f6a761..3f3d9b5c7 100644 --- a/apps/ade-cli/README.md +++ b/apps/ade-cli/README.md @@ -207,6 +207,13 @@ ade brain pin set 123456 ade brain pin clear ``` +`ade brain status` and `ade runtime status` report `starting: true` when the +brain is not answering yet but its registered service and brain process say it +is coming up. Treat that as "wait", not "restart": `ade brain restart` on a +booting brain only starts its startup clock over. `ade brain restart` likewise +reports the installer's own `starting` message instead of claiming "restarted." +when the replacement is alive but has not bound the socket yet. + `ade brain repair-credentials` runs entirely locally and never contacts the brain — the state it repairs is the one that keeps the brain from starting, so a repair that needed a running brain would be unavailable exactly when it matters. @@ -460,6 +467,8 @@ ade machines connect --project ADE ade machines hop --session chat-1 ade doctor --json ade doctor --online --text # also check the latest desktop release over the network +ade report-issue --text # print a redacted diagnostic report + a prefilled GitHub issue URL (local files only; no brain needed) +ade report-issue --open # also copy the report to the clipboard and open that issue URL in the browser ade tools status --text # pinned agent CLIs: installed version + entry path per tool, plus the machine tools root ade tools ensure --text # fetch whatever this build pins and is missing (no names = all); streams progress to stderr ade tools ensure codex --text # one tool; an unknown name is a usage error listing the pinned set @@ -742,7 +751,7 @@ status row (`ok` / `warn` / `fail`) per check. It exits non-zero when any row is `fail`. The rows are: - **App** — the installed ADE desktop version (read from the `.app` bundle on disk) against the latest known version. Latest-known comes from the on-disk `update-status.json` by default; pass `--online` to also fetch the latest release from GitHub (short timeout, best-effort). `warn` when the install is behind or missing. -- **Brain** — whether the machine brain responds on its socket, plus its version, pid, and uptime. `fail` when it is not responding or when its build identity does not match the expected runtime for this CLI/role. +- **Brain** — whether the machine brain responds on its socket, plus its version, pid, and uptime. `fail` when it is not responding or when its build identity does not match the expected runtime for this CLI/role. A brain that is not answering *yet* is reported as `warn` (`starting`) rather than `fail`: when the login service is registered and the brain process behind it is alive and younger than the young-brain window (`RUNTIME_SERVICE_YOUNG_BRAIN_MS`, 2 min), it is still coming up — first launch, cold disk, large project database — and restarting it would only reset its clock. This is the CLI's read of the same `brain_starting` state the desktop recovery screen shows; `ade runtime status` and `ade brain status` report it as a `starting: true` field with the same wording. Nothing to repair: keep waiting for the endpoint. - **Wedge history** — the last wedge that was recovered, read from the runtime dir or the brain's reported `lastWedge`. `warn` when the most recent wedge is within the last 24h. Two things write that record: the in-process loop watchdog (reported as the blocking command and how long it blocked) and the external watchdog (`ade runtime watchdog-check`), which stops a brain whose heartbeat has gone stale and is reported as how long the brain went without a beat. A brain that is wedged right now shows up as a failing **Brain** row; the heartbeat itself has no separate row because a stale heartbeat plus a live brain is exactly what the watchdog converts into a restart within a minute. - **Sync port** — the sync host port the brain bound. `ok` on the default port, `warn` when bound elsewhere (with the base-port holders it found), `fail` when the brain is up but reported no port. - **Publish health** — account-directory publish state from the brain's sync route health. `ok` when a publish succeeded recently, `fail` when it has been failing for ≥2 min, otherwise `warn`, with the slowest publish leg annotated. @@ -750,6 +759,20 @@ status row (`ok` / `warn` / `fail`) per check. It exits non-zero when any row is - **Account** — whether this machine's brain is signed in to an ADE account (and the credential source), read via the brain's `account.call status`. `warn` when signed out or unavailable. - **Credentials** — whether the shared credential store (`$ADE_HOME/secrets/credentials.json.enc`) can be read, and whether an unreadable one was set aside earlier. `fail` when it cannot be read, naming the next step: a store sealed with a key this process cannot obtain is unlocked by opening the ADE app on this computer, while anything else needs a fresh sign-in. `warn` when a quarantined file is still waiting to be restored. Unlike every other row, this one is read **straight from disk** rather than through the brain — the failure it exists for is a brain that cannot start, so a check that needed a running brain would be silent exactly when it matters. It is non-creating: it never mints a machine key or OS key material, so running the diagnostic cannot change the state it reports. `ade brain repair-credentials` acts on the same reading. +When a row fails and the checks above do not explain it, `ade doctor --text` +points at `ade report-issue`. That command is the headless counterpart to the +desktop "Report issue" button: it reads only local files — it never starts or +contacts the brain — so it still works on the machine where ADE itself will not +come up, and on Windows where there is no desktop error screen to press. It +prints a redacted diagnostic report plus a prefilled GitHub issue URL (`--open` +also opens that URL, `--json` returns `{ installId, issueUrl, report }`). + +There is no `ade recovery diagnose` / `ade recovery repair`: those are +Electron-main IPC (`ade.recovery.diagnose` / `ade.recovery.repair`) backed by +the desktop's local-runtime connection pool, which does not exist in a headless +CLI. The CLI equivalents are `ade doctor` for the diagnosis, `ade brain restart` +for the repair, and `ade brain repair-credentials` for the credential half. + Default doctor does not call provider, GitHub, or Linear networks. Every row but **Credentials** comes from the local brain over its socket; **Credentials** is a read-only inspection of the machine's own secrets directory. It never prints diff --git a/apps/ade-cli/scripts/install-runtime-rollback.test.mjs b/apps/ade-cli/scripts/install-runtime-rollback.test.mjs new file mode 100644 index 000000000..33abfe15e --- /dev/null +++ b/apps/ade-cli/scripts/install-runtime-rollback.test.mjs @@ -0,0 +1,420 @@ +/** + * Rollback contract for the POSIX runtime installer (scripts/install-runtime.sh). + * + * The script used to copy the freshly downloaded `ade` over the installed one + * and only then run `ade --version`, so a bad download left a binary that could + * not start where a working one had been. These tests pin the guarantee the + * PowerShell installer already had: nothing is promoted until the staged binary + * passes its own preflight, and a promoted binary that fails is rolled back to + * the previous one. + * + * The script is driven end to end against a fake release: a `curl` earlier on + * PATH serves files from a fixture directory, and the "runtime" is a shell + * script whose `--version` behaviour is steered by environment variables. + */ +import assert from "node:assert/strict"; +import test from "node:test"; +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptPath = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "install-runtime.sh", +); + +function releaseTarget() { + const platform = process.platform === "darwin" ? "darwin" : "linux"; + const cpu = process.arch === "arm64" ? "arm64" : "x64"; + return `${platform}-${cpu}`; +} + +/** + * `--version` fails for the staged copy when ADE_TEST_FAIL_STAGED is set, and + * for the promoted copy (the one under the install dir) when + * ADE_TEST_FAIL_INSTALLED is set. Every other subcommand -- `brain start`, + * `setup` -- succeeds silently, because neither is what these tests are about. + */ +const FAKE_ADE = `#!/bin/sh +if [ "\$1" = "--version" ]; then + # Records which file the installer actually executed for each version check, + # so a test can prove the staged preflight does not run out of \$TMPDIR. + if [ -n "\${ADE_TEST_VERSION_LOG:-}" ]; then + echo "\$0" >>"\$ADE_TEST_VERSION_LOG" + fi + # Records the runtime env the installer handed this check, so a test can + # prove the native modules are not \`dlopen\`ed out of \$TMPDIR either. + if [ -n "\${ADE_TEST_ENV_LOG:-}" ]; then + echo "\$0|\${ADE_RUNTIME_ROOT:-}|\${ADE_RUNTIME_NODE_MODULES:-}|\${NODE_PATH:-}" >>"\$ADE_TEST_ENV_LOG" + fi + case "\$0" in + */bin/ade) + # Stands in for the user hitting Ctrl-C while the promoted binary hangs + # on its own \`--version\`: the installer shell gets the same SIGINT the + # terminal would have delivered, and runs its signal handler. + if [ -n "\${ADE_TEST_INTERRUPT_INSTALLED:-}" ]; then + kill -INT "\$PPID" 2>/dev/null || true + exit 130 + fi + if [ -n "\${ADE_TEST_FAIL_INSTALLED:-}" ]; then + echo "fake ade: installed copy cannot start" >&2 + exit 3 + fi + ;; + *) + if [ -n "\${ADE_TEST_FAIL_STAGED:-}" ]; then + echo "fake ade: staged copy cannot start" >&2 + exit 3 + fi + ;; + esac + echo "9.9.9-fake" + exit 0 +fi +exit 0 +`; + +// A rename can fail for reasons the installer cannot control (a full disk, a +// permission change under it). What must not happen when the rollback's own +// `mv` fails is the cleanup then deleting the backup it just failed to put +// back. This shim fails exactly that one rename -- the previous runtime going +// back -- and is the real `mv` for everything else. +const FAKE_MV = `#!/bin/sh +case "\$1" in + *.previous) + if [ -n "\${ADE_TEST_FAIL_RUNTIME_RESTORE:-}" ]; then + echo "fake mv: cannot restore \$1" >&2 + exit 1 + fi + ;; +esac +if [ -x /bin/mv ]; then exec /bin/mv "\$@"; fi +exec /usr/bin/mv "\$@" +`; + +const FAKE_CURL = `#!/bin/sh +# Serves \$ADE_TEST_ASSET_DIR/ instead of hitting the network. +url="" +out="" +while [ "\$#" -gt 0 ]; do + case "\$1" in + -o) out="\$2"; shift 2 ;; + http*) url="\$1"; shift ;; + *) shift ;; + esac +done +[ -n "\$url" ] || exit 2 +[ -n "\$out" ] || exit 2 +cp "\$ADE_TEST_ASSET_DIR/\${url##*/}" "\$out" +`; + +function writeExecutable(filePath, contents) { + fs.writeFileSync(filePath, contents, { mode: 0o755 }); + fs.chmodSync(filePath, 0o755); +} + +function sha256(filePath) { + const runner = spawnSync("shasum", ["-a", "256", filePath], { encoding: "utf8" }); + if (runner.status === 0) return runner.stdout.trim().split(/\s+/)[0]; + return execFileSync("sha256sum", [filePath], { encoding: "utf8" }).trim().split(/\s+/)[0]; +} + +/** A fake release plus a machine that already has ADE installed on it. */ +function makeInstall({ previousBinary = "#!/bin/sh\necho previous\n" } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-install-rollback-")); + const assets = path.join(root, "assets"); + const fakeBin = path.join(root, "fakebin"); + const adeHome = path.join(root, "home", ".ade"); + const installDir = path.join(adeHome, "bin"); + const target = releaseTarget(); + const runtimeDir = path.join(adeHome, "runtime", target); + + fs.mkdirSync(assets, { recursive: true }); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.mkdirSync(installDir, { recursive: true }); + fs.mkdirSync(runtimeDir, { recursive: true }); + + // The machine's existing install: a working binary and a runtime directory + // with a file only this (older) install has. + writeExecutable(path.join(installDir, "ade"), previousBinary); + fs.writeFileSync(path.join(runtimeDir, "previous-runtime.txt"), "previous\n"); + + // The release: binary + native archive + checksum manifest. + const binaryAsset = `ade-${target}`; + const archiveAsset = `${binaryAsset}.native.tar.gz`; + writeExecutable(path.join(assets, binaryAsset), FAKE_ADE); + const archiveStage = path.join(root, "archive-stage"); + fs.mkdirSync(path.join(archiveStage, "node_modules", "fake-dep"), { recursive: true }); + fs.writeFileSync( + path.join(archiveStage, "node_modules", "fake-dep", "index.js"), + "module.exports = 1;\n", + ); + execFileSync("tar", ["-czf", path.join(assets, archiveAsset), "-C", archiveStage, "."]); + fs.writeFileSync( + path.join(assets, "SHA256SUMS"), + [binaryAsset, archiveAsset] + .map((name) => `${sha256(path.join(assets, name))} ${name}\n`) + .join(""), + ); + + writeExecutable(path.join(fakeBin, "curl"), FAKE_CURL); + writeExecutable(path.join(fakeBin, "mv"), FAKE_MV); + + return { root, assets, fakeBin, adeHome, installDir, runtimeDir, cleanup: () => fs.rmSync(root, { recursive: true, force: true }) }; +} + +function runInstaller(fixture, extraEnv = {}) { + return spawnSync("sh", [scriptPath], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${fixture.fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + HOME: path.join(fixture.root, "home"), + ADE_TEST_ASSET_DIR: fixture.assets, + ADE_HOME: fixture.adeHome, + ADE_INSTALL_DIR: fixture.installDir, + ADE_INSTALL_NO_PROMPT: "1", + ADE_INSTALL_NO_PATH: "1", + ...extraEnv, + }, + }); +} + +test("a downloaded runtime that cannot start never replaces the installed one", () => { + const fixture = makeInstall(); + try { + const result = runInstaller(fixture, { ADE_TEST_FAIL_STAGED: "1" }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /could not start/); + // The failure has to name where the evidence is and what to do next. + assert.match(result.stderr, /install-failure\.log/); + assert.match(result.stderr, /run the installer again/); + + assert.equal( + fs.readFileSync(path.join(fixture.installDir, "ade"), "utf8"), + "#!/bin/sh\necho previous\n", + ); + // Nothing was promoted, so the previous native runtime is untouched too. + assert.ok(fs.existsSync(path.join(fixture.runtimeDir, "previous-runtime.txt"))); + assert.ok(!fs.existsSync(path.join(fixture.runtimeDir, "node_modules"))); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.new"))); + // The runtime now stages next to the real one; a failed install must not + // leave either scratch directory under the ADE home. + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.new`)); + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.previous`)); + } finally { + fixture.cleanup(); + } +}); + +// A `noexec` /tmp is normal on hardened Linux hosts and in containers, and it +// cannot be simulated portably from a test. What is testable is the property +// that makes it irrelevant: the preflight executes the copy under the install +// directory, never the staged one in $TMPDIR. +test("the staged preflight runs the install-directory copy, not the one in TMPDIR", () => { + const fixture = makeInstall(); + const versionLog = path.join(fixture.adeHome, "version-checks.log"); + try { + const result = runInstaller(fixture, { ADE_TEST_VERSION_LOG: versionLog }); + + assert.equal(result.status, 0, result.stderr); + const executed = fs + .readFileSync(versionLog, "utf8") + .split("\n") + .filter(Boolean); + + // Staged preflight first, then the promoted binary. + assert.deepEqual(executed.slice(0, 2), [ + path.join(fixture.installDir, "ade.new"), + path.join(fixture.installDir, "ade"), + ]); + // Nothing was ever executed out of the $TMPDIR staging directory. + assert.ok(!executed.some((entry) => entry.includes("ade-install."))); + } finally { + fixture.cleanup(); + } +}); + +// The binary is not the only thing the preflight has to run from an +// executable filesystem: it `dlopen`s the .node modules NODE_PATH points at. +// Staging the runtime archive in $TMPDIR made the binary copy's relocation +// pointless on a `noexec` /tmp, because the native modules were still loaded +// from there. +test("the staged preflight loads native modules from the ADE home, not TMPDIR", () => { + const fixture = makeInstall(); + const envLog = path.join(fixture.adeHome, "version-env.log"); + try { + const result = runInstaller(fixture, { ADE_TEST_ENV_LOG: envLog }); + + assert.equal(result.status, 0, result.stderr); + const rows = fs.readFileSync(envLog, "utf8").split("\n").filter(Boolean); + assert.ok(rows.length >= 2, `expected a staged and a promoted check, got ${rows.length}`); + + // Nothing the installer executed saw a TMPDIR staging path, as the binary + // it ran or anywhere in its runtime env. + for (const row of rows) { + assert.ok(!row.includes("ade-install."), `TMPDIR staging path in: ${row}`); + } + + // The staged preflight runs against the runtime staged under the ADE home, + // one same-filesystem rename away from where it will be promoted; the + // second check runs against the promoted one. NODE_PATH keeps whatever the + // ambient environment had after the runtime entry, so only the first entry + // is this script's. + const parse = (row) => { + const [binary, runtimeRoot, nodeModules, nodePath] = row.split("|"); + return { binary, runtimeRoot, nodeModules, firstNodePath: nodePath.split(":")[0] }; + }; + const staged = parse(rows[0]); + assert.deepEqual(staged, { + binary: path.join(fixture.installDir, "ade.new"), + runtimeRoot: `${fixture.runtimeDir}.new`, + nodeModules: path.join(`${fixture.runtimeDir}.new`, "node_modules"), + firstNodePath: path.join(`${fixture.runtimeDir}.new`, "node_modules"), + }); + const promoted = parse(rows[1]); + assert.deepEqual(promoted, { + binary: path.join(fixture.installDir, "ade"), + runtimeRoot: fixture.runtimeDir, + nodeModules: path.join(fixture.runtimeDir, "node_modules"), + firstNodePath: path.join(fixture.runtimeDir, "node_modules"), + }); + + // Neither staging directory survives a successful install. + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.new`)); + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.previous`)); + } finally { + fixture.cleanup(); + } +}); + +test("a staged preflight failure says the existing install was not touched", () => { + const fixture = makeInstall(); + const versionLog = path.join(fixture.adeHome, "version-checks.log"); + try { + const result = runInstaller(fixture, { + ADE_TEST_FAIL_STAGED: "1", + ADE_TEST_VERSION_LOG: versionLog, + }); + + assert.notEqual(result.status, 0); + // Nothing was promoted, so "nothing was left installed" would be wrong and + // "was put back" would imply a rollback that never had to happen. + assert.match(result.stderr, /was not touched/); + assert.doesNotMatch(result.stderr, /nothing was left installed/); + assert.doesNotMatch(result.stderr, /put back/); + + // The one thing it ran was the install-directory copy, and it is gone. + assert.deepEqual( + fs.readFileSync(versionLog, "utf8").split("\n").filter(Boolean), + [path.join(fixture.installDir, "ade.new")], + ); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.new"))); + } finally { + fixture.cleanup(); + } +}); + +test("a promoted runtime that fails its version check is rolled back", () => { + const fixture = makeInstall(); + try { + const result = runInstaller(fixture, { ADE_TEST_FAIL_INSTALLED: "1" }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /could not start/); + assert.match(result.stderr, /put back/); + + assert.equal( + fs.readFileSync(path.join(fixture.installDir, "ade"), "utf8"), + "#!/bin/sh\necho previous\n", + ); + assert.ok(fs.existsSync(path.join(fixture.runtimeDir, "previous-runtime.txt"))); + assert.ok(!fs.existsSync(path.join(fixture.runtimeDir, "node_modules"))); + // The rollback copies are scratch state, not something to leave behind. + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.bak"))); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.new"))); + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.new`)); + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.previous`)); + + const log = fs.readFileSync(path.join(fixture.adeHome, "install-failure.log"), "utf8"); + assert.match(log, /installed copy cannot start/); + } finally { + fixture.cleanup(); + } +}); + +test("Ctrl-C while the promoted binary is being checked puts the old one back", () => { + const fixture = makeInstall(); + try { + const result = runInstaller(fixture, { ADE_TEST_INTERRUPT_INSTALLED: "1" }); + + assert.notEqual(result.status, 0); + // The interrupt lands after the new binary is already in place but before + // it has proved it can start, so the machine must be left on the install + // it had -- not on an unverified binary with its backup deleted. + assert.equal( + fs.readFileSync(path.join(fixture.installDir, "ade"), "utf8"), + "#!/bin/sh\necho previous\n", + ); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.bak"))); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.new"))); + // The runtime was already promoted when the interrupt landed, so putting + // the old binary back is only half a rollback: an old `ade` against the new + // native sidecar is an install that cannot start. The runtime has to go + // back with it. + assert.ok(fs.existsSync(path.join(fixture.runtimeDir, "previous-runtime.txt"))); + assert.ok(!fs.existsSync(path.join(fixture.runtimeDir, "node_modules"))); + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.previous`)); + assert.ok(!fs.existsSync(`${fixture.runtimeDir}.new`)); + } finally { + fixture.cleanup(); + } +}); + +// `restore_previous_install` removes the promoted runtime before moving the +// backup back, so a failure in that move leaves the machine with no runtime at +// all. The scratch cleanup that runs next must not then delete the backup -- +// that backup is the only copy of the runtime left on the machine. +test("a failed runtime restore keeps the backup instead of deleting the last runtime", () => { + const fixture = makeInstall(); + try { + const result = runInstaller(fixture, { + ADE_TEST_FAIL_INSTALLED: "1", + ADE_TEST_FAIL_RUNTIME_RESTORE: "1", + }); + + assert.notEqual(result.status, 0); + // The binary rolled back normally; only the runtime restore failed. + assert.equal( + fs.readFileSync(path.join(fixture.installDir, "ade"), "utf8"), + "#!/bin/sh\necho previous\n", + ); + // The runtime the machine had is still on disk, under the backup name a + // re-run (or a human) can recover it from. + assert.ok( + fs.existsSync(path.join(`${fixture.runtimeDir}.previous`, "previous-runtime.txt")), + "the only remaining runtime was deleted by the scratch cleanup", + ); + } finally { + fixture.cleanup(); + } +}); + +test("a healthy runtime is promoted and leaves no rollback state behind", () => { + const fixture = makeInstall(); + try { + const result = runInstaller(fixture); + + assert.equal(result.status, 0, result.stderr); + assert.equal(fs.readFileSync(path.join(fixture.installDir, "ade"), "utf8"), FAKE_ADE); + assert.ok(fs.existsSync(path.join(fixture.runtimeDir, "node_modules", "fake-dep"))); + assert.ok(!fs.existsSync(path.join(fixture.runtimeDir, "previous-runtime.txt"))); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.bak"))); + assert.ok(!fs.existsSync(path.join(fixture.installDir, "ade.new"))); + } finally { + fixture.cleanup(); + } +}); diff --git a/apps/ade-cli/scripts/install-runtime.ps1 b/apps/ade-cli/scripts/install-runtime.ps1 index ac5d6f2b8..8721e4d0c 100644 --- a/apps/ade-cli/scripts/install-runtime.ps1 +++ b/apps/ade-cli/scripts/install-runtime.ps1 @@ -327,13 +327,29 @@ $binaryAsset = "ade-$target.exe" $nativeAsset = "ade-$target.native.tar.gz" $runtimeDir = Join-Path $AdeHome "runtime\$target" $destinationBinary = Join-Path $InstallDir "ade.exe" +# $TEMP holds downloads and nothing else. The staged runtime, the runtime +# backup, the staged binary copy and the binary backup all live next to what +# they replace, under the ADE home. Two reasons, and both have bitten the POSIX +# installer this now mirrors (scripts/install-runtime.sh): +# - promoting and restoring have to be same-directory renames. `Move-Item` +# across volumes is a copy plus a delete, and %TEMP% is routinely on a +# different volume from the user profile (redirected TEMP, a RAM disk, a +# roaming profile), so a half-copied ade.exe or runtime tree is the broken +# install this block exists to prevent; +# - AppLocker and most EDR agents block execution out of %TEMP% outright, so +# a preflight staged there fails on every managed corporate machine. The +# binary is not the only thing that has to be executable: it loads the +# .node modules NODE_PATH points at, and those come out of the runtime +# archive. Staging the runtime under the ADE home is what makes the +# preflight test the same files the promoted install will load. $tempRoot = Join-Path ([IO.Path]::GetTempPath()) ("ade-install-" + [Guid]::NewGuid().ToString("N")) -$stagedBinary = Join-Path $tempRoot "ade.exe" +$downloadedBinary = Join-Path $tempRoot "ade.exe" $stagedArchive = Join-Path $tempRoot $nativeAsset $checksumManifest = Join-Path $tempRoot "SHA256SUMS" -$stagedRuntime = Join-Path $tempRoot "runtime" -$backupBinary = Join-Path $tempRoot "ade.previous.exe" -$backupRuntime = Join-Path $tempRoot "runtime.previous" +$stagedRuntime = "$runtimeDir.new" +$backupRuntime = "$runtimeDir.previous" +$pendingBinary = Join-Path $InstallDir "ade.new.exe" +$backupBinary = Join-Path $InstallDir "ade.bak.exe" $script:PreviousNodePath = $env:NODE_PATH $previousEnvironment = @{ ADE_HOME = $env:ADE_HOME @@ -345,19 +361,83 @@ $previousEnvironment = @{ $previousServiceWasStopped = $false $previousServiceWasRunning = $false $promotedBinary = $false +# $promotedBinary only says the new ade.exe was renamed into place. Whether it +# can actually start is a separate fact, and the window between the two is +# exactly where an abort must put the previous binary back -- so the cleanup +# gates on this flag, not on the rename. Mirrors `promoted_binary` in +# scripts/install-runtime.sh, which is set only once the check has passed. +$binaryVerified = $false $promotedRuntime = $false $preserveTempForRecovery = $false $installSucceeded = $false +# "staged" until the new binary is promoted, "installed" afterwards. The two +# stages leave the machine in different states, and a message that describes +# the wrong one sends a user looking for damage that is not there. +$installStage = "staged" + +# Plain, stage-aware note about what is on disk after a failed install. Mirrors +# `die_runtime_unusable` in scripts/install-runtime.sh. +function Write-AdeInstallStateNote( + [string]$Stage, + [bool]$RestoredPreviousBinary, + [string]$BinaryPath +) { + Clear-AdeActiveLine + if ($Stage -eq "staged") { + if (Test-Path -LiteralPath $BinaryPath -PathType Leaf) { + [Console]::Error.WriteLine("ade install: your existing ADE at $BinaryPath was not touched.") + } else { + [Console]::Error.WriteLine("ade install: nothing was left installed at $BinaryPath.") + } + } elseif ($RestoredPreviousBinary) { + [Console]::Error.WriteLine("ade install: the ADE you already had was put back, so nothing is broken.") + } elseif (Test-Path -LiteralPath $BinaryPath -PathType Leaf) { + # The rollback is best effort, so the disk is the only thing worth + # believing here: if it could not remove the binary that just failed, the + # broken one is still what runs, and saying "nothing was left installed" + # sends the user looking in the wrong place. + [Console]::Error.WriteLine("ade install: the ADE at $BinaryPath is the one that just failed to start.") + } else { + [Console]::Error.WriteLine("ade install: nothing was left installed at $BinaryPath.") + } + [Console]::Error.WriteLine( + "ade install: next: run the installer again; if it fails the same way, open an issue at https://github.com/$Repo/issues with the error below.") +} try { - New-Item -ItemType Directory -Force -Path $tempRoot, $stagedRuntime | Out-Null + # The install dir and the runtime's parent are created up front now: the + # staged runtime and the staged binary copy live inside them, not in %TEMP%. + New-Item -ItemType Directory -Force -Path $InstallDir, (Split-Path $runtimeDir -Parent), $tempRoot | Out-Null + # Leftovers from an install that was killed before its cleanup ran. A backup + # is put back before it is deleted: if the earlier run died between the two + # renames, that backup is the machine's only copy, and deleting it outright + # is how a retry would turn a recoverable abort into no install at all. + Remove-Item -LiteralPath $stagedRuntime -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $pendingBinary -Force -ErrorAction SilentlyContinue + if ((Test-Path -LiteralPath $backupRuntime) -and -not (Test-Path -LiteralPath $runtimeDir)) { + Move-Item -LiteralPath $backupRuntime -Destination $runtimeDir -Force -ErrorAction SilentlyContinue + } + if ((Test-Path -LiteralPath $backupBinary -PathType Leaf) -and + -not (Test-Path -LiteralPath $destinationBinary -PathType Leaf)) { + Move-Item -LiteralPath $backupBinary -Destination $destinationBinary -Force -ErrorAction SilentlyContinue + } + # Each backup is dropped only once the thing it backs up is actually on disk. + # The restores above are best effort, and deleting a backup whose restore + # silently failed is what would turn a recoverable abort into no install. + if (Test-Path -LiteralPath $runtimeDir) { + Remove-Item -LiteralPath $backupRuntime -Recurse -Force -ErrorAction SilentlyContinue + } + if (Test-Path -LiteralPath $destinationBinary -PathType Leaf) { + Remove-Item -LiteralPath $backupBinary -Force -ErrorAction SilentlyContinue + } + New-Item -ItemType Directory -Force -Path $stagedRuntime | Out-Null Write-AdeBanner Write-Host " Installing ADE to $AdeHome" Write-Host "" - Download-Asset $binaryAsset $stagedBinary "ADE runtime" + Download-Asset $binaryAsset $downloadedBinary "ADE runtime" Download-Asset $nativeAsset $stagedArchive "Native dependencies" Download-Asset "SHA256SUMS" $checksumManifest - Verify-Checksum $checksumManifest $binaryAsset $stagedBinary + Verify-Checksum $checksumManifest $binaryAsset $downloadedBinary Verify-Checksum $checksumManifest $nativeAsset $stagedArchive & tar.exe -xzf $stagedArchive -C $stagedRuntime @@ -366,8 +446,17 @@ try { Fail "native dependency archive is missing node_modules" } + # The preflight copy lands in $InstallDir, not %TEMP% -- see the note above. + # ade.new.exe is the scratch name the promotion below renames from, and the + # finally already removes it, so this costs nothing but the copy. + Copy-Item -LiteralPath $downloadedBinary -Destination $pendingBinary -Force + + # Preflight before anything is promoted: the new binary against the staged + # runtime, both under the ADE home. A download that cannot even print its + # version never replaces ade.exe, so the previous install is still there and + # still working. Set-ProcessRuntimeEnvironment $AdeHome $stagedRuntime - & $stagedBinary --version | Out-Null + & $pendingBinary --version | Out-Null if ($LASTEXITCODE -ne 0) { Fail "downloaded ADE runtime failed its version check" } if ((Test-Path -LiteralPath $destinationBinary -PathType Leaf) -and -not $NoService) { @@ -393,21 +482,30 @@ try { } } - New-Item -ItemType Directory -Force -Path $InstallDir, (Split-Path $runtimeDir -Parent) | Out-Null - if (Test-Path -LiteralPath $destinationBinary -PathType Leaf) { - Move-Item -LiteralPath $destinationBinary -Destination $backupBinary - } + # Every move below is a same-directory rename: $runtimeDir.new/.previous sit + # beside $runtimeDir, and ade.new.exe/ade.bak.exe beside ade.exe. Nothing + # crosses a volume, so nothing can be half-copied. if (Test-Path -LiteralPath $runtimeDir) { Move-Item -LiteralPath $runtimeDir -Destination $backupRuntime } Move-Item -LiteralPath $stagedRuntime -Destination $runtimeDir $promotedRuntime = $true - Move-Item -LiteralPath $stagedBinary -Destination $destinationBinary + # Promote the binary last, and only by rename -- of the very copy the + # preflight above just ran, so a truncated write can never be what ends up at + # ade.exe. + if (Test-Path -LiteralPath $destinationBinary -PathType Leaf) { + Move-Item -LiteralPath $destinationBinary -Destination $backupBinary + } + Move-Item -LiteralPath $pendingBinary -Destination $destinationBinary $promotedBinary = $true + $installStage = "installed" Set-ProcessRuntimeEnvironment $AdeHome $runtimeDir & $destinationBinary --version | Out-Null if ($LASTEXITCODE -ne 0) { Fail "installed ADE runtime failed its version check" } + # The new binary has now proved it runs, so the cleanup below must stop + # treating ade.exe as unverified and rolling the backup back over it. + $binaryVerified = $true if (-not $NoService) { # `brain start`, NOT `serve --install-service`. The latter registers the # service at whatever ADE_DEFAULT_ROLE happens to be, and in a fresh install @@ -434,10 +532,12 @@ try { if ($LASTEXITCODE -ne 0) { $rollbackErrors.Add("new brain service cleanup exited with code $LASTEXITCODE") } } catch { $rollbackErrors.Add("new brain service cleanup failed: $($_.Exception.Message)") } } + $restoredPreviousBinary = $false try { if ($promotedBinary) { Remove-Item -LiteralPath $destinationBinary -Force -ErrorAction Stop } if (Test-Path -LiteralPath $backupBinary -PathType Leaf) { Move-Item -LiteralPath $backupBinary -Destination $destinationBinary -Force -ErrorAction Stop + $restoredPreviousBinary = $true } } catch { $rollbackErrors.Add("binary restore failed: $($_.Exception.Message)") } try { @@ -463,15 +563,65 @@ try { } if ($rollbackErrors.Count -gt 0) { $preserveTempForRecovery = $true - throw "ADE runtime install failed ($($installError.Exception.Message)); rollback also failed: $($rollbackErrors -join '; '). Recovery files were retained at $tempRoot" + Write-AdeInstallStateNote $installStage $restoredPreviousBinary $destinationBinary + throw "ADE runtime install failed ($($installError.Exception.Message)); rollback also failed: $($rollbackErrors -join '; '). Recovery files were retained at $tempRoot, $backupBinary and $backupRuntime" } + Write-AdeInstallStateNote $installStage $restoredPreviousBinary $destinationBinary throw $installError } finally { foreach ($name in $previousEnvironment.Keys) { [Environment]::SetEnvironmentVariable($name, $previousEnvironment[$name], "Process") } if (-not $preserveTempForRecovery) { + # Scratch state only: the downloads, the staged runtime and the staged + # binary copy. The two backups go too, but only after the window in which + # either is the machine's only copy -- an abort (Ctrl-C, a throw between + # the two renames) between moving the old one aside and moving the new one + # in would otherwise leave the machine with nothing installed. Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $pendingBinary -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $stagedRuntime -Recurse -Force -ErrorAction SilentlyContinue + # The binary first, because whether the old binary goes back decides + # whether the old runtime has to go back with it. What is at ade.exe + # between its rename and its version check is an unverified binary, so an + # abort in that window restores the backup over it -- Test-Path alone would + # see a file there and leave the broken one installed. + $restoredOldBinary = $false + if (-not $binaryVerified -and (Test-Path -LiteralPath $backupBinary -PathType Leaf)) { + try { + Move-Item -LiteralPath $backupBinary -Destination $destinationBinary -Force -ErrorAction Stop + $restoredOldBinary = $true + } catch {} + } + # Before the runtime is promoted its backup is the machine's only runtime, + # so it goes back if nothing is at $runtimeDir. + if (-not $promotedRuntime) { + if ((Test-Path -LiteralPath $backupRuntime) -and -not (Test-Path -LiteralPath $runtimeDir)) { + try { + Move-Item -LiteralPath $backupRuntime -Destination $runtimeDir -Force -ErrorAction Stop + } catch {} + } + } + # After it is promoted, the old binary having just gone back is what makes + # the old runtime needed again: an old ade.exe against the new native + # sidecar is the broken install this whole block exists to prevent. + if ($promotedRuntime -and $restoredOldBinary) { + if (Test-Path -LiteralPath $backupRuntime) { + try { + Remove-Item -LiteralPath $runtimeDir -Recurse -Force -ErrorAction SilentlyContinue + Move-Item -LiteralPath $backupRuntime -Destination $runtimeDir -Force -ErrorAction Stop + } catch {} + } + } + # Same rule as the pre-install repair above: a backup is only scratch once + # what it backs up is on disk again. The restores are best effort, so a + # failed one must keep its backup rather than have it deleted underneath. + if (Test-Path -LiteralPath $runtimeDir) { + Remove-Item -LiteralPath $backupRuntime -Recurse -Force -ErrorAction SilentlyContinue + } + if ($binaryVerified -or $restoredOldBinary) { + Remove-Item -LiteralPath $backupBinary -Force -ErrorAction SilentlyContinue + } } } diff --git a/apps/ade-cli/scripts/install-runtime.sh b/apps/ade-cli/scripts/install-runtime.sh index 8c4a9f0d4..4947697c0 100644 --- a/apps/ade-cli/scripts/install-runtime.sh +++ b/apps/ade-cli/scripts/install-runtime.sh @@ -448,25 +448,210 @@ downloaded_bytes="$(( $(file_size_bytes "$tmp_dir/ade") + $(file_size_bytes "$tmp_dir/native.tar.gz") ))" -chmod 755 "$tmp_dir/ade" -cp "$tmp_dir/ade" "$dest_dir/ade" -chmod 755 "$dest_dir/ade" - -staged_runtime_dir="$tmp_dir/runtime" +staged_binary="$tmp_dir/ade" +# The staged runtime, the runtime backup, the staged binary copy and the binary +# backup all live next to what they replace, under the ADE home -- never in +# $TMPDIR. Two reasons, and both have bitten this script: +# - promoting and restoring have to be same-directory renames, which are +# atomic. A `mv` out of $TMPDIR is a copy across filesystems, and a +# half-copied `ade` or runtime is the broken install this block exists to +# prevent; +# - a /tmp mounted `noexec` (common on hardened Linux hosts and in +# containers) makes the preflight fail on every install. The binary is not +# the only thing that has to be executable: it `dlopen`s the .node modules +# NODE_PATH points at, and those come out of the runtime archive. Staging +# the runtime under the ADE home is what makes the preflight test the same +# files the promoted install will load. +staged_runtime_dir="$runtime_dir.new" staged_node_modules="$staged_runtime_dir/node_modules" -backup_runtime_dir="$tmp_dir/runtime.previous" +backup_runtime_dir="$runtime_dir.previous" +pending_binary="$dest_dir/ade.new" +backup_binary="$dest_dir/ade.bak" +promoted_runtime=0 +promoted_binary=0 +have_backup_binary=0 +# Kept out of $tmp_dir so it survives the EXIT trap: the failure message points +# a stuck user at it, and a log deleted on the way out points at nothing. +install_log="$ade_home/install-failure.log" + +# Scratch state only: the download, the staged runtime, and the staged binary +# copy. The two backups go too, but only once neither is still the machine's +# only copy of what it replaced -- an abort (Ctrl-C, SIGTERM) part-way through +# the promotion would otherwise leave the machine with no runtime at all, or +# with a restore that silently failed and its backup deleted anyway. +cleanup_install_scratch() { + rm -rf "$tmp_dir" + rm -f "$pending_binary" + rm -rf "$staged_runtime_dir" + # The binary first, because whether the old binary goes back decides whether + # the old runtime has to go back with it. This script *copies* the old `ade` + # aside, so $dest_dir/ade is never absent; the window that matters is the one + # between the new binary being renamed into place and its `--version` check + # passing. Abort in there (the obvious Ctrl-C when a bad build hangs) and what + # is installed is an unverified binary, so the backup goes back over it. Only + # after the check passes is the backup just disk. (install-runtime.ps1 uses + # Move-Item, so there the backup really is the only copy for a moment -- + # hence its own restore-if-missing branch.) + restored_old_binary=0 + if [ "$promoted_binary" -eq 0 ] && [ -e "$backup_binary" ]; then + if mv "$backup_binary" "$dest_dir/ade" 2>/dev/null; then + restored_old_binary=1 + fi + fi + # Deleting the backup once it is no longer needed keeps an abandoned + # `ade.bak` from sitting in the install dir forever -- but a restore that + # failed leaves it as the last copy of the working binary, so it stays. + if [ "$promoted_binary" -eq 1 ] || [ "$restored_old_binary" -eq 1 ]; then + rm -f "$backup_binary" + fi + + # The runtime backup has two ways of still being needed. Before the runtime + # is promoted it is the machine's only runtime, so it goes back if the + # directory it came from is empty. After the runtime is promoted but before + # the new binary passes its check, the old binary is what the restore above + # just put back -- and an old binary paired with the new native sidecar is + # the broken install this whole block exists to prevent, so the old runtime + # goes back with it. + restore_old_runtime=0 + if [ "$promoted_runtime" -eq 0 ]; then + if [ -e "$backup_runtime_dir" ] && [ ! -e "$runtime_dir" ]; then + restore_old_runtime=1 + fi + elif [ "$restored_old_binary" -eq 1 ] && [ -e "$backup_runtime_dir" ]; then + restore_old_runtime=1 + rm -rf "$runtime_dir" 2>/dev/null || true + fi + runtime_restore_failed=0 + if [ "$restore_old_runtime" -eq 1 ]; then + mv "$backup_runtime_dir" "$runtime_dir" 2>/dev/null || runtime_restore_failed=1 + fi + # A restore that failed is not the only way the backup can still be the + # machine's only runtime. `restore_previous_install` removes $runtime_dir + # before its own best-effort `mv`, so if that `mv` failed there is nothing at + # $runtime_dir at all -- and none of the flags above record it, because the + # runtime was promoted and the old binary was put back by that function + # rather than by this one. Believe the disk: the backup only goes when a + # runtime is actually installed. + if [ "$runtime_restore_failed" -eq 0 ] && [ -e "$runtime_dir" ]; then + rm -rf "$backup_runtime_dir" + fi +} + +# The EXIT trap alone is not enough: a handler for HUP/INT/TERM that does not +# exit returns to the interrupted command, so a Ctrl-C mid-download deleted the +# scratch state and then carried on against paths that no longer existed. Each +# signal cleans up once (the EXIT trap is cleared first) and aborts with that +# signal's conventional status. +trap 'cleanup_install_scratch' EXIT +trap 'trap - EXIT; cleanup_install_scratch; exit 129' HUP +trap 'trap - EXIT; cleanup_install_scratch; exit 130' INT +trap 'trap - EXIT; cleanup_install_scratch; exit 143' TERM + +# The runtime sidecar env has to be in place before *any* `--version` check: +# the binary loads its native modules through it, so a preflight run without it +# tests something the real ade never does. Called once for the staged runtime +# and again for the promoted one. +previous_node_path="${NODE_PATH:-}" +set_runtime_env() { + ADE_RUNTIME_ROOT="$1" + ADE_RUNTIME_NODE_MODULES="$1/node_modules" + NODE_PATH="$1/node_modules${previous_node_path:+:$previous_node_path}" + export ADE_RUNTIME_ROOT ADE_RUNTIME_NODE_MODULES NODE_PATH +} + +# Runs `ade --version` and keeps whatever it printed, so a failure has evidence +# instead of just an exit status. Returns the binary's status. +version_check() { + version_check_binary="$1" + version_check_stage="$2" + if : >"$install_log" 2>/dev/null; then + printf 'ade install: %s check of %s\n' "$version_check_stage" "$version_check_binary" \ + >>"$install_log" 2>/dev/null || true + "$version_check_binary" --version >>"$install_log" 2>&1 + else + "$version_check_binary" --version >/dev/null 2>&1 + fi +} + +# Puts the machine back on the install it had. Best effort by design: every +# branch here runs while something has already gone wrong, and a failed restore +# must still let the caller print the real reason rather than abort on `set -e`. +restore_previous_install() { + if [ "$have_backup_binary" -eq 1 ] && [ -e "$backup_binary" ]; then + mv "$backup_binary" "$dest_dir/ade" 2>/dev/null || true + else + # Nothing to restore means this was a first install; leaving the binary that + # just failed its own version check is worse than leaving none. + rm -f "$dest_dir/ade" 2>/dev/null || true + fi + if [ "$promoted_runtime" -eq 1 ] && [ -e "$backup_runtime_dir" ]; then + rm -rf "$runtime_dir" 2>/dev/null || true + mv "$backup_runtime_dir" "$runtime_dir" 2>/dev/null || true + fi +} + +# "staged" until the new binary is promoted, "installed" afterwards. The two +# stages leave the machine in different states, and a message that describes +# the wrong one sends a user looking for damage that is not there. +install_stage="staged" + +die_runtime_unusable() { + if [ "$install_stage" = "staged" ]; then + if [ -e "$dest_dir/ade" ]; then + printf 'ade install: your existing ADE at %s was not touched.\n' "$dest_dir/ade" >&2 + else + printf 'ade install: nothing was left installed at %s.\n' "$dest_dir/ade" >&2 + fi + elif [ "$have_backup_binary" -eq 1 ]; then + printf 'ade install: the ADE you already had was put back, so nothing is broken.\n' >&2 + elif [ -e "$dest_dir/ade" ]; then + # The rollback is best effort, so the disk is the only thing worth + # believing here: if it could not remove the binary that just failed, the + # broken one is still what runs, and saying "nothing was left installed" + # sends the user looking in the wrong place. + printf 'ade install: the ADE at %s is the one that just failed to start.\n' "$dest_dir/ade" >&2 + else + printf 'ade install: nothing was left installed at %s.\n' "$dest_dir/ade" >&2 + fi + if [ -s "$install_log" ]; then + printf 'ade install: what it printed is in %s\n' "$install_log" >&2 + fi + printf 'ade install: next: run the installer again; if it fails the same way, open an issue at https://github.com/%s/issues with that log.\n' \ + "$repo" >&2 + die "$1" +} + +chmod 755 "$staged_binary" rm -rf "$staged_runtime_dir" "$backup_runtime_dir" mkdir -p "$staged_runtime_dir" tar -xzf "$tmp_dir/native.tar.gz" -C "$staged_runtime_dir" [ -d "$staged_node_modules" ] || die "native dependency archive is missing node_modules" +# The preflight copy lands in $dest_dir, not $TMPDIR -- see the noexec note +# above. $dest_dir/ade.new is the scratch name the promotion below renames +# from, and the EXIT trap already removes it, so this costs nothing but the +# copy. +rm -f "$pending_binary" +cp "$staged_binary" "$pending_binary" +chmod 755 "$pending_binary" + +# Preflight before anything is promoted: the new binary against the staged +# runtime, both under the ADE home so neither is executed or `dlopen`ed out of +# $TMPDIR. A download that cannot even print its version never replaces +# $dest_dir/ade, so the previous install is still there and still working. +set_runtime_env "$staged_runtime_dir" +if ! version_check "$pending_binary" "staged"; then + rm -f "$pending_binary" + die_runtime_unusable "the ADE runtime that was just downloaded could not start" +fi + if [ -e "$runtime_dir" ]; then mv "$runtime_dir" "$backup_runtime_dir" fi if mv "$staged_runtime_dir" "$runtime_dir"; then - rm -rf "$backup_runtime_dir" + promoted_runtime=1 else if [ -e "$backup_runtime_dir" ]; then rm -rf "$runtime_dir" @@ -475,11 +660,38 @@ else die "failed to install ADE native runtime dependencies" fi -export ADE_RUNTIME_ROOT="$runtime_dir" -export ADE_RUNTIME_NODE_MODULES="$runtime_dir/node_modules" -export NODE_PATH="$runtime_dir/node_modules${NODE_PATH:+:$NODE_PATH}" +# Promote the binary last, and only by rename -- of the very copy the preflight +# above just ran, so a truncated write can never be the thing at $dest_dir/ade. +rm -f "$backup_binary" +if [ -e "$dest_dir/ade" ]; then + if cp "$dest_dir/ade" "$backup_binary"; then + chmod 755 "$backup_binary" 2>/dev/null || true + have_backup_binary=1 + else + rm -f "$pending_binary" + die "could not back up the existing ADE runtime at $dest_dir/ade" + fi +fi +mv "$pending_binary" "$dest_dir/ade" -"$dest_dir/ade" --version >/dev/null || die "installed ade binary failed to run" +set_runtime_env "$runtime_dir" +install_stage="installed" + +if ! version_check "$dest_dir/ade" "installed"; then + restore_previous_install + die_runtime_unusable "the newly installed ADE runtime could not start" +fi +# The new binary has now proved it runs, so the cleanup handlers must stop +# treating $dest_dir/ade as unverified and rolling the backup back over it. +promoted_binary=1 + +# Past the point of no return: the install is good, so the rollback copies are +# just disk. ~150 MB of it, which is why they are not kept around. The log goes +# with them: a file called install-failure.log left behind by a successful +# install is a false alarm waiting to be found. +rm -f "$backup_binary" +rm -rf "$backup_runtime_dir" +rm -f "$install_log" if command -v systemctl >/dev/null 2>&1 && systemctl --user show-environment >/dev/null 2>&1; then try_install_service diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index d5f1c860d..e58bea49d 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -1,9 +1,10 @@ -import { spawn } from "node:child_process"; +import { spawn, type ChildProcess } from "node:child_process"; import fs from "node:fs"; import { createServer } from "node:http"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { PAIRING_REAUTHENTICATION_REQUIRED_MESSAGE } from "./services/account/accountMachinePublisherService"; import { @@ -24,6 +25,8 @@ import { isEphemeralRuntimeSocketPath, isFailedServiceManagerResult, machineRuntimeMismatchReason, + monitorBrainSocketOwnership, + unlinkOwnedRuntimeSocket, parseCliArgs, parseSnoozeDurationMs, readRuntimeIdleExitMs, @@ -36,7 +39,9 @@ import { startHeadlessRpcSocketServer, startHeadlessRpcTcpServer, shouldAutoRegisterProjectForPlan, + formatBrainStatus, shouldBlockManualMachineRuntimeSpawn, + shouldProbeBrainStartupState, shouldEnforceMachineRuntimeBuildCompatibility, shouldAttemptDesktopSocketConnection, summarizeExecution, @@ -149,6 +154,64 @@ function writeSyncHostSingletonLock(args: { }, null, 2)}\n`, "utf8"); } +function withTsxNodeOptions(value: string | undefined): string { + const existing = value?.trim(); + return existing ? `${existing} --import tsx` : "--import tsx"; +} + +/** + * A sync-host singleton lock owned by a SAME-channel brain (same ADE_HOME), + * which the startup loop retries against forever rather than failing the brain + * outright the way a cross-channel owner does. + */ +function writeSameChannelSyncHostLock(args: { + lockPath: string; + pid: number; + port: number; + adeHome: string; +}): void { + const now = new Date().toISOString(); + fs.mkdirSync(path.dirname(args.lockPath), { recursive: true }); + fs.writeFileSync(args.lockPath, `${JSON.stringify({ + version: 1, + owner: { + id: "sync-host-squatter", + pid: args.pid, + port: args.port, + appName: "ADE", + packageChannel: null, + adeHome: args.adeHome, + serviceName: "com.ade.runtime", + socketPath: path.join(args.adeHome, "sock", "ade.sock"), + projectRoot: null, + commandLine: null, + quitCommand: `ADE_HOME='${args.adeHome}' ade brain stop --text`, + createdAt: now, + updatedAt: now, + }, + }, null, 2)}\n`, "utf8"); +} + +function readSyncHostLockOwnerPid(lockPath: string): number | null { + try { + const parsed = JSON.parse(fs.readFileSync(lockPath, "utf8")) as { + owner?: { pid?: unknown }; + }; + return typeof parsed.owner?.pid === "number" ? parsed.owner.pid : null; + } catch { + return null; + } +} + +function killChildQuietly(child: ChildProcess | null, signal: NodeJS.Signals): void { + if (!child || child.exitCode != null || child.signalCode != null) return; + try { + child.kill(signal); + } catch { + // Already gone. + } +} + describe("ADE CLI", () => { const ambientChatSessionId = process.env.ADE_CHAT_SESSION_ID; beforeEach(() => { @@ -876,6 +939,151 @@ describe("ADE CLI", () => { } }); + /** + * `ade serve` publishes its RPC socket BEFORE the mobile sync host is up. + * + * The brain used to bind `ade.sock` only after `runSyncHostStartupLoop` + * returned, which coupled desktop reachability to phone-sync hosting: a busy + * sync port band, a stale lease from a just-killed predecessor, a slow + * project scope — anything that loop retried kept the socket unpublished, + * and the desktop's service handover budget expired against a brain that was + * alive and healthy ("the background service couldn't be set up — click + * Repair", then "ADE couldn't open this project"). + * + * This drives a real `ade serve` child with the machine-wide sync-host lease + * held by a live same-channel pid, so the startup loop can never finish, and + * proves the socket is nonetheless bound, listening, and answering RPC. + * Verified to fail against the pre-reorder `cli.ts` (socket never appears). + * + * A child process rather than an in-process `runCli`: a fully started brain + * owns background services (ingress pollers, sync status refreshers) whose + * teardown races surface as unhandled rejections in the runner, and killing + * a child is the only way to end a brain the way the OS does. + */ + // DARWIN-GATE: booting a real brain needs the cr-sqlite extension, which ships + // only for macOS, so this can run nowhere else. + crdtHostIt( + "binds and serves the RPC socket while the mobile sync host is still retrying", + async () => { + const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const cliPath = path.join(packageRoot, "src", "cli.ts"); + const adeHome = fs.mkdtempSync(path.join(os.tmpdir(), "ade-serve-order-")); + const projectRoot = path.join(adeHome, "project"); + const lockPath = path.join(adeHome, "sync-host-lock.json"); + // Deliberately NOT `/sock/ade.sock`: that is the machine layout + // socket, which would make this brain the "primary" one and start the + // freshness-monitor / service-reinstall paths this test has no business + // in. + const socketPath = path.join(adeHome, "sock", "ade-order.sock"); + fs.mkdirSync(projectRoot, { recursive: true }); + + // A live pid the sync-host singleton must treat as a real owner. + const squatter = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000);"], { + stdio: "ignore", + }); + squatter.on("error", () => {}); + if (!squatter.pid) throw new Error("Failed to start the fake sync-host owner process."); + writeSameChannelSyncHostLock({ lockPath, pid: squatter.pid, port: 8802, adeHome }); + + let brain: ChildProcess | null = null; + let brainExit: { code: number | null; signal: NodeJS.Signals | null } | null = null; + let stderr = ""; + try { + brain = spawn(process.execPath, [cliPath, "serve", "--socket", socketPath], { + cwd: packageRoot, + env: { + ...process.env, + ADE_HOME: adeHome, + ADE_PROJECT_ROOT: projectRoot, + ADE_PACKAGE_CHANNEL: "", + ADE_SYNC_HOST_LOCK_PATH: lockPath, + ADE_SYNC_HOST_SINGLETON_TEST_MODE: "1", + ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1", + ADE_DISABLE_TOOLS_FETCH: "1", + NODE_OPTIONS: withTsxNodeOptions(process.env.NODE_OPTIONS), + }, + stdio: ["ignore", "ignore", "pipe"], + }); + brain.stderr?.setEncoding("utf8"); + brain.stderr?.on("data", (chunk: string) => { stderr += chunk; }); + brain.on("exit", (code, signal) => { brainExit = { code, signal }; }); + + let client: JsonRpcClient | null = null; + // 60s + 45s worst case leaves headroom inside the 150s test timeout, so + // a stuck brain fails with the diagnostic below and still runs `finally` + // (which kills the squatter) instead of being cut off by the runner. + const deadline = Date.now() + 60_000; + for (;;) { + if (brainExit) { + throw new Error( + `ADE brain exited (${JSON.stringify(brainExit)}) instead of serving:\n${stderr}`, + ); + } + if (fs.existsSync(socketPath)) { + try { + client = await JsonRpcClient.connect(socketPath); + break; + } catch { + // Not listening yet. + } + } + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for ${socketPath} to accept connections:\n${stderr}`); + } + await new Promise((resolve) => setTimeout(resolve, 150)); + } + + try { + // Bound is not enough: it has to be serving desktop RPC. + await client.request("ade/initialize", { + protocolVersion: "2025-06-18", + clientName: "serve-socket-ordering-test", + identity: { role: "external", callerId: "serve-socket-ordering-test" }, + }); + await expect(client.request("ping")).resolves.toMatchObject({ pong: true }); + } finally { + client.close(); + } + + // ...and the sync host is provably still down: the squatter keeps the + // machine-wide lease, so the startup loop is still retrying. This is + // the assertion the old ordering could not satisfy — the socket did + // not exist until that loop returned. + expect(readSyncHostLockOwnerPid(lockPath)).toBe(squatter.pid); + + // The loop is genuinely running and losing, not silently skipped. Not + // asserted at the moment of connect on purpose: the socket can be (and + // routinely is) reachable before the first attempt has even failed + // once, which is the whole point of the reorder. + const logDeadline = Date.now() + 45_000; + while (!stderr.includes("ADE brain sync host failed")) { + if (Date.now() >= logDeadline) { + throw new Error(`ADE brain never reported a sync host failure:\n${stderr}`); + } + await new Promise((resolve) => setTimeout(resolve, 150)); + } + // A sync host that never starts must not take the brain down with it, + // and must not hand the lease over either. + expect(brainExit).toBeNull(); + expect(readSyncHostLockOwnerPid(lockPath)).toBe(squatter.pid); + } finally { + // SIGKILL, not SIGTERM: a graceful brain shutdown spends seconds + // tearing down scopes, and the next test in this file spawns its own + // brain — leaving this one competing for CPU is how neighbouring + // startup waits start timing out. + const exited = brain ? new Promise((resolve) => { + if (brain?.exitCode != null || brain?.signalCode != null) resolve(); + else brain?.once("exit", () => resolve()); + }) : null; + killChildQuietly(brain, "SIGKILL"); + if (exited) await exited; + killChildQuietly(squatter, "SIGKILL"); + fs.rmSync(adeHome, { recursive: true, force: true }); + } + }, + 150_000, + ); + const posixIt = process.platform === "win32" ? it.skip : it; posixIt( "creates the headless RPC unix socket with 0600 perms and parent dir 0700", @@ -1055,6 +1263,57 @@ describe("ADE CLI", () => { })).toBe(false); }); + it("renders the brain starting verdict in --text next to the last-failure line", () => { + const text = formatBrainStatus({ + ok: false, + starting: true, + runtime: { running: false, starting: true, socketPath: "/Users/example/.ade/sock/ade.sock" }, + service: { message: "ADE login service is loaded." }, + lastFailure: "brain serve failed 2x", + }); + expect(text).toContain("nothing to repair"); + expect(text).toContain("brain serve failed 2x"); + // The same output has to read as a plain failure when nothing is coming up. + expect(formatBrainStatus({ ok: false, starting: false, runtime: { running: false } })) + .not.toContain("nothing to repair"); + }); + + it("skips the brain-starting probe inside supervisor and handover probe children", () => { + // Those children run `ade runtime status` with the install lock set. On + // Windows the probe would ask the service manager, which spawns another + // `ade runtime status` — an unbounded recursive fan-out that leaks + // descendants past the spawn timeout. + expect(shouldProbeBrainStartupState({ + socketOverride: null, + socketPath: "/Users/example/.ade/sock/ade.sock", + machineSocketPath: "/Users/example/.ade/sock/ade.sock", + env: { ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1" }, + })).toBe(false); + expect(shouldProbeBrainStartupState({ + socketOverride: null, + socketPath: "/Users/example/.ade/sock/ade.sock", + machineSocketPath: "/Users/example/.ade/sock/ade.sock", + env: {}, + })).toBe(true); + }); + + it("skips the brain-starting probe when --socket points at another runtime", () => { + expect(shouldProbeBrainStartupState({ + socketOverride: "/tmp/other.sock", + socketPath: "/tmp/other.sock", + machineSocketPath: "/Users/example/.ade/sock/ade.sock", + env: {}, + })).toBe(false); + // An override that resolves back to this machine's own brain is still the + // machine brain, so it keeps the starting verdict. + expect(shouldProbeBrainStartupState({ + socketOverride: "/Users/example/.ade/sock/ade.sock", + socketPath: "/Users/example/.ade/sock/ade.sock", + machineSocketPath: "/Users/example/.ade/sock/ade.sock", + env: {}, + })).toBe(true); + }); + it("parses runtime idle expiry with a minimum clamp", () => { expect(readRuntimeIdleExitMs({ ADE_RUNTIME_IDLE_EXIT_MS: "30000" } as NodeJS.ProcessEnv)).toBe(30_000); expect(readRuntimeIdleExitMs({ ADE_RUNTIME_IDLE_EXIT_MS: "100" } as NodeJS.ProcessEnv)).toBe(5_000); @@ -11241,3 +11500,161 @@ describe("describeLastFailureForStartupLog", () => { expect(described.endsWith("…")).toBe(true); }); }); + +describe("monitorBrainSocketOwnership", () => { + const flush = async (): Promise => { + // The monitor floors its poll interval at 250ms; outlast one tick. + await new Promise((resolve) => setTimeout(resolve, 400)); + }; + const socketPath = "/tmp/ade-ownership/ade.sock"; + + it("ends the brain when another brain replaces the socket it bound", async () => { + let inode: bigint | null = 100n; + const lost: string[] = []; + const stop = monitorBrainSocketOwnership(socketPath, (reason) => lost.push(reason), { + intervalMs: 250, + readInode: () => inode, + }); + await flush(); + expect(lost).toEqual([]); + // A rival that had already probed this path as stale unlinked our inode + // and bound its own. No EADDRINUSE ever happened; only the inode changed. + inode = 200n; + await flush(); + expect(lost).toEqual(["replaced"]); + stop(); + }); + + it("ends the brain when the socket path is removed underneath it", async () => { + let inode: bigint | null = 100n; + const lost: string[] = []; + const stop = monitorBrainSocketOwnership(socketPath, (reason) => lost.push(reason), { + intervalMs: 250, + readInode: () => inode, + }); + inode = null; + await flush(); + expect(lost).toEqual(["removed"]); + stop(); + }); + + it("stays quiet while the inode is unchanged, and reports nothing after stop", async () => { + const lost: string[] = []; + const stop = monitorBrainSocketOwnership(socketPath, (reason) => lost.push(reason), { + intervalMs: 250, + readInode: () => 100n, + }); + await flush(); + stop(); + await flush(); + expect(lost).toEqual([]); + }); + + it("is a no-op for a Windows named pipe, which has no directory entry to steal", () => { + const lost: string[] = []; + const stop = monitorBrainSocketOwnership( + String.raw`\\.\pipe\ade-runtime-abc`, + (reason) => lost.push(reason), + { intervalMs: 250, readInode: () => { throw new Error("must not stat a named pipe"); } }, + ); + stop(); + expect(lost).toEqual([]); + }); + + it("fails open when the socket reports no inode", () => { + const lost: string[] = []; + const stop = monitorBrainSocketOwnership(socketPath, (reason) => lost.push(reason), { + intervalMs: 250, + readInode: () => null, + }); + stop(); + expect(lost).toEqual([]); + }); +}); + +describe("unlinkOwnedRuntimeSocket", () => { + const socketPath = "/tmp/ade-ownership/ade.sock"; + + // Regression: the shutdown that follows a "replaced" ownership loss used to + // unlink unconditionally, deleting the rival brain's socket — the winner + // then listened on an inode nothing could reach. + it("refuses to remove a socket another process rebound", () => { + const unlinked: string[] = []; + const outcome = unlinkOwnedRuntimeSocket(socketPath, 100n, { + readInode: () => 200n, + unlink: (target) => unlinked.push(target), + }); + + expect(outcome).toBe("not_owned"); + expect(unlinked).toEqual([]); + }); + + it("removes the socket while we still own the inode we bound", () => { + const unlinked: string[] = []; + const outcome = unlinkOwnedRuntimeSocket(socketPath, 100n, { + readInode: () => 100n, + unlink: (target) => unlinked.push(target), + }); + + expect(outcome).toBe("unlinked"); + expect(unlinked).toEqual([socketPath]); + }); + + it("reports an already-gone socket without touching the filesystem", () => { + const unlinked: string[] = []; + const outcome = unlinkOwnedRuntimeSocket(socketPath, 100n, { + readInode: () => null, + unlink: (target) => unlinked.push(target), + }); + + expect(outcome).toBe("absent"); + expect(unlinked).toEqual([]); + }); + + // A failed unlink used to be reported as "absent", which is the opposite of + // what happened: the socket file is still there, still ours, and the next + // brain will probe it as stale. The caller logs the difference. + it("reports a failed unlink as failed, not absent", () => { + const outcome = unlinkOwnedRuntimeSocket(socketPath, 100n, { + readInode: () => 100n, + unlink: () => { throw new Error("EPERM"); }, + }); + + expect(outcome).toBe("failed"); + }); + + // ENOENT between the stat and the unlink is the one failure that is not a + // failure: the path is gone, which is exactly what we were trying to do. + it("reports an unlink that lost the race to ENOENT as absent", () => { + const outcome = unlinkOwnedRuntimeSocket(socketPath, 100n, { + readInode: () => 100n, + unlink: () => { + throw Object.assign(new Error("ENOENT: no such file or directory"), { code: "ENOENT" }); + }, + }); + + expect(outcome).toBe("absent"); + }); + + it("falls back to the unconditional unlink when no inode was recorded at bind time", () => { + const unlinked: string[] = []; + const outcome = unlinkOwnedRuntimeSocket(socketPath, null, { + readInode: () => 200n, + unlink: (target) => unlinked.push(target), + }); + + expect(outcome).toBe("unlinked"); + expect(unlinked).toEqual([socketPath]); + }); + + it("never unlinks a Windows named pipe, which has no directory entry", () => { + const unlinked: string[] = []; + const outcome = unlinkOwnedRuntimeSocket(String.raw`\\.\pipe\ade-runtime-abc`, 100n, { + readInode: () => { throw new Error("must not stat a named pipe"); }, + unlink: (target) => unlinked.push(target), + }); + + expect(outcome).toBe("not_owned"); + expect(unlinked).toEqual([]); + }); +}); diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index 7b5506aa5..52bdb76f9 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -33,6 +33,11 @@ import { runDoctorCommand, type DoctorRow, } from "./commands/doctor"; +import { + buildCliDiagnosticReport, + buildReportIssuePayload, + openDiagnosticIssue, +} from "./commands/reportIssue"; export { readInstalledDesktopVersion }; import { MAX_STATUS_NOTE_CHARACTERS, @@ -131,8 +136,12 @@ import { import { isCurrentProcessDescendantOfPid, resolveAdeServeCommand, + RuntimeServiceStillStartingError, type AdeServiceCommand, } from "./serviceManager/common"; +import { awaitRuntimeServiceEndpoint } from "./services/runtime/awaitRuntimeServiceEndpoint"; +import { readBrainStartupState } from "./services/runtime/brainStartupState"; +import { connectWhileServiceStarts } from "./services/runtime/connectWhileServiceStarts"; import { normalizeAdeRuntimeRole, resolveAdeDefaultRole } from "./runtimeRoles"; import { isIndefiniteSnooze, @@ -243,6 +252,7 @@ type InvocationStep = { type FormatterId = | "status" | "doctor" + | "brain-status" | "auth" | "account-auth" | "account-token" @@ -386,6 +396,7 @@ type CliPlan = | { kind: "setup"; rest: string[] } | { kind: "connect"; rest: string[] } | { kind: "doctor"; online: boolean } + | { kind: "report-issue"; open: boolean } | { kind: "serve"; rest: string[] } | { kind: "rpc-stdio"; rest: string[] } | { kind: "pty-host-worker" } @@ -684,6 +695,7 @@ const TOP_LEVEL_HELP = `${ADE_BANNER} $ ade sync web [--open] [--no-clipboard] Print (and copy) the web client pairing link + code $ ade sync status | pin generate Manage machine sync and phone pairing $ ade doctor [--online] Inspect installed app and machine-brain health + $ ade report-issue [--open] Print a redacted diagnostic report for a bug report $ ade lanes list | show | create | child Work with lanes and lane stacks $ ade git status | commit | push | stash Run ADE-aware git operations $ ade operations status | wait Poll operation/test/chat/run status @@ -12630,6 +12642,12 @@ function buildCliPlan( online: readFlag(args, ["--online"]), }; } + if (primary === "report-issue") { + return { + kind: "report-issue", + open: readFlag(args, ["--open"]), + }; + } if (primary === "auth") { const sub = firstPositional(args) ?? "status"; if (sub !== "status") @@ -15293,7 +15311,6 @@ function shouldAllowRuntimeSelfShutdown(env: NodeJS.ProcessEnv = process.env): b } class RuntimeSelfShutdownBlockedError extends Error {} -class RuntimeServiceRecoveryOwnedError extends Error {} function isLocalRuntimeSocketPath(socketPath: string): boolean { return !socketPath.startsWith("tcp://"); @@ -15368,6 +15385,32 @@ function isServiceManagedMachineRuntimeSocket(socketPath: string): boolean { && !isEphemeralRuntimeSocketPath(socketPath); } +/** + * Whether a silent socket earns the `brain_starting` probe. + * + * Two vetoes, both about not making a bad situation worse: + * + * - Supervisor and handover probes run `ade runtime status` as a CHILD process + * with `ADE_DISABLE_RUNTIME_SERVICE_INSTALL=1`. Probing the service manager + * from inside such a child is recursive on Windows: the status probe asks the + * service manager, which spawns another `ade runtime status`, which fails on + * the same silent pipe and probes again. Windows has no process groups, so + * the spawn timeout kills only the leader and leaks every descendant. + * - A `--socket` override points at a different runtime entirely. The default + * machine service's youth says nothing about it, and "still starting, nothing + * to repair" would bury that runtime's real connect error. + */ +export function shouldProbeBrainStartupState(args: { + socketOverride: string | null; + socketPath: string; + machineSocketPath: string; + env?: NodeJS.ProcessEnv; +}): boolean { + const env = args.env ?? process.env; + if (env.ADE_DISABLE_RUNTIME_SERVICE_INSTALL === "1") return false; + return !args.socketOverride || args.socketPath === args.machineSocketPath; +} + export function shouldBlockManualMachineRuntimeSpawn( socketPath: string, env: NodeJS.ProcessEnv = process.env, @@ -15403,17 +15446,22 @@ async function repairMachineRuntimeServiceConnection(args: { ); if (!result.ok) { if (serviceManagerOwnsRuntimeRecovery(result)) { - throw new RuntimeServiceRecoveryOwnedError( - `${result.message} The registered service still owns recovery for this endpoint, so ADE did not start a competing manual brain.`, - ); + throw new RuntimeServiceStillStartingError({ + kind: "recovery_owned", + installMessage: result.message, + }); } return null; } - client = await SocketJsonRpcClient.connect( - args.socketPath, - args.options.timeoutMs, - "ADE runtime endpoint", - ); + client = await connectWhileServiceStarts({ + install: result, + socketPath: args.socketPath, + connect: () => SocketJsonRpcClient.connect( + args.socketPath, + args.options.timeoutMs, + "ADE runtime endpoint", + ), + }); const runtimeInfo = await initializeMachineRuntimeDaemon( client, args.options, @@ -15439,7 +15487,7 @@ async function repairMachineRuntimeServiceConnection(args: { } catch (error) { if ( error instanceof RuntimeSelfShutdownBlockedError - || error instanceof RuntimeServiceRecoveryOwnedError + || error instanceof RuntimeServiceStillStartingError ) throw error; return null; } finally { @@ -15730,11 +15778,30 @@ async function runRuntimeCommand( client.close(); } } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + // Not answering is not the same as broken. When the service is + // registered and the brain behind it is alive and young, it is still + // coming up — the same verdict the desktop calls `brain_starting`. + // Callers keep waiting for the endpoint instead of restarting it. + const starting = shouldProbeBrainStartupState({ + socketOverride, + socketPath, + machineSocketPath: resolveMachineAdeLayout().socketPath, + }) + ? (await readBrainStartupState()).starting + : false; return { ok: false, running: false, + starting, socketPath, - message: error instanceof Error ? error.message : String(error), + // `detail` is the raw connect error on both branches, so it is always + // present: a caller reading it should never have to know which verdict + // produced the message above it. + detail, + message: starting + ? `ADE brain is still starting; it has not answered on ${socketPath} yet. Keep waiting — there is nothing to repair.` + : detail, }; } } @@ -15985,6 +16052,34 @@ async function runSetupCli( return { ok: true, detail: names.join(", ") }; }, getAccountStatus: () => readSetupAccountStatus(options), + // The installer has just registered the brain service. A brain that is + // registered and alive but not answering yet is *starting*, not broken, + // so this waits for its endpoint instead of letting the account step run + // against a socket that was never given time to open. + awaitRuntimeService: async ({ budgetMs, onStarting }) => { + const socketPath = await resolveMachineRuntimeSocketPath(options.socketPath); + return await awaitRuntimeServiceEndpoint({ + budgetMs, + onStarting, + probe: async () => { + try { + const client = await SocketJsonRpcClient.connect( + socketPath, + options.timeoutMs, + "ADE runtime endpoint", + ); + client.close(); + return true; + } catch { + return false; + } + }, + installService: async () => { + const { installRuntimeService } = await import("./serviceManager"); + return await withAdeDefaultRole("cto", () => installRuntimeService()); + }, + }); + }, // Delegates to the same `ade connect` implementation so the OAuth flow, // the service step and the machine-directory wait stay in one place. runConnect: async () => { @@ -16142,6 +16237,7 @@ async function runConnectCli( ok: install.ok, message: install.message, selfMutationBlocked: install.selfMutationBlocked, + starting: install.starting, }; }, getMachineKey: () => machineKey, @@ -16237,8 +16333,13 @@ async function runBrainCommand( // serve path clears it on a successful listen. Surface it as a plain // one-liner so `ade brain status --text` matches the Desktop recovery screen. const lastFailure = readLastFailure({ kind: "machine" }); + // Mirrors `ade runtime status`: a registered service whose young brain has + // not bound the socket yet is starting, not broken, so `ade brain status` + // does not read as a failure that wants repairing. + const starting = isRecord(runtime) && runtime.starting === true; return { ok: service.ok && (!isRecord(runtime) || runtime.ok !== false), + starting, service, runtime, sync, @@ -16282,7 +16383,12 @@ async function runBrainCommand( message: !stopped.ok ? `ADE brain restart attempted after stop warning: ${stopped.message}` : started.ok - ? "ADE brain restarted." + // `starting` means the replacement is alive but has not answered on + // the socket yet. Claiming "restarted." there reads as "ready now" + // and sends callers into a connect that is not due to succeed yet. + ? started.starting + ? started.message + : "ADE brain restarted." : started.message, }; } @@ -16651,7 +16757,7 @@ async function runServe( const [ { resolveMachineAdeLayout }, { ProjectRegistry }, - { ProjectScopeRegistry }, + { ProjectScopeRegistry, SYNC_HOST_ADOPT_TIMEOUT_MS }, { createMultiProjectRpcRequestHandler, createPersonalChatScope, @@ -17342,6 +17448,15 @@ async function runServe( } else { activeScope = await scopeRegistry.resolveActiveSyncHost(); } + if (!activeScope && scopeRegistry.getRequestedSyncHostProjectId()) { + // A null here can mean "superseded" rather than "no host"; adopt the + // switch that overtook ours instead of clobbering it with the + // projectless lease. See `adoptRequestedSyncHost`. + activeScope = await scopeRegistry.adoptRequestedSyncHost(SYNC_HOST_ADOPT_TIMEOUT_MS); + if (!activeScope) { + throw new Error("Sync host switch superseded by a concurrent project switch; retrying."); + } + } if (!activeScope && sharedSyncListener) { // Binding the shared listener IS hosting phone sync, even with no project // scope to attach to it. Take the machine-wide lease first so this path @@ -17470,58 +17585,87 @@ async function runServe( } } - if (syncEnabled) { + // The mobile sync host is started AFTER the RPC socket is bound (see + // `startSyncHostInBackground` below). It used to be awaited here, before the + // bind, which coupled every desktop connection to phone-sync hosting: a + // project scope that was slow to open, a sync port band that was busy, a + // stale lease from a just-killed predecessor — anything the startup loop + // retried — kept `ade.sock` unpublished, and the desktop's service handover + // budget expired against a brain that was alive and healthy. Nothing about + // serving desktop RPC needs the sync host up first. + let syncHostStartupFailure: unknown = null; + const startSyncHostInBackground = async (): Promise => { + if (!syncEnabled) { + // Deliberately NOT clearing the machine failure record. The bind above + // already cleared every non-`sync_host` failure, and a `--no-sync` brain + // proves nothing about whether the sync host can start. The record is + // keyed by ADE_HOME, not by socket path, so the two brains that run with + // `--no-sync` — an ephemeral runtime socket and the desktop's isolated + // runtime, both spawned precisely BECAUSE the real brain is unhealthy — + // would otherwise erase the crash-loop streak that the installer's + // young-brain veto and the `brain_crash_looping` diagnosis depend on. + return; + } try { const [{ runSyncHostStartupLoop }, { getRuntimeServiceMainPid }] = await Promise.all([ import("./services/sync/syncHostStartupLoop"), import("./serviceManager"), ]); + // This loop no longer needs a socket-liveness abort. That abort guarded + // against a rival brain taking the RPC socket while this one waited for + // sync (PR #949's zombies). The socket is bound before this loop runs + // now, so a rival that dials it finds a live owner and refuses; the bind + // itself is the claim. The one remaining way to lose the path — a rival + // unlinking a socket it proved stale, which a bound socket never is — + // is caught by `monitorBrainSocketOwnership`. await runSyncHostStartupLoop({ startSyncHost, isDone: () => done, log: (message) => process.stderr.write(`${message}\n`), getServiceMainPid: getRuntimeServiceMainPid, - // The pre-loop claim above only sees a socket that ALREADY existed. Two - // brains started together on a fresh path both pass it, then one wins - // the lease and binds while the loser waits here forever — never - // reaching its own bind check. Re-check while we wait, and only for a - // provably live owner so a probe hiccup can't make a brain quit on - // itself. - abortIf: async () => { - if (!isAdeRuntimeNamedPipePath(socketPath) && !fs.existsSync(socketPath)) return false; - return await probeLocalSocketForLiveness(socketPath) === "live"; - }, }); + // A recorded sync-host failure is cleared only once the sync host is + // really up; clearing it on the bind would reset the crash-loop counter + // on every restart of a brain that keeps dying right here. + if (!done) clearLastFailure({ kind: "machine" }); } catch (error: unknown) { + if (done) return; // Cross-channel conflict (another build's live brain owns mobile sync): - // real builds never run sync-less, so fail before publishing ade.sock. - const [{ SyncHostSingletonConflictError }, { SyncHostStartupAbortedError }] = await Promise.all([ - import("./services/sync/syncHostSingleton"), - import("./services/sync/syncHostStartupLoop"), - ]); + // real builds never run sync-less, so the brain still refuses to keep + // running. The RPC socket is already published by now; closing it is + // what `finish()` does, and the recorded failure carries the same code + // project recovery keyed on before. const message = error instanceof Error ? error.message : String(error); - if (error instanceof SyncHostStartupAbortedError) { - await disposeServeResources(); - throw Object.assign(new CliExecutionError("ADE brain socket is already in use.", { - socketPath, - cause: "Another ADE brain took this socket while this one waited for mobile sync.", - nextAction: "Stop the existing ADE brain or choose a different --socket path.", - }), { code: "socket_owned_by_other" as const }); + // The conflict class is loaded to classify the failure, and that import + // can itself reject (a torn install, a disk error). Letting it escape + // would skip `finish()` — the brain would keep serving a socket it has + // already decided to give up, and the rejection would surface as an + // unhandled one. An unclassifiable failure is still a failure. + let conflict = false; + try { + const { SyncHostSingletonConflictError } = await import("./services/sync/syncHostSingleton"); + conflict = error instanceof SyncHostSingletonConflictError; + } catch (importError: unknown) { + process.stderr.write( + `ADE brain could not classify its sync host failure: ${ + importError instanceof Error ? importError.message : String(importError) + }\n`, + ); } - if (error instanceof SyncHostSingletonConflictError) { - await disposeServeResources(); - throw new CliExecutionError("ADE brain refusing to run without mobile sync.", { + if (conflict) { + syncHostStartupFailure = new CliExecutionError("ADE brain refusing to run without mobile sync.", { cause: message, socketPath, nextAction: "Stop the other ADE brain that owns mobile sync, then start this build again.", }); + } else { + process.stderr.write(`ADE brain sync host startup loop failed: ${message}\n`); + syncHostStartupFailure = error; } - process.stderr.write(`ADE brain sync host startup loop failed: ${message}\n`); - await disposeServeResources(); - throw error; + finish(); } - } + }; fs.mkdirSync(layout.adeDir, { recursive: true, mode: 0o700 }); if (isAdeRuntimeNamedPipePath(socketPath)) { @@ -17678,8 +17822,19 @@ async function runServe( process.stderr.write( `ADE brain listening on ${socketPath}${tcpUrl ? ` and ${tcpUrl}` : ""}\n`, ); - clearLastFailure({ kind: "machine" }); serveStarted = true; + // The RPC socket is up: any recorded startup failure that was NOT about the + // sync host is over. Sync-host failures stay recorded until the sync host + // actually comes up (below), so a brain that binds and then dies on a + // cross-channel conflict every time still accumulates a crash-loop count. + if (readLastFailure({ kind: "machine" })?.component !== "sync_host") { + clearLastFailure({ kind: "machine" }); + } + // Started after the account-publisher subscription above so the lease the + // sync host takes is what starts the publisher, and after the socket is + // published so a desktop can already reach this brain while phone sync + // hosting is still coming up (or still retrying). + void startSyncHostInBackground(); // Pinned agent tools are fetched, not bundled — roughly 600 MB across the // three of them. A source checkout still resolves all three out of the repo's @@ -17757,6 +17912,18 @@ async function runServe( stopBrainFreshnessMonitor = () => freshnessMonitor.stop(); } + // Remembered so the shutdown below removes only the socket file this brain + // bound. If a rival rebinds the path, the inode there is theirs. + const boundSocketInode = readRuntimeSocketInode(socketPath); + let ownsSocketPath = !isAdeRuntimeNamedPipePath(socketPath); + const stopSocketOwnershipMonitor = monitorBrainSocketOwnership(socketPath, (reason) => { + // "replaced" means someone else's socket is at the path now; "removed" + // means there is nothing left to clean up. Either way this brain is no + // longer the owner and must not unlink on the way out. + ownsSocketPath = false; + headlessProjectLogger.warn("brain.socket_ownership_lost", { socketPath, reason }); + finish(); + }); const stopParentMonitor = monitorRuntimeParentProcess(finish); const stopIdleMonitor = monitorRuntimeIdleExit(states, finish); try { @@ -17781,16 +17948,33 @@ async function runServe( } finally { stopParentMonitor(); stopIdleMonitor(); + stopSocketOwnershipMonitor(); } for (const state of states) { stopHeadlessRpcServer(state); } await disposeServeResources(); - if (!isAdeRuntimeNamedPipePath(socketPath)) { - try { - fs.unlinkSync(socketPath); - } catch {} + if (ownsSocketPath) { + const outcome = unlinkOwnedRuntimeSocket(socketPath, boundSocketInode); + if (outcome === "not_owned") { + headlessProjectLogger.warn("brain.socket_unlink_skipped", { + socketPath, + reason: "Another process rebound this path; removing it would delete their socket.", + }); + } else if (outcome === "failed") { + headlessProjectLogger.warn("brain.socket_unlink_failed", { + socketPath, + reason: "The socket file is still ours and still present; the next brain will probe it as stale.", + }); + } + } + if (syncHostStartupFailure != null) { + // A sync-host startup failure ends the brain even though the socket was + // already published; record it like any other startup failure so project + // recovery diagnoses the conflict instead of an unexplained exit. + serveStarted = false; + throw syncHostStartupFailure; } return null; } finally { @@ -17856,6 +18040,111 @@ function isPidAlive(pid: number): boolean { } } +/** How often a bound brain re-checks that it still owns its socket path. */ +const BRAIN_SOCKET_OWNERSHIP_POLL_MS = 5_000; + +/** Inode of a socket path, or null when it is gone or unreadable. */ +export function readRuntimeSocketInode(target: string): bigint | null { + try { + return fs.statSync(target, { bigint: true }).ino; + } catch { + return null; + } +} + +/** + * Removes this brain's own socket file — and only its own. + * + * The shutdown path used to unlink unconditionally. When the ownership monitor + * fires with "replaced" a rival has already rebound the path, so that unlink + * deleted the *winner's* socket and left the machine with a live brain nothing + * could reach. Compare the inode against the one we bound before removing it. + * + * With no inode recorded at bind time (a filesystem that reports none) the old + * unconditional behaviour stands: leaking a stale socket file is worse than the + * race it would avoid. + */ +export function unlinkOwnedRuntimeSocket( + socketPath: string, + ownInode: bigint | null, + deps: { + readInode?: (target: string) => bigint | null; + unlink?: (target: string) => void; + } = {}, +): "unlinked" | "not_owned" | "absent" | "failed" { + if (isAdeRuntimeNamedPipePath(socketPath)) return "not_owned"; + const readInode = deps.readInode ?? readRuntimeSocketInode; + const unlink = deps.unlink ?? ((target: string) => fs.unlinkSync(target)); + const current = readInode(socketPath); + if (current == null) return "absent"; + if (ownInode != null && current !== ownInode) return "not_owned"; + try { + unlink(socketPath); + } catch (error) { + // ENOENT means someone else removed it between our stat and our unlink: + // the path is gone, which is the outcome "absent" describes and the + // outcome we wanted. Reporting it as "failed" would send the caller + // looking for a stale socket that does not exist. + if ((error as NodeJS.ErrnoException | null)?.code === "ENOENT") return "absent"; + // Distinct from "absent": the socket file is still there and still ours, + // so the next brain to start will probe a stale path we failed to clean up. + return "failed"; + } + return "unlinked"; +} + +/** + * A unix-domain brain can lose its own endpoint AFTER a successful `listen()`. + * The bind is preceded by a check, not a lock: `existsSync` -> await + * `assertBrainSocketUnowned` -> `unlink` -> `listen`. Two brains that both + * probe the same *stale* socket file race across that await — the first + * unlinks and binds inode X, the second (whose probe predates that bind) then + * unlinks the path->X link and binds a fresh inode Y. Neither sees + * `EADDRINUSE`, so the guard around `listen` never fires, and the first brain + * lives on listening to an inode nothing can reach: the PR #949 zombie. + * + * The sync-host loop's socket-liveness abort used to catch this incidentally, + * because the loser was still inside that loop when the socket went live. + * Binding before the sync host removed that accident, so make the check + * explicit: remember the inode we bound and end the brain if the path stops + * pointing at it. The supervisor then restarts us, and the restarted brain + * finds the rival's live socket and reports `socket_owned_by_other` instead of + * squatting silently. + * + * Windows named pipes are exempt: a pipe name is a kernel object with no + * directory entry to steal, and a bound pipe can never be probed as stale. + */ +export function monitorBrainSocketOwnership( + socketPath: string, + onLost: (reason: "removed" | "replaced") => void, + options: { + intervalMs?: number; + readInode?: (target: string) => bigint | null; + } = {}, +): () => void { + if (isAdeRuntimeNamedPipePath(socketPath)) return () => {}; + const readInode = options.readInode ?? readRuntimeSocketInode; + const ownInode = readInode(socketPath); + // No inode to compare against (a platform or filesystem that does not report + // one) means this guard cannot run. Fail open: a brain with no watchdog is + // strictly better than one that ends itself on an unreadable stat. + if (ownInode == null) return () => {}; + let done = false; + const timer = setInterval(() => { + if (done) return; + const current = readInode(socketPath); + if (current === ownInode) return; + done = true; + clearInterval(timer); + onLost(current == null ? "removed" : "replaced"); + }, Math.max(250, options.intervalMs ?? BRAIN_SOCKET_OWNERSHIP_POLL_MS)); + timer.unref?.(); + return () => { + done = true; + clearInterval(timer); + }; +} + function monitorRuntimeParentProcess(onGone: () => void): () => void { const parentPid = readRuntimeParentPid(); if (parentPid == null || parentPid === process.pid) return () => {}; @@ -20273,6 +20562,49 @@ function formatAccountMachines(value: unknown): string { ].join("\n"); } +/** + * `ade brain status` and `ade runtime status` in --text mode. + * + * `starting` — the CLI's read of the desktop's `brain_starting` state — only + * means something if a human can see it, so it is printed here with the same + * wording as the `ade doctor` Brain row, next to the last-failure line it has + * to be read against. Accepts both shapes: `ade brain status` wraps the runtime + * result, `ade runtime status` is that result. + */ +function brainStatusFormatter(rest: string[]): FormatterId | undefined { + const sub = rest.find((arg) => arg !== "--" && !arg.startsWith("-")) ?? "status"; + return sub === "status" || sub === "show" ? "brain-status" : undefined; +} + +export function formatBrainStatus(value: unknown): string { + const result = isRecord(value) ? value : {}; + const runtime = isRecord(result.runtime) ? result.runtime : result; + const service = isRecord(result.service) ? result.service : null; + const starting = result.starting === true || runtime.starting === true; + return renderKeyValues("ADE brain", [ + ["ok", result.ok], + ["endpoint", runtime.running === true ? "running" : "not responding"], + ["socket", runtime.socketPath], + ["version", runtime.version], + ["pid", runtime.pid], + [ + "starting", + starting + ? "yes \u00b7 the background service is up and its brain is coming up; nothing to repair" + : null, + ], + ["channel", runtime.packageChannel], + ["build", runtime.buildHash], + ["role", runtime.defaultRole], + ["project", runtime.projectRoot], + ["service", service ? service.message : null], + ["port", result.port], + ["connected peers", result.connectedPeers], + ["last failure", result.lastFailure], + ["message", starting ? null : result.message], + ]); +} + function formatTextOutput( value: unknown, formatter: FormatterId | undefined, @@ -20293,12 +20625,14 @@ function formatTextOutput( ["workspace", isRecord(value) ? value.workspaceRoot : null], ["socket", isRecord(value) ? value.socketPath : null], ]); + case "brain-status": + return formatBrainStatus(value); case "doctor": { const doctorRows = isRecord(value) && Array.isArray(value.rows) ? value.rows.filter(isRecord) : []; if (doctorRows.length > 0) { - return renderTable( + const table = renderTable( ["check", "status", "detail"], doctorRows.map((row) => [ asString(row.label) ?? asString(row.key) ?? "Unknown", @@ -20311,6 +20645,11 @@ function formatTextOutput( ]), "No health checks were returned.", ); + // A failing row that these checks cannot explain is the case + // `ade report-issue` exists for: it reads local files only, so it still + // works on the machine where the brain will not come up. + if (!doctorRows.some((row) => row.status === "fail")) return table; + return `${table}\n\nIf a failure above is unexplained, run \`ade report-issue\` and file the printed report.`; } const project = isRecord(value) && isRecord(value.project) ? value.project : {}; @@ -21920,6 +22259,34 @@ async function runCli( throw error; } } + if (plan.kind === "report-issue") { + const { projectRoot } = resolveRoots(parsed.options); + const built = buildCliDiagnosticReport({ + surface: "cli", + projectRoot: fs.existsSync(path.join(projectRoot, ".ade")) ? projectRoot : null, + cliVersion: VERSION, + }); + // Copy-then-open, the same order the desktop button uses: the issue + // template asks the user to paste the report from their clipboard, so + // opening it without copying first sends them to a form with nothing to + // paste. Both steps are best effort and the report is printed regardless. + const openedIssue = plan.open ? await openDiagnosticIssue(built) : null; + if (parsed.options.text) { + const clipboardNote = openedIssue?.copied ? "\n(the report is on your clipboard)" : ""; + return { + output: `${built.report}\nFile the issue at:\n${built.issueUrl}${clipboardNote}\n`, + exitCode: 0, + }; + } + return { + output: formatOutput( + buildReportIssuePayload(built, openedIssue), + parsed.options, + undefined, + ), + exitCode: 0, + }; + } if (plan.kind === "doctor") { const result = await runDoctorCommand(plan.online, parsed.options, { resolveMachineRuntimeSocketPath, @@ -21945,14 +22312,14 @@ async function runCli( if (plan.kind === "runtime") { const result = await runRuntimeCommand(plan.rest, parsed.options); return { - output: formatOutput(result, parsed.options, undefined), + output: formatOutput(result, parsed.options, brainStatusFormatter(plan.rest)), exitCode: isRecord(result) && result.ok === false ? 1 : 0, }; } if (plan.kind === "brain") { const result = await runBrainCommand(plan.rest, parsed.options); return { - output: formatOutput(result, parsed.options, undefined), + output: formatOutput(result, parsed.options, brainStatusFormatter(plan.rest)), exitCode: isRecord(result) && result.ok === false ? 1 : 0, }; } diff --git a/apps/ade-cli/src/commands/connect.test.ts b/apps/ade-cli/src/commands/connect.test.ts index d13fda95b..2155f0398 100644 --- a/apps/ade-cli/src/commands/connect.test.ts +++ b/apps/ade-cli/src/commands/connect.test.ts @@ -291,6 +291,25 @@ describe("runConnectCommand", () => { expect(result.steps[1]).toMatchObject({ id: "service", state: "ok" }); }); + it("says the service is still starting instead of claiming it is running", async () => { + // `ok: true, starting: true`: registered, brain alive, socket not answering + // yet. Neither a failure nor "running" — the step has to say which. + const { deps } = makeDeps({ + service: { installed: false, running: false }, + installService: async () => ({ + ok: true, + starting: true, + message: "the background service (pid 4242) is still starting.", + }), + }); + const result = await runConnectCommand([], deps); + + expect(result.ok).toBe(true); + expect(result.steps[1]).toMatchObject({ id: "service", state: "ok" }); + expect(result.steps[1].detail).toContain("still starting"); + expect(result.steps[1].detail).not.toContain("installed and running"); + }); + it("reports a real service install failure with a foreground fallback", async () => { const { deps } = makeDeps({ service: { installed: false, running: false }, diff --git a/apps/ade-cli/src/commands/connect.ts b/apps/ade-cli/src/commands/connect.ts index 0ab7401c6..68f5ead35 100644 --- a/apps/ade-cli/src/commands/connect.ts +++ b/apps/ade-cli/src/commands/connect.ts @@ -68,6 +68,12 @@ export type ConnectServiceInstallResult = { message?: string | null; /** Set when the caller is running inside the very brain it tried to mutate. */ selfMutationBlocked?: boolean; + /** + * The service is registered and its brain is alive, but it had not answered + * on the socket yet. Installed, not running — reporting it as "running" is a + * claim the install did not make. + */ + starting?: boolean; }; export type ConnectMachine = { @@ -423,7 +429,16 @@ export async function runConnectCommand( const install = await deps.installService(); if (install.ok) { service = await deps.getServiceStatus(); - pushStep({ id: "service", state: "ok", detail: `${mechanism} installed and running` }); + pushStep({ + id: "service", + state: "ok", + // A `starting` install registered the service and left a live brain + // that has not answered yet. It is not running *now*, and saying so + // is what sends people looking for a fault that does not exist. + detail: install.starting + ? `${mechanism} installed — the background service is still starting` + : `${mechanism} installed and running`, + }); } else if (install.selfMutationBlocked) { // Running inside the brain it would replace. Not a failure: the service // is already doing its job. diff --git a/apps/ade-cli/src/commands/doctor.test.ts b/apps/ade-cli/src/commands/doctor.test.ts index ba7939d45..537e72e65 100644 --- a/apps/ade-cli/src/commands/doctor.test.ts +++ b/apps/ade-cli/src/commands/doctor.test.ts @@ -192,6 +192,32 @@ describe("doctor row evaluation", () => { ]); }); + it("reports a young registered brain as starting, and a dead one as failing", () => { + const starting = healthyInput(); + starting.brain = { + ...starting.brain, + running: false, + error: "connect ENOENT /tmp/ade.sock", + starting: true, + startingAgeMs: 12_000, + }; + const startingRow = evaluateDoctorRows(starting).find((row) => row.key === "brain"); + expect(startingRow?.status).toBe("warn"); + expect(startingRow?.detail).toContain("starting"); + expect(startingRow?.detail).not.toContain("connect ENOENT"); + + const dead = healthyInput(); + dead.brain = { + ...dead.brain, + running: false, + error: "connect ENOENT /tmp/ade.sock", + starting: false, + }; + const deadRow = evaluateDoctorRows(dead).find((row) => row.key === "brain"); + expect(deadRow?.status).toBe("fail"); + expect(deadRow?.detail).toContain("not responding"); + }); + it("names the credential store's two bad states with the right next step", () => { const lockedOut = healthyInput(); lockedOut.credentials = { diff --git a/apps/ade-cli/src/commands/doctor.ts b/apps/ade-cli/src/commands/doctor.ts index ed2be21fe..c0f110d75 100644 --- a/apps/ade-cli/src/commands/doctor.ts +++ b/apps/ade-cli/src/commands/doctor.ts @@ -21,6 +21,7 @@ import { type CredentialStoreHealth, } from "../services/credentials/credentialStore"; import { resolveMachineAdeLayout } from "../services/projects/machineLayout"; +import { readBrainStartupState } from "../services/runtime/brainStartupState"; import { DEFAULT_SYNC_HOST_PORT } from "../services/sync/syncProtocol"; import type { SyncListenerPortDiagnosis, @@ -45,6 +46,16 @@ export type DoctorRow = { export type DoctorBrainInput = { running: boolean; + /** + * The brain is not answering, but its service is registered and the brain + * process behind it is alive and young — it is still coming up. Reported as + * a warning, not a failure: this is the CLI's read of the desktop's + * `brain_starting` recovery state, and repairing here would only restart a + * booting brain. See `services/runtime/brainStartupState`. + */ + starting?: boolean; + /** Age of the starting brain in ms when known, for the row's wording. */ + startingAgeMs?: number | null; version: string | null; buildHash: string | null; pid: number | null; @@ -556,6 +567,18 @@ function appRow(input: DoctorInput["app"]): DoctorRow { function brainRow(input: DoctorBrainInput): DoctorRow { if (!input.running) { + // A registered, alive, young brain that has not bound its socket yet is + // starting, not broken. Calling that a failure is what sent people into a + // Repair that killed the booting brain and started the race over. + if (input.starting) { + const age = input.startingAgeMs != null ? ` (${compactDuration(input.startingAgeMs)} so far)` : ""; + return { + key: "brain", + label: "Brain", + status: "warn", + detail: `starting · the background service is up and its brain is coming up${age}; nothing to repair`, + }; + } return { key: "brain", label: "Brain", @@ -879,6 +902,11 @@ export async function runDoctorCommand( } const wedge = readBrainLoopWatchdogLastWedge(layout.runtimeDir) ?? brainProbe.runtimeLastWedge; + // Asked only of a brain that did not answer: a responding brain is running, + // never starting. + const startupState = brainProbe.brain.running + ? null + : await readBrainStartupState(); const input: DoctorInput = { nowMs, app: { @@ -887,7 +915,13 @@ export async function runDoctorCommand( path: installedApp.path, online, }, - brain: brainProbe.brain, + brain: startupState + ? { + ...brainProbe.brain, + starting: startupState.starting, + startingAgeMs: startupState.ageMs, + } + : brainProbe.brain, wedge, syncPort, portDiagnoses, diff --git a/apps/ade-cli/src/commands/reportIssue.test.ts b/apps/ade-cli/src/commands/reportIssue.test.ts new file mode 100644 index 000000000..422c0ce96 --- /dev/null +++ b/apps/ade-cli/src/commands/reportIssue.test.ts @@ -0,0 +1,153 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + buildCliDiagnosticReport, + buildReportIssuePayload, + openDiagnosticIssue, +} from "./reportIssue"; + +const tempDirs: string[] = []; + +function adeHome(analytics: Record | null): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "ade-report-issue-")); + tempDirs.push(dir); + fs.mkdirSync(path.join(dir, "secrets"), { recursive: true }); + if (analytics) { + fs.writeFileSync( + path.join(dir, "secrets", "product-analytics.json"), + JSON.stringify(analytics), + "utf8", + ); + } + return dir; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe("buildCliDiagnosticReport", () => { + it("reports the PostHog distinct_id and a prefilled issue URL", () => { + const built = buildCliDiagnosticReport({ + env: { ADE_HOME: adeHome({ identifiedUserHash: "hash-1", anonymousId: "anon-1" }) }, + cliVersion: "1.2.60", + now: () => new Date("2026-08-16T09:30:00.000Z"), + }); + + expect(built.installId).toBe("hash-1"); + expect(built.issueUrl.startsWith("https://github.com/")).toBe(true); + expect(built.issueUrl).toContain("/issues/new"); + expect(built.report).toContain("1.2.60"); + }); + + it("falls back to the anonymous id, then to 'unknown', without throwing", () => { + const anonymous = buildCliDiagnosticReport({ + env: { ADE_HOME: adeHome({ anonymousId: "anon-2", installationId: "install-2" }) }, + }); + // `installationId` is a different identifier: no PostHog event is attributed + // to it, so it must never be reported as the install id. + expect(anonymous.installId).toBe("anon-2"); + + const missing = buildCliDiagnosticReport({ env: { ADE_HOME: adeHome(null) } }); + expect(missing.installId).toBe("unknown"); + expect(missing.report.length).toBeGreaterThan(0); + }); + + it("omits the install id when analytics is switched off", () => { + const built = buildCliDiagnosticReport({ + env: { ADE_HOME: adeHome({ identifiedUserHash: "hash-3", anonymousId: "anon-3", enabled: false }) }, + }); + + expect(built.installId).toBe("unknown"); + expect(built.report).not.toContain("hash-3"); + expect(built.report).not.toContain("anon-3"); + }); + + it("omits the install id when the opt-out marker is on disk", () => { + const home = adeHome({ identifiedUserHash: "hash-4", anonymousId: "anon-4" }); + fs.writeFileSync(path.join(home, "secrets", "product-analytics.json.disabled"), "disabled\n", "utf8"); + + const built = buildCliDiagnosticReport({ env: { ADE_HOME: home } }); + + expect(built.installId).toBe("unknown"); + expect(built.report).not.toContain("hash-4"); + }); +}); + +describe("openDiagnosticIssue", () => { + it("puts the report on the clipboard before it opens the issue template", async () => { + // The template says "paste the report from your clipboard", so the copy has + // to have happened by the time the browser is opened -- otherwise the user + // lands on a form asking for something that is not on their clipboard. + const order: string[] = []; + let copiedText: string | null = null; + + const result = await openDiagnosticIssue( + { report: "REPORT BODY", issueUrl: "https://github.com/acme/ade/issues/new?title=x" }, + { + copy: (text) => { + order.push("copy"); + copiedText = text; + return true; + }, + open: async (url) => { + order.push(`open:${url}`); + }, + }, + ); + + expect(order).toEqual(["copy", "open:https://github.com/acme/ade/issues/new?title=x"]); + expect(copiedText).toBe("REPORT BODY"); + expect(result).toEqual({ copied: true, opened: true }); + }); + + it("still opens the issue when the machine has no clipboard, and survives a browserless box", async () => { + const result = await openDiagnosticIssue( + { report: "REPORT BODY", issueUrl: "https://github.com/acme/ade/issues/new" }, + { + copy: () => { + throw new Error("no pbcopy here"); + }, + open: async () => { + throw new Error("no browser here"); + }, + }, + ); + + expect(result).toEqual({ copied: false, opened: false }); + }); +}); + +describe("buildReportIssuePayload", () => { + const built = { + report: "REPORT BODY", + issueUrl: "https://github.com/acme/ade/issues/new", + installId: "install-abc", + }; + + it("tells a scripted caller whether --open actually reached the clipboard", () => { + // `--open` copies the report and then opens a template that says "paste the + // report from your clipboard". A box with no clipboard binary silently + // skips the copy, so the JSON has to say so -- otherwise the only way to + // find out is a user pasting nothing into a GitHub issue. + expect(buildReportIssuePayload(built, { copied: true })).toEqual({ + ok: true, + installId: "install-abc", + issueUrl: "https://github.com/acme/ade/issues/new", + copied: true, + report: "REPORT BODY", + }); + expect(buildReportIssuePayload(built, { copied: false }).copied).toBe(false); + }); + + it("reports nothing copied when --open was not asked for", () => { + const payload = buildReportIssuePayload(built, null); + + expect(payload.copied).toBe(false); + // The rest of the contract is unchanged: same keys, same `ok`. + expect(payload.ok).toBe(true); + expect(payload.issueUrl).toBe(built.issueUrl); + }); +}); diff --git a/apps/ade-cli/src/commands/reportIssue.ts b/apps/ade-cli/src/commands/reportIssue.ts new file mode 100644 index 000000000..8ccde0bfc --- /dev/null +++ b/apps/ade-cli/src/commands/reportIssue.ts @@ -0,0 +1,173 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + buildDiagnosticIssueUrl, + buildDiagnosticReport, +} from "../services/diagnostics/diagnosticReport"; +import { + collectMachineDiagnosticSources, + readDiagnosticJsonFile, +} from "../services/diagnostics/diagnosticSources"; +import { copyToClipboard } from "../lib/clipboard"; +import { openExternalUrl } from "../lib/externalLinks"; + +/** + * Headless counterpart to the desktop "Report issue" button. Reads only local + * files — it never starts or contacts the brain — so it still works on the + * machine where ADE itself will not come up, and on Windows where there is no + * desktop error screen to press. + */ + +export type ReportIssueOptions = { + surface?: string; + projectRoot?: string | null; + cliVersion?: string | null; + env?: NodeJS.ProcessEnv; + now?: () => Date; +}; + +export type ReportIssueResult = { + report: string; + issueUrl: string; + installId: string; +}; + +/** + * The same PostHog `distinct_id` the desktop reports, read without writing. + * + * Null whenever analytics is off — the `.disabled` marker the desktop writes, + * or `enabled: false` in the state itself. No event carries the id then, so it + * correlates to nothing, and printing it into a report the user is about to + * paste into a public issue is the opposite of the choice they made. + */ +function readInstallId(secretsDir: string): string | null { + const statePath = path.join(secretsDir, "product-analytics.json"); + if (fs.existsSync(`${statePath}.disabled`)) return null; + const state = readDiagnosticJsonFile(statePath); + if (!state || typeof state !== "object") return null; + const record = state as Record; + if (record.enabled === false) return null; + // Only the two keys PostHog actually uses as `distinct_id`; `installationId` + // is a different identifier and would make the CLI report an id no event in + // PostHog is ever attributed to. + for (const key of ["identifiedUserHash", "anonymousId"]) { + const value = record[key]; + if (typeof value === "string" && value.trim()) return value.trim(); + } + return null; +} + +export function buildCliDiagnosticReport(options: ReportIssueOptions = {}): ReportIssueResult { + const env = options.env ?? process.env; + const at = options.now?.() ?? new Date(); + const projectRoot = options.projectRoot?.trim() || null; + const surface = options.surface?.trim() || "cli"; + + const sources = collectMachineDiagnosticSources({ + env, + projectRoot, + includeProjectCliLog: true, + }); + const installId = readInstallId(sources.layout.secretsDir) ?? "unknown"; + + const report = buildDiagnosticReport({ + generatedAt: at.toISOString(), + app: { + version: options.cliVersion ?? null, + packageChannel: env.ADE_PACKAGE_CHANNEL?.trim() || null, + isPackaged: null, + platform: process.platform, + arch: process.arch, + osRelease: os.release(), + nodeVersion: process.versions.node ?? null, + timezoneOffsetMinutes: -at.getTimezoneOffset(), + }, + identity: { installId }, + context: { + surface, + headline: null, + code: null, + technicalDetail: null, + projectRoot, + }, + state: sources.state, + storage: sources.storage, + logs: sources.logs, + notes: sources.notes, + redaction: sources.redaction, + }); + + return { + report, + installId, + issueUrl: buildDiagnosticIssueUrl({ + surface, + appVersion: options.cliVersion ?? null, + platform: process.platform, + arch: process.arch, + installId, + redaction: sources.redaction, + }), + }; +} + +/** + * The side effects of `ade report-issue --open`, in the order the desktop + * "Report issue" button does them: the report goes on the clipboard, *then* + * the prefilled GitHub issue opens. + * + * The order is the whole point. The template the URL carries says "paste the + * report from your clipboard" — the URL itself only holds a stub, because a + * full report does not fit in a query string. A flow that opens that form + * without copying anything sends the user to a page asking for something that + * is not there (and whatever unrelated text they had copied is what they would + * paste). Both steps are best effort: a box with no clipboard binary and no + * browser still gets the whole report on stdout. + */ +export async function openDiagnosticIssue( + built: Pick, + deps: { + copy?: (text: string) => boolean; + open?: (url: string) => Promise; + } = {}, +): Promise<{ copied: boolean; opened: boolean }> { + const copy = deps.copy ?? copyToClipboard; + const open = deps.open ?? openExternalUrl; + let copied = false; + try { + copied = copy(built.report); + } catch { + copied = false; + } + let opened = false; + try { + await open(built.issueUrl); + opened = true; + } catch { + // Headless boxes have no browser; the caller still prints the URL. + opened = false; + } + return { copied, opened }; +} + +/** + * The `--json` shape of `ade report-issue`. `copied` is here because `--open` + * has two side effects, and a script that asked for machine-readable output + * could not tell whether the second one happened: on a box with no clipboard + * binary the report is only on stdout, and a caller that assumed otherwise + * would tell its user to paste something that is not there. Without `--open` + * nothing is copied, so it is simply false. + */ +export function buildReportIssuePayload( + built: Pick, + side: { copied: boolean } | null, +): { ok: true; installId: string; issueUrl: string; copied: boolean; report: string } { + return { + ok: true, + installId: built.installId, + issueUrl: built.issueUrl, + copied: side?.copied ?? false, + report: built.report, + }; +} diff --git a/apps/ade-cli/src/commands/setup.test.ts b/apps/ade-cli/src/commands/setup.test.ts index c845be9b9..5b49437b8 100644 --- a/apps/ade-cli/src/commands/setup.test.ts +++ b/apps/ade-cli/src/commands/setup.test.ts @@ -521,6 +521,64 @@ describe("runSetupCommand", () => { expect(chunks.join("")).toContain("What's left"); }); + it("waits out a still-starting background service instead of calling it broken", async () => { + // The regression this pins: a brain that is registered, alive and simply + // not answering yet used to surface as a failed install ("installed but + // not running" / "sign-in didn't finish") with a Repair button attached. + const chunks: string[] = []; + let budget = 0; + const result = await runSetupCommand(["--continue", "--no-desktop"], deps({ + reporter: reporter(chunks), + awaitRuntimeService: async ({ budgetMs, onStarting }) => { + budget = budgetMs; + onStarting(); + return { ready: false, starting: true, detail: "still starting" }; + }, + getAccountStatus: async () => ({ signedIn: false, identity: null }), + runConnect: async () => ({ ok: false, detail: "sign-in didn't finish" }), + verify: async () => ({ + ok: false, + detail: "the ADE brain is not running", + nextAction: "ade brain start", + }), + })); + + expect(budget).toBeGreaterThanOrEqual(60_000); + const account = result.steps.find((s) => s.id === "account"); + expect(account?.state).toBe("skipped"); + expect(account?.detail).toContain("still starting"); + expect(account?.nextAction).toBe("ade connect"); + // No step failed, so the install is not reported as one. + expect(result.ok).toBe(true); + const output = chunks.join(""); + expect(output).toContain("Starting ADE's background service"); + expect(output).not.toContain("the ADE brain is not running"); + }); + + it("still reports a real failure once the background service is answering", async () => { + const chunks: string[] = []; + const result = await runSetupCommand(["--continue", "--no-desktop"], deps({ + reporter: reporter(chunks), + awaitRuntimeService: async () => ({ + ready: true, + starting: false, + detail: "background service is running", + }), + getAccountStatus: async () => ({ signedIn: false, identity: null }), + runConnect: async () => ({ ok: false, detail: "sign-in didn't finish" }), + verify: async () => ({ + ok: false, + detail: "sign-in didn't finish", + nextAction: "ade connect", + }), + })); + + expect(result.ok).toBe(false); + expect(result.steps.find((s) => s.id === "account")?.state).toBe("failed"); + // Nothing was waited on, so nothing announced a wait. + expect(chunks.join("")).not.toContain("Starting ADE's background service"); + }); + it("fails verification for a signed-in machine that never reached the account", async () => { // The exact state a clean install lands in: brain healthy, account signed // in, machine absent from the account directory because no project is diff --git a/apps/ade-cli/src/commands/setup.ts b/apps/ade-cli/src/commands/setup.ts index 0def961e4..3cc9ca90b 100644 --- a/apps/ade-cli/src/commands/setup.ts +++ b/apps/ade-cli/src/commands/setup.ts @@ -17,6 +17,7 @@ * step and the run continues, because a failed sign-in must not cost the user * the desktop app. Nothing here reports success it did not achieve. */ +import { RUNTIME_SERVICE_START_WAIT_MS } from "../serviceManager/runtimeServiceBudgets"; import { SetupReporter, detectTerminalCapabilities, @@ -95,6 +96,27 @@ export type SetupStepResult = { nextAction?: string; }; +/** Outcome of waiting for the machine brain's endpoint to answer. */ +export type SetupServiceReadiness = { + /** The endpoint answered. */ + ready: boolean; + /** + * The service is registered with a live brain that has simply not answered + * yet. Nothing downstream may call that a failed install. + */ + starting: boolean; + detail: string; +}; + +/** + * How long a brain the service installer reported as `starting` gets to answer + * before setup stops waiting on it. The desktop's own post-install wait, by + * definition -- a cold machine opening a large project database routinely needs + * more than the ten seconds that used to be on offer, and reporting that as + * "installed but not running" sent people hunting a fault that did not exist. + */ +export const SETUP_SERVICE_START_BUDGET_MS = RUNTIME_SERVICE_START_WAIT_MS; + export type SetupDeps = { platform?: NodeJS.Platform; env?: NodeJS.ProcessEnv; @@ -106,6 +128,16 @@ export type SetupDeps = { ensureAgentTools: ( onProgress: (progress: SetupProgress) => void, ) => Promise; + /** + * Registers the machine brain service if needed and waits for its endpoint. + * Optional so callers that do not own a brain (tests, `--no-*` paths) can + * omit it; `onStarting` fires once, the first time the wait actually begins, + * so a fast install prints no line at all. + */ + awaitRuntimeService?: (args: { + budgetMs: number; + onStarting: () => void; + }) => Promise; getAccountStatus: () => Promise; runConnect: () => Promise; readInstalledDesktop: () => { version: string | null; path: string | null }; @@ -298,6 +330,30 @@ export async function runSetupCommand( } reporter.completeStep(toolsStep); + // --- background service ---------------------------------------------------- + // Everything below needs a brain that answers. A brain that is still coming + // up is not a broken one, so this waits it out rather than letting the + // account step fail against an endpoint that was never given time to open. + let serviceStarting = false; + if (deps.awaitRuntimeService) { + try { + let announced = false; + const readiness = await deps.awaitRuntimeService({ + budgetMs: SETUP_SERVICE_START_BUDGET_MS, + onStarting: () => { + if (announced) return; + announced = true; + reporter.line(" Starting ADE's background service..."); + }, + }); + serviceStarting = readiness.starting && !readiness.ready; + } catch { + // The wait is a courtesy, not a gate: its failure must not cost the user + // the account and desktop steps that follow. + serviceStarting = false; + } + } + // --- step: account --------------------------------------------------------- try { await runAccountStep({ step: accountStep, ask, interactive, deps }); @@ -306,6 +362,14 @@ export async function runSetupCommand( accountStep.detail = describeError(error); accountStep.nextAction = "ade connect"; } + // A brain that is still starting is the reason this step could not finish, + // and "sign-in didn't finish" is not a true account of that. Say what is + // actually happening and leave the same recovery command. + if (serviceStarting && accountStep.state === "failed") { + accountStep.state = "skipped"; + accountStep.detail = "ADE's background service is still starting"; + accountStep.nextAction = "ade connect"; + } reporter.completeStep(accountStep); // --- step: desktop app ----------------------------------------------------- @@ -348,7 +412,9 @@ export async function runSetupCommand( const nextAction = result.nextAction ?? "ade connect"; const alreadyOffered = accountStep.state === "skipped" && accountStep.nextAction === nextAction; - if (!result.ok && accountStep.state !== "failed" && !alreadyOffered) { + // `serviceStarting` is the one case where a failed check is not a failed + // install: the brain is alive and coming up, and the installer said so. + if (!result.ok && !serviceStarting && accountStep.state !== "failed" && !alreadyOffered) { accountStep.state = "failed"; accountStep.detail = result.detail; accountStep.nextAction = nextAction; @@ -357,6 +423,12 @@ export async function runSetupCommand( verified = false; } + if (serviceStarting) { + reporter.line( + " ADE's background service is still starting. Give it a moment, then run `ade connect`.", + ); + } + const totals: SetupTotals = { elapsedMs: options.elapsedMs + (now() - startedAt), downloadedBytes, diff --git a/apps/desktop/src/main/services/shared/externalLinks.test.ts b/apps/ade-cli/src/lib/externalLinks.test.ts similarity index 100% rename from apps/desktop/src/main/services/shared/externalLinks.test.ts rename to apps/ade-cli/src/lib/externalLinks.test.ts diff --git a/apps/ade-cli/src/lib/externalLinks.ts b/apps/ade-cli/src/lib/externalLinks.ts new file mode 100644 index 000000000..5629ea9d4 --- /dev/null +++ b/apps/ade-cli/src/lib/externalLinks.ts @@ -0,0 +1,87 @@ +import { execFile } from "node:child_process"; +import { + resolveTrustedWindowsTool, + trustedWindowsToolKernelPath, +} from "./trustedWindowsTools"; + +const ALLOWED_EXTERNAL_URL_SCHEMES = new Set(["http:", "https:", "mailto:"]); +const OPEN_TIMEOUT_MS = 5_000; + +export function normalizeExternalUrl(url: string | undefined | null): string | null { + const raw = typeof url === "string" ? url.trim() : ""; + if (!raw) return null; + + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + throw new Error("Invalid URL"); + } + + if (!ALLOWED_EXTERNAL_URL_SCHEMES.has(parsed.protocol)) { + throw new Error("Only http(s) and mailto: URLs are allowed."); + } + + return parsed.toString(); +} + +function execFileOpen(file: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile(file, args, { timeout: OPEN_TIMEOUT_MS, windowsHide: true }, (error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + +function windowsRundll32Path(): string { + try { + return resolveTrustedWindowsTool("rundll32"); + } catch { + // Cross-platform unit tests mock `process.platform` to win32 on macOS/Linux, + // where the kernel SystemRoot alias cannot be canonicalized. The kernel + // path is still the command we would spawn; execFile is mocked in those tests. + return trustedWindowsToolKernelPath("rundll32"); + } +} + +function openWithPlatformHelper(url: string): Promise { + if (process.platform === "darwin") { + return execFileOpen("/usr/bin/open", [url]); + } + if (process.platform === "win32") { + return execFileOpen(windowsRundll32Path(), ["url.dll,FileProtocolHandler", url]); + } + return execFileOpen("/usr/bin/xdg-open", [url]); +} + +async function openWithElectronShell(url: string): Promise { + try { + // Dual-runtime: ADE CLI bundles this module, and a static + // `import { shell } from "electron"` crashes headless startup with + // "does not provide an export named 'shell'". Load Electron only when the + // OS opener failed and we are actually in the desktop process. + const electron = await import("electron"); + if (electron.shell?.openExternal) { + await electron.shell.openExternal(url); + return; + } + } catch { + // Not running inside Electron. + } + throw new Error("No external URL opener is available."); +} + +export async function openExternalUrl(url: string | undefined | null): Promise { + const normalized = normalizeExternalUrl(url); + if (!normalized) return; + + try { + await openWithPlatformHelper(normalized); + } catch { + await openWithElectronShell(normalized); + } +} diff --git a/apps/ade-cli/src/serviceManager/common.test.ts b/apps/ade-cli/src/serviceManager/common.test.ts index af4a2a63f..dd2545c6e 100644 --- a/apps/ade-cli/src/serviceManager/common.test.ts +++ b/apps/ade-cli/src/serviceManager/common.test.ts @@ -1,4 +1,6 @@ import fs from "node:fs"; +import net from "node:net"; +import { EventEmitter } from "node:events"; import { createHash } from "node:crypto"; import { spawnSync as spawnChildSync } from "node:child_process"; import os from "node:os"; @@ -9,6 +11,7 @@ import { buildWindowsParentPidQueryArgs, buildWindowsProcessCommandLineQueryArgs, listStaleChannelServePids, + parsePsElapsedMs, isCurrentProcessDescendantOfPid, isStaleChannelServeCommandLine, PARENT_PID_UNKNOWN, @@ -55,8 +58,15 @@ import { renderLaunchdPlist, uninstallLaunchdService, } from "./installLaunchd"; -import { resolveWatchdogServiceName } from "./installLaunchdWatchdog"; -import { installSystemdService, renderSystemdEnvironment, renderSystemdUnit, servicePath as systemdServicePath } from "./installSystemd"; +import { requestAdeRuntimeShutdown } from "./runtimeShutdownRequest"; +import { + installLaunchdWatchdogAgent, + renderWatchdogLaunchdPlist, + resolveWatchdogServiceName, + uninstallLaunchdWatchdogAgent, + watchdogCommand, + watchdogLaunchAgentPath, +} from "./installLaunchdWatchdog"; import { isWindowsTaskStateRunning } from "./installWindows"; const originalArgv = [...process.argv]; @@ -737,6 +747,21 @@ describe("launchd service rendering", () => { }); }); +describe("parsePsElapsedMs", () => { + it("reads every `ps -o etime=` shape", () => { + expect(parsePsElapsedMs("00:05")).toBe(5_000); + expect(parsePsElapsedMs(" 12:34\n")).toBe((12 * 60 + 34) * 1_000); + expect(parsePsElapsedMs("01:02:03")).toBe(((1 * 60 + 2) * 60 + 3) * 1_000); + expect(parsePsElapsedMs("2-01:02:03")).toBe((((2 * 24 + 1) * 60 + 2) * 60 + 3) * 1_000); + }); + + it("fails open on anything it does not recognise", () => { + expect(parsePsElapsedMs("")).toBeNull(); + expect(parsePsElapsedMs("garbage")).toBeNull(); + expect(parsePsElapsedMs("1:2:3:4")).toBeNull(); + }); +}); + describe("launchd service install", () => { const serviceCommand: AdeServiceCommand = { command: "/Applications/ADE.app/Contents/MacOS/ade", @@ -747,6 +772,10 @@ describe("launchd service install", () => { deps: NonNullable[0]>, ) => installLaunchdService({ responsivenessProbe: () => true, + // Unknown age by default, so a running-but-quiet agent takes the restart + // path these tests were written for; the young-brain tests inject an age. + pidElapsedMs: () => null, + recentCrashLoop: () => false, ...deps, }); @@ -875,6 +904,199 @@ describe("launchd service install", () => { }); }); + it("reports a live replacement that has not answered yet as starting, not failed", async () => { + const homeDir = makeTempHome("ade-launchd-handover-starting-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + // The handover poll sees launchd's replacement child running. + { status: 0, stdout: "state = running\npid = 4321\n", stderr: "" }, + ]); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + responsivenessProbe: () => false, + handoverPidAlive: (pid) => pid === 4321, + handoverTimeoutMs: 0, + }); + + expect(result).toMatchObject({ + ok: true, + starting: true, + action: "install", + }); + expect(result.failureStep).toBeUndefined(); + expect(result.message).toContain("still starting"); + }); + + it("waits for a young unresponsive brain instead of restarting it", async () => { + const homeDir = makeTempHome("ade-launchd-young-brain-"); + const servicePath = launchAgentPath(homeDir); + fs.mkdirSync(path.dirname(servicePath), { recursive: true }); + fs.writeFileSync(servicePath, renderLaunchdPlist(serviceCommand, homeDir), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = runningAgentSpawn(calls, 1234); + // Not answering on the first probe, answering once waited for. + const responsivenessProbe = vi.fn() + .mockReturnValueOnce(false) + .mockReturnValueOnce(true); + const kill = vi.fn(); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + env: { ...process.env, ADE_FORCE_RUNTIME_SERVICE_RESTART: "1" }, + responsivenessProbe, + pidElapsedMs: () => 5_000, + handoverPidAlive: () => true, + terminateDeps: { kill, pidAlive: () => true }, + }); + + expect(result.ok).toBe(true); + expect(result.starting).toBeUndefined(); + expect(kill).not.toHaveBeenCalled(); + // No unload/load of the brain agent: only the watchdog is (re)armed. + expect(calls.filter((call) => call.command === "launchctl" && call.args[1] === servicePath)).toEqual([]); + }); + + it("restarts a young quiet brain anyway when the machine is crash-looping", async () => { + const homeDir = makeTempHome("ade-launchd-young-crashloop-"); + const servicePath = launchAgentPath(homeDir); + fs.mkdirSync(path.dirname(servicePath), { recursive: true }); + fs.writeFileSync(servicePath, renderLaunchdPlist(serviceCommand, homeDir), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 0, stdout: "state = running\npid = 1234\n", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + ]); + const kill = vi.fn(); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + env: { ...process.env, ADE_FORCE_RUNTIME_SERVICE_RESTART: "1" }, + responsivenessProbe: () => true, + pidElapsedMs: () => 5_000, + recentCrashLoop: () => true, + currentPid: 9999, + parentPid: () => null, + terminateDeps: { kill, pidAlive: () => false }, + }); + + expect(result).toMatchObject({ ok: true, restarted: true }); + expect(calls.map((call) => call.args[0])).toContain("load"); + }); + + it("does not restart a young brain that answers, even when a restart was forced", async () => { + const homeDir = makeTempHome("ade-launchd-young-answering-"); + const servicePath = launchAgentPath(homeDir); + fs.mkdirSync(path.dirname(servicePath), { recursive: true }); + fs.writeFileSync(servicePath, renderLaunchdPlist(serviceCommand, homeDir), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = runningAgentSpawn(calls, 1234); + const kill = vi.fn(); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + env: { ...process.env, ADE_FORCE_RUNTIME_SERVICE_RESTART: "1" }, + responsivenessProbe: () => true, + pidElapsedMs: () => 5_000, + handoverPidAlive: () => true, + terminateDeps: { kill, pidAlive: () => true }, + }); + + expect(result.ok).toBe(true); + // Not a restart: the trust-reset caller must see that and try again later. + expect(result.restarted).toBeUndefined(); + expect(kill).not.toHaveBeenCalled(); + }); + + it("returns starting for a young brain that is still quiet after the wait", async () => { + const homeDir = makeTempHome("ade-launchd-young-brain-quiet-"); + const servicePath = launchAgentPath(homeDir); + fs.mkdirSync(path.dirname(servicePath), { recursive: true }); + fs.writeFileSync(servicePath, renderLaunchdPlist(serviceCommand, homeDir), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = runningAgentSpawn(calls, 1234); + const kill = vi.fn(); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + responsivenessProbe: () => false, + pidElapsedMs: () => 5_000, + handoverPidAlive: () => true, + handoverTimeoutMs: 0, + terminateDeps: { kill, pidAlive: () => true }, + }); + + expect(result).toMatchObject({ ok: true, starting: true }); + expect(kill).not.toHaveBeenCalled(); + }); + + // Regression: the young-brain wait and the real handover used to share one + // install-wide deadline. A young brain that died late in its wait left the + // restart that followed with ~0 ms, so a replacement launchd had not named + // yet was reported as a `replacement_pid` handover failure. + it("gives the restart a full handover window after a young brain dies mid-wait", async () => { + const homeDir = makeTempHome("ade-launchd-young-died-"); + const servicePath = launchAgentPath(homeDir); + fs.mkdirSync(path.dirname(servicePath), { recursive: true }); + fs.writeFileSync(servicePath, renderLaunchdPlist(serviceCommand, homeDir), "utf8"); + + let loadSeen = false; + let printsAfterLoad = 0; + const spawnSync: ServiceManagerSpawnSync = (command, args) => { + if (command === "launchctl" && args[0] === "load") { + loadSeen = true; + return { status: 0, stdout: "", stderr: "" }; + } + if (command === "launchctl" && args[0] === "print") { + if (!loadSeen) return { status: 0, stdout: "state = running\npid = 1234\n", stderr: "" }; + // launchd has not named the replacement yet for the first few polls. + printsAfterLoad += 1; + if (printsAfterLoad <= 3) return { status: 1, stdout: "", stderr: "not found" }; + return { status: 0, stdout: "state = running\npid = 5678\n", stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; + }; + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + responsivenessProbe: () => false, + // Young enough to wait for; it dies during the wait. + pidElapsedMs: () => 5_000, + handoverPidAlive: (pid) => pid !== 1234, + handoverTimeoutMs: 300, + handoverPollMs: 10, + terminateDeps: { kill: vi.fn(), pidAlive: () => false }, + // This test's spawn stub answers only `launchctl`, so the ancestry probe + // would read an empty parent list and, on a host whose backend reports + // "unknown" for that, fail safe into the self-mutation block. Inject the + // chain like every other install test here: this case is about handover + // windows, not about who our parent is. + currentPid: 9999, + parentPid: () => null, + }); + + expect(result).toMatchObject({ ok: true, starting: true, restarted: true }); + expect(result.message).toContain("5678"); + }); + it("reloads an unchanged running launch agent when a packaged trust reset requests it", async () => { const homeDir = makeTempHome("ade-launchd-trust-reset-"); const servicePath = launchAgentPath(homeDir); @@ -1253,144 +1475,268 @@ describe("launchd service install", () => { }); }); -describe("systemd service rendering", () => { - it("renders the user service path under the home directory", () => { - expect(systemdServicePath("/home/example")).toBe( - path.join("/home/example", ".config", "systemd", "user", `${ADE_RUNTIME_SERVICE_NAME}.service`), - ); +/** launchd that keeps reporting one running agent child, whatever is asked of it. */ +function runningAgentSpawn( + calls: Array<{ command: string; args: string[] }>, + pid: number, +): ServiceManagerSpawnSync { + return (command, args) => { + calls.push({ command, args }); + if (command === "launchctl" && args[0] === "print") { + return { status: 0, stdout: `state = running\npid = ${pid}\n`, stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; + }; +} + +function spawnSequence( + calls: Array<{ command: string; args: string[] }>, + results: ServiceManagerProcessResult[], +): ServiceManagerSpawnSync { + let loadSeen = false; + return (command, args) => { + const next = results.shift(); + if ( + !next + && loadSeen + && command === "launchctl" + && args[0] === "print" + ) { + return { status: 0, stdout: "state = running\npid = 7777\n", stderr: "" }; + } + calls.push({ command, args }); + if (command === "launchctl" && args[0] === "load") loadSeen = true; + return next ?? { status: 0, stdout: "", stderr: "" }; + }; +} + +const watchdogServiceCommand: AdeServiceCommand = { + command: "/usr/local/bin/node", + args: ["/opt/ade/cli.cjs", "serve"], + env: { ADE_HOME: "/Users/example/.ade" }, +}; + +describe("resolveWatchdogServiceName", () => { + it("keeps each channel on its own watchdog", () => { + expect(resolveWatchdogServiceName("com.ade.runtime")).toBe("com.ade.watchdog"); + expect(resolveWatchdogServiceName("com.ade.runtime.beta")).toBe("com.ade.watchdog.beta"); + expect(resolveWatchdogServiceName("com.example.custom")).toBe("com.example.custom.watchdog"); }); +}); - it("renders unit content with quoted ExecStart and escaped environment values", () => { - const unit = renderSystemdUnit({ - command: "/opt/ADE CLI/node", - args: ["/opt/ade/cli.cjs", "serve"], +describe("watchdogCommand", () => { + it("runs the same binary the brain was installed from", () => { + expect(watchdogCommand(watchdogServiceCommand)).toEqual({ + command: "/usr/local/bin/node", + args: ["/opt/ade/cli.cjs", "runtime", "watchdog-check"], env: { - NODE_PATH: "/tmp/100%/node modules", - ADE_HOME: "/home/example/ade path\\with\"quotes", + ADE_HOME: "/Users/example/.ade", + ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1", }, }); - - expect(unit).toContain("Description=ADE runtime service"); - expect(unit).toContain("Type=simple"); - expect(unit).toContain("ExecStart='/opt/ADE CLI/node' '/opt/ade/cli.cjs' 'serve'"); - expect(unit).toContain("Restart=always"); - expect(unit).toContain("Environment=\"NODE_PATH=/tmp/100%%/node modules\""); - expect(unit).toContain("Environment=\"ADE_HOME=/home/example/ade path\\\\with\\\"quotes\""); - expect(unit).toContain("WantedBy=default.target"); }); - it("quotes systemd environment assignments for whitespace, backslashes, quotes, and percent signs", () => { - expect(renderSystemdEnvironment("NODE_PATH", "C:\\ADE deps\\100% \"runtime\"")).toBe( - "Environment=\"NODE_PATH=C:\\\\ADE deps\\\\100%% \\\"runtime\\\"\"", - ); + it("appends the check when the command has no serve argument", () => { + expect(watchdogCommand({ command: "/opt/ade/ade", args: [] }).args) + .toEqual(["runtime", "watchdog-check"]); }); }); -describe("systemd service install", () => { - const serviceCommand: AdeServiceCommand = { - command: "/opt/ade/bin/ade", - args: ["serve"], - env: { NODE_PATH: "/opt/ade/node_modules" }, - }; +describe("renderWatchdogLaunchdPlist", () => { + it("runs on an interval and never keeps itself alive", () => { + const plist = renderWatchdogLaunchdPlist({ + command: watchdogServiceCommand, + homeDir: "/Users/example", + }); + expect(plist).toContain("com.ade.watchdog"); + expect(plist).toContain("StartInterval"); + expect(plist).toContain("60"); + expect(plist).toContain("watchdog-check"); + // KeepAlive would make launchd respawn a one-shot check in a tight loop. + expect(plist).not.toContain("KeepAlive"); + }); - it("writes the user unit and enables it immediately", () => { - const homeDir = makeTempHome("ade-systemd-install-"); - const targetPath = systemdServicePath(homeDir); + it("refuses an interval short enough to thrash", () => { + const plist = renderWatchdogLaunchdPlist({ + command: watchdogServiceCommand, + homeDir: "/Users/example", + startIntervalSeconds: 1, + }); + expect(plist).toContain("15"); + }); +}); + +describe("installLaunchdWatchdogAgent", () => { + it("writes and loads the agent", () => { + const homeDir = makeTempHome("ade-watchdog-home-"); const calls: Array<{ command: string; args: string[] }> = []; - const spawnSync = spawnSequence(calls, [ - { status: 0, stdout: "", stderr: "" }, - { status: 0, stdout: "", stderr: "" }, - ]); + const result = installLaunchdWatchdogAgent({ + command: watchdogServiceCommand, + homeDir, + spawnSync: spawnSequence(calls, []), + }); - const result = installSystemdService({ command: serviceCommand, spawnSync, homeDir }); + const servicePath = watchdogLaunchAgentPath(homeDir); + expect(result.installed).toBe(true); + expect(fs.existsSync(servicePath)).toBe(true); + expect(calls.map((call) => call.args[0])).toEqual(["unload", "load"]); + }); - expect(result).toMatchObject({ - ok: true, - serviceName: ADE_RUNTIME_SERVICE_NAME, - action: "install", - path: targetPath, + it("reports a load failure instead of claiming the agent is armed", () => { + const homeDir = makeTempHome("ade-watchdog-home-"); + const result = installLaunchdWatchdogAgent({ + command: watchdogServiceCommand, + homeDir, + spawnSync: (command, args) => + args[0] === "load" + ? { status: 1, stdout: "", stderr: "Load failed" } + : { status: 0, stdout: "", stderr: "" }, }); - expect(fs.readFileSync(targetPath, "utf8")).toBe(renderSystemdUnit(serviceCommand)); - expect(calls).toEqual([ - { command: "systemctl", args: ["--user", "daemon-reload"] }, - { command: "systemctl", args: ["--user", "enable", "--now", `${ADE_RUNTIME_SERVICE_NAME}.service`] }, - { command: "systemctl", args: ["--user", "restart", `${ADE_RUNTIME_SERVICE_NAME}.service`] }, - ]); + expect(result.installed).toBe(false); }); - it("does not enable when daemon-reload fails", () => { - const homeDir = makeTempHome("ade-systemd-reload-fail-"); - const calls: Array<{ command: string; args: string[] }> = []; - const spawnSync = spawnSequence(calls, [ - { status: 1, stdout: "", stderr: "reload failed" }, - ]); + it("removes the agent with the brain it guards", () => { + const homeDir = makeTempHome("ade-watchdog-home-"); + installLaunchdWatchdogAgent({ + command: watchdogServiceCommand, + homeDir, + spawnSync: spawnSequence([], []), + }); + const servicePath = watchdogLaunchAgentPath(homeDir); + expect(fs.existsSync(servicePath)).toBe(true); - const result = installSystemdService({ command: serviceCommand, spawnSync, homeDir }); + const calls: Array<{ command: string; args: string[] }> = []; + uninstallLaunchdWatchdogAgent({ homeDir, spawnSync: spawnSequence(calls, []) }); - expect(result.ok).toBe(false); - expect(result.message).toBe("reload failed"); - expect(calls).toEqual([ - { command: "systemctl", args: ["--user", "daemon-reload"] }, - ]); + expect(fs.existsSync(servicePath)).toBe(false); + expect(calls.map((call) => call.args[0])).toEqual(["bootout", "unload"]); }); +}); - it("surfaces enable failures after a successful reload", () => { - const homeDir = makeTempHome("ade-systemd-enable-fail-"); - const calls: Array<{ command: string; args: string[] }> = []; - const spawnSync = spawnSequence(calls, [ - { status: 0, stdout: "", stderr: "" }, - { status: 1, stdout: "", stderr: "enable failed" }, - ]); +/** + * A stand-in for the brain's JSON-RPC endpoint. `replies` maps a method to the + * result it answers with; anything absent is simply not answered, which is how + * a wedged brain behaves. `closeOnShutdown` models the brain whose orderly exit + * drops the socket before its own response gets out. + */ +function fakeEndpoint( + replies: Record, + options: { closeOnShutdown?: boolean } = {}, +): { + socket: net.Socket; + written: string[]; +} { + const written: string[] = []; + const socket = new EventEmitter() as unknown as net.Socket & { destroy: () => void }; + let buffer = ""; + (socket as unknown as { write: unknown }).write = (payload: string) => { + written.push(payload); + buffer += payload; + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline < 0) break; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + const message = JSON.parse(line) as { id: number; method: string }; + if (options.closeOnShutdown && message.method === "shutdown") { + queueMicrotask(() => socket.emit("close")); + continue; + } + if (!(message.method in replies)) continue; + queueMicrotask(() => { + socket.emit( + "data", + Buffer.from(`${JSON.stringify({ jsonrpc: "2.0", id: message.id, result: replies[message.method] })}\n`), + ); + }); + } + return true; + }; + (socket as unknown as { destroy: () => void }).destroy = () => {}; + queueMicrotask(() => socket.emit("connect")); + return { socket, written }; +} - const result = installSystemdService({ command: serviceCommand, spawnSync, homeDir }); +describe("requestAdeRuntimeShutdown", () => { + const socketPath = String.raw`\\.\pipe\ade-runtime-stable-0123456789abcdef`; - expect(result.ok).toBe(false); - expect(result.message).toBe("enable failed"); - expect(calls.map((call) => call.args)).toEqual([ - ["--user", "daemon-reload"], - ["--user", "enable", "--now", `${ADE_RUNTIME_SERVICE_NAME}.service`], - ]); + it("identifies the endpoint before asking it to leave", async () => { + const endpoint = fakeEndpoint({ "runtime/info": { pid: 4242 }, shutdown: {} }); + const result = await requestAdeRuntimeShutdown({ + pid: 4242, + socketPath, + connect: () => endpoint.socket, + }); + expect(result).toEqual({ requested: true }); + const methods = endpoint.written.map((line) => (JSON.parse(line) as { method: string }).method); + expect(methods).toEqual(["runtime/info", "shutdown"]); }); - it("surfaces restart failures after enabling the user unit", () => { - const homeDir = makeTempHome("ade-systemd-restart-fail-"); - const calls: Array<{ command: string; args: string[] }> = []; - const spawnSync = spawnSequence(calls, [ - { status: 0, stdout: "", stderr: "" }, - { status: 0, stdout: "", stderr: "" }, - { status: 1, stdout: "", stderr: "restart failed" }, - ]); + it("refuses to shut down a pid the endpoint does not belong to", async () => { + // A pid scraped from a port diagnosis or a supervisor record can have been + // recycled; shutting down whoever happens to answer would be a stranger. + const endpoint = fakeEndpoint({ "runtime/info": { pid: 999 }, shutdown: {} }); + const result = await requestAdeRuntimeShutdown({ + pid: 4242, + socketPath, + connect: () => endpoint.socket, + }); + expect(result.requested).toBe(false); + const methods = endpoint.written.map((line) => (JSON.parse(line) as { method: string }).method); + expect(methods).toEqual(["runtime/info"]); + }); - const result = installSystemdService({ command: serviceCommand, spawnSync, homeDir }); + /** + * The orderly exit this path asks for tears down the brain's listening + * socket, which races the JSON-RPC response back to us. Reading the close as + * a refusal would send the caller to `taskkill /F` and cut the flush short. + */ + it("treats a close after the request as the shutdown taking effect", async () => { + const endpoint = fakeEndpoint({ "runtime/info": { pid: 4242 } }, { closeOnShutdown: true }); + const result = await requestAdeRuntimeShutdown({ + pid: 4242, + socketPath, + connect: () => endpoint.socket, + }); + expect(result).toEqual({ requested: true }); + }); - expect(result.ok).toBe(false); - expect(result.message).toBe("restart failed"); - expect(calls.map((call) => call.args)).toEqual([ - ["--user", "daemon-reload"], - ["--user", "enable", "--now", `${ADE_RUNTIME_SERVICE_NAME}.service`], - ["--user", "restart", `${ADE_RUNTIME_SERVICE_NAME}.service`], - ]); + it("reports a close before the request as the endpoint hanging up", async () => { + const endpoint = fakeEndpoint({}); + queueMicrotask(() => endpoint.socket.emit("close")); + const result = await requestAdeRuntimeShutdown({ + pid: 4242, + socketPath, + connect: () => endpoint.socket, + }); + expect(result).toEqual({ + requested: false, + reason: "the runtime endpoint closed before it could be asked to stop", + }); }); -}); + it("gives up on a wedged endpoint instead of hanging the caller", async () => { + const endpoint = fakeEndpoint({}); + const result = await requestAdeRuntimeShutdown({ + pid: 4242, + socketPath, + timeoutMs: 250, + connect: () => endpoint.socket, + }); + expect(result).toEqual({ + requested: false, + reason: "the runtime endpoint did not answer within 250ms", + }); + }); -function spawnSequence( - calls: Array<{ command: string; args: string[] }>, - results: ServiceManagerProcessResult[], -): ServiceManagerSpawnSync { - let loadSeen = false; - return (command, args) => { - const next = results.shift(); - if ( - !next - && loadSeen - && command === "launchctl" - && args[0] === "print" - ) { - return { status: 0, stdout: "state = running\npid = 7777\n", stderr: "" }; - } - calls.push({ command, args }); - if (command === "launchctl" && args[0] === "load") loadSeen = true; - return next ?? { status: 0, stdout: "", stderr: "" }; - }; -} + it("never dials a tcp runtime endpoint", async () => { + const result = await requestAdeRuntimeShutdown({ + pid: 4242, + socketPath: "tcp://127.0.0.1:9999?token=secret", + connect: () => { + throw new Error("must not connect"); + }, + }); + expect(result.requested).toBe(false); + }); +}); diff --git a/apps/ade-cli/src/serviceManager/common.ts b/apps/ade-cli/src/serviceManager/common.ts index e19040f92..823cc240d 100644 --- a/apps/ade-cli/src/serviceManager/common.ts +++ b/apps/ade-cli/src/serviceManager/common.ts @@ -18,8 +18,61 @@ export type ServiceManagerResult = { selfMutationBlocked?: boolean; /** Typed install verification stage for callers that need repair diagnostics. */ failureStep?: "predecessor_exit" | "replacement_pid" | "replacement_responsive"; + /** + * The service is registered and its brain process is alive, but it had not + * answered on the socket when the install's wait budget ran out. That is a + * brain still coming up (first launch, slow disk, big project database), not + * a broken one — the platform supervisor keeps it, and callers should keep + * waiting for the endpoint rather than restart it. + */ + starting?: boolean; + /** + * The install actually (re)started the service child. Absent/false when the + * install was a no-op (already running and answering) or when it chose to + * wait for a young brain instead of restarting it — callers that need a + * restart to have happened (the one-time trust reset) check this rather than + * `ok`. + */ + restarted?: boolean; }; +// Re-exported so the many existing `from "./common"` importers keep working; +// the definitions live in runtimeServiceBudgets.ts with the rest of the +// lifecycle budgets they have to stay ordered against. +export { + RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS, + RUNTIME_SERVICE_START_WAIT_MS, + RUNTIME_SERVICE_YOUNG_BRAIN_MS, + WINDOWS_HANDOVER_TIMEOUT_MS, +} from "./runtimeServiceBudgets"; + +/** + * Parses `ps -o etime=` output (`[[dd-]hh:]mm:ss`) into milliseconds. + * Returns null for anything it does not recognise so callers fail open. + */ +export function parsePsElapsedMs(raw: string): number | null { + const text = raw.trim(); + const match = text.match(/^(?:(\d+)-)?(?:(\d+):)?(\d{1,2}):(\d{2})$/); + if (!match) return null; + const days = Number(match[1] ?? 0); + const hours = Number(match[2] ?? 0); + const minutes = Number(match[3]); + const seconds = Number(match[4]); + if (![days, hours, minutes, seconds].every((value) => Number.isFinite(value))) return null; + return (((days * 24 + hours) * 60 + minutes) * 60 + seconds) * 1_000; +} + +/** Milliseconds a POSIX process has been alive, or null when unknown. */ +export function readPidElapsedMs( + pid: number, + run: ServiceManagerSpawnSync = spawnSync, +): number | null { + if (!Number.isFinite(pid) || pid <= 0) return null; + const result = run("ps", ["-o", "etime=", "-p", String(pid)], { encoding: "utf8" }); + if (result.status !== 0) return null; + return parsePsElapsedMs(processOutputRaw(result)); +} + /** * A replacement that reached its readiness phase is already registered with * the platform supervisor. That supervisor owns subsequent retries; starting @@ -75,7 +128,7 @@ function processOutputText(result: ServiceManagerProcessResult): string { } /** Untrimmed stdout, for line-oriented output whose first line matters. */ -function processOutputRaw(result: ServiceManagerProcessResult): string { +export function processOutputRaw(result: ServiceManagerProcessResult): string { if (typeof result.stdout === "string") return result.stdout; if (Buffer.isBuffer(result.stdout)) return result.stdout.toString("utf8"); return ""; @@ -792,3 +845,48 @@ function streamToText(value: string | Buffer | null | undefined): string { export function serviceManagerResultText(result: ServiceManagerProcessResult): string { return streamToText(result.stderr) || streamToText(result.stdout); } + +/** + * The registered service owns a live brain that has not answered on its socket + * yet. + * + * Every path that could otherwise reach `spawnDaemon` has to see this and + * stop: an unmanaged brain on a supervised socket is a second brain, and the + * user's actual problem is only that the first one is still starting. The TUI + * client and the machine-daemon path had a class each for exactly this, which + * meant two places to keep the "do not spawn a rival" rule. + */ +export type RuntimeServiceStillStartingReason = + /** The socket has not answered yet; nothing has been waited out. */ + | { kind: "not_answered"; socketPath: string; installMessage?: string | null } + /** The installer failed but the registered service still owns recovery. */ + | { kind: "recovery_owned"; installMessage: string } + /** A `starting` install outlasted the caller's own connect budget. */ + | { kind: "wait_exhausted"; socketPath: string; installMessage: string }; + +export function runtimeServiceStillStartingMessage( + reason: RuntimeServiceStillStartingReason, +): string { + if (reason.kind === "recovery_owned") { + return `${reason.installMessage} The registered service still owns recovery for this endpoint, ` + + "so ADE did not start a competing manual brain."; + } + if (reason.kind === "wait_exhausted") { + return `${reason.installMessage} It had still not answered on ${reason.socketPath} when ADE stopped waiting, ` + + "so ADE did not start a competing manual brain."; + } + const detail = reason.installMessage?.trim() ? ` (${reason.installMessage.trim()})` : ""; + return "ADE's background service is still starting — try again in a moment." + + ` It had not answered on ${reason.socketPath} yet${detail}` + + ", so ADE did not start a second brain alongside it."; +} + +export class RuntimeServiceStillStartingError extends Error { + readonly reason: RuntimeServiceStillStartingReason; + + constructor(reason: RuntimeServiceStillStartingReason) { + super(runtimeServiceStillStartingMessage(reason)); + this.name = "RuntimeServiceStillStartingError"; + this.reason = reason; + } +} diff --git a/apps/ade-cli/src/serviceManager/installLaunchd.ts b/apps/ade-cli/src/serviceManager/installLaunchd.ts index 534327328..28d27166b 100644 --- a/apps/ade-cli/src/serviceManager/installLaunchd.ts +++ b/apps/ade-cli/src/serviceManager/installLaunchd.ts @@ -7,8 +7,10 @@ import { type AdeServiceCommand, isCurrentProcessDescendantOfPid, listStaleChannelServePids, + readPidElapsedMs, resolveAdeServeCliScriptPath, resolveAdeServeCommand, + RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS, serviceManagerResultText, type ServiceManagerResult, type ServiceManagerSpawnSync, @@ -21,6 +23,13 @@ import { installLaunchdWatchdogAgent, uninstallLaunchdWatchdogAgent, } from "./installLaunchdWatchdog"; +import { + awaitServiceHandover, + awaitYoungBrainStart, + defaultResponsivenessProbe, + type HandoverWaitDeps, + serviceHandoverSleep, +} from "./serviceHandover"; import { detectSyncHostSingletonConflict, formatSyncHostSingletonConflictMessage, @@ -47,6 +56,10 @@ type LaunchdServiceManagerDeps = { handoverTimeoutMs?: number; handoverPollMs?: number; handoverPidAlive?: (pid: number) => boolean; + /** Age of a live service pid; tests inject it, production asks `ps`. */ + pidElapsedMs?: (pid: number, run: ServiceManagerSpawnSync) => number | null; + /** Whether the brain has recorded a fresh streak of startup failures; tests inject it. */ + recentCrashLoop?: () => boolean; sleep?: (ms: number) => Promise; }; @@ -237,50 +250,6 @@ function launchdTerminateDeps(deps: TerminatePidDeps | undefined): TerminatePidD return { ...deps, platform: deps?.platform ?? "darwin" }; } -async function sleepAsync(ms: number): Promise { - // Awaited lifecycle delays must stay referenced: in a standalone CLI the - // handover polling can be the only pending work, and an unref'd timer lets - // the process exit mid-repair (before SIGKILL escalation / launchctl load). - await new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -function runtimeStatusArgs(command: AdeServiceCommand, socketPath: string): string[] { - const args = [...command.args]; - const serveIndex = args.lastIndexOf("serve"); - if (serveIndex >= 0) { - args.splice(serveIndex, 1, "runtime", "status"); - } else { - args.push("runtime", "status"); - } - args.push("--socket", socketPath, "--timeout", "1500", "--text"); - return args; -} - -function defaultResponsivenessProbe(args: { - socketPath: string; - timeoutMs: number; - command: AdeServiceCommand; -}): boolean { - const env = { - ...process.env, - ...(args.command.env ?? {}), - ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1", - }; - const result = spawnSync( - args.command.command, - runtimeStatusArgs(args.command, args.socketPath), - { - encoding: "utf8", - env, - timeout: args.timeoutMs, - stdio: "ignore", - }, - ); - return result.status === 0 && !result.error; -} - function handoverFailure( servicePath: string, failureStep: NonNullable, @@ -321,7 +290,7 @@ export async function installLaunchdService( : null; const plistUnchanged = existingPlist === plist; const forceRestart = env.ADE_FORCE_RUNTIME_SERVICE_RESTART === "1"; - const loaded = getLoadedLaunchdState(run); + let loaded = getLoadedLaunchdState(run); if (!forceRestart && plistUnchanged && loaded?.running === true) { if ( deps.probeResponsiveness === false @@ -340,6 +309,61 @@ export async function installLaunchdService( }; } } + const isAlive = deps.handoverPidAlive ?? deps.terminateDeps?.pidAlive ?? pidAlive; + const sleep = deps.sleep ?? serviceHandoverSleep; + const timeoutMs = Math.max(0, deps.handoverTimeoutMs ?? RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS); + const pollMs = Math.max(10, deps.handoverPollMs ?? 100); + const pidElapsedMs = deps.pidElapsedMs ?? readPidElapsedMs; + + const handoverWait: HandoverWaitDeps = { + readSupervisedPid: () => { + const replacement = getLoadedLaunchdState(run); + return replacement?.running === true ? replacement.pid : null; + }, + isAlive, + probeResponsiveness, + socketPath, + command, + sleep, + pollMs, + }; + + const young = await awaitYoungBrainStart({ + definitionUnchanged: plistUnchanged, + supervisedPid: loaded?.pid, + running: loaded?.running === true, + adeHome, + recentCrashLoop: deps.recentCrashLoop, + run, + pidElapsedMs, + timeoutMs, + wait: handoverWait, + }); + if (young.kind === "responsive") { + installLaunchdWatchdogAgent({ command, homeDir, spawnSync: run }); + return { + ok: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: servicePath, + message: "ADE service launchd service is already installed; its background service finished starting.", + }; + } + if (young.kind === "starting") { + installLaunchdWatchdogAgent({ command, homeDir, spawnSync: run }); + return { + ok: true, + starting: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: servicePath, + message: `ADE service launchd service is installed; the background service (pid ${young.pid}) is still starting.`, + }; + } + if (young.kind === "died") { + // The young child died while we waited: fall through and (re)start it. + loaded = getLoadedLaunchdState(run); + } const selfBlock = selfServiceMutationBlock({ action: "install", loadedPid: loaded?.pid, @@ -414,30 +438,14 @@ export async function installLaunchdService( }; } const oldPid = loaded?.pid ?? null; - const isAlive = deps.handoverPidAlive ?? deps.terminateDeps?.pidAlive ?? pidAlive; - const sleep = deps.sleep ?? sleepAsync; - const timeoutMs = Math.max(0, deps.handoverTimeoutMs ?? 10_000); - const pollMs = Math.max(10, deps.handoverPollMs ?? 100); - const deadline = Date.now() + timeoutMs; - let predecessorGone = oldPid == null || !isAlive(oldPid); - let replacementPid: number | null = null; - let replacementResponsive = false; - do { - predecessorGone = oldPid == null || !isAlive(oldPid); - const replacement = getLoadedLaunchdState(run); - replacementPid = replacement?.running === true ? replacement.pid : null; - const replacementDiffers = replacementPid != null && replacementPid !== oldPid; - if (predecessorGone && replacementDiffers) { - replacementResponsive = probeResponsiveness({ - socketPath, - timeoutMs: Math.min(1_500, Math.max(1, deadline - Date.now())), - command, - }); - if (replacementResponsive) break; - } - if (Date.now() >= deadline) break; - await sleep(Math.min(pollMs, Math.max(1, deadline - Date.now()))); - } while (Date.now() <= deadline); + // A fresh full budget: the young-brain wait above may have spent all of its + // own, and the real handover is the one whose outcome decides whether this + // install reports `starting` or a failure. + const { predecessorGone, replacementPid, replacementResponsive } = await awaitServiceHandover( + handoverWait, + oldPid, + timeoutMs, + ); if (!predecessorGone) { return handoverFailure( @@ -454,6 +462,25 @@ export async function installLaunchdService( ); } if (!replacementResponsive) { + // `replacementPid != null` first: `isAlive` is injectable, and a test (or + // a future caller) that answers `true` unconditionally must not turn "no + // replacement at all" into a `starting` result the caller waits on. + if (replacementPid != null && isAlive(replacementPid)) { + // launchd owns a live replacement that has not answered yet. That is a + // slow start, not a failed install; the supervisor keeps the child and + // the caller keeps waiting for the endpoint. Reporting this as a failure + // was what made every slow machine read as a broken one. + installLaunchdWatchdogAgent({ command, homeDir, spawnSync: run }); + return { + ok: true, + starting: true, + restarted: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: servicePath, + message: `ADE service launchd service installed; the background service (pid ${replacementPid}) is still starting after ${timeoutMs}ms.`, + }; + } return handoverFailure( servicePath, "replacement_responsive", @@ -470,6 +497,7 @@ export async function installLaunchdService( }); return { ok: true, + restarted: true, serviceName: ADE_RUNTIME_SERVICE_NAME, action: "install", path: servicePath, diff --git a/apps/ade-cli/src/serviceManager/installLaunchdWatchdog.test.ts b/apps/ade-cli/src/serviceManager/installLaunchdWatchdog.test.ts deleted file mode 100644 index a9c20a552..000000000 --- a/apps/ade-cli/src/serviceManager/installLaunchdWatchdog.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { describe, expect, it } from "vitest"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { - installLaunchdWatchdogAgent, - renderWatchdogLaunchdPlist, - resolveWatchdogServiceName, - uninstallLaunchdWatchdogAgent, - watchdogCommand, - watchdogLaunchAgentPath, -} from "./installLaunchdWatchdog"; -import type { AdeServiceCommand } from "./common"; - -const serviceCommand: AdeServiceCommand = { - command: "/usr/local/bin/node", - args: ["/opt/ade/cli.cjs", "serve"], - env: { ADE_HOME: "/Users/example/.ade" }, -}; - -function tempHome(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), "ade-watchdog-home-")); -} - -function recordingSpawn(calls: Array<{ command: string; args: string[] }>) { - return (command: string, args: string[]) => { - calls.push({ command, args }); - return { status: 0, stdout: "", stderr: "" }; - }; -} - -describe("resolveWatchdogServiceName", () => { - it("keeps each channel on its own watchdog", () => { - expect(resolveWatchdogServiceName("com.ade.runtime")).toBe("com.ade.watchdog"); - expect(resolveWatchdogServiceName("com.ade.runtime.beta")).toBe("com.ade.watchdog.beta"); - expect(resolveWatchdogServiceName("com.example.custom")).toBe("com.example.custom.watchdog"); - }); -}); - -describe("watchdogCommand", () => { - it("runs the same binary the brain was installed from", () => { - expect(watchdogCommand(serviceCommand)).toEqual({ - command: "/usr/local/bin/node", - args: ["/opt/ade/cli.cjs", "runtime", "watchdog-check"], - env: { - ADE_HOME: "/Users/example/.ade", - ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1", - }, - }); - }); - - it("appends the check when the command has no serve argument", () => { - expect(watchdogCommand({ command: "/opt/ade/ade", args: [] }).args) - .toEqual(["runtime", "watchdog-check"]); - }); -}); - -describe("renderWatchdogLaunchdPlist", () => { - it("runs on an interval and never keeps itself alive", () => { - const plist = renderWatchdogLaunchdPlist({ - command: serviceCommand, - homeDir: "/Users/example", - }); - expect(plist).toContain("com.ade.watchdog"); - expect(plist).toContain("StartInterval"); - expect(plist).toContain("60"); - expect(plist).toContain("watchdog-check"); - // KeepAlive would make launchd respawn a one-shot check in a tight loop. - expect(plist).not.toContain("KeepAlive"); - }); - - it("refuses an interval short enough to thrash", () => { - const plist = renderWatchdogLaunchdPlist({ - command: serviceCommand, - homeDir: "/Users/example", - startIntervalSeconds: 1, - }); - expect(plist).toContain("15"); - }); -}); - -describe("installLaunchdWatchdogAgent", () => { - it("writes and loads the agent", () => { - const homeDir = tempHome(); - const calls: Array<{ command: string; args: string[] }> = []; - const result = installLaunchdWatchdogAgent({ - command: serviceCommand, - homeDir, - spawnSync: recordingSpawn(calls), - }); - - const servicePath = watchdogLaunchAgentPath(homeDir); - expect(result.installed).toBe(true); - expect(fs.existsSync(servicePath)).toBe(true); - expect(calls.map((call) => call.args[0])).toEqual(["unload", "load"]); - }); - - it("reports a load failure instead of claiming the agent is armed", () => { - const homeDir = tempHome(); - const result = installLaunchdWatchdogAgent({ - command: serviceCommand, - homeDir, - spawnSync: (command, args) => - args[0] === "load" - ? { status: 1, stdout: "", stderr: "Load failed" } - : { status: 0, stdout: "", stderr: "" }, - }); - expect(result.installed).toBe(false); - }); - - it("removes the agent with the brain it guards", () => { - const homeDir = tempHome(); - installLaunchdWatchdogAgent({ - command: serviceCommand, - homeDir, - spawnSync: recordingSpawn([]), - }); - const servicePath = watchdogLaunchAgentPath(homeDir); - expect(fs.existsSync(servicePath)).toBe(true); - - const calls: Array<{ command: string; args: string[] }> = []; - uninstallLaunchdWatchdogAgent({ homeDir, spawnSync: recordingSpawn(calls) }); - - expect(fs.existsSync(servicePath)).toBe(false); - expect(calls.map((call) => call.args[0])).toEqual(["bootout", "unload"]); - }); -}); diff --git a/apps/ade-cli/src/serviceManager/installSystemd.test.ts b/apps/ade-cli/src/serviceManager/installSystemd.test.ts new file mode 100644 index 000000000..2f165abe6 --- /dev/null +++ b/apps/ade-cli/src/serviceManager/installSystemd.test.ts @@ -0,0 +1,318 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + ADE_RUNTIME_SERVICE_NAME, + type AdeServiceCommand, + type ServiceManagerProcessResult, + type ServiceManagerSpawnSync, +} from "./common"; +import { + getSystemdUnitState, + installSystemdService, + parseSystemdShowOutput, + renderSystemdEnvironment, + renderSystemdUnit, + servicePath as systemdServicePath, +} from "./installSystemd"; + +const tempDirs: string[] = []; + +afterEach(() => { + while (tempDirs.length > 0) { + const dir = tempDirs.pop(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +function makeTempHome(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +describe("systemd service rendering", () => { + it("renders the user service path under the home directory", () => { + expect(systemdServicePath("/home/example")).toBe( + path.join("/home/example", ".config", "systemd", "user", `${ADE_RUNTIME_SERVICE_NAME}.service`), + ); + }); + + it("renders unit content with quoted ExecStart and escaped environment values", () => { + const unit = renderSystemdUnit({ + command: "/opt/ADE CLI/node", + args: ["/opt/ade/cli.cjs", "serve"], + env: { + NODE_PATH: "/tmp/100%/node modules", + ADE_HOME: "/home/example/ade path\\with\"quotes", + }, + }); + + expect(unit).toContain("Description=ADE runtime service"); + expect(unit).toContain("Type=simple"); + expect(unit).toContain("ExecStart='/opt/ADE CLI/node' '/opt/ade/cli.cjs' 'serve'"); + expect(unit).toContain("Restart=always"); + expect(unit).toContain("Environment=\"NODE_PATH=/tmp/100%%/node modules\""); + expect(unit).toContain("Environment=\"ADE_HOME=/home/example/ade path\\\\with\\\"quotes\""); + expect(unit).toContain("WantedBy=default.target"); + }); + + it("quotes systemd environment assignments for whitespace, backslashes, quotes, and percent signs", () => { + expect(renderSystemdEnvironment("NODE_PATH", "C:\\ADE deps\\100% \"runtime\"")).toBe( + "Environment=\"NODE_PATH=C:\\\\ADE deps\\\\100%% \\\"runtime\\\"\"", + ); + }); +}); + +type SystemdShowState = { activeState: string; mainPid: number }; + +/** + * A fake `systemctl` that answers `show` from a script of unit states (the last + * entry repeats) and lets individual subcommands be failed. Everything the + * systemd installer runs goes through `systemctl`, so one stub covers it. + */ +function systemdSpawn(options: { + calls: Array<{ command: string; args: string[] }>; + states: SystemdShowState[]; + failures?: Record; +}): ServiceManagerSpawnSync { + const states = [...options.states]; + return (command, args) => { + options.calls.push({ command, args }); + if (command !== "systemctl") return { status: 0, stdout: "", stderr: "" }; + const subcommand = args[1]; + if (subcommand === "show") { + const state = states.length > 1 ? states.shift()! : states[0]; + return { + status: 0, + stdout: `ActiveState=${state.activeState}\nMainPID=${state.mainPid}\n`, + stderr: "", + }; + } + const failure = options.failures?.[subcommand ?? ""]; + if (failure) return failure; + return { status: 0, stdout: "", stderr: "" }; + }; +} + +const INACTIVE: SystemdShowState = { activeState: "inactive", mainPid: 0 }; + +function systemdCallArgs(calls: Array<{ command: string; args: string[] }>): string[][] { + return calls.filter((call) => call.args[1] !== "show").map((call) => call.args); +} + +describe("systemd service install", () => { + const serviceCommand: AdeServiceCommand = { + command: "/opt/ade/bin/ade", + args: ["serve"], + env: { NODE_PATH: "/opt/ade/node_modules" }, + }; + const unitName = `${ADE_RUNTIME_SERVICE_NAME}.service`; + + function installDeps(homeDir: string, spawnSync: ServiceManagerSpawnSync, overrides: Record = {}) { + return { + command: serviceCommand, + spawnSync, + homeDir, + env: {} as NodeJS.ProcessEnv, + handoverTimeoutMs: 200, + handoverPollMs: 10, + sleep: async () => {}, + recentCrashLoop: () => false, + ...overrides, + }; + } + + it("writes the user unit, enables it, and waits for the replacement to answer", async () => { + const homeDir = makeTempHome("ade-systemd-install-"); + const targetPath = systemdServicePath(homeDir); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [INACTIVE, { activeState: "active", mainPid: 4242 }], + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync, { + responsivenessProbe: () => true, + handoverPidAlive: () => true, + })); + + expect(result).toMatchObject({ + ok: true, + restarted: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + }); + expect(result.starting).toBeUndefined(); + expect(fs.readFileSync(targetPath, "utf8")).toBe(renderSystemdUnit(serviceCommand)); + expect(systemdCallArgs(calls)).toEqual([ + ["--user", "daemon-reload"], + ["--user", "enable", "--now", unitName], + ["--user", "restart", unitName], + ]); + }); + + it("reports a live replacement that has not answered yet as starting, not failed", async () => { + const homeDir = makeTempHome("ade-systemd-starting-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [INACTIVE, { activeState: "active", mainPid: 4242 }], + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync, { + responsivenessProbe: () => false, + handoverPidAlive: () => true, + })); + + expect(result).toMatchObject({ ok: true, starting: true, restarted: true }); + expect(result.failureStep).toBeUndefined(); + expect(result.message).toContain("pid 4242"); + }); + + it("still fails the install when the replacement died without answering", async () => { + const homeDir = makeTempHome("ade-systemd-dead-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [INACTIVE, { activeState: "active", mainPid: 4242 }], + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync, { + responsivenessProbe: () => false, + handoverPidAlive: (pid: number) => pid !== 4242, + })); + + expect(result).toMatchObject({ ok: false, failureStep: "replacement_responsive" }); + expect(result.starting).toBeUndefined(); + }); + + it("waits for a young unresponsive brain instead of restarting it", async () => { + const homeDir = makeTempHome("ade-systemd-young-"); + const targetPath = systemdServicePath(homeDir); + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + fs.writeFileSync(targetPath, renderSystemdUnit(serviceCommand), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ calls, states: [{ activeState: "active", mainPid: 909 }] }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync, { + responsivenessProbe: () => false, + handoverPidAlive: () => true, + pidElapsedMs: () => 5_000, + })); + + expect(result).toMatchObject({ ok: true, starting: true }); + expect(result.restarted).toBeUndefined(); + // The whole point: nothing was restarted out from under the booting brain. + expect(systemdCallArgs(calls)).toEqual([]); + }); + + it("restarts a young brain when a fresh crash-loop record vetoes the wait", async () => { + const homeDir = makeTempHome("ade-systemd-crashloop-"); + const targetPath = systemdServicePath(homeDir); + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + fs.writeFileSync(targetPath, renderSystemdUnit(serviceCommand), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [{ activeState: "active", mainPid: 909 }, { activeState: "active", mainPid: 910 }], + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync, { + responsivenessProbe: () => false, + handoverPidAlive: (pid: number) => pid !== 909, + pidElapsedMs: () => 5_000, + recentCrashLoop: () => true, + })); + + expect(result).toMatchObject({ ok: true, starting: true, restarted: true }); + expect(systemdCallArgs(calls)).toContainEqual(["--user", "restart", unitName]); + }); + + it("does not restart an unchanged unit whose brain already answers", async () => { + const homeDir = makeTempHome("ade-systemd-noop-"); + const targetPath = systemdServicePath(homeDir); + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + fs.writeFileSync(targetPath, renderSystemdUnit(serviceCommand), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ calls, states: [{ activeState: "active", mainPid: 909 }] }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync, { + responsivenessProbe: () => true, + })); + + expect(result).toMatchObject({ ok: true }); + expect(result.restarted).toBeUndefined(); + expect(systemdCallArgs(calls)).toEqual([]); + }); + + it("treats a unit systemd reports as activating as a live brain, not a dead one", () => { + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ calls, states: [{ activeState: "activating", mainPid: 77 }] }); + expect(getSystemdUnitState(spawnSync)).toEqual({ active: true, mainPid: 77 }); + }); + + it("reads MainPID=0 as no pid", () => { + expect(parseSystemdShowOutput("ActiveState=inactive\nMainPID=0\n").get("MainPID")).toBe("0"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ calls, states: [INACTIVE] }); + expect(getSystemdUnitState(spawnSync)).toEqual({ active: false, mainPid: null }); + }); + + it("does not enable when daemon-reload fails", async () => { + const homeDir = makeTempHome("ade-systemd-reload-fail-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [INACTIVE], + failures: { "daemon-reload": { status: 1, stdout: "", stderr: "reload failed" } }, + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync)); + + expect(result.ok).toBe(false); + expect(result.message).toBe("reload failed"); + expect(systemdCallArgs(calls)).toEqual([["--user", "daemon-reload"]]); + }); + + it("surfaces enable failures after a successful reload", async () => { + const homeDir = makeTempHome("ade-systemd-enable-fail-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [INACTIVE], + failures: { enable: { status: 1, stdout: "", stderr: "enable failed" } }, + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync)); + + expect(result.ok).toBe(false); + expect(result.message).toBe("enable failed"); + expect(systemdCallArgs(calls)).toEqual([ + ["--user", "daemon-reload"], + ["--user", "enable", "--now", unitName], + ]); + }); + + it("surfaces restart failures after enabling the user unit", async () => { + const homeDir = makeTempHome("ade-systemd-restart-fail-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = systemdSpawn({ + calls, + states: [INACTIVE], + failures: { restart: { status: 1, stdout: "", stderr: "restart failed" } }, + }); + + const result = await installSystemdService(installDeps(homeDir, spawnSync)); + + expect(result.ok).toBe(false); + expect(result.message).toBe("restart failed"); + expect(systemdCallArgs(calls)).toEqual([ + ["--user", "daemon-reload"], + ["--user", "enable", "--now", unitName], + ["--user", "restart", unitName], + ]); + }); +}); diff --git a/apps/ade-cli/src/serviceManager/installSystemd.ts b/apps/ade-cli/src/serviceManager/installSystemd.ts index 28a0bbcb2..6dd4c2c82 100644 --- a/apps/ade-cli/src/serviceManager/installSystemd.ts +++ b/apps/ade-cli/src/serviceManager/installSystemd.ts @@ -5,18 +5,43 @@ import path from "node:path"; import { ADE_RUNTIME_SERVICE_NAME, type AdeServiceCommand, + isPidAlive, + processOutputRaw, + readPidElapsedMs, renderCommand, resolveAdeServeCommand, + RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS, serviceManagerResultText, type ServiceManagerResult, type ServiceManagerSpawnSync, type ServiceManagerStatusResult, } from "./common"; +import { + awaitServiceHandover, + awaitYoungBrainStart, + defaultResponsivenessProbe, + type HandoverWaitDeps, + type PidElapsedMsLookup, + type ResponsivenessProbe, + serviceHandoverSleep, +} from "./serviceHandover"; type SystemdServiceManagerDeps = { command?: AdeServiceCommand; spawnSync?: ServiceManagerSpawnSync; homeDir?: string; + env?: NodeJS.ProcessEnv; + /** Defaults on. Repair-only callers may skip the preflight RPC probe. */ + probeResponsiveness?: boolean; + responsivenessProbe?: ResponsivenessProbe; + handoverTimeoutMs?: number; + handoverPollMs?: number; + handoverPidAlive?: (pid: number) => boolean; + /** Age of a live service pid; tests inject it, production asks `ps`. */ + pidElapsedMs?: PidElapsedMsLookup; + /** Whether the brain has recorded a fresh streak of startup failures; tests inject it. */ + recentCrashLoop?: () => boolean; + sleep?: (ms: number) => Promise; }; export function servicePath(homeDir = os.homedir()): string { @@ -57,12 +82,163 @@ WantedBy=default.target `; } -export function installSystemdService(deps: SystemdServiceManagerDeps = {}): ServiceManagerResult { +export type SystemdUnitState = { active: boolean; mainPid: number | null }; + +/** + * Parses `systemctl show`'s `KEY=value` output. Values are taken verbatim after + * the first `=`; systemd never wraps these two properties. + */ +export function parseSystemdShowOutput(output: string): Map { + const properties = new Map(); + for (const line of output.split(/\r?\n/)) { + const separator = line.indexOf("="); + if (separator <= 0) continue; + properties.set(line.slice(0, separator).trim(), line.slice(separator + 1).trim()); + } + return properties; +} + +/** + * One `systemctl show` round trip for both properties the handover needs. + * `null` means the query itself failed (no systemd user bus, unit unknown to + * this systemd) — distinct from "known and inactive", which is `active:false`. + */ +export function getSystemdUnitState( + run: ServiceManagerSpawnSync, + unitName = serviceUnitName(), +): SystemdUnitState | null { + const show = run( + "systemctl", + ["--user", "show", unitName, "-p", "ActiveState", "-p", "MainPID"], + { encoding: "utf8" }, + ); + if (show.status !== 0) return null; + const properties = parseSystemdShowOutput(processOutputRaw(show)); + const activeState = properties.get("ActiveState"); + if (activeState == null) return null; + const rawPid = Number(properties.get("MainPID") ?? "0"); + const mainPid = Number.isFinite(rawPid) && rawPid > 0 ? Math.floor(rawPid) : null; + // `activating` is systemd's own "still coming up" and must not be read as + // dead: restarting a unit in that state is exactly the booting-brain kill + // this whole change exists to stop. + return { active: activeState === "active" || activeState === "activating", mainPid }; +} + +function handoverFailure( + targetPath: string, + failureStep: NonNullable, + message: string, +): ServiceManagerResult { + return { + ok: false, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + failureStep, + message, + }; +} + +/** + * Linux parity with the launchd installer. Headless brains, `install.sh` + * installs and remote runtimes all land here, and before this they got the + * pre-#1102 behaviour: write the unit, `enable --now`, `restart`, report + * success the instant systemd accepted the restart — no proof the replacement + * ever answered, and no way to tell a caller "installed, still starting". + * A remote bootstrap then dialled a socket that was not up yet and read a + * healthy-but-slow brain as a broken one. + */ +export async function installSystemdService( + deps: SystemdServiceManagerDeps = {}, +): Promise { const run = deps.spawnSync ?? spawnSync; - const targetPath = servicePath(deps.homeDir); + const env = deps.env ?? process.env; + const homeDir = deps.homeDir ?? os.homedir(); + const targetPath = servicePath(homeDir); const command = deps.command ?? resolveAdeServeCommand(); - fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + const adeHome = command.env?.ADE_HOME?.trim() || env.ADE_HOME?.trim() || path.join(homeDir, ".ade"); + const socketPath = path.join(adeHome, "sock", "ade.sock"); + const unitName = serviceUnitName(); + const probeResponsiveness = deps.responsivenessProbe ?? defaultResponsivenessProbe; + const isAlive = deps.handoverPidAlive ?? isPidAlive; + const sleep = deps.sleep ?? serviceHandoverSleep; + const timeoutMs = Math.max(0, deps.handoverTimeoutMs ?? RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS); + const pollMs = Math.max(10, deps.handoverPollMs ?? 100); + const pidElapsedMs = deps.pidElapsedMs ?? readPidElapsedMs; + const unit = renderSystemdUnit(command); + const existingUnit = fs.existsSync(targetPath) ? fs.readFileSync(targetPath, "utf8") : null; + const unitUnchanged = existingUnit === unit; + const forceRestart = env.ADE_FORCE_RUNTIME_SERVICE_RESTART === "1"; + + let state = getSystemdUnitState(run, unitName); + + // The unit is current and systemd already has a live, answering child. + // Restarting it would only interrupt a working brain. + if (!forceRestart && unitUnchanged && state?.active === true) { + if ( + deps.probeResponsiveness === false + || probeResponsiveness({ socketPath, timeoutMs: 1_500, command }) + ) { + return { + ok: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + message: "ADE service systemd user service is already installed and running.", + }; + } + } + + const handoverWait: HandoverWaitDeps = { + readSupervisedPid: () => { + const replacement = getSystemdUnitState(run, unitName); + return replacement?.active === true ? replacement.mainPid : null; + }, + isAlive, + probeResponsiveness, + socketPath, + command, + sleep, + pollMs, + }; + + const young = await awaitYoungBrainStart({ + definitionUnchanged: unitUnchanged, + supervisedPid: state?.mainPid, + running: state?.active === true, + adeHome, + recentCrashLoop: deps.recentCrashLoop, + run, + pidElapsedMs, + timeoutMs, + wait: handoverWait, + }); + if (young.kind === "responsive") { + return { + ok: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + message: "ADE service systemd user service is already installed; its background service finished starting.", + }; + } + if (young.kind === "starting") { + return { + ok: true, + starting: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + message: `ADE service systemd user service is installed; the background service (pid ${young.pid}) is still starting.`, + }; + } + if (young.kind === "died") { + // The young child died while we waited: fall through and (re)start it. + state = getSystemdUnitState(run, unitName); + } + + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); fs.writeFileSync(targetPath, unit, "utf8"); const reload = run("systemctl", ["--user", "daemon-reload"], { encoding: "utf8" }); if (reload.status !== 0) { @@ -74,28 +250,78 @@ export function installSystemdService(deps: SystemdServiceManagerDeps = {}): Ser message: serviceManagerResultText(reload) || "systemctl daemon-reload failed.", }; } - const unitName = serviceUnitName(); const enable = run("systemctl", ["--user", "enable", "--now", unitName], { encoding: "utf8" }); - if (enable.status === 0) { - const restart = run("systemctl", ["--user", "restart", unitName], { encoding: "utf8" }); - if (restart.status !== 0) { + if (enable.status !== 0) { + return { + ok: false, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + message: serviceManagerResultText(enable) || "systemctl enable --now failed.", + }; + } + const oldPid = state?.mainPid ?? null; + const restart = run("systemctl", ["--user", "restart", unitName], { encoding: "utf8" }); + if (restart.status !== 0) { + return { + ok: false, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: targetPath, + message: serviceManagerResultText(restart) || "systemctl restart failed.", + }; + } + + // A fresh full budget: the young-brain wait above may have spent all of its + // own, and the real handover is the one whose outcome decides whether this + // install reports `starting` or a failure. + const { predecessorGone, replacementPid, replacementResponsive } = await awaitServiceHandover( + handoverWait, + oldPid, + timeoutMs, + ); + if (!predecessorGone) { + return handoverFailure( + targetPath, + "predecessor_exit", + `ADE service handover failed because predecessor pid ${oldPid} is still alive.`, + ); + } + if (replacementPid == null || replacementPid === oldPid) { + return handoverFailure( + targetPath, + "replacement_pid", + `ADE service handover failed because systemd did not report a distinct replacement pid (old ${oldPid ?? "none"}, new ${replacementPid ?? "none"}).`, + ); + } + if (!replacementResponsive) { + if (isAlive(replacementPid)) { + // systemd owns a live replacement that has not answered yet. A slow + // start, not a failed install: the supervisor keeps the child and the + // caller keeps waiting for the endpoint. return { - ok: false, + ok: true, + starting: true, + restarted: true, serviceName: ADE_RUNTIME_SERVICE_NAME, action: "install", path: targetPath, - message: serviceManagerResultText(restart) || "systemctl restart failed.", + message: `ADE service systemd user service installed; the background service (pid ${replacementPid}) is still starting after ${timeoutMs}ms.`, }; } + return handoverFailure( + targetPath, + "replacement_responsive", + `ADE service handover failed because replacement pid ${replacementPid} did not initialize over ${socketPath} within ${timeoutMs}ms.`, + ); } return { - ok: enable.status === 0, + ok: true, + restarted: true, serviceName: ADE_RUNTIME_SERVICE_NAME, action: "install", path: targetPath, - message: enable.status === 0 - ? "ADE service systemd user service installed." - : serviceManagerResultText(enable) || "systemctl enable --now failed.", + message: "ADE service systemd user service installed.", }; } diff --git a/apps/ade-cli/src/serviceManager/installWindows.test.ts b/apps/ade-cli/src/serviceManager/installWindows.test.ts index bd91631b0..14addeb74 100644 --- a/apps/ade-cli/src/serviceManager/installWindows.test.ts +++ b/apps/ade-cli/src/serviceManager/installWindows.test.ts @@ -1,5 +1,5 @@ import fs from "node:fs"; -import { spawnSync as spawnChildSync } from "node:child_process"; +import { spawn, spawnSync, spawnSync as spawnChildSync } from "node:child_process"; import os from "node:os"; import path from "node:path"; import { resolveMachineAdeLayout } from "../services/projects/machineLayout"; @@ -40,8 +40,14 @@ import { windowsSchtasksCommand, windowsTaskkillCommand, WINDOWS_TASK_ACTION_FIELD_SEPARATOR, + buildWindowsRuntimeQueryArgs, renderWindowsServiceLauncher, } from "./installWindows"; +import { + BRAIN_HEARTBEAT_INTERVAL_MS, + BRAIN_HEARTBEAT_STALE_MS, +} from "../services/runtime/brainHeartbeat"; +import { waitForWindowsRuntimeReadiness } from "./windowsSupervisor"; const tempDirs: string[] = []; @@ -476,6 +482,156 @@ describe("Windows background service helpers", () => { }); }); + it("reports a supervised brain that has not answered yet as starting, not failed", async () => { + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 3, stdout: "", stderr: "" }, + { status: 3, stdout: "", stderr: "" }, + { status: 1, stdout: "", stderr: "ERROR: value not found" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "The operation completed successfully.", stderr: "" }, + { status: 0, stdout: "1234", stderr: "" }, + ]); + const launcherPath = path.join(makeTempHome("ade-windows-service-starting-"), "brain-service.ps1"); + + const result = await installWindowsService({ + command: serviceCommand, + launcherPath, + readPidRecord: immediateReadiness.readPidRecord, + // The probe's Win32_Process identity check confirmed the recorded pid IS + // our supervisor; it is the brain behind it that has not answered. + readinessProbe: () => ({ + ready: false, + supervised: true, + diagnostic: "Runtime PID 5678 has not bound the pipe yet.", + }), + handoverTimeoutMs: 0, + serviceName, + spawnSync, + userName: taskUser, + }); + + expect(result).toMatchObject({ + ok: true, + starting: true, + serviceName, + action: "install", + path: taskName, + }); + expect(result.failureStep).toBeUndefined(); + expect(result.message).toContain("still starting"); + }); + + it("fails, not starting, when the recorded supervisor pid is not ours", async () => { + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 3, stdout: "", stderr: "" }, + { status: 3, stdout: "", stderr: "" }, + { status: 1, stdout: "", stderr: "ERROR: value not found" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "The operation completed successfully.", stderr: "" }, + { status: 0, stdout: "1234", stderr: "" }, + ]); + const launcherPath = path.join(makeTempHome("ade-windows-service-dead-sup-"), "brain-service.ps1"); + + const result = await installWindowsService({ + command: serviceCommand, + launcherPath, + readPidRecord: immediateReadiness.readPidRecord, + // Recycled pid: alive, but `Win32_Process` says it is not a powershell + // running our launcher. `pidAlive` would call this "our brain, starting"; + // only the identity check can tell, and it says no. + readinessProbe: () => ({ + ready: false, + supervised: false, + diagnostic: "Supervisor PID 1234 is stale or belongs to another process.", + }), + handoverTimeoutMs: 0, + pidAlive: () => true, + serviceName, + spawnSync, + userName: taskUser, + }); + + expect(result).toMatchObject({ ok: false, failureStep: "replacement_responsive" }); + expect(result.starting).toBeUndefined(); + }); + + it("waits for a young unresponsive brain instead of replacing it", async () => { + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + // legacy task lookups (none) + { status: 3, stdout: "", stderr: "" }, + { status: 3, stdout: "", stderr: "" }, + ]); + const launcherPath = path.join(makeTempHome("ade-windows-service-young-"), "brain-service.ps1"); + const pidPath = `${launcherPath}.pid.json`; + // Pre-render the launcher exactly as the install would, so it reads as unchanged. + const machineLayout = resolveMachineAdeLayout( + { ...process.env, ...(serviceCommand.env ?? {}) }, + "win32", + ); + fs.mkdirSync(path.dirname(launcherPath), { recursive: true }); + fs.writeFileSync(launcherPath, `\uFEFF${renderWindowsServiceLauncher(serviceCommand, { + pidPath, + logPath: `${launcherPath}.log`, + heartbeatPath: path.win32.join(machineLayout.runtimeDir, "heartbeat.json"), + wedgeBreadcrumbPath: path.win32.join(machineLayout.runtimeDir, "event-loop-wedge.json"), + })}`, "utf8"); + const youngRecord = { ...readyPidRecord, runtimeStartedAtMs: Date.now() - 5_000 }; + const readinessProbe = vi.fn() + .mockReturnValueOnce({ ready: false, diagnostic: "not yet" }) + .mockReturnValue({ ready: true, diagnostic: "ready" }); + + const result = await installWindowsService({ + command: serviceCommand, + launcherPath, + pidPath, + readPidRecord: () => youngRecord, + readinessProbe, + handoverTimeoutMs: 5_000, + handoverPollMs: 10, + pidAlive: () => true, + serviceName, + spawnSync, + userName: taskUser, + }); + + expect(result).toMatchObject({ ok: true, action: "install" }); + expect(result.restarted).toBeUndefined(); + // No run-key rewrite, no supervisor start: the brain was left alone. + expect(calls.some((call) => call.args.some((arg) => /Add|\/Run|Start-Process/i.test(arg)))).toBe(false); + }); + + it("still fails the install when the supervisor never publishes a brain at all", async () => { + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 3, stdout: "", stderr: "" }, + { status: 3, stdout: "", stderr: "" }, + { status: 1, stdout: "", stderr: "ERROR: value not found" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "The operation completed successfully.", stderr: "" }, + { status: 0, stdout: "1234", stderr: "" }, + ]); + const launcherPath = path.join(makeTempHome("ade-windows-service-no-brain-"), "brain-service.ps1"); + + const result = await installWindowsService({ + command: serviceCommand, + launcherPath, + readPidRecord: () => null, + readinessProbe: () => ({ ready: false, diagnostic: "unused" }), + handoverTimeoutMs: 0, + serviceName, + spawnSync, + userName: taskUser, + }); + + expect(result).toMatchObject({ + ok: false, + failureStep: "replacement_responsive", + }); + }); + it("ends and replaces a running channel task before starting the repaired runtime", async () => { const calls: Array<{ command: string; args: string[] }> = []; const spawnSync = spawnSequence(calls, [ @@ -1123,3 +1279,285 @@ describe("Windows background service helpers", () => { })).toMatch(/^C:\\Users\\arul\\\.ade-beta\\runtime\\brain-service-[a-f0-9]{12}\.ps1$/i); }); }); + +describe("Windows runtime supervisor", () => { + it("renders bounded restart state for both child exits and launch failures", () => { + const script = renderWindowsServiceLauncher({ + command: "C:\\Program Files\\ADE\\ade.exe", + args: ["C:\\Program Files\\ADE\\resources\\ade-cli\\cli.cjs", "serve"], + }, { + pidPath: "C:\\Users\\arul\\.ade-beta\\runtime\\brain.pid.json", + initialRestartDelayMs: 250, + maxRestartDelayMs: 5_000, + healthyRuntimeMs: 30_000, + }); + + expect(script).toContain("while ($true)"); + expect(script).toContain("$initialRestartDelayMs = 250"); + expect(script).toContain("$maxRestartDelayMs = 5000"); + expect(script).toContain("lastLaunchError = $lastLaunchError"); + expect(script).toContain("} catch {"); + expect(script).toContain("Start-Sleep -Milliseconds ([int]$restartDelayMs)"); + }); + + it("reads legacy and current PID records with bounded diagnostics", () => { + const pidPath = path.join(makeTempHome("ade-windows-supervisor-"), "brain.pid.json"); + fs.writeFileSync(pidPath, JSON.stringify({ supervisorPid: 101, runtimePid: 202 }), "utf8"); + expect(readWindowsServicePidRecord({ pidPath })).toEqual({ + supervisorPid: 101, + runtimePid: 202, + runtimeStartedAtMs: null, + restartCount: 0, + lastExitCode: null, + lastExitAt: null, + nextRestartAt: null, + lastLaunchError: null, + sessionBound: null, + }); + + fs.writeFileSync(pidPath, JSON.stringify({ + supervisorPid: 101, + runtimePid: null, + restartCount: 3, + lastLaunchError: "x".repeat(800), + }), "utf8"); + expect(readWindowsServicePidRecord({ pidPath })).toMatchObject({ + runtimePid: null, + restartCount: 3, + lastLaunchError: "x".repeat(512), + }); + }); + + it("binds runtime PID inspection to the executable, entrypoint, and serve command", () => { + const args = buildWindowsRuntimeQueryArgs(202, { + command: "C:\\Program Files\\ADE\\ade.exe", + args: ["C:\\Program Files\\ADE\\resources\\ade-cli\\cli.cjs", "serve"], + }); + const query = args.at(-1) ?? ""; + expect(query).toContain("ProcessId = 202"); + expect(query).toContain("C:\\Program Files\\ADE\\ade.exe"); + expect(query).toContain("C:\\Program Files\\ADE\\resources\\ade-cli\\cli.cjs"); + expect(query).toContain("matchesServe"); + }); + + it("waits asynchronously for semantic readiness without blocking the caller", async () => { + const sleepStarted: number[] = []; + const wait = waitForWindowsRuntimeReadiness({ + command: { command: "C:\\ADE\\ade.exe", args: ["serve"] }, + launcherPath: "C:\\ADE\\brain-service.ps1", + pidPath: "C:\\ADE\\brain.pid.json", + socketPath: "\\\\.\\pipe\\ade-test", + spawnSync, + readPidRecord: () => null, + // Comfortably more than one poll: the helper computes `remaining` from a + // deadline captured a few statements earlier, so a budget close to + // `pollMs` lets a stalled runner skip the first sleep and turn the + // synchronous `sleepStarted` assertion into a flake. Every iteration is + // free here (`readPidRecord` always answers null). + timeoutMs: 60, + pollMs: 10, + sleep: async (ms) => { + sleepStarted.push(ms); + await new Promise((resolve) => setTimeout(resolve, ms)); + }, + }); + + expect(wait).toBeInstanceOf(Promise); + expect(sleepStarted).toEqual([10]); + await expect(wait).resolves.toMatchObject({ + ready: false, + diagnostic: expect.stringContaining("did not publish a PID record"), + }); + }); + + it("reports `supervised` from the probe's identity check, never from pid liveness", async () => { + const record = { + supervisorPid: 4321, + runtimePid: 5678, + runtimeStartedAtMs: Date.now(), + restartCount: 0, + lastExitCode: null, + lastExitAt: null, + nextRestartAt: null, + lastLaunchError: null, + sessionBound: null, + }; + const wait = (supervised: boolean | undefined) => waitForWindowsRuntimeReadiness({ + command: { command: "C:\\ADE\\ade.exe", args: ["serve"] }, + launcherPath: "C:\\ADE\\brain-service.ps1", + pidPath: "C:\\ADE\\brain.pid.json", + socketPath: "\\\\.\\pipe\\ade-test", + spawnSync, + readPidRecord: () => record, + readinessProbe: () => ({ ready: false, supervised, diagnostic: "not yet" }), + timeoutMs: 0, + pollMs: 10, + }); + + // The recorded pids are alive as far as `process.kill(pid, 0)` is + // concerned in every one of these cases -- what differs is whether + // `Win32_Process` says the pid is a powershell running OUR launcher. Only + // that answer may promote a failed install to "still starting". + await expect(wait(true)).resolves.toMatchObject({ ready: false, supervised: true }); + await expect(wait(false)).resolves.toMatchObject({ ready: false, supervised: false }); + // A probe that says nothing is unknown, and unknown is not healthy. + await expect(wait(undefined)).resolves.toMatchObject({ ready: false, supervised: false }); + }); + + (process.platform === "win32" ? it : it.skip)( + "keeps supervising a missing executable and publishes launch-error backoff diagnostics", + async () => { + const dir = makeTempHome("ade-windows-supervisor-"); + const launcherPath = path.join(dir, "brain-service.ps1"); + const pidPath = `${launcherPath}.pid.json`; + fs.writeFileSync(launcherPath, `\uFEFF${renderWindowsServiceLauncher({ + command: path.join(dir, "missing-ade.exe"), + args: ["serve"], + }, { + pidPath, + initialRestartDelayMs: 100, + maxRestartDelayMs: 200, + })}`, "utf8"); + const supervisor = spawn(windowsPowerShellCommand(), [ + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + launcherPath, + ], { stdio: "ignore", windowsHide: true }); + try { + // The supervisor is a real detached PowerShell process, so this waits on + // powershell.exe cold start plus two full launch-failure backoff cycles. + // A 5s budget is a coin flip on a loaded Windows CI runner, where the + // record simply had not been written yet and the assertion below read + // null. Widen the patience; the assertion itself is unchanged. + const deadline = Date.now() + 45_000; + let record = readWindowsServicePidRecord({ pidPath }); + while ((!record?.lastLaunchError || record.restartCount < 2) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + record = readWindowsServicePidRecord({ pidPath }); + } + expect(record).toMatchObject({ + supervisorPid: supervisor.pid, + runtimePid: null, + restartCount: expect.any(Number), + lastLaunchError: expect.any(String), + nextRestartAt: expect.any(String), + }); + expect(record?.restartCount).toBeGreaterThanOrEqual(2); + } finally { + if (supervisor.pid) { + spawnSync("taskkill.exe", ["/PID", String(supervisor.pid), "/T", "/F"], { + encoding: "utf8", + windowsHide: true, + }); + } + } + }, + 60_000, + ); +}); + +describe("windows supervisor wedge guard", () => { + const command = { + command: "C:\\ade\\node.exe", + args: ["C:\\ade\\cli.cjs", "serve"], + env: { ADE_HOME: "C:\\Users\\example\\.ade" }, + }; + + it("waits in slices and stops a brain that stopped beating", () => { + const script = renderWindowsServiceLauncher(command, { + pidPath: "C:\\ade\\launcher.pid.json", + heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", + wedgeBreadcrumbPath: "C:\\ade\\runtime\\event-loop-wedge.json", + }); + + // An unbounded WaitForExit is exactly what makes a wedge invisible. + expect(script).not.toContain("$process.WaitForExit()\r\n $lastExitCode"); + expect(script).toContain("while (-not $process.WaitForExit($heartbeatPollMs))"); + expect(script).toContain("Test-BrainWedged $process.Id"); + expect(script).toContain("Write-WedgeBreadcrumb $wedgeAgeMs"); + // Kill($true) is .NET Core only; the supervisor must run under PS 5.1. + expect(script).toContain("$process.Kill()"); + expect(script).not.toContain("$process.Kill($true)"); + }); + + it("stops the wedged brain's whole process tree, through an absolute taskkill", () => { + const script = renderWindowsServiceLauncher(command, { + pidPath: "C:\\ade\\launcher.pid.json", + heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", + wedgeBreadcrumbPath: "C:\\ade\\runtime\\event-loop-wedge.json", + }); + + // Absolute System32 path, never a bare `taskkill` off PATH. On a real + // Windows host the resolver returns the verified filesystem form + // (C:\Windows\System32\taskkill.exe); elsewhere it falls back to the + // kernel GLOBALROOT form — both end in System32\taskkill.exe. + expect(script).toMatch(/\$taskkillPath = '[^']*System32\\taskkill\.exe'/i); + expect(script).toContain("& $taskkillPath '/PID' $process.Id '/T' '/F'"); + // Kill() alone orphans ConPTYs and agent CLIs, so the tree kill must come + // first and Kill() must only mop up what taskkill could not. + const taskkillAt = script.indexOf("& $taskkillPath"); + const killAt = script.indexOf("$process.Kill()"); + expect(taskkillAt).toBeGreaterThan(-1); + expect(taskkillAt).toBeLessThan(killAt); + expect(script).toContain("if (-not $process.HasExited) { $process.Kill() }"); + // Bounded, and never a bare WaitForExit(): if taskkill was unresolvable and + // Kill() threw, an unbounded wait parks the supervisor on the wedge forever. + expect(script).not.toContain("$process.WaitForExit()"); + const waitAt = script.indexOf("if ($process.WaitForExit(30000)) { break }", killAt); + expect(waitAt).toBeGreaterThan(killAt); + expect(script).toContain("did not exit after the kill"); + // Leaving the wait loop is conditional on the process being GONE. An + // unconditional break after a timed-out kill would start a second brain + // beside an unkillable one, both wanting the same ports and worktrees. + expect(script).not.toMatch(/WaitForExit\(30000\)[^\r\n]*[\r\n]+\s*break/); + const wedgeRetryAt = script.indexOf("retrying on the next heartbeat check", waitAt); + expect(wedgeRetryAt).toBeGreaterThan(waitAt); + // The bounded wait can fall through with the process still alive, and + // `.ExitCode` throws on a live process -- which would surface the wedge as + // a launch failure. Read it only once the process has actually exited. + expect(script).toContain( + "if ($process.HasExited) { $lastExitCode = $process.ExitCode } else { $lastExitCode = $null }", + ); + // ...and never as an unguarded statement of its own. + expect(script).not.toMatch(/(?:^|[\r\n])\s*\$lastExitCode = \$process\.ExitCode/); + }); + + it("keeps the beat interval and stale threshold bound to the brain's own", () => { + const script = renderWindowsServiceLauncher(command, { + pidPath: "C:\\ade\\launcher.pid.json", + heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", + }); + expect(script).toContain(`$heartbeatStaleMs = ${BRAIN_HEARTBEAT_STALE_MS}`); + expect(script).toContain(`$heartbeatPollMs = ${BRAIN_HEARTBEAT_INTERVAL_MS}`); + }); + + it("only judges a beat that belongs to the child it started", () => { + const script = renderWindowsServiceLauncher(command, { + pidPath: "C:\\ade\\launcher.pid.json", + heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", + wedgeBreadcrumbPath: "C:\\ade\\runtime\\event-loop-wedge.json", + }); + expect(script).toContain("if ([int]$beat.pid -ne $runtimePid) { return $null }"); + expect(script).toContain("if ($ageMs -le $heartbeatStaleMs) { return $null }"); + }); + + it("keeps its old exit-only behaviour when no heartbeat path is configured", () => { + const script = renderWindowsServiceLauncher(command, { + pidPath: "C:\\ade\\launcher.pid.json", + }); + expect(script).toContain("$heartbeatPath = $null"); + expect(script).toContain("if ([string]::IsNullOrEmpty($heartbeatPath)) { return $null }"); + }); + + it("refuses a stale threshold short enough to fire on an ordinary gap", () => { + const script = renderWindowsServiceLauncher(command, { + pidPath: "C:\\ade\\launcher.pid.json", + heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", + heartbeatStaleMs: 500, + }); + expect(script).toContain("$heartbeatStaleMs = 30000"); + }); +}); diff --git a/apps/ade-cli/src/serviceManager/installWindows.ts b/apps/ade-cli/src/serviceManager/installWindows.ts index 6b01d1bdb..606e7d340 100644 --- a/apps/ade-cli/src/serviceManager/installWindows.ts +++ b/apps/ade-cli/src/serviceManager/installWindows.ts @@ -17,7 +17,9 @@ import { resolveAdeServeCliScriptPath, resolveAdeServeCommand, resolveRuntimeServiceName, + RUNTIME_SERVICE_YOUNG_BRAIN_MS, serviceManagerResultText, + WINDOWS_HANDOVER_TIMEOUT_MS, terminatePidGracefullyAsync, type ServiceManagerResult, type ServiceManagerSpawnSync, @@ -116,6 +118,14 @@ type WindowsServiceManagerDeps = { handoverTimeoutMs?: number; handoverPollMs?: number; sleep?: (ms: number) => Promise; + /** + * Process liveness for the supervisor/brain pids in the record; tests inject + * it. Only ever a cheap gate on ENTERING the young-brain wait -- the + * authority on whether a recorded pid is really ours is the readiness + * probe's `Win32_Process` identity check, which runs inside that wait and + * decides whether the install may report `starting`. + */ + pidAlive?: (pid: number) => boolean; }; function resolvedServiceName( @@ -780,14 +790,22 @@ async function installWindowsServiceImpl( machineLayout.runtimeDir, BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE, ); + const renderedLauncher = `\uFEFF${renderWindowsServiceLauncher(serviceCommand, { + pidPath, + logPath, + heartbeatPath, + wedgeBreadcrumbPath, + })}`; + let existingLauncher: string | null = null; + try { + existingLauncher = fs.readFileSync(launcherPath, "utf8"); + } catch { + existingLauncher = null; + } + const launcherUnchanged = existingLauncher === renderedLauncher; try { fs.mkdirSync(path.dirname(launcherPath), { recursive: true }); - fs.writeFileSync(launcherPath, `\uFEFF${renderWindowsServiceLauncher(serviceCommand, { - pidPath, - logPath, - heartbeatPath, - wedgeBreadcrumbPath, - })}`, { + fs.writeFileSync(launcherPath, renderedLauncher, { encoding: "utf8", mode: 0o600, }); @@ -817,6 +835,56 @@ async function installWindowsServiceImpl( message: legacyRemoval.message, }; } + // launchd parity for the young-brain wait: an unchanged launcher whose + // supervisor and brain are alive, started less than RUNTIME_SERVICE_YOUNG_BRAIN_MS + // ago and never restarted, is a brain still coming up. Wait for it instead of + // ending and replacing it — every Repair used to kill exactly that brain. + const readPidRecord = deps.readPidRecord + ?? ((target: string) => readWindowsServicePidRecord({ pidPath: target })); + const isAlive = deps.pidAlive ?? isPidAlive; + const youngRecord = launcherUnchanged ? readPidRecord(pidPath) : null; + if ( + youngRecord + && youngRecord.restartCount === 0 + && youngRecord.runtimePid != null + && youngRecord.runtimeStartedAtMs != null + && Date.now() - youngRecord.runtimeStartedAtMs < RUNTIME_SERVICE_YOUNG_BRAIN_MS + && isAlive(youngRecord.supervisorPid) + && isAlive(youngRecord.runtimePid) + ) { + const youngReadiness = await waitForWindowsRuntimeReadiness({ + command: serviceCommand, + launcherPath, + pidPath, + socketPath, + spawnSync: run, + readPidRecord, + readinessProbe: deps.readinessProbe ?? defaultWindowsRuntimeReadiness, + timeoutMs: deps.handoverTimeoutMs ?? WINDOWS_HANDOVER_TIMEOUT_MS, + pollMs: deps.handoverPollMs ?? 100, + sleep: deps.sleep, + }); + if (youngReadiness.ready) { + return { + ok: true, + serviceName, + action: "install", + path: taskName, + message: "ADE per-user startup entry is already installed; the channel brain finished starting.", + }; + } + if (youngReadiness.supervised) { + return { + ok: true, + starting: true, + serviceName, + action: "install", + path: taskName, + message: `ADE per-user startup entry is installed; the channel brain is still starting on ${socketPath}: ${youngReadiness.diagnostic}`, + }; + } + // The young brain died while we waited: fall through and replace it. + } const currentRemoval = removeWindowsTaskIfPresent( run, taskName, @@ -888,8 +956,6 @@ async function installWindowsServiceImpl( message: `ADE per-user startup entry was installed, but the background service failed to start: ${startFailure}`, }; } - const readPidRecord = deps.readPidRecord - ?? ((target: string) => readWindowsServicePidRecord({ pidPath: target })); const readiness = await waitForWindowsRuntimeReadiness({ command: serviceCommand, launcherPath, @@ -898,10 +964,31 @@ async function installWindowsServiceImpl( spawnSync: run, readPidRecord, readinessProbe: deps.readinessProbe ?? defaultWindowsRuntimeReadiness, - timeoutMs: deps.handoverTimeoutMs ?? 15_000, + // Shorter than the POSIX budget on purpose: the Windows install + // already spends several PowerShell round-trips before this wait, and the + // whole child is bounded by `RUNTIME_SERVICE_START_WAIT_MS` (90s). A + // supervised brain that is not ready by then is reported as `starting`, + // not as a failure. + timeoutMs: deps.handoverTimeoutMs ?? WINDOWS_HANDOVER_TIMEOUT_MS, pollMs: deps.handoverPollMs ?? 100, sleep: deps.sleep, }); + if (!readiness.ready && readiness.supervised) { + // Same contract as launchd: a supervised brain that has not answered yet + // is still starting, not broken. The supervisor keeps it; the caller keeps + // waiting for the endpoint. + return { + ok: true, + starting: true, + restarted: true, + serviceName, + action: "install", + path: taskName, + message: + `ADE per-user startup entry installed; the channel brain is still starting on ${socketPath}: ` + + readiness.diagnostic, + }; + } if (!readiness.ready) { return { ok: false, @@ -920,6 +1007,7 @@ async function installWindowsServiceImpl( const sessionBound = readPidRecord(pidPath)?.sessionBound === true; return { ok: true, + restarted: true, serviceName, action: "install", path: taskName, diff --git a/apps/ade-cli/src/serviceManager/runtimeServiceBudgets.ts b/apps/ade-cli/src/serviceManager/runtimeServiceBudgets.ts new file mode 100644 index 000000000..640c58654 --- /dev/null +++ b/apps/ade-cli/src/serviceManager/runtimeServiceBudgets.ts @@ -0,0 +1,55 @@ +/** + * Every timeout in the brain's start/handover lifecycle, in one place. + * + * These numbers only mean anything relative to each other: the desktop's wait + * for the endpoint has to outlast the installer's handover budget, and the + * young-brain window has to outlast a cold start. They used to be spelled as + * bare literals in seven files, so tuning one silently broke that ordering. + * + * Lives in the CLI because the CLI owns the service installers; the desktop + * imports from here the same way it imports the installers themselves. + */ + +/** + * How long a freshly (re)started brain gets to answer before the installer + * stops waiting and reports it as `starting` instead of ready. Generous on + * purpose: this used to be 10s, and a brain that legitimately took longer on a + * cold machine was reported as a failed install, which the desktop turned into + * "couldn't be set up" plus a Repair that killed the brain and started the + * same race over. + */ +export const RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS = 30_000; + +/** + * A brain younger than this that is not answering yet is presumed to still be + * starting. Installers leave it alone and wait for it instead of restarting it: + * restarting a booting brain only resets its clock, and doing so on every + * Repair click is how a slow machine could never finish starting one. + * + * Also the desktop's "still starting, not broken" window. + */ +export const RUNTIME_SERVICE_YOUNG_BRAIN_MS = 120_000; + +/** + * The Windows supervisor's handover budget. Shorter than the POSIX one because + * the supervisor is a process this installer starts and watches directly, + * rather than a service handed to launchd/systemd and observed through it. + */ +export const WINDOWS_HANDOVER_TIMEOUT_MS = 15_000; + +/** + * How long a caller waits for the service endpoint to come up. + * + * Deliberately larger than RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS: an installer + * that gives up and reports `starting` has NOT failed, and the caller has to + * still be waiting when the brain finishes. Two handover budgets (the + * young-brain wait plus the real restart) fit inside this with room to spare. + */ +export const RUNTIME_SERVICE_START_WAIT_MS = 90_000; + +/** + * How long a caller keeps dialing a brain the installer reported as `starting` + * before it gives up. Twice the handover budget: the installer stopped waiting, + * the supervisor did not, and the socket appears the moment the brain is up. + */ +export const RUNTIME_SERVICE_STARTING_CONNECT_WAIT_MS = 60_000; diff --git a/apps/ade-cli/src/serviceManager/runtimeShutdownRequest.test.ts b/apps/ade-cli/src/serviceManager/runtimeShutdownRequest.test.ts deleted file mode 100644 index e83f8e312..000000000 --- a/apps/ade-cli/src/serviceManager/runtimeShutdownRequest.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import net from "node:net"; -import { EventEmitter } from "node:events"; -import { describe, expect, it } from "vitest"; -import { requestAdeRuntimeShutdown } from "./runtimeShutdownRequest"; - -/** - * A stand-in for the brain's JSON-RPC endpoint. `replies` maps a method to the - * result it answers with; anything absent is simply not answered, which is how - * a wedged brain behaves. `closeOnShutdown` models the brain whose orderly exit - * drops the socket before its own response gets out. - */ -function fakeEndpoint( - replies: Record, - options: { closeOnShutdown?: boolean } = {}, -): { - socket: net.Socket; - written: string[]; -} { - const written: string[] = []; - const socket = new EventEmitter() as unknown as net.Socket & { destroy: () => void }; - let buffer = ""; - (socket as unknown as { write: unknown }).write = (payload: string) => { - written.push(payload); - buffer += payload; - for (;;) { - const newline = buffer.indexOf("\n"); - if (newline < 0) break; - const line = buffer.slice(0, newline); - buffer = buffer.slice(newline + 1); - const message = JSON.parse(line) as { id: number; method: string }; - if (options.closeOnShutdown && message.method === "shutdown") { - queueMicrotask(() => socket.emit("close")); - continue; - } - if (!(message.method in replies)) continue; - queueMicrotask(() => { - socket.emit( - "data", - Buffer.from(`${JSON.stringify({ jsonrpc: "2.0", id: message.id, result: replies[message.method] })}\n`), - ); - }); - } - return true; - }; - (socket as unknown as { destroy: () => void }).destroy = () => {}; - queueMicrotask(() => socket.emit("connect")); - return { socket, written }; -} - -describe("requestAdeRuntimeShutdown", () => { - const socketPath = String.raw`\\.\pipe\ade-runtime-stable-0123456789abcdef`; - - it("identifies the endpoint before asking it to leave", async () => { - const endpoint = fakeEndpoint({ "runtime/info": { pid: 4242 }, shutdown: {} }); - const result = await requestAdeRuntimeShutdown({ - pid: 4242, - socketPath, - connect: () => endpoint.socket, - }); - expect(result).toEqual({ requested: true }); - const methods = endpoint.written.map((line) => (JSON.parse(line) as { method: string }).method); - expect(methods).toEqual(["runtime/info", "shutdown"]); - }); - - it("refuses to shut down a pid the endpoint does not belong to", async () => { - // A pid scraped from a port diagnosis or a supervisor record can have been - // recycled; shutting down whoever happens to answer would be a stranger. - const endpoint = fakeEndpoint({ "runtime/info": { pid: 999 }, shutdown: {} }); - const result = await requestAdeRuntimeShutdown({ - pid: 4242, - socketPath, - connect: () => endpoint.socket, - }); - expect(result.requested).toBe(false); - const methods = endpoint.written.map((line) => (JSON.parse(line) as { method: string }).method); - expect(methods).toEqual(["runtime/info"]); - }); - - /** - * The orderly exit this path asks for tears down the brain's listening - * socket, which races the JSON-RPC response back to us. Reading the close as - * a refusal would send the caller to `taskkill /F` and cut the flush short. - */ - it("treats a close after the request as the shutdown taking effect", async () => { - const endpoint = fakeEndpoint({ "runtime/info": { pid: 4242 } }, { closeOnShutdown: true }); - const result = await requestAdeRuntimeShutdown({ - pid: 4242, - socketPath, - connect: () => endpoint.socket, - }); - expect(result).toEqual({ requested: true }); - }); - - it("reports a close before the request as the endpoint hanging up", async () => { - const endpoint = fakeEndpoint({}); - queueMicrotask(() => endpoint.socket.emit("close")); - const result = await requestAdeRuntimeShutdown({ - pid: 4242, - socketPath, - connect: () => endpoint.socket, - }); - expect(result).toEqual({ - requested: false, - reason: "the runtime endpoint closed before it could be asked to stop", - }); - }); - - it("gives up on a wedged endpoint instead of hanging the caller", async () => { - const endpoint = fakeEndpoint({}); - const result = await requestAdeRuntimeShutdown({ - pid: 4242, - socketPath, - timeoutMs: 250, - connect: () => endpoint.socket, - }); - expect(result).toEqual({ - requested: false, - reason: "the runtime endpoint did not answer within 250ms", - }); - }); - - it("never dials a tcp runtime endpoint", async () => { - const result = await requestAdeRuntimeShutdown({ - pid: 4242, - socketPath: "tcp://127.0.0.1:9999?token=secret", - connect: () => { - throw new Error("must not connect"); - }, - }); - expect(result.requested).toBe(false); - }); -}); diff --git a/apps/ade-cli/src/serviceManager/serviceHandover.ts b/apps/ade-cli/src/serviceManager/serviceHandover.ts new file mode 100644 index 000000000..f5360d91c --- /dev/null +++ b/apps/ade-cli/src/serviceManager/serviceHandover.ts @@ -0,0 +1,239 @@ +import { spawnSync } from "node:child_process"; +import { + type AdeServiceCommand, + RUNTIME_SERVICE_YOUNG_BRAIN_MS, + type ServiceManagerSpawnSync, +} from "./common"; +import { + LAST_FAILURE_CRASH_LOOP_WINDOW_MS, + readLastFailure, +} from "../../../desktop/src/main/services/runtime/lastFailureStore"; + +/** + * Handover primitives shared by every platform installer (launchd, systemd, + * Windows). They were born inside `installLaunchd.ts`; systemd needs the exact + * same semantics, and two copies of "is this brain young enough to wait for" + * would drift the moment one of them is tuned. + */ + +export type ResponsivenessProbe = (args: { + socketPath: string; + timeoutMs: number; + command: AdeServiceCommand; +}) => boolean; + +export type PidElapsedMsLookup = ( + pid: number, + run: ServiceManagerSpawnSync, +) => number | null; + +export async function serviceHandoverSleep(ms: number): Promise { + // Awaited lifecycle delays must stay referenced: in a standalone CLI the + // handover polling can be the only pending work, and an unref'd timer lets + // the process exit mid-repair (before SIGKILL escalation / service load). + await new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +export function runtimeStatusArgs(command: AdeServiceCommand, socketPath: string): string[] { + const args = [...command.args]; + const serveIndex = args.lastIndexOf("serve"); + if (serveIndex >= 0) { + args.splice(serveIndex, 1, "runtime", "status"); + } else { + args.push("runtime", "status"); + } + args.push("--socket", socketPath, "--timeout", "1500", "--text"); + return args; +} + +/** + * The probe is a whole `ade runtime status` child process: Node/Electron + * start-up plus loading the CLI bundle BEFORE it can even dial the socket. Its + * kill timeout therefore has to be the socket wait plus a start-up allowance; + * when the two were equal, a machine where the CLI took longer than the socket + * budget to start killed every probe before it could answer, and a perfectly + * healthy brain read as "never responsive". + */ +export const RESPONSIVENESS_PROBE_STARTUP_ALLOWANCE_MS = 8_000; + +/** + * How often the handover wait spends a probe child. Each probe is a full CLI + * process; on the slow machines this wait exists for, probing every poll tick + * would compete with the very brain it is waiting on. + */ +export const RESPONSIVENESS_PROBE_INTERVAL_MS = 750; + +export const defaultResponsivenessProbe: ResponsivenessProbe = (args) => { + const env = { + ...process.env, + ...(args.command.env ?? {}), + ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1", + }; + const result = spawnSync( + args.command.command, + runtimeStatusArgs(args.command, args.socketPath), + { + encoding: "utf8", + env, + timeout: args.timeoutMs + RESPONSIVENESS_PROBE_STARTUP_ALLOWANCE_MS, + stdio: "ignore", + }, + ); + return result.status === 0 && !result.error; +}; + +/** + * A fresh failure streak recorded by the brain itself (`last-failure.json`, + * the same record project recovery and the startup backoff read). Two or more + * failures inside the crash-loop window means the supervisor is respawning a + * brain that dies, and its youth is not a reason to wait for it. + * + * Scoped to the ADE home this install is FOR, not the installer's own: a + * desktop on one channel installing another channel's service would otherwise + * read the wrong `last-failure.json`. + */ +export function recentCrashLoopForAdeHome(adeHome: string): boolean { + const report = readLastFailure({ kind: "machine", env: { ADE_HOME: adeHome } }); + if (!report || report.count < 2) return false; + const firstAt = Date.parse(report.firstAt); + return Number.isFinite(firstAt) && Date.now() - firstAt <= LAST_FAILURE_CRASH_LOOP_WINDOW_MS; +} + +/** + * A running service child that is younger than the young-brain window and not + * answering yet is presumed to still be starting. Unknown age (the age lookup + * failed) counts as NOT young, so a wedged brain is never mistaken for a + * booting one — this fails toward restarting, never toward waiting forever. + */ +export function isYoungBrain( + pid: number | null | undefined, + run: ServiceManagerSpawnSync, + elapsedMs: PidElapsedMsLookup, +): boolean { + if (!pid) return false; + const elapsed = elapsedMs(pid, run); + return elapsed != null && elapsed < RUNTIME_SERVICE_YOUNG_BRAIN_MS; +} + +// --------------------------------------------------------------------------- +// The handover wait itself +// --------------------------------------------------------------------------- + +/** Reads the supervisor's current child pid, or null when it has none. */ +export type SupervisedPidReader = () => number | null; + +export type HandoverWaitDeps = { + /** `launchctl print` / `systemctl show` — whatever names the live child. */ + readSupervisedPid: SupervisedPidReader; + isAlive: (pid: number) => boolean; + probeResponsiveness: ResponsivenessProbe; + socketPath: string; + command: AdeServiceCommand; + sleep: (ms: number) => Promise; + pollMs: number; +}; + +export type HandoverWaitOutcome = { + predecessorGone: boolean; + replacementPid: number | null; + replacementResponsive: boolean; +}; + +/** + * Waits for a distinct replacement child to answer on the socket. + * + * Used twice per install: once with no predecessor (the young-brain wait) and + * once for the real restart. Each call gets its own `timeoutMs` budget rather + * than sharing one install-wide deadline — a young brain that dies late in its + * wait used to leave the real handover ~0 ms, so the restart that followed was + * reported as a `replacement_pid` failure instead of `starting`. + */ +export async function awaitServiceHandover( + deps: HandoverWaitDeps, + oldPid: number | null, + timeoutMs: number, +): Promise { + const deadline = Date.now() + Math.max(0, timeoutMs); + let predecessorGone = oldPid == null || !deps.isAlive(oldPid); + let replacementPid: number | null = null; + let replacementResponsive = false; + let lastProbeAt = 0; + do { + predecessorGone = oldPid == null || !deps.isAlive(oldPid); + replacementPid = deps.readSupervisedPid(); + const replacementDiffers = replacementPid != null && replacementPid !== oldPid; + // Each probe is a full CLI child process; on the slow machines this wait + // exists for, running one every poll tick would compete with the very + // brain it is waiting on. Poll the supervisor cheaply, probe at a slower + // cadence. + if ( + predecessorGone + && replacementDiffers + && Date.now() - lastProbeAt >= RESPONSIVENESS_PROBE_INTERVAL_MS + ) { + lastProbeAt = Date.now(); + replacementResponsive = deps.probeResponsiveness({ + socketPath: deps.socketPath, + timeoutMs: Math.min(1_500, Math.max(1, deadline - Date.now())), + command: deps.command, + }); + if (replacementResponsive) break; + } + if (Date.now() >= deadline) break; + await deps.sleep(Math.min(deps.pollMs, Math.max(1, deadline - Date.now()))); + } while (Date.now() <= deadline); + return { predecessorGone, replacementPid, replacementResponsive }; +} + +export type YoungBrainWaitOutcome = + /** Not a young-brain situation at all; go straight to the restart. */ + | { kind: "skipped" } + /** The child finished starting and answered. Nothing to restart. */ + | { kind: "responsive" } + /** Still starting, still alive. The caller reports `starting`. */ + | { kind: "starting"; pid: number } + /** It died while we waited; fall through and (re)start the service. */ + | { kind: "died" }; + +/** + * The "is this brain young enough to wait for, and did waiting help?" decision, + * identical on launchd and systemd. + * + * A live supervised child behind an unchanged service definition that is not + * answering yet, and is young, is still starting — first launch, cold disk, big + * project database. Restarting it only resets its clock, which is what made a + * Repair click kill the very brain that was seconds from ready. + * + * A brain that keeps dying is always "young" (the supervisor just respawned + * it), so a recorded failure streak vetoes the wait: that brain needs the + * restart and the crash-loop diagnosis, not more patience. + */ +export async function awaitYoungBrainStart(args: { + /** The plist/unit on disk already matches what we would write. */ + definitionUnchanged: boolean; + /** The supervisor reports a running child, and this is its pid. */ + supervisedPid: number | null | undefined; + running: boolean; + adeHome: string; + recentCrashLoop?: () => boolean; + run: ServiceManagerSpawnSync; + pidElapsedMs: PidElapsedMsLookup; + timeoutMs: number; + wait: HandoverWaitDeps; +}): Promise { + if (!args.definitionUnchanged || !args.running) return { kind: "skipped" }; + const crashLooping = args.recentCrashLoop + ? args.recentCrashLoop() + : recentCrashLoopForAdeHome(args.adeHome); + if (crashLooping) return { kind: "skipped" }; + if (!isYoungBrain(args.supervisedPid, args.run, args.pidElapsedMs)) return { kind: "skipped" }; + + const young = await awaitServiceHandover(args.wait, null, args.timeoutMs); + if (young.replacementResponsive) return { kind: "responsive" }; + if (young.replacementPid != null && args.wait.isAlive(young.replacementPid)) { + return { kind: "starting", pid: young.replacementPid }; + } + return { kind: "died" }; +} diff --git a/apps/ade-cli/src/serviceManager/windowsSupervisor.test.ts b/apps/ade-cli/src/serviceManager/windowsSupervisor.test.ts deleted file mode 100644 index 0820b9734..000000000 --- a/apps/ade-cli/src/serviceManager/windowsSupervisor.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import fs from "node:fs"; -import { spawn, spawnSync } from "node:child_process"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; -import { - buildWindowsRuntimeQueryArgs, - readWindowsServicePidRecord, - windowsPowerShellCommand, -} from "./installWindows"; -import { - BRAIN_HEARTBEAT_INTERVAL_MS, - BRAIN_HEARTBEAT_STALE_MS, -} from "../services/runtime/brainHeartbeat"; -import { - renderWindowsServiceLauncher, - waitForWindowsRuntimeReadiness, -} from "./windowsSupervisor"; - -const tempDirs: string[] = []; - -afterEach(() => { - for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); - } -}); - -function tempDir(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "ade-windows-supervisor-")); - tempDirs.push(dir); - return dir; -} - -describe("Windows runtime supervisor", () => { - it("renders bounded restart state for both child exits and launch failures", () => { - const script = renderWindowsServiceLauncher({ - command: "C:\\Program Files\\ADE\\ade.exe", - args: ["C:\\Program Files\\ADE\\resources\\ade-cli\\cli.cjs", "serve"], - }, { - pidPath: "C:\\Users\\arul\\.ade-beta\\runtime\\brain.pid.json", - initialRestartDelayMs: 250, - maxRestartDelayMs: 5_000, - healthyRuntimeMs: 30_000, - }); - - expect(script).toContain("while ($true)"); - expect(script).toContain("$initialRestartDelayMs = 250"); - expect(script).toContain("$maxRestartDelayMs = 5000"); - expect(script).toContain("lastLaunchError = $lastLaunchError"); - expect(script).toContain("} catch {"); - expect(script).toContain("Start-Sleep -Milliseconds ([int]$restartDelayMs)"); - }); - - it("reads legacy and current PID records with bounded diagnostics", () => { - const pidPath = path.join(tempDir(), "brain.pid.json"); - fs.writeFileSync(pidPath, JSON.stringify({ supervisorPid: 101, runtimePid: 202 }), "utf8"); - expect(readWindowsServicePidRecord({ pidPath })).toEqual({ - supervisorPid: 101, - runtimePid: 202, - runtimeStartedAtMs: null, - restartCount: 0, - lastExitCode: null, - lastExitAt: null, - nextRestartAt: null, - lastLaunchError: null, - sessionBound: null, - }); - - fs.writeFileSync(pidPath, JSON.stringify({ - supervisorPid: 101, - runtimePid: null, - restartCount: 3, - lastLaunchError: "x".repeat(800), - }), "utf8"); - expect(readWindowsServicePidRecord({ pidPath })).toMatchObject({ - runtimePid: null, - restartCount: 3, - lastLaunchError: "x".repeat(512), - }); - }); - - it("binds runtime PID inspection to the executable, entrypoint, and serve command", () => { - const args = buildWindowsRuntimeQueryArgs(202, { - command: "C:\\Program Files\\ADE\\ade.exe", - args: ["C:\\Program Files\\ADE\\resources\\ade-cli\\cli.cjs", "serve"], - }); - const query = args.at(-1) ?? ""; - expect(query).toContain("ProcessId = 202"); - expect(query).toContain("C:\\Program Files\\ADE\\ade.exe"); - expect(query).toContain("C:\\Program Files\\ADE\\resources\\ade-cli\\cli.cjs"); - expect(query).toContain("matchesServe"); - }); - - it("waits asynchronously for semantic readiness without blocking the caller", async () => { - const sleepStarted: number[] = []; - const wait = waitForWindowsRuntimeReadiness({ - command: { command: "C:\\ADE\\ade.exe", args: ["serve"] }, - launcherPath: "C:\\ADE\\brain-service.ps1", - pidPath: "C:\\ADE\\brain.pid.json", - socketPath: "\\\\.\\pipe\\ade-test", - spawnSync, - readPidRecord: () => null, - timeoutMs: 12, - pollMs: 10, - sleep: async (ms) => { - sleepStarted.push(ms); - await new Promise((resolve) => setTimeout(resolve, ms)); - }, - }); - - expect(wait).toBeInstanceOf(Promise); - expect(sleepStarted).toEqual([10]); - await expect(wait).resolves.toMatchObject({ - ready: false, - diagnostic: expect.stringContaining("did not publish a PID record"), - }); - }); - - (process.platform === "win32" ? it : it.skip)( - "keeps supervising a missing executable and publishes launch-error backoff diagnostics", - async () => { - const dir = tempDir(); - const launcherPath = path.join(dir, "brain-service.ps1"); - const pidPath = `${launcherPath}.pid.json`; - fs.writeFileSync(launcherPath, `\uFEFF${renderWindowsServiceLauncher({ - command: path.join(dir, "missing-ade.exe"), - args: ["serve"], - }, { - pidPath, - initialRestartDelayMs: 100, - maxRestartDelayMs: 200, - })}`, "utf8"); - const supervisor = spawn(windowsPowerShellCommand(), [ - "-NoProfile", - "-NonInteractive", - "-ExecutionPolicy", - "Bypass", - "-File", - launcherPath, - ], { stdio: "ignore", windowsHide: true }); - try { - // The supervisor is a real detached PowerShell process, so this waits on - // powershell.exe cold start plus two full launch-failure backoff cycles. - // A 5s budget is a coin flip on a loaded Windows CI runner, where the - // record simply had not been written yet and the assertion below read - // null. Widen the patience; the assertion itself is unchanged. - const deadline = Date.now() + 45_000; - let record = readWindowsServicePidRecord({ pidPath }); - while ((!record?.lastLaunchError || record.restartCount < 2) && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 20)); - record = readWindowsServicePidRecord({ pidPath }); - } - expect(record).toMatchObject({ - supervisorPid: supervisor.pid, - runtimePid: null, - restartCount: expect.any(Number), - lastLaunchError: expect.any(String), - nextRestartAt: expect.any(String), - }); - expect(record?.restartCount).toBeGreaterThanOrEqual(2); - } finally { - if (supervisor.pid) { - spawnSync("taskkill.exe", ["/PID", String(supervisor.pid), "/T", "/F"], { - encoding: "utf8", - windowsHide: true, - }); - } - } - }, - 60_000, - ); -}); - -describe("windows supervisor wedge guard", () => { - const command = { - command: "C:\\ade\\node.exe", - args: ["C:\\ade\\cli.cjs", "serve"], - env: { ADE_HOME: "C:\\Users\\example\\.ade" }, - }; - - it("waits in slices and stops a brain that stopped beating", () => { - const script = renderWindowsServiceLauncher(command, { - pidPath: "C:\\ade\\launcher.pid.json", - heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", - wedgeBreadcrumbPath: "C:\\ade\\runtime\\event-loop-wedge.json", - }); - - // An unbounded WaitForExit is exactly what makes a wedge invisible. - expect(script).not.toContain("$process.WaitForExit()\r\n $lastExitCode"); - expect(script).toContain("while (-not $process.WaitForExit($heartbeatPollMs))"); - expect(script).toContain("Test-BrainWedged $process.Id"); - expect(script).toContain("Write-WedgeBreadcrumb $wedgeAgeMs"); - // Kill($true) is .NET Core only; the supervisor must run under PS 5.1. - expect(script).toContain("$process.Kill()"); - expect(script).not.toContain("$process.Kill($true)"); - }); - - it("stops the wedged brain's whole process tree, through an absolute taskkill", () => { - const script = renderWindowsServiceLauncher(command, { - pidPath: "C:\\ade\\launcher.pid.json", - heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", - wedgeBreadcrumbPath: "C:\\ade\\runtime\\event-loop-wedge.json", - }); - - // Absolute System32 path, never a bare `taskkill` off PATH. On a real - // Windows host the resolver returns the verified filesystem form - // (C:\Windows\System32\taskkill.exe); elsewhere it falls back to the - // kernel GLOBALROOT form — both end in System32\taskkill.exe. - expect(script).toMatch(/\$taskkillPath = '[^']*System32\\taskkill\.exe'/i); - expect(script).toContain("& $taskkillPath '/PID' $process.Id '/T' '/F'"); - // Kill() alone orphans ConPTYs and agent CLIs, so the tree kill must come - // first and Kill() must only mop up what taskkill could not. - const taskkillAt = script.indexOf("& $taskkillPath"); - const killAt = script.indexOf("$process.Kill()"); - expect(taskkillAt).toBeGreaterThan(-1); - expect(taskkillAt).toBeLessThan(killAt); - expect(script).toContain("if (-not $process.HasExited) { $process.Kill() }"); - // Bounded, and never a bare WaitForExit(): if taskkill was unresolvable and - // Kill() threw, an unbounded wait parks the supervisor on the wedge forever. - expect(script).not.toContain("$process.WaitForExit()"); - const waitAt = script.indexOf("if ($process.WaitForExit(30000)) { break }", killAt); - expect(waitAt).toBeGreaterThan(killAt); - expect(script).toContain("did not exit after the kill"); - // Leaving the wait loop is conditional on the process being GONE. An - // unconditional break after a timed-out kill would start a second brain - // beside an unkillable one, both wanting the same ports and worktrees. - expect(script).not.toMatch(/WaitForExit\(30000\)[^\r\n]*[\r\n]+\s*break/); - const wedgeRetryAt = script.indexOf("retrying on the next heartbeat check", waitAt); - expect(wedgeRetryAt).toBeGreaterThan(waitAt); - // The bounded wait can fall through with the process still alive, and - // `.ExitCode` throws on a live process -- which would surface the wedge as - // a launch failure. Read it only once the process has actually exited. - expect(script).toContain( - "if ($process.HasExited) { $lastExitCode = $process.ExitCode } else { $lastExitCode = $null }", - ); - // ...and never as an unguarded statement of its own. - expect(script).not.toMatch(/(?:^|[\r\n])\s*\$lastExitCode = \$process\.ExitCode/); - }); - - it("keeps the beat interval and stale threshold bound to the brain's own", () => { - const script = renderWindowsServiceLauncher(command, { - pidPath: "C:\\ade\\launcher.pid.json", - heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", - }); - expect(script).toContain(`$heartbeatStaleMs = ${BRAIN_HEARTBEAT_STALE_MS}`); - expect(script).toContain(`$heartbeatPollMs = ${BRAIN_HEARTBEAT_INTERVAL_MS}`); - }); - - it("only judges a beat that belongs to the child it started", () => { - const script = renderWindowsServiceLauncher(command, { - pidPath: "C:\\ade\\launcher.pid.json", - heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", - wedgeBreadcrumbPath: "C:\\ade\\runtime\\event-loop-wedge.json", - }); - expect(script).toContain("if ([int]$beat.pid -ne $runtimePid) { return $null }"); - expect(script).toContain("if ($ageMs -le $heartbeatStaleMs) { return $null }"); - }); - - it("keeps its old exit-only behaviour when no heartbeat path is configured", () => { - const script = renderWindowsServiceLauncher(command, { - pidPath: "C:\\ade\\launcher.pid.json", - }); - expect(script).toContain("$heartbeatPath = $null"); - expect(script).toContain("if ([string]::IsNullOrEmpty($heartbeatPath)) { return $null }"); - }); - - it("refuses a stale threshold short enough to fire on an ordinary gap", () => { - const script = renderWindowsServiceLauncher(command, { - pidPath: "C:\\ade\\launcher.pid.json", - heartbeatPath: "C:\\ade\\runtime\\heartbeat.json", - heartbeatStaleMs: 500, - }); - expect(script).toContain("$heartbeatStaleMs = 30000"); - }); -}); diff --git a/apps/ade-cli/src/serviceManager/windowsSupervisor.ts b/apps/ade-cli/src/serviceManager/windowsSupervisor.ts index 6fbac477c..1503bcc9b 100644 --- a/apps/ade-cli/src/serviceManager/windowsSupervisor.ts +++ b/apps/ade-cli/src/serviceManager/windowsSupervisor.ts @@ -82,6 +82,21 @@ export type WindowsServicePidRecord = { export type WindowsRuntimeReadiness = { ready: boolean; diagnostic: string; + /** + * A process that is verifiably OUR supervisor -- the recorded pid is alive + * AND is a powershell running THIS launcher, per + * `buildWindowsSupervisorQueryArgs` -- owns a brain that has not answered + * yet. Callers treat that as "still starting" rather than a failed install. + * + * Deliberately not a bare `process.kill(pid, 0)` liveness probe: the pid + * comes from an on-disk record, a pid that outlived its record can have been + * recycled by an unrelated process, and `isPidAlive` reports EPERM (a pid + * owned by somebody else entirely) as alive. Answering "still starting" for + * a recycled pid turns a genuinely failed install into an `ok: true` the + * caller waits on and never repairs, so the identity check the readiness + * probe already performs is what decides this. + */ + supervised?: boolean; }; export type WindowsRuntimeReadinessProbe = (args: { @@ -499,18 +514,25 @@ export const defaultWindowsRuntimeReadiness: WindowsRuntimeReadinessProbe = (arg { encoding: "utf8", windowsHide: true }, ); if (supervisor.status !== 0) { + // Not supervised: either the pid is gone/recycled (3, 4) or the query + // itself could not answer. An unanswerable query is never reported as + // "still starting" -- unknown must not read as healthy. return { ready: false, + supervised: false, diagnostic: supervisor.status === 3 || supervisor.status === 4 ? `Supervisor PID ${pidRecord.supervisorPid} is stale or belongs to another process.` : serviceManagerResultText(supervisor) || `Unable to inspect supervisor PID ${pidRecord.supervisorPid}.`, }; } + // Past this point the recorded supervisor is verifiably ours, so every + // not-ready answer below is a brain still coming up under a live supervisor. if (pidRecord.runtimePid == null) { const restart = pidRecord.nextRestartAt ? `; restart scheduled for ${pidRecord.nextRestartAt}` : ""; const launchError = pidRecord.lastLaunchError ? ` Last launch error: ${pidRecord.lastLaunchError}.` : ""; return { ready: false, + supervised: true, diagnostic: `Supervisor PID ${pidRecord.supervisorPid} is running, but the ADE brain is between restart attempts${restart}.${launchError}`, }; } @@ -522,6 +544,7 @@ export const defaultWindowsRuntimeReadiness: WindowsRuntimeReadinessProbe = (arg if (runtime.status !== 0) { return { ready: false, + supervised: true, diagnostic: runtime.status === 3 || runtime.status === 4 ? `Runtime PID ${pidRecord.runtimePid} is stale or does not match this channel executable.` : serviceManagerResultText(runtime) || `Unable to inspect runtime PID ${pidRecord.runtimePid}.`, @@ -540,6 +563,7 @@ export const defaultWindowsRuntimeReadiness: WindowsRuntimeReadinessProbe = (arg if (status.status !== 0) { return { ready: false, + supervised: true, diagnostic: serviceManagerResultText(status) || `Runtime PID ${pidRecord.runtimePid} has not initialized on ${args.socketPath}.`, }; @@ -555,11 +579,13 @@ export const defaultWindowsRuntimeReadiness: WindowsRuntimeReadinessProbe = (arg } return { ready: false, + supervised: true, diagnostic: `Runtime endpoint responded with PID ${String(payload.pid ?? "unknown")}; expected ${pidRecord.runtimePid}.`, }; } catch { return { ready: false, + supervised: true, diagnostic: `Runtime PID ${pidRecord.runtimePid} returned an invalid readiness payload.`, }; } @@ -635,6 +661,7 @@ export async function waitForWindowsRuntimeReadiness(args: { const readinessProbe = args.readinessProbe ?? defaultWindowsRuntimeReadiness; const sleep = args.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); let diagnostic = "The Windows brain supervisor did not publish a PID record."; + let supervised = false; do { const pidRecord = readPidRecord(args.pidPath); if (pidRecord) { @@ -646,11 +673,15 @@ export async function waitForWindowsRuntimeReadiness(args: { spawnSync: args.spawnSync, }); if (result.ready) return result; + // A record alone is not a supervisor. `supervised` is whatever the + // probe's identity check concluded about the recorded pid, so a stale + // record naming a recycled pid never reads as "still starting". + supervised = result.supervised === true; diagnostic = result.diagnostic; } const remaining = deadline - Date.now(); if (remaining <= 0) break; await sleep(Math.min(Math.max(10, args.pollMs), remaining)); } while (Date.now() <= deadline); - return { ready: false, diagnostic }; + return { ready: false, diagnostic, supervised }; } diff --git a/apps/ade-cli/src/services/diagnostics/diagnosticReport.test.ts b/apps/ade-cli/src/services/diagnostics/diagnosticReport.test.ts new file mode 100644 index 000000000..9fd8948bd --- /dev/null +++ b/apps/ade-cli/src/services/diagnostics/diagnosticReport.test.ts @@ -0,0 +1,393 @@ +import { describe, expect, it } from "vitest"; +import { + buildDiagnosticIssueUrl, + buildDiagnosticReport, + projectPathLabel, + redactDiagnosticText, + tailLogText, +} from "./diagnosticReport"; + +const CONTEXT = { + homeDir: "/Users/ada", + username: "ada", + hostname: "adas-macbook-pro.local", + projectRoots: ["/Users/ada/Projects/Photon"], +}; + +describe("redactDiagnosticText", () => { + it("collapses every spelling of the home directory", () => { + const input = [ + "/Users/ada/.ade/runtime/brain.jsonl", + "C:\\Users\\ada\\AppData\\Roaming\\ADE", + "C:\\\\Users\\\\ada\\\\AppData", + "file:///Users/ada/Library/Logs", + "%2FUsers%2Fada%2FDownloads", + "/home/ada/.ade", + ].join("\n"); + + const out = redactDiagnosticText(input, CONTEXT); + + expect(out).not.toMatch(/ada/i); + expect(out).toContain("~/.ade/runtime/brain.jsonl"); + expect(out).toContain("~\\AppData\\Roaming\\ADE"); + expect(out).toContain("~/Library/Logs"); + }); + + it("labels the project by name and hash instead of its path", () => { + const out = redactDiagnosticText( + "opening /Users/ada/Projects/Photon/.ade/ade.db", + CONTEXT, + ); + const label = projectPathLabel("/Users/ada/Projects/Photon"); + + expect(out).toBe(`opening ${label}/.ade/ade.db`); + expect(label).toMatch(/^$/); + // Same project, same label: two reports about one project correlate. + expect(projectPathLabel("/Users/ada/Projects/Photon")).toBe(label); + expect(projectPathLabel("/Users/bob/Projects/Photon")).not.toBe(label); + }); + + it("removes emails and every credential shape we have seen in logs", () => { + // Assembled from its segments at runtime so secret scanners do not flag + // this synthetic credential as a real leaked JWT. + const jwtFixture = ["eyJhbGciOiJIUzI1NiJ9", "eyJzdWIiOiIxIn0", "abcdefghijklmnop"].join("."); + const input = [ + "signed in as ada.lovelace+ade@example.com", + "authorization: Bearer sk-ant-api03-AAAABBBBCCCCDDDDEEEEFFFF", + `token=${jwtFixture}`, + "github token ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + "posthog phc_ABCDEFGHIJKLMNOPQRSTUVWXYZ01", + "GET /pair?token=8e1f2a3b4c5d6e7f&mode=relay", + 'cookie: "0123456789abcdef0123456789abcdef0123"', + "https://ada:hunter2@relay.example.com/v1", + ].join("\n"); + + const out = redactDiagnosticText(input, CONTEXT); + + // The email rule runs before the userinfo one, so `ada:hunter2@relay…` is + // eaten as an address whole — the host goes with it and the line is left + // as `https://:/v1`. + expect(out).not.toContain("example.com"); + expect(out).not.toMatch(/hunter2|ghp_ABCDEF|phc_ABCDEF|eyJhbGciOiJ/); + expect(out).not.toMatch(/sk-ant-api03/); + expect(out).not.toContain("8e1f2a3b4c5d6e7f"); + expect(out).not.toContain("0123456789abcdef0123456789abcdef0123"); + expect(out).toContain(""); + expect(out).toContain(""); + expect(out).toContain(""); + expect(out).toContain(""); + // Structure survives: a maintainer can still tell which line was which. + expect(out).toContain("github token"); + expect(out).toContain("mode=relay"); + }); + + it("redacts every token prefix ADE itself accepts, not just the classic PAT", () => { + // Split so a secret scanner cannot read these synthetic keys as real ones. + const finePat = `github_pat_${"11ABCDEFG0abcdefghij"}_${"KLMNOPQRSTUVWXYZ0123456789abcdef"}`; + const input = [ + `fine-grained ${finePat}`, + `google ${"AIza"}SyABCDEFGHIJKLMNOPQRSTUVWXYZ0123456`, + `xai ${"xai-"}ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789`, + `groq ${"gsk_"}ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789`, + ].join("\n"); + + const out = redactDiagnosticText(input, CONTEXT); + + expect(out).not.toContain(finePat); + expect(out).not.toMatch(/AIzaSy|xai-ABCDEF|gsk_ABCDEF/); + expect(out).toContain("fine-grained "); + // Still idempotent with the added prefixes. + expect(redactDiagnosticText(out, CONTEXT)).toBe(out); + }); + + it("keeps model ids and snake_case identifiers that only look like keys", () => { + // Redaction that eats ordinary log text costs the maintainer the exact + // lines they were sent the report to read. None of these are key shapes: + // Together keys are `tgp_v1_...`, Mistral keys carry no prefix at all, and + // a Groq key is `gsk_` plus a run of base62 with no underscores in it. + const input = [ + "model mistral-small-2503-instruct-v1 selected", + "counter tg_message_delivery_attempt_count=4", + "error gsk_missing_runtime_module_error_here", + ].join("\n"); + + const out = redactDiagnosticText(input, CONTEXT); + + expect(out).toBe(input); + }); + + it("hides the domain when the account name is the email local part", () => { + // `ada` is both the OS account and the local part of the address. If the + // account rule ran first it would leave `@company.example`, which the + // email rule can no longer match — shipping the employer's domain. + const out = redactDiagnosticText("signed in as ada@company.example", CONTEXT); + + expect(out).not.toContain("company.example"); + expect(out).toBe("signed in as "); + }); + + it("keeps loopback addresses and drops routable ones", () => { + const input = [ + "brain answering on 127.0.0.1:8787", + "peer 192.168.1.44 connected", + "relay 2606:4700:4700::1111 handshake", + "local ::1 ok", + "chrome 140.0.7339.207", + "at 2026-08-16T09:30:00.123Z", + ].join("\n"); + + const out = redactDiagnosticText(input, CONTEXT); + + expect(out).toContain("127.0.0.1:8787"); + expect(out).toContain("::1 ok"); + expect(out).not.toContain("192.168.1.44"); + expect(out).not.toContain("2606:4700"); + expect(out).toContain(""); + // A four-part version is not an address, and a timestamp is not IPv6. + expect(out).toContain("chrome 140.0.7339.207"); + expect(out).toContain("2026-08-16T09:30:00.123Z"); + }); + + it("removes this machine's name and any tailnet name", () => { + const out = redactDiagnosticText( + "adas-macbook-pro.local reached mini-desktop.tailnet-cafe.ts.net", + CONTEXT, + ); + + expect(out).not.toContain("adas-macbook-pro"); + expect(out).not.toContain("ts.net"); + expect(out).toContain(""); + expect(out).toContain(""); + }); + + it("is idempotent, so re-redacting a stored report changes nothing", () => { + const input = [ + "/Users/ada/Projects/Photon/.ade/ade.db", + "ada@example.com Bearer sk-live-AAAABBBBCCCCDDDD", + "peer 10.0.0.7 via adas-macbook-pro.local", + ].join("\n"); + + const once = redactDiagnosticText(input, CONTEXT); + expect(redactDiagnosticText(once, CONTEXT)).toBe(once); + }); +}); + +describe("tailLogText", () => { + it("keeps the newest lines within both caps", () => { + const lines = Array.from({ length: 500 }, (_, index) => `line-${index}`); + const out = tailLogText(lines.join("\n"), { maxLines: 5, maxBytes: 32 * 1024 }); + + expect(out.split("\n")).toEqual(["line-495", "line-496", "line-497", "line-498", "line-499"]); + }); + + it("honours the byte cap even when the line cap allows more", () => { + const lines = Array.from({ length: 40 }, () => "x".repeat(100)); + const out = tailLogText(lines.join("\n"), { maxLines: 40, maxBytes: 250 }); + + expect(Buffer.byteLength(out, "utf8")).toBeLessThanOrEqual(250); + }); +}); + +describe("buildDiagnosticReport", () => { + function build() { + return buildDiagnosticReport({ + generatedAt: "2026-08-16T09:30:00.000Z", + app: { + version: "1.2.61", + packageChannel: "stable", + isPackaged: true, + platform: "darwin", + arch: "arm64", + osRelease: "25.3.0", + osProductVersion: "26.1", + electronVersion: "38.0.0", + nodeVersion: "22.14.0", + chromeVersion: "140.0.7339.207", + timezoneOffsetMinutes: -420, + }, + identity: { + installId: "ade_0123456789abcdef0123456789abcdef", + accountHash: "a1b2c3d4e5f6", + }, + context: { + surface: "project_recovery", + headline: "ADE couldn't open this project", + code: "db_integrity", + technicalDetail: "sqlite disk image is malformed at /Users/ada/Projects/Photon/.ade/ade.db", + projectRoot: "/Users/ada/Projects/Photon", + }, + state: { + localRuntimeStatus: { connectionState: "disconnected", runtimeMode: "primary", pid: 4321 }, + machineLastFailure: { code: "db_integrity", message: "cannot open /Users/ada/.ade/ade.db" }, + }, + storage: [{ label: "ADE home", path: "/Users/ada/.ade", freeBytes: 5 * 1024 ** 3, totalBytes: 500 * 1024 ** 3 }], + logs: [ + { + label: "Brain", + path: "/Users/ada/.ade/runtime/brain.jsonl", + text: '{"msg":"peer 192.168.1.9 for ada@example.com"}', + }, + { label: "Desktop updates", path: "/Users/ada/Library/ade-update.jsonl", error: "(not present)" }, + ], + notes: ["doctor: not run"], + redaction: CONTEXT, + }); + } + + it("includes every section a maintainer needs, keyed by the install id", () => { + const report = build(); + + for (const heading of [ + "# ADE diagnostic report", + "## What happened", + "## Install", + "## Technical detail", + "## Runtime status", + "## Last failure (machine)", + "## Disk space", + "## Logs", + "## Notes", + ]) { + expect(report).toContain(heading); + } + // The correlation key must survive redaction — it is the whole point. + expect(report).toContain("ade_0123456789abcdef0123456789abcdef"); + expect(report).toContain("Install id (PostHog distinct_id)"); + expect(report).toContain("Account hash: a1b2c3d4e5f6"); + expect(report).toContain("1.2.61"); + expect(report).toContain("db_integrity"); + expect(report).toContain("5.0 GB free of 500.0 GB"); + }); + + it("redacts the assembled document, not just the log lines", () => { + const report = build(); + + expect(report).not.toMatch(/\/Users\/ada/); + expect(report).not.toContain("ada@example.com"); + expect(report).not.toContain("192.168.1.9"); + expect(report).not.toContain("adas-macbook-pro"); + expect(report).toContain(projectPathLabel("/Users/ada/Projects/Photon")); + // Redaction already ran, so running it again is a no-op. + expect(redactDiagnosticText(report, CONTEXT)).toBe(report); + }); +}); + +describe("redactDiagnosticText idempotency", () => { + // Regression: `user` and `host` are ordinary account and machine names on + // Windows and in containers, and the name rules matched them again inside + // the ``/`` they had just written — `<>`, then `<<>>`. + it("leaves its own placeholders alone when the name IS the placeholder word", () => { + const context = { username: "user", hostname: "host" }; + const once = redactDiagnosticText("user@host started on host as user", context); + + expect(once).toContain(""); + expect(once).toContain(""); + expect(once).not.toContain("<<"); + expect(redactDiagnosticText(once, context)).toBe(once); + }); + + // Regression: the first fix for the above excluded `<` and `>` from the name + // boundaries, which silently suppressed the match for every bracketed name a + // log actually writes — `user=` and `Host: ` shipped the real + // account and machine name in the report. + it("still redacts a bracketed name instead of mistaking it for a placeholder", () => { + const context = { username: "ada", hostname: "buildbox" }; + const once = redactDiagnosticText("user= host= peer ", context); + + expect(once).not.toMatch(/ada/i); + expect(once).not.toContain("buildbox"); + expect(once).toContain("user="); + expect(once).toContain("host="); + // Brackets are re-emitted, never doubled around a placeholder. + expect(once).not.toContain("<>"); + expect(once).not.toContain("<>"); + expect(once).toBe("user= host= peer <@x>"); + expect(redactDiagnosticText(once, context)).toBe(once); + }); +}); + +describe("redactDiagnosticText hostnames", () => { + // Regression: the machine-name rule used to hand an already-assembled + // pattern to a helper that escapes what it is given, so the word-boundary + // lookarounds became literal text and no hostname was ever replaced. + it("replaces a suffix-less hostname and its mixed-case spellings", () => { + const context = { hostname: "ARUL-DESKTOP" }; + const input = [ + "bound socket on ARUL-DESKTOP", + "peer arul-desktop refused the handover", + "Arul-Desktop:7777", + ].join("\n"); + + const out = redactDiagnosticText(input, context); + + expect(out).not.toMatch(/arul-desktop/i); + expect(out).toBe(["bound socket on ", "peer refused the handover", ":7777"].join("\n")); + }); + + it("replaces both the fqdn and its short form, longest first", () => { + const out = redactDiagnosticText( + "adas-macbook-pro.local and adas-macbook-pro both answered", + { hostname: "adas-macbook-pro.local" }, + ); + + expect(out).toBe(" and both answered"); + }); + + it("leaves an unrelated word that merely contains the hostname alone", () => { + const out = redactDiagnosticText("superhostname-extra", { hostname: "hostname" }); + expect(out).toBe("superhostname-extra"); + }); +}); + +describe("buildDiagnosticIssueUrl", () => { + it("targets the ADE repo with a title and a compact stub body", () => { + const url = buildDiagnosticIssueUrl({ + surface: "project_recovery", + headline: "ADE couldn't open this project", + code: "db_integrity", + appVersion: "1.2.61", + platform: "darwin", + arch: "arm64", + installId: "ade_0123456789abcdef0123456789abcdef", + }); + const parsed = new URL(url); + + expect(parsed.origin + parsed.pathname).toBe("https://github.com/arul28/ADE/issues/new"); + expect(parsed.searchParams.get("title")).toBe("[report] ADE couldn't open this project (db_integrity)"); + expect(parsed.searchParams.get("body")).toContain("clipboard"); + expect(parsed.searchParams.get("body")).toContain("ade_0123456789abcdef0123456789abcdef"); + // Well under GitHub's URL ceiling: the full report rides the clipboard. + expect(url.length).toBeLessThan(2_000); + }); + + // Regression: the headline is renderer-supplied (an update failure message + // carries OS paths) and used to reach the issue title and stub body raw. + it("redacts the caller-supplied headline in the title and the stub body", () => { + const url = buildDiagnosticIssueUrl({ + surface: "auto_update", + headline: "Update failed writing /Users/ada/Library/Application Support/ADE on adas-macbook-pro.local", + code: "update_write_failed", + appVersion: "1.2.61", + platform: "darwin", + arch: "arm64", + installId: "ade_0123456789abcdef0123456789abcdef", + redaction: CONTEXT, + }); + const parsed = new URL(url); + const title = parsed.searchParams.get("title") ?? ""; + const body = parsed.searchParams.get("body") ?? ""; + + expect(title).not.toContain("/Users/ada"); + expect(title).not.toMatch(/adas-macbook-pro/i); + expect(title).toContain("~/Library/Application Support/ADE"); + expect(title).toContain(""); + expect(body).not.toContain("/Users/ada"); + // The URL itself is still a normal encoded URL, not a redacted string. + expect(parsed.origin + parsed.pathname).toBe("https://github.com/arul28/ADE/issues/new"); + }); + + it("leaves the title untouched when no redaction context is supplied", () => { + const url = buildDiagnosticIssueUrl({ surface: "cli", headline: "plain headline" }); + expect(new URL(url).searchParams.get("title")).toBe("[report] plain headline"); + }); +}); diff --git a/apps/ade-cli/src/services/diagnostics/diagnosticReport.ts b/apps/ade-cli/src/services/diagnostics/diagnosticReport.ts new file mode 100644 index 000000000..107287627 --- /dev/null +++ b/apps/ade-cli/src/services/diagnostics/diagnosticReport.ts @@ -0,0 +1,600 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +/** + * Diagnostic-report assembly and redaction, shared by the desktop + * "Report issue" button and the headless `ade report-issue` command. + * + * Assembly and redaction are pure — they touch neither Electron nor the + * network, and collection lives in the callers — so both a Markdown report and + * its redaction can be unit-tested with synthetic inputs on every platform. + * The one exception is {@link writeDiagnosticReportFile}, which lives here so + * every surface writes the report with the same owner-only permissions. + */ + +/** Where a maintainer files the issue this report is attached to. */ +export const DIAGNOSTIC_ISSUE_REPO = "arul28/ADE"; + +/** GitHub rejects very long URLs (~8 KB); the body in the URL stays a stub. */ +export const ISSUE_URL_MAX_LENGTH = 6_000; + +/** Per-log-file tail caps. Newest lines are kept, oldest dropped. */ +export const LOG_TAIL_MAX_LINES = 120; +export const LOG_TAIL_MAX_BYTES = 32 * 1024; + +export type DiagnosticRedactionContext = { + homeDir?: string | null; + username?: string | null; + hostname?: string | null; + /** Absolute project roots collapsed to ``. */ + projectRoots?: readonly (string | null | undefined)[]; +}; + +export type DiagnosticLogTail = { + label: string; + /** Displayed after redaction, so an absolute path is fine here. */ + path: string; + text?: string | null; + error?: string | null; +}; + +export type DiagnosticVolumeSpace = { + label: string; + path: string; + freeBytes: number | null; + totalBytes: number | null; +}; + +export type DiagnosticReportContext = { + /** Which screen the user pressed the button on. */ + surface: string; + headline?: string | null; + code?: string | null; + technicalDetail?: string | null; + projectRoot?: string | null; +}; + +export type DiagnosticReportInput = { + generatedAt: string; + app: { + version: string | null; + packageChannel?: string | null; + isPackaged?: boolean | null; + platform: string; + arch: string; + osRelease?: string | null; + /** macOS `sw_vers -productVersion`, when it could be read. */ + osProductVersion?: string | null; + electronVersion?: string | null; + nodeVersion?: string | null; + chromeVersion?: string | null; + timezoneOffsetMinutes?: number | null; + }; + identity: { + /** PostHog `distinct_id` for this installation (`ade_…`). */ + installId?: string | null; + /** Truncated one-way hash of the signed-in account id; never the email. */ + accountHash?: string | null; + machineKeyFingerprint?: string | null; + }; + context: DiagnosticReportContext; + /** JSON blobs rendered verbatim (after redaction). */ + state?: { + localRuntimeStatus?: unknown; + machineLastFailure?: unknown; + projectLastFailure?: unknown; + lastWedge?: unknown; + recoveryDiagnosis?: unknown; + updateTransaction?: unknown; + }; + storage?: readonly DiagnosticVolumeSpace[]; + logs?: readonly DiagnosticLogTail[]; + /** Free-form operational notes, e.g. "doctor: not run". */ + notes?: readonly string[]; + redaction?: DiagnosticRedactionContext; +}; + +// --------------------------------------------------------------------------- +// Redaction +// --------------------------------------------------------------------------- + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Every textual spelling of an absolute path we might meet in a log line: + * native separators, JSON-escaped backslashes, percent-encoded, and `file://`. + */ +function pathSpellings(absolutePath: string): string[] { + const posix = absolutePath.replace(/\\/g, "/"); + const win = absolutePath.replace(/\//g, "\\"); + const spellings = new Set([ + absolutePath, + posix, + win, + win.replace(/\\/g, "\\\\"), + encodeURIComponent(posix), + posix.replace(/\//g, "%2F"), + posix.replace(/\//g, "%2f"), + ]); + return [...spellings].filter((value) => value.length > 2); +} + +function alternationRegExp(values: readonly string[], flags = "gi"): RegExp | null { + const parts = [...new Set(values)].filter(Boolean).sort((a, b) => b.length - a.length); + if (parts.length === 0) return null; + return new RegExp(parts.map(escapeRegExp).join("|"), flags); +} + +/** + * Stable, non-reversible label for a project directory: the maintainer can + * correlate two reports about the same project without learning where it is. + */ +export function projectPathLabel(projectRoot: string): string { + const normalized = projectRoot.replace(/[\\/]+$/, ""); + const name = normalized.split(/[\\/]/).filter(Boolean).pop() ?? "project"; + const digest = createHash("sha256").update(normalized).digest("hex").slice(0, 6); + return ``; +} + +function isLoopbackIpv4(value: string): boolean { + return value === "0.0.0.0" || value.startsWith("127."); +} + +function isPlausibleIpv4(value: string): boolean { + const octets = value.split("."); + if (octets.length !== 4) return false; + return octets.every((octet) => { + if (!/^\d{1,3}$/.test(octet)) return false; + return Number(octet) <= 255; + }); +} + +/** + * Word boundaries for the name rules. The surrounding angle brackets are + * *captured* rather than excluded: an account or machine literally named + * `user` or `host` — both common on Windows and in containers — otherwise + * matched inside the ``/`` this function had just produced, and + * every further pass added another pair of brackets. Excluding `<`/`>` from + * the boundaries instead would be worse than the bug it fixed: it would also + * suppress the match for a genuine `user=` or `Host: ` log + * line, shipping the real name in the report. Re-emitting the brackets keeps + * the idempotency this module promises (and asserts in its tests) while still + * redacting bracketed names. + */ +const NAME_BOUNDARY_START = "(?` → ``, `ada` → ``, `` → ``. */ +function bracketAwarePlaceholder(placeholder: string) { + return (_match: string, open: string, close: string): string => + open === "<" && close === ">" ? placeholder : `${open}${placeholder}${close}`; +} + +/** + * Strips everything that could identify the machine or its owner. Applied to + * the whole report as the final step, so a section added later cannot leak by + * forgetting to call it. + * + * Idempotent: the placeholders it writes (`~`, ``, ``, …) never match + * any of its own patterns, so re-running it is a no-op. + */ +export function redactDiagnosticText( + text: string, + context: DiagnosticRedactionContext = {}, +): string { + if (!text) return ""; + let out = text; + + // 1. Project roots first — otherwise the home-dir rule below rewrites their + // prefix and the recognizable `` label can no longer be formed. + for (const root of context.projectRoots ?? []) { + const trimmed = typeof root === "string" ? root.trim() : ""; + if (!trimmed) continue; + const label = projectPathLabel(trimmed); + const pattern = alternationRegExp(pathSpellings(trimmed)); + if (pattern) out = out.replace(pattern, label); + } + + // 2. This machine's home directory, in every spelling. + const homeDir = context.homeDir?.trim(); + if (homeDir) { + const pattern = alternationRegExp(pathSpellings(homeDir)); + if (pattern) out = out.replace(pattern, "~"); + } + + // 3. Any other user's home directory shape, so a path we did not anticipate + // (another account, a copied log) still cannot name a person. + out = out + .replace(/(?:%2F|\/)(?:Users|home)(?:%2F|\/)[^/\\\s"'`,;:)\]}]+/gi, "~") + .replace(/[A-Za-z]:\\{1,2}Users\\{1,2}[^\\/\s"'`,;:)\]}]+/gi, "~") + .replace(/[A-Za-z]%3A(?:%5C)+Users(?:%5C)+[^%\s"'`,;:)\]}]+/gi, "~"); + + // 4. Emails, before both the account-name rule and the token blobs. Before + // the names because an account name is very often the local part of its + // owner's address (`ada` / `ada@company.com`): rewriting that first would + // leave `@company.com`, which the email pattern can no longer match + // (`>` is not a local-part character) — so the employer's domain would + // ship in the report. Before the tokens so an address is not eaten as a + // secret. + out = out.replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, ""); + + // 5. The OS account name wherever it appears on its own. + const username = context.username?.trim(); + if (username && username.length >= 3) { + out = out.replace( + new RegExp( + `${NAME_BOUNDARY_START}(?)${NAME_BOUNDARY_END}`, + "gi", + ), + bracketAwarePlaceholder(""), + ); + } + + // 6. Credentials. Prefixed forms first, then key-adjacent blobs. + out = out + .replace(/\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{4,}(?:\.[A-Za-z0-9_-]+)?/g, "") + .replace(/\b(Bearer|Basic|Token)\s+[A-Za-z0-9._~+/=-]{8,}/gi, "$1 ") + .replace(/\b(?:sk|pk|rk)-[A-Za-z0-9_-]{10,}/g, "") + .replace(/\bgh[pousr]_[A-Za-z0-9]{16,}/g, "") + // Fine-grained GitHub PATs are their own format, not a fifth member of the + // `gh?_` family: the prefix is a whole word and the body carries + // underscores, so neither the rule above nor the key/value rule below sees + // one that appears on its own in a log line. ADE accepts these everywhere + // it accepts a classic PAT, so an unredacted one is a live credential in a + // report the user is being told to paste into a public issue. + .replace(/\bgithub_pat_[A-Za-z0-9_]{20,}/g, "") + .replace(/\bph[cx]_[A-Za-z0-9_-]{16,}/g, "") + // The remaining real prefixes ADE stores keys for (Google AI, xAI, Groq). + // Anthropic/OpenAI/DeepSeek/OpenRouter/Moonshot are all `sk-`, already + // covered above. Together (`tgp_v1_...`) and Mistral (an unprefixed + // 32-char blob) have no prefix worth matching on: `tg_`/`mistral-` are not + // key shapes at all, and as patterns they ate ordinary log text + // (`tg_message_delivery_attempt_count`, the model id + // `mistral-small-2503-instruct-v1`) -- the real keys are long enough for + // the key-adjacent `{32,}` rule below. Groq keys are `gsk_` plus a run of + // base62, so the body deliberately excludes `_`: without that, any long + // enough `gsk_`-prefixed snake_case identifier disappeared too. + .replace(/\bAIza[A-Za-z0-9_-]{20,}/g, "") + .replace(/\bxai-[A-Za-z0-9]{20,}/g, "") + .replace(/\bgsk_[A-Za-z0-9]{32,}/g, "") + .replace(/\bxox[abposr]-[A-Za-z0-9-]{10,}/g, "") + .replace( + /([?&](?:token|key|api[_-]?key|access[_-]?token|secret|password|pin|sig|signature|code|auth)=)[^&\s"'`<>]+/gi, + "$1", + ) + .replace( + /((?:token|secret|password|passwd|authorization|cookie|api[_-]?key|apikey|access[_-]?key)"?\s*[:=]\s*"?)([A-Za-z0-9+/=_-]{32,})/gi, + "$1", + ); + + // 7. URL userinfo (`https://user:pass@host`). + out = out.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+(?::[^/\s@]*)?@/gi, "$1@"); + + // 8. Addresses. Loopback stays: "the brain answered on 127.0.0.1" is the + // fact a maintainer needs, and it identifies nobody. + out = out.replace(/\b\d{1,3}(?:\.\d{1,3}){3}\b/g, (match) => { + if (!isPlausibleIpv4(match)) return match; + return isLoopbackIpv4(match) ? match : ""; + }); + out = out.replace(/(?"); + out = out.replace( + /(? { + if (match === "::1" || match === "::") return match; + // Without a digit this is far more likely to be `Namespace::member` + // than an address, and mangling code identifiers costs real signal. + return /\d/.test(match) ? "" : match; + }, + ); + + // 9. Machine and tailnet names. + const hostname = context.hostname?.trim(); + if (hostname && hostname.length >= 3) { + const short = hostname.split(".")[0] ?? ""; + // Built here rather than through `alternationRegExp`, which escapes what it + // is given: escaping an already-assembled pattern turns the word-boundary + // lookarounds into literal text and the rule silently matches nothing. + const names = [...new Set([hostname, ...(short.length >= 3 ? [short] : [])])] + .filter(Boolean) + .sort((a, b) => b.length - a.length); + if (names.length > 0) { + // Boundaries and brackets hoisted outside the alternation: inside it, + // each branch would need its own copy and the capture-group numbering + // would shift with every extra name. + const pattern = new RegExp( + `${NAME_BOUNDARY_START}( escapeRegExp(name)).join("|")})(>?)${NAME_BOUNDARY_END}`, + "gi", + ); + out = out.replace(pattern, bracketAwarePlaceholder("")); + } + } + out = out + .replace(/(?") + .replace(/(?"); + + return out; +} + +// --------------------------------------------------------------------------- +// Report assembly +// --------------------------------------------------------------------------- + +/** Keeps the newest lines of a log, bounded by both line count and bytes. */ +export function tailLogText( + text: string, + limits: { maxLines?: number; maxBytes?: number } = {}, +): string { + const maxLines = limits.maxLines ?? LOG_TAIL_MAX_LINES; + const maxBytes = limits.maxBytes ?? LOG_TAIL_MAX_BYTES; + const lines = text.split(/\r?\n/); + while (lines.length > 0 && lines[lines.length - 1] === "") lines.pop(); + let kept = lines.slice(Math.max(0, lines.length - maxLines)); + while (kept.length > 1 && Buffer.byteLength(kept.join("\n"), "utf8") > maxBytes) { + kept = kept.slice(1); + } + let joined = kept.join("\n"); + if (Buffer.byteLength(joined, "utf8") > maxBytes) { + joined = Buffer.from(joined, "utf8").subarray(-maxBytes).toString("utf8").replace(/^�+/, ""); + } + return joined; +} + +function formatBytes(bytes: number | null | undefined): string { + if (bytes == null || !Number.isFinite(bytes)) return "unknown"; + const gb = bytes / 1024 ** 3; + if (gb >= 1) return `${gb.toFixed(1)} GB`; + return `${Math.round(bytes / 1024 ** 2)} MB`; +} + +function bullet(label: string, value: unknown): string { + const text = + value == null || value === "" + ? "unknown" + : typeof value === "string" + ? value + : String(value); + return `- ${label}: ${text}`; +} + +/** Omits the line entirely when there is nothing to say, instead of "unknown". */ +function optionalBullet(label: string, value: unknown): string | null { + if (value == null || value === "") return null; + return bullet(label, value); +} + +function lines(...values: (string | null)[]): string { + return values.filter((value): value is string => value != null).join("\n"); +} + +/** + * Wraps `body` in a fence longer than any backtick run inside it, so a log line + * that itself contains ``` cannot break out of the block. + */ +function fence(body: string, language = ""): string { + const longestRun = [...body.matchAll(/`+/g)].reduce((max, match) => Math.max(max, match[0].length), 0); + const marker = "`".repeat(Math.max(3, longestRun + 1)); + return [marker + language, body, marker].join("\n"); +} + +function jsonBlock(value: unknown): string { + try { + return fence(JSON.stringify(value, null, 2) ?? "null", "json"); + } catch { + return fence("(unserializable)", ""); + } +} + +function section(title: string, body: string | null): string | null { + if (!body || !body.trim()) return null; + return `## ${title}\n\n${body.trim()}`; +} + +/** + * Renders the Markdown report, then redacts the whole thing. Redaction runs on + * the assembled text rather than per-field on purpose: a future section cannot + * leak by forgetting to opt in. + */ +export function buildDiagnosticReport(input: DiagnosticReportInput): string { + const { app, identity, context } = input; + const projectRoots = [ + context.projectRoot, + ...(input.redaction?.projectRoots ?? []), + ].filter((value): value is string => typeof value === "string" && value.trim().length > 0); + + const parts: (string | null)[] = []; + + parts.push("# ADE diagnostic report"); + + parts.push( + section( + "What happened", + lines( + bullet("Surface", context.surface), + optionalBullet("Headline", context.headline), + optionalBullet("Recovery code", context.code), + bullet("Generated", input.generatedAt), + ), + ), + ); + + const osLine = [app.osProductVersion, app.osRelease].filter(Boolean).join(" / ") || null; + parts.push( + section( + "Install", + lines( + bullet("ADE version", app.version), + optionalBullet("Channel", app.packageChannel), + optionalBullet("Packaged", app.isPackaged == null ? null : String(app.isPackaged)), + bullet("Platform", `${app.platform} ${app.arch}`), + optionalBullet("OS", osLine), + optionalBullet("Electron", app.electronVersion), + optionalBullet("Node", app.nodeVersion), + optionalBullet("Chrome", app.chromeVersion), + optionalBullet( + "Timezone offset (min)", + app.timezoneOffsetMinutes == null ? null : String(app.timezoneOffsetMinutes), + ), + // The one line that makes the whole report correlatable: this is the + // PostHog `distinct_id` for anonymous events from this installation. + bullet("Install id (PostHog distinct_id)", identity.installId), + optionalBullet("Account hash", identity.accountHash), + optionalBullet("Machine key fingerprint", identity.machineKeyFingerprint), + bullet("Project", context.projectRoot ? projectPathLabel(context.projectRoot) : "none"), + ), + ), + ); + + parts.push(section("Technical detail", context.technicalDetail ? fence(context.technicalDetail) : null)); + + const state = input.state ?? {}; + parts.push(section("Runtime status", state.localRuntimeStatus == null ? null : jsonBlock(state.localRuntimeStatus))); + parts.push(section("Recovery diagnosis", state.recoveryDiagnosis == null ? null : jsonBlock(state.recoveryDiagnosis))); + parts.push(section("Last failure (machine)", state.machineLastFailure == null ? null : jsonBlock(state.machineLastFailure))); + parts.push(section("Last failure (project)", state.projectLastFailure == null ? null : jsonBlock(state.projectLastFailure))); + parts.push(section("Last wedge", state.lastWedge == null ? null : jsonBlock(state.lastWedge))); + parts.push(section("Update transaction", state.updateTransaction == null ? null : jsonBlock(state.updateTransaction))); + + const storage = input.storage ?? []; + parts.push( + section( + "Disk space", + storage.length + ? storage + .map((entry) => `- ${entry.label} (${entry.path}): ${formatBytes(entry.freeBytes)} free of ${formatBytes(entry.totalBytes)}`) + .join("\n") + : null, + ), + ); + + const logs = input.logs ?? []; + parts.push( + section( + "Logs", + logs.length + ? logs + .map((log) => { + const heading = `### ${log.label}\n\n\`${log.path}\``; + if (log.error) return `${heading}\n\n${log.error}`; + const text = (log.text ?? "").trim(); + if (!text) return `${heading}\n\n(empty)`; + return `${heading}\n\n${fence(text)}`; + }) + .join("\n\n") + : null, + ), + ); + + parts.push( + section("Notes", (input.notes ?? []).length ? (input.notes ?? []).map((note) => `- ${note}`).join("\n") : null), + ); + + const rendered = parts.filter((part): part is string => Boolean(part)).join("\n\n") + "\n"; + + return redactDiagnosticText(rendered, { + ...input.redaction, + projectRoots, + }); +} + +// --------------------------------------------------------------------------- +// GitHub issue URL +// --------------------------------------------------------------------------- + +export type DiagnosticIssueUrlInput = { + surface: string; + headline?: string | null; + code?: string | null; + appVersion?: string | null; + platform?: string | null; + arch?: string | null; + installId?: string | null; + repo?: string; + /** + * Same context the report body is redacted with. The headline is caller + * supplied (an update failure message, a runtime error) and routinely + * carries absolute paths, so the title and stub body get the same treatment + * as the report before they are put in a URL. + */ + redaction?: DiagnosticRedactionContext; +}; + +export function diagnosticIssueTitle(input: DiagnosticIssueUrlInput): string { + const headline = input.headline?.trim(); + const base = headline || `Problem on the ${input.surface.replace(/_/g, " ")} screen`; + const suffix = input.code?.trim() ? ` (${input.code.trim()})` : ""; + const title = `[report] ${base}${suffix}`; + return (input.redaction ? redactDiagnosticText(title, input.redaction) : title).slice(0, 180); +} + +/** + * The URL carries only a stub: the full report is on the clipboard, because a + * GitHub issue URL stops working somewhere north of 8 KB. + */ +export function diagnosticIssueBody(input: DiagnosticIssueUrlInput): string { + const body = [ + "", + "", + "**What I was doing:**", + "", + "", + "---", + "", + bullet("Surface", input.surface), + ...(input.code ? [bullet("Recovery code", input.code)] : []), + bullet("ADE version", input.appVersion), + bullet("Platform", [input.platform, input.arch].filter(Boolean).join(" ") || null), + bullet("Install id", input.installId), + "", + "Paste the full diagnostic report from your clipboard here:", + "", + ].join("\n"); + return input.redaction ? redactDiagnosticText(body, input.redaction) : body; +} + +export function buildDiagnosticIssueUrl(input: DiagnosticIssueUrlInput): string { + const repo = input.repo?.trim() || DIAGNOSTIC_ISSUE_REPO; + const url = new URL(`https://github.com/${repo}/issues/new`); + url.searchParams.set("title", diagnosticIssueTitle(input)); + url.searchParams.set("labels", "bug"); + const body = diagnosticIssueBody(input); + url.searchParams.set("body", body); + if (url.toString().length <= ISSUE_URL_MAX_LENGTH) return url.toString(); + // Degrade to a title-only link rather than handing the browser a URL the + // GitHub frontend will reject outright. + const short = new URL(`https://github.com/${repo}/issues/new`); + short.searchParams.set("title", diagnosticIssueTitle(input)); + return short.toString(); +} + +/** `2026-08-16T09-30-00-000Z-project_recovery.md`, safe on every filesystem. */ +export function diagnosticReportFileName(surface: string, at: Date): string { + const stamp = at.toISOString().replace(/[:.]/g, "-"); + const slug = surface.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "report"; + return `${stamp}-${slug}.md`; +} + +/** Convenience for callers that already know the directory. */ +export function diagnosticReportFilePath(dir: string, surface: string, at: Date): string { + return path.join(dir, diagnosticReportFileName(surface, at)); +} + +/** + * Writes the report with an owner-only directory and file. Best effort: a + * read-only or full disk must not turn "report a bug" into a second bug, and + * the issue URL is still usable on its own. + * + * Shared by every surface (desktop button, `ade report-issue`, the TUI) so the + * 0o700/0o600 modes are stated in exactly one place. + */ +export function writeDiagnosticReportFile(filePath: string, report: string): boolean { + try { + fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 }); + fs.writeFileSync(filePath, report, { encoding: "utf8", mode: 0o600 }); + return true; + } catch { + return false; + } +} diff --git a/apps/ade-cli/src/services/diagnostics/diagnosticSources.ts b/apps/ade-cli/src/services/diagnostics/diagnosticSources.ts new file mode 100644 index 000000000..211b986e6 --- /dev/null +++ b/apps/ade-cli/src/services/diagnostics/diagnosticSources.ts @@ -0,0 +1,166 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + tailLogText, + type DiagnosticLogTail, + type DiagnosticRedactionContext, + type DiagnosticVolumeSpace, +} from "./diagnosticReport"; +import { resolveMachineAdeLayout } from "../projects/machineLayout"; +import { resolveWindowsSupervisorLogPath } from "../../serviceManager/installWindows"; + +/** + * Everything a diagnostic report reads off this machine. + * + * `diagnosticReport.ts` renders and redacts; this reads. The desktop's + * "Report issue" button and `ade report-issue` need the same logs, the same + * volume pair and the same redaction context, and they had two copies of all + * of it — so a log added to one report never appeared in the other. + * + * Electron-only inputs (the app's own jsonl logs, `readVolumeSpace`, the typed + * last-failure store) stay in the desktop service and are layered on top. + */ + +/** Reads the last 512 KB of a file and tails it. Never throws. */ +export function readLogTail(label: string, filePath: string): DiagnosticLogTail { + try { + const stat = fs.statSync(filePath); + if (!stat.isFile()) return { label, path: filePath, error: "(not a file)" }; + const readBytes = Math.min(stat.size, 512 * 1024); + const handle = fs.openSync(filePath, "r"); + try { + const buffer = Buffer.alloc(readBytes); + fs.readSync(handle, buffer, 0, readBytes, Math.max(0, stat.size - readBytes)); + return { label, path: filePath, text: tailLogText(buffer.toString("utf8")) }; + } finally { + fs.closeSync(handle); + } + } catch (error) { + const code = error && typeof error === "object" && "code" in error + ? String((error as { code?: unknown }).code) + : ""; + return { label, path: filePath, error: code === "ENOENT" ? "(not present)" : "(could not be read)" }; + } +} + +/** Parses a JSON file, or null for anything that is missing or malformed. */ +export function readDiagnosticJsonFile(filePath: string): unknown { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown; + } catch { + return null; + } +} + +/** Free space via `statfs`. The desktop passes its own Electron-aware reader. */ +export function readVolumeViaStatfs(label: string, dirPath: string): DiagnosticVolumeSpace | null { + try { + const stats = fs.statfsSync(dirPath, { bigint: true }); + return { + label, + path: dirPath, + freeBytes: Number(stats.bavail * stats.bsize), + totalBytes: Number(stats.blocks * stats.bsize), + }; + } catch { + return null; + } +} + +/** `os.userInfo()` throws when the uid has no passwd entry (slim containers). */ +export function readOsUsername(): string | null { + try { + return os.userInfo().username; + } catch { + return null; + } +} + +/** + * The one note every report carries: neither entry point starts the brain to + * collect a report, because the machine where the brain will not start is + * exactly where the report is needed. + */ +export const DIAGNOSTIC_COLLECTION_NOTES = [ + "doctor: not run (the report is collected without starting the background service)", +] as const; + +export function diagnosticRedactionContext( + projectRoot: string | null, +): DiagnosticRedactionContext { + return { + homeDir: os.homedir(), + username: readOsUsername(), + hostname: os.hostname(), + // Only the project root is collapsed to a `` label; the ADE + // home is already reduced to `~/.ade` by the home-directory rule, and + // labelling it would hide which channel's home this machine uses. + projectRoots: projectRoot ? [projectRoot] : [], + }; +} + +export type MachineDiagnosticSourceOptions = { + env?: NodeJS.ProcessEnv; + projectRoot?: string | null; + /** + * The CLI's own transcript log. The desktop has richer logs of its own and + * does not write this one. + */ + includeProjectCliLog?: boolean; + /** Overridden by the desktop, which reads volumes through Electron's helper. */ + readVolume?: (label: string, dirPath: string) => DiagnosticVolumeSpace | null; +}; + +export type MachineDiagnosticSources = { + layout: ReturnType; + logs: DiagnosticLogTail[]; + storage: DiagnosticVolumeSpace[]; + state: { + machineLastFailure: unknown; + projectLastFailure: unknown; + lastWedge: unknown; + }; + notes: string[]; + redaction: DiagnosticRedactionContext; +}; + +export function collectMachineDiagnosticSources( + options: MachineDiagnosticSourceOptions = {}, +): MachineDiagnosticSources { + const env = options.env ?? process.env; + const projectRoot = options.projectRoot?.trim() || null; + const layout = resolveMachineAdeLayout(env); + const readVolume = options.readVolume ?? readVolumeViaStatfs; + + const logs: DiagnosticLogTail[] = []; + if (process.platform === "win32") { + logs.push(readLogTail("Background service supervisor", resolveWindowsSupervisorLogPath({ env }))); + } else { + logs.push(readLogTail("Background service (stderr)", path.join(layout.runtimeDir, "launchd.err.log"))); + } + logs.push(readLogTail("Brain", path.join(layout.runtimeDir, "brain.jsonl"))); + if (options.includeProjectCliLog && projectRoot) { + logs.push(readLogTail("ADE CLI", path.join(projectRoot, ".ade", "transcripts", "logs", "ade-cli.jsonl"))); + } + + const storage = [ + readVolume("ADE home", layout.adeDir), + projectRoot ? readVolume("Project", projectRoot) : null, + ].filter((entry): entry is DiagnosticVolumeSpace => entry != null); + + return { + layout, + logs, + storage, + state: { + machineLastFailure: readDiagnosticJsonFile(path.join(layout.runtimeDir, "last-failure.json")), + projectLastFailure: projectRoot + ? readDiagnosticJsonFile(path.join(projectRoot, ".ade", "runtime", "last-failure.json")) + : null, + lastWedge: readDiagnosticJsonFile(path.join(layout.runtimeDir, "last-wedge.json")), + }, + notes: [...DIAGNOSTIC_COLLECTION_NOTES], + redaction: diagnosticRedactionContext(projectRoot), + }; +} diff --git a/apps/ade-cli/src/services/projects/projectScope.test.ts b/apps/ade-cli/src/services/projects/projectScope.test.ts index 57f6c77d7..c83ba0720 100644 --- a/apps/ade-cli/src/services/projects/projectScope.test.ts +++ b/apps/ade-cli/src/services/projects/projectScope.test.ts @@ -709,6 +709,92 @@ describe("ProjectScopeRegistry", () => { }); }); +describe("ProjectScopeRegistry.adoptRequestedSyncHost", () => { + beforeEach(() => { + createAdeRuntimeMock.mockReset(); + createAdeRuntimeMock.mockImplementation(async () => ({ dispose: vi.fn() })); + }); + + /** + * `get()` is stubbed throughout: opening a real scope boots a whole sync + * runtime, and what is under test here is only the waiting policy. + */ + const stubScopeFor = (scopes: ProjectScopeRegistry, projectId: string) => { + const scope = { registryProjectId: projectId } as unknown as Awaited< + ReturnType + >; + vi.spyOn(scopes, "get").mockResolvedValue(scope); + return scope; + }; + + it("adopts a sync host that lands while it is waiting", async () => { + const { registry, first } = createRegistry(); + const scopes = new ProjectScopeRegistry(registry, { syncRuntime: { enabled: true } }); + stubScopeFor(scopes, first.projectId); + let clock = 0; + let elapsedWhenSet = -1; + + const adopted = await scopes.adoptRequestedSyncHost(30_000, { + now: () => clock, + pollMs: 250, + sleep: () => { + clock += 250; + // A concurrent switch completes a second into our wait. + if (clock >= 1_000 && elapsedWhenSet < 0) { + elapsedWhenSet = clock; + (scopes as unknown as { syncHostProjectId: string | null }).syncHostProjectId = first.projectId; + } + return Promise.resolve(); + }, + }); + + expect(adopted?.registryProjectId).toBe(first.projectId); + expect(elapsedWhenSet).toBe(1_000); + }); + + it("returns null when nothing lands inside the budget, so the caller retries", async () => { + const { registry } = createRegistry(); + const scopes = new ProjectScopeRegistry(registry, { syncRuntime: { enabled: true } }); + let clock = 0; + let polls = 0; + + const adopted = await scopes.adoptRequestedSyncHost(1_000, { + now: () => clock, + pollMs: 250, + sleep: () => { + polls += 1; + clock += 250; + return Promise.resolve(); + }, + }); + + expect(adopted).toBeNull(); + expect(polls).toBe(4); + }); + + it("returns immediately when a host is already active", async () => { + const { registry, first } = createRegistry(); + const scopes = new ProjectScopeRegistry(registry, { syncRuntime: { enabled: true } }); + stubScopeFor(scopes, first.projectId); + (scopes as unknown as { syncHostProjectId: string | null }).syncHostProjectId = first.projectId; + const sleep = vi.fn(() => Promise.resolve()); + + const adopted = await scopes.adoptRequestedSyncHost(30_000, { sleep }); + + expect(adopted?.registryProjectId).toBe(first.projectId); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("does not wait at all with a zero budget", async () => { + const { registry } = createRegistry(); + const scopes = new ProjectScopeRegistry(registry, { syncRuntime: { enabled: true } }); + const sleep = vi.fn(() => Promise.resolve()); + + expect(await scopes.adoptRequestedSyncHost(0, { sleep })).toBeNull(); + expect(sleep).not.toHaveBeenCalled(); + }); +}); + describe("markActiveHostProjectOpen", () => { it("marks the current sync host open even when a stale project is first", () => { const catalog = [ diff --git a/apps/ade-cli/src/services/projects/projectScope.ts b/apps/ade-cli/src/services/projects/projectScope.ts index 2bf72d31b..9617d8a6a 100644 --- a/apps/ade-cli/src/services/projects/projectScope.ts +++ b/apps/ade-cli/src/services/projects/projectScope.ts @@ -10,6 +10,12 @@ type SwitchSyncHostOptions = { const SYNC_HOST_COLD_BOOT_TIMEOUT_MS = 60_000; const SYNC_HOST_INITIALIZE_TIMEOUT_MS = 30_000; const SYNC_HOST_CONFIGURE_TIMEOUT_MS = 10_000; +/** + * How long the brain waits for a sync-host switch that superseded its own to + * land before it gives up and retries the whole resolution. + */ +export const SYNC_HOST_ADOPT_TIMEOUT_MS = 30_000; +const SYNC_HOST_ADOPT_POLL_MS = 250; class SyncHostPhaseTimeoutError extends Error {} @@ -181,6 +187,45 @@ export class ProjectScopeRegistry { return this.syncHostProjectId; } + /** + * The project a caller most recently asked to host sync, whether or not that + * switch has completed. `switchSyncHost` returns null both for "superseded by + * a newer switch" and for a genuinely absent host, and the brain's startup + * loop must not read the former as "no project, host projectless". + */ + getRequestedSyncHostProjectId(): ProjectId | null { + return this.latestSyncHostTransitionProjectId; + } + + /** + * Waits for a sync-host switch that superseded ours to land, and adopts its + * result. + * + * `switchSyncHost` returning null when a request is outstanding means + * "superseded": the RPC socket is published before the brain's startup loop + * runs, so a desktop that connected meanwhile may have requested its own + * switch and bumped the transition past ours. That is a project host in + * progress, not the absence of one — taking the projectless lease now would + * clobber it. Returns null if nothing lands inside the budget, which the + * caller retries. + */ + async adoptRequestedSyncHost( + timeoutMs: number, + deps: { sleep?: (ms: number) => Promise; now?: () => number; pollMs?: number } = {}, + ): Promise { + const now = deps.now ?? Date.now; + const sleep = deps.sleep + ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const pollMs = Math.max(1, deps.pollMs ?? SYNC_HOST_ADOPT_POLL_MS); + const deadline = now() + Math.max(0, timeoutMs); + while (now() < deadline) { + const activeId = this.getActiveSyncHostProjectId(); + if (activeId) return await this.get(activeId); + await sleep(pollMs); + } + return null; + } + async resolveActiveSyncHost(): Promise { if (!this.options.syncRuntime?.enabled) return null; const existingHostId = this.syncHostProjectId; diff --git a/apps/ade-cli/src/services/runtime/awaitRuntimeServiceEndpoint.test.ts b/apps/ade-cli/src/services/runtime/awaitRuntimeServiceEndpoint.test.ts new file mode 100644 index 000000000..fc4b7f75d --- /dev/null +++ b/apps/ade-cli/src/services/runtime/awaitRuntimeServiceEndpoint.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it, vi } from "vitest"; +import { awaitRuntimeServiceEndpoint } from "./awaitRuntimeServiceEndpoint"; + +const instantSleep = () => Promise.resolve(); + +/** A probe that fails `failures` times and answers on the next call. */ +function probeAfter(failures: number): { probe: () => Promise; calls: () => number } { + let calls = 0; + return { + probe: () => { + calls += 1; + return Promise.resolve(calls > failures); + }, + calls: () => calls, + }; +} + +describe("awaitRuntimeServiceEndpoint", () => { + it("never installs anything when the brain already answers", async () => { + const installService = vi.fn(); + const onStarting = vi.fn(); + + const result = await awaitRuntimeServiceEndpoint({ + probe: () => Promise.resolve(true), + installService, + onStarting, + budgetMs: 10_000, + sleep: instantSleep, + }); + + expect(result).toEqual({ ready: true, starting: false, detail: "background service is running" }); + expect(installService).not.toHaveBeenCalled(); + expect(onStarting).not.toHaveBeenCalled(); + }); + + it("keeps dialing after the install until the brain answers", async () => { + const { probe, calls } = probeAfter(4); + const onStarting = vi.fn(); + + const result = await awaitRuntimeServiceEndpoint({ + probe, + installService: () => Promise.resolve({ ok: true, message: "installed" }), + onStarting, + budgetMs: 10_000, + sleep: instantSleep, + pollMs: 1, + }); + + expect(result.ready).toBe(true); + expect(calls()).toBe(5); + // Announced once, as soon as the service is supervised. + expect(onStarting).toHaveBeenCalledTimes(1); + }); + + // The whole point of the helper: a supervised brain that outlasts the budget + // is `starting`, not a failure. Reporting it as one is what turned a slow + // machine into "couldn't be set up". + it("reports a brain that outlasts the budget as starting, not failed", async () => { + let clock = 0; + const result = await awaitRuntimeServiceEndpoint({ + probe: () => Promise.resolve(false), + installService: () => Promise.resolve({ ok: true, message: "installed" }), + onStarting: () => {}, + budgetMs: 100, + sleep: () => { + clock += 50; + return Promise.resolve(); + }, + now: () => clock, + pollMs: 50, + }); + + expect(result).toEqual({ + ready: false, + starting: true, + detail: "background service is still starting", + }); + }); + + it("waits on an install that reported `starting` rather than treating it as failed", async () => { + const { probe } = probeAfter(2); + const onStarting = vi.fn(); + + const result = await awaitRuntimeServiceEndpoint({ + probe, + installService: () => Promise.resolve({ + ok: false, + starting: true, + message: "the background service is still starting", + }), + onStarting, + budgetMs: 10_000, + sleep: instantSleep, + pollMs: 1, + }); + + expect(result.ready).toBe(true); + expect(onStarting).toHaveBeenCalledTimes(1); + }); + + it("returns the installer's own message when the install genuinely failed", async () => { + const onStarting = vi.fn(); + + const result = await awaitRuntimeServiceEndpoint({ + probe: () => Promise.resolve(false), + installService: () => Promise.resolve({ ok: false, message: "launchctl load failed." }), + onStarting, + budgetMs: 10_000, + sleep: instantSleep, + }); + + expect(result).toEqual({ ready: false, starting: false, detail: "launchctl load failed." }); + expect(onStarting).not.toHaveBeenCalled(); + }); + + it("still gives the endpoint one chance with a zero budget", async () => { + const { probe, calls } = probeAfter(1); + + const result = await awaitRuntimeServiceEndpoint({ + probe, + installService: () => Promise.resolve({ ok: true, message: "installed" }), + onStarting: () => {}, + budgetMs: 0, + sleep: instantSleep, + }); + + // One probe before the install, one after; the second answers. + expect(result.ready).toBe(true); + expect(calls()).toBe(2); + }); +}); diff --git a/apps/ade-cli/src/services/runtime/awaitRuntimeServiceEndpoint.ts b/apps/ade-cli/src/services/runtime/awaitRuntimeServiceEndpoint.ts new file mode 100644 index 000000000..7bc99a384 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/awaitRuntimeServiceEndpoint.ts @@ -0,0 +1,67 @@ +/** + * "Is the brain answering, and if not, install the service and wait for it." + * + * Setup's single hardest question, and the one place a wrong answer is most + * expensive: reporting "not running" against a socket that was simply never + * given time to open is what turned every slow machine into a broken one. + * + * The policy is here, free of the CLI's globals, so it can be tested against a + * probe that answers on the Nth call rather than against a real brain. + */ + +export type RuntimeServiceEndpointOutcome = { + ready: boolean; + /** Supervised and coming up: not ready, but explicitly not a failure. */ + starting: boolean; + detail: string; +}; + +export type AwaitRuntimeServiceEndpointDeps = { + /** Dials the endpoint once. Resolves false for any connect failure. */ + probe: () => Promise; + /** Runs the platform service installer. Only called when the probe fails. */ + installService: () => Promise<{ ok: boolean; starting?: boolean; message: string }>; + /** Called once the service is registered and supervised. */ + onStarting: () => void; + /** How long to keep dialing after a successful (or `starting`) install. */ + budgetMs: number; + sleep?: (ms: number) => Promise; + pollMs?: number; + now?: () => number; +}; + +const DEFAULT_POLL_MS = 500; + +export async function awaitRuntimeServiceEndpoint( + deps: AwaitRuntimeServiceEndpointDeps, +): Promise { + const now = deps.now ?? Date.now; + const sleep = deps.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const pollMs = Math.max(1, deps.pollMs ?? DEFAULT_POLL_MS); + const running: RuntimeServiceEndpointOutcome = { + ready: true, + starting: false, + detail: "background service is running", + }; + + if (await deps.probe()) return running; + + const install = await deps.installService(); + // `starting` is a live supervised brain that had not answered inside the + // installer's own budget — a slow start, not a failed install. + if (!install.ok && !install.starting) { + return { ready: false, starting: false, detail: install.message }; + } + + // Registered and supervised from here on, whether or not the install itself + // waited long enough to see it answer. + deps.onStarting(); + const deadline = now() + Math.max(0, deps.budgetMs); + for (;;) { + if (await deps.probe()) return running; + if (now() >= deadline) { + return { ready: false, starting: true, detail: "background service is still starting" }; + } + await sleep(pollMs); + } +} diff --git a/apps/ade-cli/src/services/runtime/brainStartupState.test.ts b/apps/ade-cli/src/services/runtime/brainStartupState.test.ts new file mode 100644 index 000000000..0137598cc --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainStartupState.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it, vi } from "vitest"; +import { + describeWindowsStartupProbe, + readBrainStartupState, + type BrainStartupProbe, +} from "./brainStartupState"; +import type { + WindowsServicePidRecord, + WindowsSupervisorState, +} from "../../serviceManager/windowsSupervisor"; + +function deps(overrides: { + installed?: boolean | null; + running?: boolean | null; + pid?: number | null; + ageMs?: number | null; + crashLooping?: boolean; +} = {}) { + return { + platform: "darwin" as NodeJS.Platform, + getServiceStatus: async () => ({ + installed: overrides.installed === undefined ? true : overrides.installed, + running: overrides.running === undefined ? true : overrides.running, + }), + getServiceMainPid: async () => (overrides.pid === undefined ? 4242 : overrides.pid), + readBrainAgeMs: async () => (overrides.ageMs === undefined ? 5_000 : overrides.ageMs), + hasRecentCrashLoop: async () => overrides.crashLooping === true, + youngBrainMs: 120_000, + }; +} + +function windowsProbe(overrides: Partial = {}) { + return { + platform: "win32" as NodeJS.Platform, + readWindowsStartupProbe: async (): Promise => ({ + installed: true, + running: false, + supervised: true, + ageMs: 5_000, + ...overrides, + }), + hasRecentCrashLoop: async () => false, + youngBrainMs: 120_000, + }; +} + +const NOW = 1_800_000_000_000; + +function runningSupervisor( + record: Partial = {}, +): WindowsSupervisorState { + return { + state: "running", + running: true, + pid: 4242, + record: { + supervisorPid: 4242, + runtimePid: 4343, + runtimeStartedAtMs: NOW, + restartCount: 0, + lastExitCode: null, + lastExitAt: null, + nextRestartAt: null, + lastLaunchError: null, + sessionBound: false, + ...record, + }, + error: null, + diagnostic: null, + }; +} + +describe("readBrainStartupState", () => { + it("calls a registered service whose brain is young 'starting'", async () => { + await expect(readBrainStartupState(deps())).resolves.toMatchObject({ + starting: true, + ageMs: 5_000, + }); + }); + + it("stops calling it starting once the brain outlives the young window", async () => { + await expect(readBrainStartupState(deps({ ageMs: 130_000 }))).resolves.toMatchObject({ + starting: false, + }); + }); + + it("is not starting when the service is absent or stopped", async () => { + await expect(readBrainStartupState(deps({ installed: false }))).resolves.toMatchObject({ + starting: false, + }); + await expect(readBrainStartupState(deps({ running: false }))).resolves.toMatchObject({ + starting: false, + }); + }); + + it("fails closed when the service running state is unknown", async () => { + // A status probe that could not tell answers null. Treating that as + // "supervised" pairs an unknown service with a young pid and reports a + // broken brain as a starting one, which suppresses the repair the user + // needs. + const state = await readBrainStartupState(deps({ running: null })); + expect(state.starting).toBe(false); + expect(state.serviceRunning).toBe(null); + }); + + it("fails closed when the age cannot be read or a probe throws", async () => { + await expect(readBrainStartupState(deps({ ageMs: null }))).resolves.toMatchObject({ + starting: false, + }); + await expect(readBrainStartupState({ + platform: "darwin", + getServiceStatus: async () => { + throw new Error("systemctl missing"); + }, + })).resolves.toMatchObject({ starting: false }); + }); + + // A crash-looping brain is respawned every few seconds, so it is ALWAYS young: + // without this veto `ade doctor` would report "starting, nothing to repair" + // forever on a permanently broken machine. + it("refuses to call a crash-looping brain starting even while it is young", async () => { + await expect( + readBrainStartupState(deps({ crashLooping: true })), + ).resolves.toMatchObject({ starting: false, ageMs: 5_000 }); + }); + + it("does not spend the crash-loop probe on a brain that is not young anyway", async () => { + const hasRecentCrashLoop = vi.fn(async () => false); + await readBrainStartupState({ ...deps({ ageMs: 130_000 }), hasRecentCrashLoop }); + expect(hasRecentCrashLoop).not.toHaveBeenCalled(); + }); + + describe("windows", () => { + it("calls a young single-start supervised brain starting", async () => { + await expect(readBrainStartupState(windowsProbe())).resolves.toMatchObject({ + starting: true, + ageMs: 5_000, + serviceInstalled: true, + }); + }); + + it("is not starting when no supervisor of ours is running", async () => { + await expect( + readBrainStartupState(windowsProbe({ supervised: false, ageMs: null, installed: true })), + ).resolves.toMatchObject({ starting: false }); + }); + + it("reads youth from the supervisor record, one start only", () => { + // Mirrors the installer's own young-brain predicate: a first start whose + // runtime pid is alive is young; anything the supervisor has already + // restarted is a crash loop wearing a fresh timestamp. + const young = describeWindowsStartupProbe({ + supervisor: runningSupervisor({ runtimeStartedAtMs: NOW - 5_000 }), + isAlive: () => true, + nowMs: NOW, + }); + expect(young).toMatchObject({ supervised: true, ageMs: 5_000 }); + + const restarted = describeWindowsStartupProbe({ + supervisor: runningSupervisor({ restartCount: 2, runtimeStartedAtMs: NOW - 5_000 }), + isAlive: () => true, + nowMs: NOW, + }); + expect(restarted).toMatchObject({ supervised: true, ageMs: null }); + + const deadRuntime = describeWindowsStartupProbe({ + supervisor: runningSupervisor({ runtimeStartedAtMs: NOW - 5_000 }), + isAlive: () => false, + nowMs: NOW, + }); + expect(deadRuntime).toMatchObject({ supervised: true, ageMs: null }); + }); + + it("is not supervised when the recorded supervisor is not verifiably ours", () => { + expect(describeWindowsStartupProbe({ + supervisor: { + state: "stopped", + running: false, + pid: null, + record: null, + error: null, + diagnostic: "no record", + }, + isAlive: () => true, + // No pid record leaves "installed" unknown, not false: this probe never + // reads the HKCU Run entry, which can be registered on its own. + })).toMatchObject({ supervised: false, installed: null, ageMs: null }); + }); + + it("never asks the status command whether the brain answered", async () => { + const getServiceStatus = vi.fn(async () => ({ installed: true, running: false })); + await expect( + readBrainStartupState({ ...windowsProbe(), getServiceStatus }), + ).resolves.toMatchObject({ starting: true }); + expect(getServiceStatus).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/ade-cli/src/services/runtime/brainStartupState.ts b/apps/ade-cli/src/services/runtime/brainStartupState.ts new file mode 100644 index 000000000..a72c543ff --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainStartupState.ts @@ -0,0 +1,201 @@ +import { + readPidElapsedMs, + RUNTIME_SERVICE_YOUNG_BRAIN_MS, +} from "../../serviceManager/common"; +import type { WindowsSupervisorState } from "../../serviceManager/windowsSupervisor"; + +/** + * The CLI's read of the desktop's `brain_starting` recovery state. + * + * A brain that does not answer on its socket is not automatically broken: when + * the service is registered and its brain process is alive but younger than the + * young-brain window, it is still coming up (first launch, cold disk, large + * project database). The desktop reaches this verdict through its connection + * pool (`ProjectRecoveryService.diagnose` -> `brain_starting`); the CLI has no + * pool, so it asks the platform service manager the same two questions — + * is the service registered and supervised, and how old is its brain — and + * applies the same `RUNTIME_SERVICE_YOUNG_BRAIN_MS` bound. + * + * Time-bounded for the same reason the desktop's is: without the age check a + * brain that wedged during boot would read as "starting" forever. And + * crash-loop-vetoed for the reason the age check alone cannot cover: a + * supervisor that respawns a dying brain every few seconds keeps producing a + * young process forever, so youth stops being evidence of progress. This is + * exactly the veto `awaitYoungBrainStart` already applies before `isYoungBrain`. + */ +export type BrainStartupState = { + /** Registered, alive, young, and not crash-looping: waiting is the right move. */ + starting: boolean; + /** Age of the service's brain process in ms, or null when unknown. */ + ageMs: number | null; + serviceInstalled: boolean | null; + serviceRunning: boolean | null; +}; + +/** + * What a platform probe reports about the service behind a silent socket. + * `supervised` is the platform's own answer to "is a supervisor we recognize + * running a brain right now", which is NOT the same question as the status + * command's `running` on Windows (there it means "the brain already answered", + * which is false by construction everywhere this module is called). + */ +export type BrainStartupProbe = { + installed: boolean | null; + running: boolean | null; + supervised: boolean; + ageMs: number | null; +}; + +export type BrainStartupStateDeps = { + getServiceStatus?: () => Promise<{ installed: boolean | null; running: boolean | null }>; + getServiceMainPid?: () => Promise; + readBrainAgeMs?: (pid: number | null) => Promise; + /** Windows-only probe; ignored on the POSIX supervisors. */ + readWindowsStartupProbe?: () => Promise; + /** The `last-failure.json` crash-loop veto, scoped to this install's ADE home. */ + hasRecentCrashLoop?: () => Promise; + youngBrainMs?: number; + platform?: NodeJS.Platform; +}; + +async function defaultGetServiceStatus(): Promise<{ + installed: boolean | null; + running: boolean | null; +}> { + const { getRuntimeServiceStatus } = await import("../../serviceManager"); + const status = getRuntimeServiceStatus(); + return { installed: status.installed, running: status.running }; +} + +async function defaultGetServiceMainPid(): Promise { + const { getRuntimeServiceMainPid } = await import("../../serviceManager"); + return getRuntimeServiceMainPid(); +} + +async function defaultReadBrainAgeMs(pid: number | null): Promise { + return pid == null ? null : readPidElapsedMs(pid); +} + +/** + * Windows has no launchd/systemd "the supervisor has a live child" answer, and + * its `ServiceManagerStatusResult.running` means "the brain answered on the + * pipe" — which is false by construction here. So this asks the supervisor + * record directly, and gates youth on the same predicate the Windows installer + * uses before it decides to wait for a booting brain instead of replacing it: + * a first start (never restarted) whose runtime pid is still alive. + */ +export function describeWindowsStartupProbe(args: { + supervisor: WindowsSupervisorState; + isAlive: (pid: number) => boolean; + nowMs?: number; +}): BrainStartupProbe { + const { supervisor } = args; + const record = supervisor.record; + // Absent record means "we cannot tell": the HKCU Run entry can be registered + // without a pid record, and this probe never reads that key. Report unknown + // rather than claiming the service is not installed. + const installed = supervisor.state === "error" || record == null ? null : true; + if (!supervisor.running || !record) { + return { installed, running: supervisor.running, supervised: false, ageMs: null }; + } + const startedAtMs = record.runtimeStartedAtMs; + const eligible = record.restartCount === 0 + && record.runtimePid != null + && startedAtMs != null + && args.isAlive(record.runtimePid); + return { + installed, + running: supervisor.running, + supervised: true, + ageMs: eligible && startedAtMs != null + ? Math.max(0, (args.nowMs ?? Date.now()) - startedAtMs) + : null, + }; +} + +async function defaultReadWindowsStartupProbe(): Promise { + const [ + { resolveWindowsServiceLauncherPath, resolveWindowsServicePidPath }, + { queryWindowsSupervisor }, + { isPidAlive }, + { spawnSync }, + ] = await Promise.all([ + import("../../serviceManager/installWindows"), + import("../../serviceManager/windowsSupervisor"), + import("../../serviceManager/common"), + import("node:child_process"), + ]); + return describeWindowsStartupProbe({ + supervisor: queryWindowsSupervisor({ + spawnSync, + launcherPath: resolveWindowsServiceLauncherPath(), + pidPath: resolveWindowsServicePidPath(), + }), + isAlive: isPidAlive, + }); +} + +async function defaultHasRecentCrashLoop(): Promise { + const [{ recentCrashLoopForAdeHome }, { resolveMachineAdeLayout }] = await Promise.all([ + import("../../serviceManager/serviceHandover"), + import("../projects/machineLayout"), + ]); + return recentCrashLoopForAdeHome(resolveMachineAdeLayout().adeDir); +} + +/** + * Call this only when the brain did NOT answer — a responding brain is running, + * never "starting". + */ +export async function readBrainStartupState( + deps: BrainStartupStateDeps = {}, +): Promise { + const youngBrainMs = deps.youngBrainMs ?? RUNTIME_SERVICE_YOUNG_BRAIN_MS; + const platform = deps.platform ?? process.platform; + let installed: boolean | null = null; + let running: boolean | null = null; + let ageMs: number | null = null; + let supervised = false; + try { + if (platform === "win32") { + const probe = await (deps.readWindowsStartupProbe ?? defaultReadWindowsStartupProbe)(); + installed = probe.installed; + running = probe.running; + supervised = probe.supervised; + ageMs = probe.ageMs; + } else { + const status = await (deps.getServiceStatus ?? defaultGetServiceStatus)(); + installed = status.installed; + running = status.running; + // No registered service, or a registered one the supervisor is not + // running: nothing is coming up, so this is a real failure and stays one. + // `running === true`, not `!== false`: an indeterminate probe answers + // null, and pairing "we could not tell" with a young pid is exactly how a + // broken brain gets reported as a starting one. Same fail-closed rule the + // catch below applies to a probe that threw. + supervised = installed === true && running === true; + if (supervised) { + const pid = await (deps.getServiceMainPid ?? defaultGetServiceMainPid)(); + ageMs = await (deps.readBrainAgeMs ?? defaultReadBrainAgeMs)(pid); + } + } + if (supervised && ageMs != null && ageMs < youngBrainMs) { + // A supervisor respawning a brain that dies produces a young process on + // every loop, so the age bound alone never expires. The recorded failure + // streak is the only thing that tells the two apart. + if (await (deps.hasRecentCrashLoop ?? defaultHasRecentCrashLoop)()) { + return { starting: false, ageMs, serviceInstalled: installed, serviceRunning: running }; + } + } + } catch { + // Any probe failure fails closed to "not starting": reporting a brain as + // starting when we cannot tell would hide a genuinely dead one. + return { starting: false, ageMs: null, serviceInstalled: installed, serviceRunning: running }; + } + return { + starting: supervised && ageMs != null && ageMs < youngBrainMs, + ageMs, + serviceInstalled: installed, + serviceRunning: running, + }; +} diff --git a/apps/ade-cli/src/services/runtime/connectWhileServiceStarts.ts b/apps/ade-cli/src/services/runtime/connectWhileServiceStarts.ts new file mode 100644 index 000000000..a11c8ccad --- /dev/null +++ b/apps/ade-cli/src/services/runtime/connectWhileServiceStarts.ts @@ -0,0 +1,52 @@ +import { + RuntimeServiceStillStartingError, + type ServiceManagerResult, +} from "../../serviceManager/common"; +import { RUNTIME_SERVICE_STARTING_CONNECT_WAIT_MS } from "../../serviceManager/runtimeServiceBudgets"; + +/** + * Dials the endpoint of a service that was just installed, allowing for one + * that reported `starting`. + * + * `starting` means the supervisor has a live brain that had not answered + * inside the installer's own budget. Keep dialing — the socket appears the + * moment the brain finishes coming up — instead of failing on the first + * connect and leaving the caller to spawn a rival brain on a supervised + * socket. + */ +export async function connectWhileServiceStarts(args: { + install: Pick; + socketPath: string; + connect: () => Promise; + waitMs?: number; + sleep?: (ms: number) => Promise; + pollMs?: number; + now?: () => number; +}): Promise { + const now = args.now ?? Date.now; + const sleep = args.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const pollMs = Math.max(1, args.pollMs ?? 500); + const waitMs = args.waitMs ?? RUNTIME_SERVICE_STARTING_CONNECT_WAIT_MS; + const deadline = now() + (args.install.starting ? Math.max(0, waitMs) : 0); + for (;;) { + try { + return await args.connect(); + } catch (error) { + if (now() >= deadline) { + // A `starting` install left a live, supervised brain behind. Even if it + // outlasted our wait, the supervisor still owns that endpoint, so + // reporting a plain connect failure — which lets the caller spawn an + // unmanaged rival on the same socket — is exactly the wrong recovery. + if (args.install.starting) { + throw new RuntimeServiceStillStartingError({ + kind: "wait_exhausted", + socketPath: args.socketPath, + installMessage: args.install.message, + }); + } + throw error; + } + await sleep(pollMs); + } + } +} diff --git a/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts b/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts index 1c80f2cc5..c52ad6dea 100644 --- a/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts +++ b/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { runSyncHostStartupLoop, SyncHostStartupAbortedError } from "./syncHostStartupLoop"; +import { runSyncHostStartupLoop } from "./syncHostStartupLoop"; import { SyncHostSingletonConflictError, type SyncHostSingletonOwner, @@ -220,27 +220,9 @@ describe("runSyncHostStartupLoop", () => { expect(attempts).toBe(1); }); - // The loop retries forever on purpose so sync recovers when a rival brain - // exits. That is also how a brain that lost its socket became an immortal - // zombie — it never reached its own bind check. - it("aborts instead of retrying forever once another brain owns the socket", async () => { - let attempts = 0; - let socketTaken = false; - await expect(runSyncHostStartupLoop({ - startSyncHost: () => { - attempts += 1; - if (attempts >= 2) socketTaken = true; - return Promise.reject(new Error("sync host unavailable")); - }, - isDone: () => false, - log: () => {}, - sleep: instantSleep, - abortIf: () => socketTaken, - })).rejects.toBeInstanceOf(SyncHostStartupAbortedError); - expect(attempts).toBe(2); - }); - - it("keeps retrying while the socket is still ours", async () => { + // The loop retries forever on purpose so mobile sync recovers when a rival + // brain exits. + it("keeps retrying until the sync host comes up", async () => { let attempts = 0; await runSyncHostStartupLoop({ startSyncHost: () => { @@ -251,7 +233,6 @@ describe("runSyncHostStartupLoop", () => { isDone: () => false, log: () => {}, sleep: instantSleep, - abortIf: () => false, }); expect(attempts).toBe(3); }); diff --git a/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts b/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts index 0ca6f3cff..5f753ba05 100644 --- a/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts +++ b/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts @@ -13,14 +13,6 @@ export type SyncHostStartupLoopDeps = { // same-channel sibling; everything else waits, so two recovering brains can // never kill each other in a loop. getServiceMainPid?: () => number | null; - /** - * Checked before every retry. This loop retries forever by design so mobile - * sync recovers when a rival brain exits — but a brain that will never win - * the lease AND has lost its RPC socket to someone else has no reason to - * exist, and staying in the loop is how it becomes an immortal zombie - * (we found 18 stacked on one dev socket). Returning true aborts the loop. - */ - abortIf?: () => boolean | Promise; kill?: (pid: number, signal: NodeJS.Signals | number) => void; pidAlive?: (pid: number) => boolean; sleep?: (ms: number) => Promise; @@ -73,14 +65,6 @@ async function terminatePidAsync( } } -/** Thrown when `abortIf` asks the loop to stop; the caller decides how to exit. */ -export class SyncHostStartupAbortedError extends Error { - constructor() { - super("ADE brain sync host startup was aborted."); - this.name = "SyncHostStartupAbortedError"; - } -} - // Keeps retrying mobile sync host startup until it succeeds or the brain // shuts down. Same-channel conflicts are transient by nature (update races, // restart overlap, a stale sibling about to be evicted), so they retry: @@ -128,7 +112,6 @@ export async function runSyncHostStartupLoop(deps: SyncHostStartupLoopDeps): Pro throw error; } if (deps.maxAttempts != null && attempt >= deps.maxAttempts) return; - if (await deps.abortIf?.()) throw new SyncHostStartupAbortedError(); await sleep(slowRetryDelayMs); continue; } @@ -146,7 +129,6 @@ export async function runSyncHostStartupLoop(deps: SyncHostStartupLoopDeps): Pro } } if (deps.maxAttempts != null && attempt >= deps.maxAttempts) return; - if (await deps.abortIf?.()) throw new SyncHostStartupAbortedError(); await sleep(attempt <= fastRetryCount ? fastRetryDelayMs : slowRetryDelayMs); } } diff --git a/apps/ade-cli/src/tuiClient/__tests__/appPolling.test.tsx b/apps/ade-cli/src/tuiClient/__tests__/appPolling.test.tsx index dbf58be7f..40c496adf 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/appPolling.test.tsx +++ b/apps/ade-cli/src/tuiClient/__tests__/appPolling.test.tsx @@ -7,6 +7,7 @@ import type { LaneSummary } from "../../../../desktop/src/shared/types/lanes"; import type { BufferedEvent } from "../../eventBuffer"; import type { AdeCodeConnection, ProjectLaunchContext } from "../types"; import { captureTuiProductAnalytics, deriveTuiAnalyticsScreen } from "../productAnalytics"; +import { RuntimeServiceStillStartingError } from "../../serviceManager/common"; const mocks = vi.hoisted(() => ({ connectToAde: vi.fn(), @@ -512,6 +513,28 @@ describe("AdeCodeApp polling", () => { await unmountApp(instance); }); + it("shows a waiting state, not a failure, while the background service is starting", async () => { + // The desktop stopped calling a slow-starting brain a broken one; the TUI + // must not keep telling the same user that ADE Code failed to start. + mocks.connectToAde.mockImplementation(async () => { + throw new RuntimeServiceStillStartingError({ + kind: "not_answered", + socketPath: "/tmp/ade/sock/ade.sock", + installMessage: "ADE brain service is registered and starting", + }); + }); + + const instance = await renderApp(); + + await waitForFrame(instance, "ADE's background service is starting"); + const frame = stripAnsi(instance.frames.join("\n")); + expect(frame).toContain("there is nothing to do"); + expect(frame).toContain("r retry now"); + expect(frame).not.toContain("ADE Code failed to start"); + + await unmountApp(instance); + }); + it("renders remote startup failures as recoverable connection loss", async () => { mocks.connectToAde.mockImplementation(async () => { throw new Error( diff --git a/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts b/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts index 8833e0289..da1c416e2 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts @@ -29,6 +29,7 @@ import { } from "../eventDedup"; import type { AgentChatEventEnvelope } from "../../../../desktop/src/shared/types/chat"; import type { ProjectLaunchContext } from "../types"; +import type { ServiceManagerResult } from "../../serviceManager/common"; const childProcess = vi.hoisted(() => { // `pid` stays undefined by default so the spawn record is a no-op for the @@ -41,7 +42,9 @@ const childProcess = vi.hoisted(() => { }); const runtimeService = vi.hoisted(() => ({ - installRuntimeService: vi.fn(() => ({ + // Typed as the real result so a test can set `starting`/`failureStep` — the + // two fields that decide whether a rival brain may be spawned. + installRuntimeService: vi.fn((): ServiceManagerResult => ({ ok: false, serviceName: "com.ade.runtime", action: "install" as const, @@ -915,6 +918,59 @@ describe("connectToAde embedded mode", () => { expect(client.close).toHaveBeenCalledTimes(1); }); + it("refuses to spawn a rival brain while the installed service is still starting", async () => { + // The installer left a live, supervised brain that had not answered yet. + // Falling through to spawnDaemon here is what put a second, unmanaged brain + // on the socket the service already owns. + useMissingMachineSocket(); + runtimeService.installRuntimeService.mockReturnValue({ + ok: true, + starting: true, + serviceName: "com.ade.runtime", + action: "install", + path: "/tmp/com.ade.runtime.plist", + message: "ADE brain service is registered and starting", + }); + vi.spyOn(JsonRpcClient, "connect").mockRejectedValue( + Object.assign(new Error("connect ENOENT"), { code: "ENOENT" }), + ); + + vi.useFakeTimers(); + try { + const attempt = connectToAde({ project, preferServiceRepair: true }); + const rejection = expect(attempt).rejects.toThrow( + /background service is still starting/, + ); + // Long enough to outlast the whole `starting` retry budget. + await vi.advanceTimersByTimeAsync(180_000); + await rejection; + } finally { + vi.useRealTimers(); + } + + expect(childProcess.spawn).not.toHaveBeenCalled(); + }); + + it("refuses to spawn a rival brain when the failed install left a registered replacement", async () => { + useMissingMachineSocket(); + runtimeService.installRuntimeService.mockReturnValue({ + ok: false, + failureStep: "replacement_responsive", + serviceName: "com.ade.runtime", + action: "install", + path: "/tmp/com.ade.runtime.plist", + message: "replacement did not answer in time", + }); + vi.spyOn(JsonRpcClient, "connect").mockRejectedValue( + Object.assign(new Error("connect ENOENT"), { code: "ENOENT" }), + ); + + await expect(connectToAde({ project, preferServiceRepair: true })).rejects.toThrow( + /background service is still starting/, + ); + expect(childProcess.spawn).not.toHaveBeenCalled(); + }); + it("keeps the script entrypoint argv shape when a CLI script is resolved", async () => { const socketPath = useMissingMachineSocket(); const entrypointDir = fs.mkdtempSync( diff --git a/apps/ade-cli/src/tuiClient/__tests__/reportIssue.test.ts b/apps/ade-cli/src/tuiClient/__tests__/reportIssue.test.ts new file mode 100644 index 000000000..fc49a90de --- /dev/null +++ b/apps/ade-cli/src/tuiClient/__tests__/reportIssue.test.ts @@ -0,0 +1,64 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildTuiDiagnosticReport } from "../reportIssue"; + +const tempDirs: string[] = []; + +function tempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + while (tempDirs.length) { + const dir = tempDirs.pop(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("buildTuiDiagnosticReport", () => { + it("writes an owner-only report and points the pane at it", () => { + const adeHome = tempDir("ade-home-"); + const reportsDir = path.join(adeHome, "diagnostic-reports"); + const built = buildTuiDiagnosticReport({ + projectRoot: null, + env: { ADE_HOME: adeHome, ADE_CLI_VERSION: "9.9.9" }, + now: () => new Date("2026-08-16T09:30:00.000Z"), + reportsDir, + }); + + expect(built.filePath).toBe( + path.join(reportsDir, "2026-08-16T09-30-00-000Z-ade-code.md"), + ); + const stat = fs.statSync(built.filePath!); + // The report carries machine state; it must not be world-readable. + expect(stat.mode & 0o077).toBe(0); + expect(fs.readFileSync(built.filePath!, "utf8")).toContain("9.9.9"); + + // The pane body is the only thing the user sees, so it has to carry both + // ways of acting on the report. + expect(built.body).toContain(built.filePath!); + expect(built.body).toContain(built.issueUrl); + }); + + it("still yields an issue URL when the report cannot be written", () => { + const adeHome = tempDir("ade-home-"); + const blocked = path.join(adeHome, "blocked"); + // A file where the directory should be: mkdir fails, and reporting a bug + // must not itself fail. + fs.writeFileSync(blocked, "not a directory"); + + const built = buildTuiDiagnosticReport({ + projectRoot: null, + env: { ADE_HOME: adeHome }, + reportsDir: path.join(blocked, "reports"), + }); + + expect(built.filePath).toBeNull(); + expect(built.issueUrl).toMatch(/^https:\/\/github\.com\//); + expect(built.body).toContain(built.issueUrl); + }); +}); diff --git a/apps/ade-cli/src/tuiClient/app.tsx b/apps/ade-cli/src/tuiClient/app.tsx index 7565d608e..ae64d2113 100644 --- a/apps/ade-cli/src/tuiClient/app.tsx +++ b/apps/ade-cli/src/tuiClient/app.tsx @@ -166,6 +166,10 @@ import type { SnoozeDurationKey } from "../../../desktop/src/renderer/lib/sessio import { buildHelpIndex, buildHelpRows, flattenHelpRows, pushRecent } from "./helpIndex"; import { hasFirstUserMessage, isPlanMode } from "./planMode"; import { connectToAde, INTERACTIVE_PROJECT_REGISTRATION } from "./connection"; +// Imported from the service manager rather than re-exported through +// ./connection: several suites mock ./connection with a partial factory, and a +// startup screen must not depend on an export those mocks have to remember. +import { RuntimeServiceStillStartingError } from "../serviceManager/common"; import { captureTuiProductAnalytics, deriveTuiAnalyticsScreen } from "./productAnalytics"; import { WorkSessionsPane } from "./components/WorkSessionsPane"; import { @@ -3500,6 +3504,12 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, const [promptCursor, setPromptCursor] = useState(0); const [backgroundLaunchStatus, setBackgroundLaunchStatus] = useState(null); const [error, setError] = useState(null); + /** + * The last connect failed only because a supervised brain had not answered + * yet. Nothing is broken and nothing needs repairing, so the startup screen + * says so and keeps waiting instead of showing a red failure. + */ + const [startupServiceStarting, setStartupServiceStarting] = useState(false); const [contextPercent, setContextPercent] = useState(null); const [tokenSummary, setTokenSummary] = useState(null); const [statusLineStats, setStatusLineStats] = useState(null); @@ -8303,6 +8313,7 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, connectionRetryTimerRef.current = null; } setError(null); + setStartupServiceStarting(false); setMode("connecting"); setConnectionRetrySeq((seq) => seq + 1); }, []); @@ -8315,6 +8326,7 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, } setMode("connecting"); setError(null); + setStartupServiceStarting(false); void (async () => { try { const conn = await connectToAde({ project, forceEmbedded, requireSocket, socketPath, preferServiceRepair, remote: remoteLaunch, projectRegistration: INTERACTIVE_PROJECT_REGISTRATION }); @@ -8382,6 +8394,7 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, heartbeatRef.current = null; const message = err instanceof Error ? err.message : String(err); setError(message); + setStartupServiceStarting(err instanceof RuntimeServiceStillStartingError); setMode("connecting"); connectionRetryTimerRef.current = setTimeout(() => { connectionRetryTimerRef.current = null; @@ -10429,6 +10442,30 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, return; } + if (name === "/report-issue") { + // Deliberately above the `!conn` gate: the report reads local files only, + // so it still answers while the runtime is unreachable — the state a bug + // report is most worth filing from. + try { + const { buildTuiDiagnosticReport } = await import("./reportIssue"); + const built = buildTuiDiagnosticReport({ projectRoot: project.projectRoot }); + setRightPane({ kind: "details", title: "Report issue", body: built.body }); + } catch (error) { + setRightPane({ + kind: "details", + title: "Report issue", + body: [ + "The report could not be built.", + "", + error instanceof Error ? error.message : String(error), + "", + "Run ade report-issue --open in any terminal instead.", + ].join("\n"), + }); + } + return; + } + if (!conn) { if (name === "/help") { renderHelpPane("", 0, helpRecentsRef.current); @@ -17245,6 +17282,23 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, if (error && !connection) { const remoteLabel = project.remoteLabel?.trim() || "the remote computer"; + // A supervised brain that has not answered yet is not a failure. Mirror the + // desktop's `brain_starting` recovery copy: say what is happening, promise + // it opens on its own, and offer nothing to repair. + if (startupServiceStarting) { + return ( + + ADE's background service is starting + + This can take a minute the first time or right after an update. + ADE Code opens as soon as it is ready — there is nothing to do. + + + Waiting automatically · r retry now · Ctrl+C quit + + + ); + } return ( @@ -17261,6 +17315,9 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, Retrying automatically · r retry now · Ctrl+C quit + + Run ade report-issue --open in another terminal to prepare a report you can post. Personal information is removed. + ); } diff --git a/apps/ade-cli/src/tuiClient/commands.ts b/apps/ade-cli/src/tuiClient/commands.ts index 1e92791c3..2436f418d 100644 --- a/apps/ade-cli/src/tuiClient/commands.ts +++ b/apps/ade-cli/src/tuiClient/commands.ts @@ -144,6 +144,9 @@ export const BUILTIN_COMMANDS: BuiltinCommand[] = [ { name: "/keybindings", description: "Show Claude-compatible keybinding config diagnostics", placement: "right", argumentHint: "[open]", category: "System" }, { name: "/statusline", description: "Show Claude-compatible status line config", placement: "right", category: "System" }, { name: "/doctor", description: "Show ADE Code and Claude-compat diagnostics", placement: "right", category: "System" }, + // The terminal counterpart of the desktop "Report issue" button. Local-only, + // like `ade report-issue`, so it still works when the brain is the problem. + { name: "/report-issue", description: "Build a redacted diagnostic report for a bug report", placement: "right", category: "System" }, { name: "/model", description: "Open the model, reasoning, and permission picker", placement: "right", category: "Model" }, { name: "/effort", description: "Open the reasoning-effort picker", placement: "right", category: "Model" }, { name: "/system", description: "Show system and runtime details", placement: "right", category: "System" }, diff --git a/apps/ade-cli/src/tuiClient/connection.ts b/apps/ade-cli/src/tuiClient/connection.ts index b5c1fee8d..8501271be 100644 --- a/apps/ade-cli/src/tuiClient/connection.ts +++ b/apps/ade-cli/src/tuiClient/connection.ts @@ -17,6 +17,8 @@ import type { AdeCodeConnection, ProjectLaunchContext, RuntimeEventGapMetadata } import type { AgentChatEventEnvelope } from "../../../desktop/src/shared/types/chat"; import type { BufferedEvent } from "../eventBuffer"; import { resolveAdeDefaultRole } from "../runtimeRoles"; +import { RUNTIME_SERVICE_STARTING_CONNECT_WAIT_MS } from "../serviceManager/runtimeServiceBudgets"; +import { RuntimeServiceStillStartingError } from "../serviceManager/common"; type RpcResponseEnvelope = | T @@ -128,6 +130,14 @@ type CreateEmbeddedRpcRequestHandler = (args: { const DAEMON_CONNECT_RETRY_INITIAL_DELAY_MS = 50; const DAEMON_CONNECT_RETRY_MAX_DELAY_MS = 200; +/** + * Attempts to allow a brain the installer reported as `starting`. Derived from + * the shared budget rather than spelled as a count, so this and the machine + * CLI's own wait for the same case cannot drift apart. + */ +const STARTING_BRAIN_CONNECT_ATTEMPTS = Math.ceil( + RUNTIME_SERVICE_STARTING_CONNECT_WAIT_MS / DAEMON_CONNECT_RETRY_MAX_DELAY_MS, +); const MULTI_PROJECT_RUNTIME_METHODS = new Set([ "ade/initialize", @@ -919,14 +929,40 @@ export async function connectToAde(args: { }); const repairService = async (): Promise => { if (!preferServiceRepair) return null; + const [{ installRuntimeService }, { serviceManagerOwnsRuntimeRecovery }] = + await Promise.all([ + import("../serviceManager"), + import("../serviceManager/common"), + ]); + let result: Awaited>; try { - const { installRuntimeService } = await import("../serviceManager"); - const result = await withAdeDefaultRole("cto", () => installRuntimeService()); - if (!result.ok) return null; - return await tryDaemon(25); + result = await withAdeDefaultRole("cto", () => installRuntimeService()); } catch { return null; } + if (!result.ok) { + // The replacement reached its readiness phase, so it is registered with + // the platform supervisor even though the install reported failure. + // That supervisor owns the retries; an unmanaged daemon on the same + // socket is a rival brain, not a recovery. + if (serviceManagerOwnsRuntimeRecovery(result)) { + throw new RuntimeServiceStillStartingError({ kind: "not_answered", socketPath: machineSocketPath, installMessage: result.message }); + } + return null; + } + // `starting` means the supervisor has a live brain that had not + // answered inside the installer's budget. The default 25 attempts is + // ~5s — far too short for the case the flag exists to describe, and + // giving up early drops through to spawnDaemon, i.e. a second, + // unmanaged brain on the socket the service already owns. + try { + return await tryDaemon(result.starting ? STARTING_BRAIN_CONNECT_ATTEMPTS : 25); + } catch { + // A successful install means a service now owns this endpoint, + // whether or not it had answered yet. Returning null here is what let + // the caller start a competing manual brain on a supervised socket. + throw new RuntimeServiceStillStartingError({ kind: "not_answered", socketPath: machineSocketPath, installMessage: result.message }); + } }; try { if (!fs.existsSync(machineSocketPath)) { @@ -940,6 +976,9 @@ export async function connectToAde(args: { } return await tryDaemon(1); } catch (firstError) { + // A supervised brain that is still coming up must not be answered with a + // second one: the fallback below this catch spawns exactly that. + if (firstError instanceof RuntimeServiceStillStartingError) throw firstError; if (firstError instanceof StaleAdeSocketError) { // tryDaemon ran with shutdownOnStale, so that brain was just asked to // exit. Its pid can outlive the request by a moment, and if it was one diff --git a/apps/ade-cli/src/tuiClient/reportIssue.ts b/apps/ade-cli/src/tuiClient/reportIssue.ts new file mode 100644 index 000000000..5f1bfc1c8 --- /dev/null +++ b/apps/ade-cli/src/tuiClient/reportIssue.ts @@ -0,0 +1,80 @@ +import path from "node:path"; +import { buildCliDiagnosticReport } from "../commands/reportIssue"; +import { + diagnosticReportFilePath, + writeDiagnosticReportFile, +} from "../services/diagnostics/diagnosticReport"; +import { resolveMachineAdeLayout } from "../services/projects/machineLayout"; + +/** + * `/report-issue` for the TUI — the terminal counterpart of the desktop + * "Report issue" button and of `ade report-issue`. + * + * Like both of those it reads only local files: it never asks the brain for + * anything, so it still answers on a machine where the brain is the problem. + * The report is redacted in {@link buildCliDiagnosticReport} (private paths, + * account names, emails, addresses and tokens) before it is written anywhere. + */ + +declare const __ADE_VERSION__: string | undefined; + +/** + * The TUI bundle carries the same `__ADE_VERSION__` define as the `ade` + * entrypoint, so a report filed from `ade code` names the same build. Running + * from source (tests, `npm run dev:code`) has no define and falls back to the + * env var the packaged runtime sets. + */ +export function resolveTuiCliVersion(env: NodeJS.ProcessEnv = process.env): string | null { + const bundled = typeof __ADE_VERSION__ === "string" ? __ADE_VERSION__.trim() : ""; + if (bundled && bundled !== "0.0.0") return bundled; + return env.ADE_CLI_VERSION?.trim() || bundled || null; +} + +export type TuiDiagnosticReport = { + /** Narrow-pane summary rendered by the `details` right pane. */ + body: string; + /** Where the full report landed, or null when it could not be written. */ + filePath: string | null; + issueUrl: string; + installId: string; +}; + +export function buildTuiDiagnosticReport(args: { + projectRoot: string | null; + env?: NodeJS.ProcessEnv; + now?: () => Date; + /** Overrides the directory the report is written to (tests). */ + reportsDir?: string; +}): TuiDiagnosticReport { + const env = args.env ?? process.env; + const at = args.now?.() ?? new Date(); + const surface = "ade_code"; + const built = buildCliDiagnosticReport({ + surface, + projectRoot: args.projectRoot, + cliVersion: resolveTuiCliVersion(env), + env, + now: () => at, + }); + const reportsDir = args.reportsDir + ?? path.join(resolveMachineAdeLayout(env).adeDir, "diagnostic-reports"); + const filePath = diagnosticReportFilePath(reportsDir, surface, at); + const written = writeDiagnosticReportFile(filePath, built.report); + const body = [ + "A diagnostic report has been prepared.", + "Private paths, account names, emails and tokens are removed before it is written.", + "", + written ? "Saved to:" : "It could not be saved to disk, so paste it from the issue page instead.", + written ? filePath : null, + "", + "File the issue at:", + built.issueUrl, + "", + `Install id: ${built.installId}`, + "", + "If ADE Code will not start at all, run ade report-issue --open in any terminal — it reads local files only.", + ] + .filter((line): line is string => line !== null) + .join("\n"); + return { body, filePath: written ? filePath : null, issueUrl: built.issueUrl, installId: built.installId }; +} diff --git a/apps/desktop/scripts/windows-release-contract.test.mjs b/apps/desktop/scripts/windows-release-contract.test.mjs index 70fd148b9..ebe0c4029 100644 --- a/apps/desktop/scripts/windows-release-contract.test.mjs +++ b/apps/desktop/scripts/windows-release-contract.test.mjs @@ -386,6 +386,86 @@ test("standalone Windows runtime signing uses only canonical credentials and val assert.doesNotMatch(windowsRuntimeSigner, /Write-Output.*(?:AZURE_CLIENT_SECRET|expectedSubject)/); }); +// The PowerShell installer used to stage the binary, the extracted runtime and +// both rollback backups under %TEMP%, run its preflight from there, and promote +// with a cross-volume `Move-Item` (a copy plus a delete, so an interrupted +// promotion leaves a half-written ade.exe). %TEMP% is also where AppLocker and +// most EDR agents block execution outright, so the preflight failed on managed +// machines before it could test anything. The POSIX installer was fixed first +// (apps/ade-cli/scripts/install-runtime.sh, and its +// install-runtime-rollback.test.mjs); these assertions pin the same model on +// the Windows side, which has no runnable test harness off a Windows host. +// Behaviour on a real machine is covered by the CI Windows job, which parses +// the script and runs the standalone-installer gate in +// windows-uninstall-cleanup.test.mjs. +test("the Windows installer stages, preflights and promotes under the ADE home, not %TEMP%", () => { + const installer = fs.readFileSync( + path.join(repoRoot, "apps", "ade-cli", "scripts", "install-runtime.ps1"), + "utf8", + ); + + // Scratch paths sit next to what they replace, so every promotion and every + // restore is a same-directory rename. + assert.match(installer, /\$stagedRuntime = "\$runtimeDir\.new"/); + assert.match(installer, /\$backupRuntime = "\$runtimeDir\.previous"/); + assert.match(installer, /\$pendingBinary = Join-Path \$InstallDir "ade\.new\.exe"/); + assert.match(installer, /\$backupBinary = Join-Path \$InstallDir "ade\.bak\.exe"/); + // %TEMP% holds downloads only: no staged runtime, no backup, no exec target. + assert.doesNotMatch(installer, /Join-Path \$tempRoot "runtime/); + assert.doesNotMatch(installer, /Join-Path \$tempRoot "ade\.(?:previous|bak)\.exe"/); + + // The preflight runs the install-directory copy against the staged runtime, + // so it exercises exactly the files the promoted install will load. + assert.match(installer, /Copy-Item -LiteralPath \$downloadedBinary -Destination \$pendingBinary -Force/); + assert.match( + installer, + /Set-ProcessRuntimeEnvironment \$AdeHome \$stagedRuntime\s*\n\s*& \$pendingBinary --version/, + ); + // The downloaded copy in %TEMP% is never executed. + assert.doesNotMatch(installer, /& \$downloadedBinary/); + + // Promotion order: runtime first, binary last, each by rename. + assert.match( + installer, + /Move-Item -LiteralPath \$stagedRuntime -Destination \$runtimeDir\s*\n\s*\$promotedRuntime = \$true/, + ); + assert.match( + installer, + /Move-Item -LiteralPath \$pendingBinary -Destination \$destinationBinary\s*\n\s*\$promotedBinary = \$true/, + ); + + // The finally clears all four scratch paths, and puts a backup back first + // when an abort landed between the two renames -- deleting it there is what + // would leave the machine with nothing installed. + const cleanup = installer.slice(installer.indexOf("if (-not $preserveTempForRecovery) {")); + for (const scratch of ["$tempRoot", "$pendingBinary", "$stagedRuntime", "$backupRuntime", "$backupBinary"]) { + assert.ok( + cleanup.includes(`Remove-Item -LiteralPath ${scratch}`), + `installer cleanup must remove ${scratch}`, + ); + } + assert.match( + cleanup, + /-not \$promotedRuntime[\s\S]{0,200}Move-Item -LiteralPath \$backupRuntime -Destination \$runtimeDir/, + ); + // `$binaryVerified`, not `$promotedBinary`: between the rename and the + // version check what sits at ade.exe is an unverified binary, so an abort in + // that window must still put the backup back over it. + assert.match( + cleanup, + /-not \$binaryVerified[\s\S]{0,200}Move-Item -LiteralPath \$backupBinary -Destination \$destinationBinary/, + ); + assert.match(installer, /\$binaryVerified = \$true/); + + // Stage-aware failure messages, matching `die_runtime_unusable` in the sh + // script: a preflight failure must not claim a rollback that never happened. + assert.match(installer, /\$installStage = "staged"/); + assert.match(installer, /\$installStage = "installed"/); + assert.match(installer, /was not touched\./); + assert.match(installer, /the ADE you already had was put back, so nothing is broken\./); + assert.match(installer, /nothing was left installed at \$BinaryPath\./); +}); + test("standalone Windows release assets remain behind the publication gate", () => { const publish = jobBlock(releasePublishWorkflow, "publish-release", null); const runtimeBuild = jobBlock(releaseWorkflow, "build-runtime-binaries", "build-results"); diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 14d335186..0cbbca1a9 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -47,6 +47,7 @@ import { selectWindowForProjectNavigation, } from "./services/deeplinks/projectNavigationWindowSelection"; import { registerIpc } from "./services/ipc/registerIpc"; +import { AttemptedProjectRoots } from "./services/ipc/knownProjectRoots"; import { createFileLogger } from "./services/logging/logger"; import { createProductAnalyticsService, @@ -507,6 +508,14 @@ const defaultEnabledBackgroundTaskFlags = new Set([ // a burst of near-simultaneous opens from each passing the cap before any has // begun. // --------------------------------------------------------------------------- +/** + * Every project open/switch attempt is recorded here, successful or not, so the + * recovery and diagnostics IPC handlers can accept the root the user just + * picked even when its first open failed before it could reach the + * recent-projects list. See `services/ipc/knownProjectRoots.ts`. + */ +const attemptedProjectRoots = new AttemptedProjectRoots(); + const RECONCILE_GLOBAL_MAX = 1; let reconcileActiveOrScheduled = 0; const pendingReconciles: Array<() => void> = []; @@ -1694,7 +1703,12 @@ app.whenReady().then(async () => { const status = localRuntimePool.getStatus().serviceInstall; if (status.state === "installed") { markMachineStateMigrationComplete({ layout: machineAdeLayout }); - if (machineTrustResetRestartRequired) { + // The reset is complete only once the brain has actually been + // replaced. A forced install may legitimately decline to restart a + // brain that is still starting up (it waits for it instead), and + // that brain loaded the pre-reset files; leaving the marker unset + // makes the next launch restart it for real. + if (machineTrustResetRestartRequired && status.restarted === true) { markMachineTrustResetComplete(machineAdeLayout); } } @@ -5898,7 +5912,19 @@ app.whenReady().then(async () => { try { const resolveStartedAt = Date.now(); repoRoot = normalizeProjectRoot(await resolveRepoRoot(selectedPath)); // require a real git repo for onboarding. + // INVARIANT: a root is recorded as "attempted" only once it has been + // proven to be a real git repository on disk — `resolveRepoRoot` throws + // otherwise. The registry widens what a renderer may later name in + // diagnostics/recovery calls, so recording an unvalidated string would + // let a renderer launder any path on the machine into a known root by + // calling `projectOpenRepo` with it first. Still recorded BEFORE the + // init steps below, which are exactly the ones that fail with coded + // errors (disk_full, db_integrity, brain_not_installed) on a first open + // that never reaches the recent-projects list — the recovery screen for + // that root is what this registry exists to keep working. + attemptedProjectRoots.record(repoRoot); if (repoRoot !== normalizeProjectRoot(selectedPath)) { + attemptedProjectRoots.record(selectedPath); pendingRepoRootCleanup = authorizePendingWindowProjectRoot(windowId, repoRoot); } // Kick off base-ref detection IN PARALLEL with the existing-context @@ -7523,6 +7549,7 @@ app.whenReady().then(async () => { createWindow: openAdeWindow, closeWindow: closeAdeWindow, switchProjectFromDialog, + attemptedProjectRoots, closeCurrentProject, closeProjectByPath, globalStatePath, diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts index a4db61c5d..8d6a0f5d1 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts @@ -124,6 +124,9 @@ const ANALYTICS_ONLY_ACTIONS = new Set([ // settle-tuple write that had to be reconciled through the chokepoint. "settle_teardown_residue", "settle_remote_write_reconciled", + // One coarse fact per "Report issue" press: whether the GitHub issue page + // opened. Never the surface it was pressed on, and never the report itself. + "issue_report", ]); const EVENT_PROPERTY_KEYS: Record> = { diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts b/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts index b450f458f..1debac419 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts @@ -1264,6 +1264,45 @@ describe("product analytics producers", () => { })).not.toHaveProperty("count_bucket"); }); + it("keeps the Report-issue outcome through the sanitizer and nothing else", () => { + // "Report issue" is the one control on ADE's error screens, so the only + // product question is whether pressing it reaches GitHub. `action` is + // allowlisted separately from the event's key list, so a new coarse action + // that is not registered ships anonymous rather than not at all. + expect(sanitizeProductAnalyticsProperties("ade_feature_used", { + feature: "connections", + action: "issue_report", + outcome: "opened", + })).toEqual({ feature: "connections", action: "issue_report", outcome: "opened" }); + + expect(sanitizeProductAnalyticsProperties("ade_feature_used", { + feature: "connections", + action: "issue_report", + outcome: "failed", + })).toMatchObject({ outcome: "failed" }); + + // The report itself, the screen it came from and the install id it carries + // are for the local file and the clipboard. None of them may ride along on + // the event, whether they arrive under a known key or an invented one. + const leaky = sanitizeProductAnalyticsProperties("ade_feature_used", { + feature: "connections", + action: "issue_report", + outcome: "opened", + surface: "project_recovery", + code: "db_integrity", + install_id: "ade_0123456789abcdef0123456789abcdef", + headline: "ADE couldn't open /Users/ada/photon", + }); + expect(leaky).toEqual({ feature: "connections", action: "issue_report", outcome: "opened" }); + + // An outcome outside the closed set is dropped, not passed through. + expect(sanitizeProductAnalyticsProperties("ade_feature_used", { + feature: "connections", + action: "issue_report", + outcome: "ENOSPC: no space left on device", + })).not.toHaveProperty("outcome"); + }); + it("maps automation completion and failed chat turns into canonical bounded outcomes", () => { const captures: ProductAnalyticsCapture[] = []; const analytics = settledAnalytics(captures); diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsService.ts b/apps/desktop/src/main/services/analytics/productAnalyticsService.ts index 4399e8f2b..45cc2d74e 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsService.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsService.ts @@ -1189,6 +1189,26 @@ export function createProductAnalyticsService(args: ProductAnalyticsServiceArgs) return new Date(state.enabledSinceMs).toISOString(); }, hashProjectId: (value: string) => opaqueId("project", value), + /** + * The id PostHog sees as `distinct_id` for anonymous events from this + * installation. Surfaced so a diagnostic report a user files by hand can be + * matched to the events this machine already sent; it is a random + * per-install token, not a device or account identifier. + * + * Null unless events are actually being sent under it. Two reasons: until + * something loads the persisted file this service is holding a freshly + * minted in-memory id that no event has ever carried (a correlation id + * that correlates to nothing), and when the user has analytics off there + * is nothing to correlate against — putting the identifier in a report + * they are about to paste into a public issue is the opposite of the + * choice they made. `getStatus()` is the reader that loads durable state + * on both the configured and unconfigured paths. + */ + getDistinctId: (): string | null => { + const status = getStatus(); + if (!status.effective) return null; + return state.identifiedUserHash ?? state.anonymousId; + }, installationIdForTesting: () => state.installationId, identifiedUserHashForTesting: () => state.identifiedUserHash, }; diff --git a/apps/desktop/src/main/services/diagnostics/diagnosticReportService.test.ts b/apps/desktop/src/main/services/diagnostics/diagnosticReportService.test.ts new file mode 100644 index 000000000..773dc89c6 --- /dev/null +++ b/apps/desktop/src/main/services/diagnostics/diagnosticReportService.test.ts @@ -0,0 +1,138 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, describe, expect, it, vi } from "vitest"; +import { collectDiagnosticReport } from "./diagnosticReportService"; + +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "ade-diag-report-")); + +afterAll(() => { + fs.rmSync(tempRoot, { recursive: true, force: true }); +}); + +function deps() { + return { + appVersion: "1.2.3", + packageChannel: null, + isPackaged: false, + userDataPath: path.join(tempRoot, "userData"), + reportsDir: path.join(tempRoot, "reports"), + installId: "install-abc", + }; +} + +describe("collectDiagnosticReport", () => { + // Regression: when the renderer named a project root main did not recognise, + // the handler silently substituted the currently open project, so a report + // about a failed open carried a different project's logs and diagnosis. It + // now degrades to machine-level state — and has to SAY so, or the reader + // draws conclusions from an absence they were never told about. + it("renders caller notes about a degraded report alongside the machine ones", async () => { + const { report } = await collectDiagnosticReport(deps(), { + surface: "project_recovery", + projectRoot: null, + extraNotes: ["requested project root was not recognised; machine-level state only"], + }); + + expect(report).toContain("## Notes"); + expect(report).toContain("- requested project root was not recognised; machine-level state only"); + }); + + // The other half of the same regression: refusing the renderer's root has to + // mean the report is genuinely machine-scoped. If the collector still ran a + // project diagnosis, the note would say "machine-level state only" over a + // body that quietly carried the open project's recovery verdict. + it("runs no project diagnosis at all for a machine-level report", async () => { + const diagnoseProject = vi.fn(async () => ({ state: "healthy" })); + + const machineLevel = await collectDiagnosticReport( + { ...deps(), diagnoseProject }, + { surface: "project_recovery", projectRoot: null }, + ); + + expect(diagnoseProject).not.toHaveBeenCalled(); + expect(machineLevel.report).not.toContain("healthy"); + + // ...and the project-scoped path is still wired, so the assertion above is + // about the null root rather than a diagnosis that never runs. + const projectRoot = path.join(tempRoot, "photon"); + fs.mkdirSync(projectRoot, { recursive: true }); + const scoped = await collectDiagnosticReport( + { ...deps(), diagnoseProject }, + { surface: "project_recovery", projectRoot }, + ); + + expect(diagnoseProject).toHaveBeenCalledTimes(1); + expect(diagnoseProject).toHaveBeenCalledWith(projectRoot); + expect(scoped.report).toContain("healthy"); + }); + + it("still returns when the runtime never answers", async () => { + // Both optional steps talk to the subsystem the user is reporting as + // broken. A step that never settles used to hold the whole report, leaving + // the "Report issue" button spinning forever. + const projectRoot = fs.mkdtempSync(path.join(tempRoot, "project-")); + const { report } = await collectDiagnosticReport( + { + ...deps(), + stepTimeoutMs: 20, + getLocalRuntimeStatus: () => new Promise(() => {}), + diagnoseProject: () => new Promise(() => {}), + }, + { surface: "project_recovery", projectRoot }, + ); + + expect(report).toContain("## Notes"); + }); + + // The step deadline is a race, and losing a race does not cancel a timer. + // Every report used to leave one pending 8s timer per optional step behind + // it -- unref'd, so it held nothing open, but still a handle the process is + // carrying and enough to hang a fake-timer test that runs after it. + it("cancels the step deadline once the step has answered", async () => { + const projectRoot = fs.mkdtempSync(path.join(tempRoot, "project-")); + vi.useFakeTimers(); + try { + await collectDiagnosticReport( + { + ...deps(), + stepTimeoutMs: 60_000, + getLocalRuntimeStatus: async () => ({ state: "running" }), + diagnoseProject: async () => ({ state: "healthy" }), + }, + { surface: "project_recovery", projectRoot }, + ); + + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + it("still returns when a collection step throws synchronously", async () => { + const projectRoot = fs.mkdtempSync(path.join(tempRoot, "project-")); + const { report } = await collectDiagnosticReport( + { + ...deps(), + getLocalRuntimeStatus: () => { + throw new Error("runtime module is not loaded"); + }, + diagnoseProject: () => { + throw new Error("recovery service is gone"); + }, + }, + { surface: "project_recovery", projectRoot }, + ); + + expect(report).toContain("## Notes"); + }); + + it("omits the notes line when there is nothing to say", async () => { + const { report } = await collectDiagnosticReport(deps(), { + surface: "project_recovery", + projectRoot: null, + }); + + expect(report).not.toContain("requested project root was not recognised"); + }); +}); diff --git a/apps/desktop/src/main/services/diagnostics/diagnosticReportService.ts b/apps/desktop/src/main/services/diagnostics/diagnosticReportService.ts new file mode 100644 index 000000000..176bb1ce5 --- /dev/null +++ b/apps/desktop/src/main/services/diagnostics/diagnosticReportService.ts @@ -0,0 +1,240 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import { + buildDiagnosticIssueUrl, + buildDiagnosticReport, + diagnosticReportFilePath, + type DiagnosticLogTail, + type DiagnosticReportContext, + type DiagnosticVolumeSpace, +} from "../../../../../ade-cli/src/services/diagnostics/diagnosticReport"; +import { + collectMachineDiagnosticSources, + readLogTail, +} from "../../../../../ade-cli/src/services/diagnostics/diagnosticSources"; +import { readVolumeSpace } from "../storage/volume"; +import { readLastFailure } from "../runtime/lastFailureStore"; + +export { + buildDiagnosticIssueUrl, + buildDiagnosticReport, + redactDiagnosticText, + /** Writes the report next to the app's other user data. Best effort. */ + writeDiagnosticReportFile, +} from "../../../../../ade-cli/src/services/diagnostics/diagnosticReport"; + +export type DiagnosticReportRequest = DiagnosticReportContext & { + /** Verbatim `UpdateTransactionResult` (or anything JSON) from the caller. */ + updateTransaction?: unknown; + /** + * Caller-supplied notes appended to the machine-collected ones — how a + * handler explains a degraded report, e.g. a project root it could not + * recognise and therefore did not collect project state for. + */ + extraNotes?: readonly string[]; +}; + +export type DiagnosticReportDeps = { + appVersion: string | null; + packageChannel: string | null; + isPackaged: boolean; + /** `app.getPath("userData")` — where the desktop's own jsonl logs live. */ + userDataPath: string; + /** Directory the written report file goes in. */ + reportsDir: string; + installId: string | null; + /** Raw account user id; hashed here and never stored or sent verbatim. */ + accountUserId?: string | null; + /** Project `.ade/logs` directory for the open project, when there is one. */ + projectLogsDir?: string | null; + getLocalRuntimeStatus?: () => Promise | unknown; + diagnoseProject?: (projectRoot: string) => Promise; + /** Deadline for each optional step above. Test seam; defaults to 8s. */ + stepTimeoutMs?: number; + env?: NodeJS.ProcessEnv; + now?: () => Date; +}; + +export type DiagnosticReportResult = { + report: string; + filePath: string; + issueUrl: string; + installId: string; +}; + +/** Truncated one-way hash: correlatable across reports, never reversible. */ +export function hashAccountUserId(userId: string | null | undefined): string | null { + const trimmed = userId?.trim(); + if (!trimmed) return null; + return createHash("sha256").update(`ade-account:${trimmed}`).digest("hex").slice(0, 12); +} + +function readMacProductVersion(): Promise { + if (process.platform !== "darwin") return Promise.resolve(null); + return new Promise((resolve) => { + try { + execFile("/usr/bin/sw_vers", ["-productVersion"], { timeout: 2_000 }, (error, stdout) => { + resolve(error ? null : stdout.trim() || null); + }); + } catch { + resolve(null); + } + }); +} + +function volumeEntry(label: string, dirPath: string): DiagnosticVolumeSpace | null { + const space = readVolumeSpace(dirPath); + if (!space) return null; + return { label, path: dirPath, freeBytes: space.freeBytes, totalBytes: space.totalBytes }; +} + +/** Deadline for one optional collection step. */ +const DIAGNOSTIC_STEP_TIMEOUT_MS = 8_000; + +/** + * Runs one optional step and always settles: a rejection, a synchronous throw + * and a promise that never answers all collapse to null. Whatever the step + * would have contributed is simply absent from the report -- far better than a + * report the user can never get. + */ +function bestEffortStep( + run: () => Promise | T, + timeoutMs = DIAGNOSTIC_STEP_TIMEOUT_MS, +): Promise { + let timer: ReturnType | null = null; + return Promise.race([ + Promise.resolve() + .then(run) + .catch(() => null), + new Promise((resolve) => { + // Unref'd so an outstanding step can never hold the process open. + timer = setTimeout(() => resolve(null), timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + }), + // Losing the race does not cancel the timer, so a step that answers first + // would otherwise leave a handle per report alive for the full deadline. + ]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + +/** + * Gathers everything the report needs from this machine and renders it. Every + * step is best-effort: a missing log or a runtime that will not answer must + * never stop a user from filing an issue. + */ +export async function collectDiagnosticReport( + deps: DiagnosticReportDeps, + request: DiagnosticReportRequest, +): Promise { + const env = deps.env ?? process.env; + const at = deps.now?.() ?? new Date(); + const projectRoot = request.projectRoot?.trim() || null; + // Logs, volumes, notes and the redaction context are the same set the + // headless `ade report-issue` collects; the Electron-only extras below are + // the only thing this report adds. + const sources = collectMachineDiagnosticSources({ + env, + projectRoot, + readVolume: volumeEntry, + }); + + // Both optional steps ask the very subsystem the user is reporting as broken + // -- the local runtime, and a recovery diagnosis that probes the brain's + // socket. A step that never settles would hold `Promise.all` forever and + // leave the "Report issue" button spinning, which is exactly the outcome + // this collector promises can never happen. A synchronous throw out of + // either one is caught here for the same reason. + const [osProductVersion, localRuntimeStatus, recoveryDiagnosis] = await Promise.all([ + readMacProductVersion().catch(() => null), + bestEffortStep(() => deps.getLocalRuntimeStatus?.() ?? null, deps.stepTimeoutMs), + projectRoot && deps.diagnoseProject + ? bestEffortStep(() => deps.diagnoseProject?.(projectRoot) ?? null, deps.stepTimeoutMs) + : Promise.resolve(null), + ]); + + const logs: DiagnosticLogTail[] = [...sources.logs]; + logs.push(readLogTail("Desktop local runtime", path.join(deps.userDataPath, "local-runtime.jsonl"))); + logs.push(readLogTail("Desktop updates", path.join(deps.userDataPath, "ade-update.jsonl"))); + if (deps.projectLogsDir) { + logs.push(readLogTail("Desktop main", path.join(deps.projectLogsDir, "main.jsonl"))); + } + + // The typed store rather than the raw file the CLI falls back to: main owns + // the writer, so it can read the record's real shape. + const machineLastFailure = (() => { + try { + return readLastFailure({ kind: "machine", env }); + } catch { + return null; + } + })(); + const projectLastFailure = projectRoot + ? (() => { + try { + return readLastFailure({ kind: "project", projectRoot }); + } catch { + return null; + } + })() + : null; + + const installId = deps.installId?.trim() || "unknown"; + const redaction = sources.redaction; + + const report = buildDiagnosticReport({ + generatedAt: at.toISOString(), + app: { + version: deps.appVersion, + packageChannel: deps.packageChannel, + isPackaged: deps.isPackaged, + platform: process.platform, + arch: process.arch, + osRelease: os.release(), + osProductVersion, + electronVersion: process.versions.electron ?? null, + nodeVersion: process.versions.node ?? null, + chromeVersion: process.versions.chrome ?? null, + timezoneOffsetMinutes: -at.getTimezoneOffset(), + }, + identity: { + installId, + accountHash: hashAccountUserId(deps.accountUserId), + }, + context: { + surface: request.surface, + headline: request.headline ?? null, + code: request.code ?? null, + technicalDetail: request.technicalDetail ?? null, + projectRoot, + }, + state: { + localRuntimeStatus: localRuntimeStatus ?? null, + recoveryDiagnosis: recoveryDiagnosis ?? null, + machineLastFailure, + projectLastFailure, + lastWedge: sources.state.lastWedge, + updateTransaction: request.updateTransaction ?? null, + }, + storage: sources.storage, + logs, + notes: [...sources.notes, ...(request.extraNotes ?? [])], + redaction, + }); + + const filePath = diagnosticReportFilePath(deps.reportsDir, request.surface, at); + const issueUrl = buildDiagnosticIssueUrl({ + surface: request.surface, + headline: request.headline ?? null, + code: request.code ?? null, + appVersion: deps.appVersion, + platform: process.platform, + arch: process.arch, + installId, + redaction, + }); + + return { report, filePath, issueUrl, installId }; +} diff --git a/apps/desktop/src/main/services/ipc/knownProjectRoots.test.ts b/apps/desktop/src/main/services/ipc/knownProjectRoots.test.ts new file mode 100644 index 000000000..d044cbf03 --- /dev/null +++ b/apps/desktop/src/main/services/ipc/knownProjectRoots.test.ts @@ -0,0 +1,132 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { AttemptedProjectRoots, resolveKnownProjectRoot } from "./knownProjectRoots"; + +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "ade-known-roots-")); +const openProject = path.join(tempRoot, "open-project"); +const recentProject = path.join(tempRoot, "recent-project"); +const stranger = path.join(tempRoot, "someone-elses-folder"); +for (const dir of [openProject, recentProject, stranger]) { + fs.mkdirSync(dir, { recursive: true }); +} + +afterAll(() => { + fs.rmSync(tempRoot, { recursive: true, force: true }); +}); + +const sources = { + openProjectRoot: openProject, + recentProjectRoots: [recentProject], +}; + +describe("resolveKnownProjectRoot", () => { + it("accepts the open project and a recent project", () => { + expect(resolveKnownProjectRoot(openProject, sources)).toBe(openProject); + expect(resolveKnownProjectRoot(recentProject, sources)).toBe(recentProject); + }); + + // Regression: the diagnostics and recovery IPC handlers only trimmed the + // renderer's string, so any directory on the machine could be diagnosed + // (logs and volume space read) or repaired (on-disk state mutated). + it("rejects a directory that is not a project this machine knows", () => { + expect(resolveKnownProjectRoot(stranger, sources)).toBeNull(); + expect(resolveKnownProjectRoot("/etc", sources)).toBeNull(); + }); + + it("rejects traversal that lands outside a known project", () => { + expect(resolveKnownProjectRoot(path.join(openProject, "..", "someone-elses-folder"), sources)).toBeNull(); + expect(resolveKnownProjectRoot(`${openProject}/../../..`, sources)).toBeNull(); + }); + + it("accepts a non-canonical spelling of a known project and returns the canonical one", () => { + const noisy = path.join(openProject, ".", "sub", ".."); + expect(resolveKnownProjectRoot(noisy, sources)).toBe(openProject); + expect(resolveKnownProjectRoot(`${openProject}${path.sep}`, sources)).toBe(openProject); + }); + + it("rejects empty and non-string input", () => { + expect(resolveKnownProjectRoot("", sources)).toBeNull(); + expect(resolveKnownProjectRoot(" ", sources)).toBeNull(); + expect(resolveKnownProjectRoot(null, sources)).toBeNull(); + expect(resolveKnownProjectRoot(undefined, sources)).toBeNull(); + }); + + it("accepts nothing when no project is open and there are no recents", () => { + expect(resolveKnownProjectRoot(openProject, {})).toBeNull(); + expect(resolveKnownProjectRoot(openProject, { openProjectRoot: null, recentProjectRoots: [] })).toBeNull(); + }); + + // Regression: the recovery screen is put on screen BY a failed open, and a + // folder whose first open failed is never written to the recent-projects + // list (that write only happens after a successful init). Refusing it made + // Repair a dead end on exactly the folder it exists for. + it("accepts a root that only ever failed to open", () => { + expect(resolveKnownProjectRoot(stranger, sources)).toBeNull(); + expect( + resolveKnownProjectRoot(stranger, { ...sources, attemptedProjectRoots: [stranger] }), + ).toBe(stranger); + // Still nothing else: the widening is one folder, not the filesystem. + expect( + resolveKnownProjectRoot(tempRoot, { ...sources, attemptedProjectRoots: [stranger] }), + ).toBeNull(); + }); + + it("folds case on win32 so a drive-letter mismatch is not a rejection", () => { + const known = String.raw`C:\Users\ada\project`; + const requested = String.raw`c:\users\ada\project`; + expect(resolveKnownProjectRoot(requested, { openProjectRoot: known }, "win32")).toBe(known); + expect(resolveKnownProjectRoot(requested, { openProjectRoot: known }, "linux")).toBeNull(); + expect( + resolveKnownProjectRoot(String.raw`C:\Users\ada\other`, { openProjectRoot: known }, "win32"), + ).toBeNull(); + }); + + it("resolves a symlink to a known project", (ctx) => { + const link = path.join(tempRoot, "link-to-open"); + try { + fs.symlinkSync(openProject, link, "dir"); + } catch { + // Windows without developer mode has no symlink privilege. Skip loudly + // rather than `return`, which reads as a pass and hides the gap. + ctx.skip(); + return; + } + expect(fs.lstatSync(link).isSymbolicLink(), "symlink setup").toBe(true); + expect(resolveKnownProjectRoot(link, sources)).toBe(openProject); + }); +}); + +describe("AttemptedProjectRoots", () => { + it("remembers a root until it expires", () => { + let now = 1_000; + const roots = new AttemptedProjectRoots(10, 500, () => now); + roots.record("/a"); + expect(roots.list()).toEqual(["/a"]); + now += 499; + expect(roots.list()).toEqual(["/a"]); + now += 2; + expect(roots.list()).toEqual([]); + }); + + it("keeps only the newest entries and re-ages a repeat attempt", () => { + let now = 1_000; + const roots = new AttemptedProjectRoots(2, 10_000, () => now); + roots.record("/a"); + roots.record("/b"); + // Re-recording /a moves it to the newest slot, so /b is the one evicted. + roots.record("/a"); + roots.record("/c"); + expect(roots.list()).toEqual(["/a", "/c"]); + }); + + it("ignores empty input", () => { + const roots = new AttemptedProjectRoots(); + roots.record(""); + roots.record(" "); + roots.record(null); + roots.record(undefined); + expect(roots.list()).toEqual([]); + }); +}); diff --git a/apps/desktop/src/main/services/ipc/knownProjectRoots.ts b/apps/desktop/src/main/services/ipc/knownProjectRoots.ts new file mode 100644 index 000000000..7b2bb77b2 --- /dev/null +++ b/apps/desktop/src/main/services/ipc/knownProjectRoots.ts @@ -0,0 +1,125 @@ +import fs from "node:fs"; +import path from "node:path"; +import { pathsEqual } from "../shared/pathCompare"; + +/** + * Validation for renderer-supplied project roots. + * + * Diagnostics reads logs and volume space at the path it is given; recovery + * mutates on-disk state there. Both used to accept whatever string the + * renderer sent, trimmed. Trimming is not validation: a compromised or simply + * buggy renderer could point either at any directory on the machine. + * + * The rule is that a renderer may only name a project main already knows — + * the one that is open, one in the recent-projects list, or one main itself + * just tried to open. That last source is what keeps the recovery screen + * working: a folder whose FIRST open failed (disk full, db integrity, brain + * not installed) never reaches the recent-projects list, because that list is + * only written after a successful init. Without it, the one root the recovery + * screen exists to repair is the one root it would be refused. + */ + +export type KnownProjectRootSources = { + /** `getCtx().project.rootPath`, when a project is open. */ + openProjectRoot?: string | null; + /** Local entries from the recent-projects list; remote ones have no root. */ + recentProjectRoots?: readonly (string | null | undefined)[]; + /** + * Roots main recently attempted to open, successfully or not. See + * {@link AttemptedProjectRoots}. + */ + attemptedProjectRoots?: readonly (string | null | undefined)[]; +}; + +/** + * Resolves symlinks when it can, and falls back to `path.resolve` when it + * cannot — a project on a volume that is not mounted right now has no + * realpath, and refusing it would be a regression on a path that is otherwise + * legitimate. The result is therefore normalized, not guaranteed canonical, so + * comparisons still have to go through {@link pathsEqual} for case folding. + */ +export function canonicalProjectPath(value: string): string { + const resolved = path.resolve(value); + try { + return fs.realpathSync.native(resolved); + } catch { + return resolved; + } +} + +/** + * Bounded, expiring record of the roots main has tried to open. + * + * Bounded and expiring because it widens what a renderer may name: an entry is + * a directory the user themselves picked moments ago, and it stops being one + * shortly after. Insertion order is the eviction order, with a re-attempt + * moving its root back to the newest slot. + */ +export class AttemptedProjectRoots { + private readonly entries = new Map(); + + constructor( + private readonly limit = 10, + private readonly ttlMs = 30 * 60 * 1_000, + private readonly now: () => number = Date.now, + ) {} + + /** Records an open/switch attempt for `root`. Ignores empty input. */ + record(root: string | null | undefined): void { + const trimmed = typeof root === "string" ? root.trim() : ""; + if (!trimmed) return; + // Delete first so a re-attempt moves to the end of the insertion order + // rather than keeping its original (about-to-be-evicted) slot. + this.entries.delete(trimmed); + this.entries.set(trimmed, this.now()); + this.prune(); + while (this.entries.size > this.limit) { + const oldest = this.entries.keys().next(); + if (oldest.done) break; + this.entries.delete(oldest.value); + } + } + + /** The still-live attempts, oldest first. */ + list(): string[] { + this.prune(); + return [...this.entries.keys()]; + } + + private prune(): void { + const cutoff = this.now() - this.ttlMs; + for (const [root, at] of this.entries) { + if (at <= cutoff) this.entries.delete(root); + } + } +} + +/** + * Returns the known root that `requested` refers to — in the registry's own + * spelling, which is what the user picked and what the recent-projects list + * shows, NOT the canonical form the comparison runs on — or null when it is not + * a project this machine knows about. Callers that need an absolute path + * resolve it themselves. + */ +export function resolveKnownProjectRoot( + requested: string | null | undefined, + sources: KnownProjectRootSources, + platform: NodeJS.Platform = process.platform, +): string | null { + const trimmed = typeof requested === "string" ? requested.trim() : ""; + if (!trimmed) return null; + const target = canonicalProjectPath(trimmed); + const known: string[] = []; + const openRoot = sources.openProjectRoot?.trim(); + if (openRoot) known.push(openRoot); + for (const list of [sources.recentProjectRoots, sources.attemptedProjectRoots]) { + for (const root of list ?? []) { + if (typeof root === "string" && root.trim()) known.push(root); + } + } + // `pathsEqual` rather than `===`: realpath only case-normalizes a path that + // exists, so on Windows and macOS two spellings of the same live directory + // still differ whenever either side skipped the realpath (unmounted volume, + // permission error) — and a case-only mismatch would read as "unknown". + return known.find((candidate) => pathsEqual(canonicalProjectPath(candidate), target, platform)) ?? null; +} diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index d2c0368a1..c2d09998b 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -19,6 +19,8 @@ import type { DiskPressureMonitor, DiskPressureSnapshot } from "../storage/diskP import path from "node:path"; import { fileURLToPath } from "node:url"; import { IPC } from "../../../shared/ipc"; +import { resolveKnownProjectRoot } from "./knownProjectRoots"; +import type { AttemptedProjectRoots } from "./knownProjectRoots"; import { redactIpcArgsForChannel } from "./ipcChannelRedaction"; import type { AttentionItem, @@ -795,6 +797,15 @@ import { quoteWindowsCmdArg } from "../shared/processExecution"; import { probeLocalhostPort } from "../probeLocalhostPort"; import type { ProcessRegistryService } from "../runtime/processRegistryService"; import { openExternalUrl } from "../shared/externalLinks"; +import { resolveAdeLayout } from "../../../shared/adeLayout"; +import { + collectDiagnosticReport, + writeDiagnosticReportFile, +} from "../diagnostics/diagnosticReportService"; +import type { + DiagnosticReportPayload, + DiagnosticReportRequestPayload, +} from "../../../shared/types/diagnostics"; const APP_RESOURCE_USAGE_CACHE_MS = 900; let appResourceUsageCache: { @@ -1647,6 +1658,7 @@ export function registerIpc({ openAttentionItem, getCurrentAccountOwnerId, accountAttentionClient, + attemptedProjectRoots, }: { getCtx: () => AppContext; getResourceUsageContexts?: () => AppContext[]; @@ -1669,6 +1681,14 @@ export function registerIpc({ createWindow?: (args?: { projectRoot?: string | null }) => Promise<{ windowId: number | null; project: ProjectInfo | null }>; closeWindow?: (windowId: number | null) => Promise<{ closed: boolean }>; switchProjectFromDialog: (selectedPath: string) => Promise; + /** + * Roots main has tried to open. Read-only here: main.ts is the single + * writer (it records a root only after resolving it to a real repository), + * and this module only consults `.list()` when validating a + * renderer-supplied root. See `knownProjectRoots.ts` for why a merely + * *attempted* root has to count as known. + */ + attemptedProjectRoots?: AttemptedProjectRoots; closeCurrentProject: () => Promise; closeProjectByPath: (projectRoot: string) => Promise; globalStatePath: string; @@ -1872,6 +1892,12 @@ export function registerIpc({ // custom properties from thrown errors, so we re-throw with the code // prepended to the message. Renderer matches on the prefix. const surfaceCodedError = (error: unknown, meta?: { rootPath?: string }): never => { + // Deliberately does NOT record `meta.rootPath` into `attemptedProjectRoots`. + // Every caller that supplies a root got there through + // `switchProjectFromDialog`, which already recorded the resolved repo root + // after validating it, so a write here would be dead — and it would make + // the registry two-writer, with this one accepting a root that was never + // proven to exist. Reads still go through `attemptedProjectRoots.list()`. if (error instanceof Error) { const code = (error as Error & { code?: unknown }).code; if (typeof code === "string" && code.length > 0) { @@ -4562,22 +4588,157 @@ export function registerIpc({ } }); + /** + * The open project's root, or null when there is no project context at all. + * The surfaces that most need to file an issue are projectless — a renderer + * crash on startup, Connections on a fresh install — and `getCtx()` throws + * there, which turned "report this" into no report at all. + */ + const openProjectRootOrNull = (): string | null => { + try { + return getCtx().project.rootPath ?? null; + } catch { + return null; + } + }; + + /** + * The renderer names a project root; main decides whether that is a project + * it knows. See `knownProjectRoots.ts` for why trimming is not enough, and + * why a root that only ever FAILED to open still counts as known. + */ + const resolveRequestedProjectRoot = (requested: string): string | null => { + let recentProjectRoots: string[] = []; + try { + recentProjectRoots = (readGlobalState(globalStatePath).recentProjects ?? []) + // Remote entries have no local root to diagnose or repair. + .filter((entry) => !entry.remote) + .map((entry) => entry.rootPath); + } catch { + // A missing or corrupt global state leaves only the open project, which + // is the safe subset — never a reason to widen what is accepted. + } + return resolveKnownProjectRoot(requested, { + openProjectRoot: openProjectRootOrNull(), + recentProjectRoots, + attemptedProjectRoots: attemptedProjectRoots?.list(), + }); + }; + ipcMain.handle(IPC.recoveryDiagnose, async (_event, arg: { projectRoot: string }): Promise => { - const projectRoot = typeof arg?.projectRoot === "string" ? arg.projectRoot.trim() : ""; - if (!projectRoot) throw new Error("Project root path is required."); + const requested = typeof arg?.projectRoot === "string" ? arg.projectRoot.trim() : ""; + if (!requested) throw new Error("Project root path is required."); + const projectRoot = resolveRequestedProjectRoot(requested); + if (!projectRoot) throw new Error("That folder is not a project ADE has open or has opened before."); if (!projectRecoveryService) throw new Error("Project recovery is unavailable in this runtime mode."); return await projectRecoveryService.diagnose(projectRoot); }); - // Return the complete ordered step array with the final report. The current - // alert only needs one result, so it does not need a separate event lifecycle. - ipcMain.handle(IPC.recoveryRepair, async (_event, arg: { projectRoot: string }): Promise => { - const projectRoot = typeof arg?.projectRoot === "string" ? arg.projectRoot.trim() : ""; - if (!projectRoot) throw new Error("Project root path is required."); + // The complete ordered step array comes back with the final report; each + // step is ALSO pushed to the calling window as it finishes. A repair can + // legitimately wait a minute or more for the background service to answer, + // and a spinner with no steps for that long reads as a hang. + ipcMain.handle(IPC.recoveryRepair, async (event, arg: { projectRoot: string }): Promise => { + const requested = typeof arg?.projectRoot === "string" ? arg.projectRoot.trim() : ""; + if (!requested) throw new Error("Project root path is required."); + const projectRoot = resolveRequestedProjectRoot(requested); + if (!projectRoot) throw new Error("That folder is not a project ADE has open or has opened before."); if (!projectRecoveryService) throw new Error("Project recovery is unavailable in this runtime mode."); - return await projectRecoveryService.repair(projectRoot); + return await projectRecoveryService.repair(projectRoot, { + onStep: (step) => { + if (event.sender.isDestroyed()) return; + event.sender.send(IPC.recoveryRepairStep, { projectRoot, step }); + }, + }); }); + /** + * Assembles the redacted diagnostic report for whichever error screen asked. + * Read-only and best effort: a missing log, a wedged brain or a runtime mode + * without a recovery service must never stop someone from filing an issue, + * so every optional input degrades to "unknown" rather than throwing. + */ + const buildDiagnosticsReport = async ( + arg: DiagnosticReportRequestPayload | undefined, + ) => { + const surface = typeof arg?.surface === "string" && arg.surface.trim() ? arg.surface.trim() : "unknown"; + const requestedRoot = typeof arg?.projectRoot === "string" ? arg.projectRoot.trim() : ""; + const resolvedRoot = requestedRoot ? resolveRequestedProjectRoot(requestedRoot) : null; + // A root the renderer named but main does not recognise is dropped rather + // than quietly swapped for the currently open project: substituting one + // would put another project's logs, volumes and recovery diagnosis under a + // report about this failure. Reporting stays possible either way — the + // report degrades to machine-level state and says so. + const rootWasRejected = Boolean(requestedRoot) && !resolvedRoot; + const projectRoot = rootWasRejected + ? null + : resolvedRoot ?? openProjectRootOrNull(); + return await collectDiagnosticReport( + { + appVersion: app.getVersion(), + packageChannel: normalizeAppPackageChannel(process.env.ADE_PACKAGE_CHANNEL), + isPackaged: app.isPackaged, + userDataPath: app.getPath("userData"), + reportsDir: path.join(app.getPath("userData"), "diagnostic-reports"), + installId: productAnalyticsService?.getDistinctId() ?? null, + accountUserId: getCurrentAccountOwnerId?.() ?? null, + projectLogsDir: projectRoot ? resolveAdeLayout(projectRoot).logsDir : null, + getLocalRuntimeStatus: () => localRuntimeConnectionPool?.getStatus() ?? null, + diagnoseProject: projectRecoveryService + ? (root: string) => projectRecoveryService.diagnose(root) + : undefined, + }, + { + surface, + headline: typeof arg?.headline === "string" ? arg.headline.slice(0, 300) : null, + code: typeof arg?.code === "string" ? arg.code.slice(0, 120) : null, + technicalDetail: typeof arg?.technicalDetail === "string" ? arg.technicalDetail.slice(0, 16_000) : null, + projectRoot, + extraNotes: rootWasRejected + ? ["requested project root was not recognised; machine-level state only"] + : undefined, + }, + ); + }; + + ipcMain.handle( + IPC.diagnosticsOpenIssue, + async (_event, arg: DiagnosticReportRequestPayload): Promise => { + const result = await buildDiagnosticsReport(arg); + const written = writeDiagnosticReportFile(result.filePath, result.report); + let copied = false; + try { + clipboard.writeText(result.report); + copied = true; + } catch { + copied = false; + } + let opened = false; + try { + await openExternalUrl(result.issueUrl); + opened = true; + } catch { + opened = false; + } + productAnalyticsService?.capture({ + event: "ade_feature_used", + surface: "desktop", + properties: { feature: "connections", action: "issue_report", outcome: opened ? "opened" : "failed" }, + projectId: null, + dedupeKey: `issue_report:${opened ? "opened" : "failed"}`, + minimumIntervalMs: 60 * 60 * 1_000, + }); + return { + report: result.report, + filePath: written ? result.filePath : "", + issueUrl: result.issueUrl, + installId: result.installId, + copied, + opened, + }; + }, + ); + ipcMain.handle(IPC.projectStateGetSnapshot, async (): Promise => { const ctx = getCtx(); if (!ctx.adeProjectService) throw new Error("Project state service unavailable."); diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts index f713f2608..22e1fb640 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts @@ -966,6 +966,30 @@ describe("local runtime connection pool", () => { pool.dispose(); }); + it("keeps the service startup streak when only an app-owned runtime connected", () => { + // The streak measures how long the MACHINE service has failed to answer. + // An isolated or spawned runtime is what the desktop falls back to because + // the service did not answer, so landing on one must not reset the marker + // -- doing so made every 60s isolated-recovery install restart the streak, + // and a long-broken service kept reading as a brain that is just starting. + const pool = new LocalRuntimeConnectionPool("1.2.3", { + debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn(), + } as never, {}); + const internals = pool as unknown as { + serviceStartupStreakStartedAt: string | null; + noteConnectedRuntime: (isMachineService: boolean) => void; + }; + + internals.serviceStartupStreakStartedAt = "2026-08-16T09:00:00.000Z"; + internals.noteConnectedRuntime(false); + expect(internals.serviceStartupStreakStartedAt).toBe("2026-08-16T09:00:00.000Z"); + + internals.noteConnectedRuntime(true); + expect(internals.serviceStartupStreakStartedAt).toBeNull(); + + pool.dispose(); + }); + it("parses structured service manager output for settings status", () => { expect(parseRuntimeServiceManagerOutput(JSON.stringify({ ok: false, @@ -979,7 +1003,16 @@ describe("local runtime connection pool", () => { ok: false, path: "/Users/admin/Library/LaunchAgents/com.ade.runtime.plist", message: "launchctl failed", + starting: false, + restarted: false, }); + expect(parseRuntimeServiceManagerOutput(JSON.stringify({ + ok: true, + action: "install", + starting: true, + path: "/Users/admin/Library/LaunchAgents/com.ade.runtime.plist", + message: "still starting", + }))?.starting).toBe(true); expect(parseRuntimeServiceManagerOutput("not json")).toBeNull(); }); diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts index b5ef9c51b..2cf2193b8 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts @@ -43,6 +43,7 @@ import { coerceProjects } from "../remoteRuntime/remoteBootstrap"; import type { Logger } from "../logging/logger"; import { getRuntimeServiceStatus, type ServiceManagerStatusResult } from "../../../../../ade-cli/src/serviceManager"; import { ADE_RUNTIME_SERVICE_NAME as RUNTIME_SERVICE_NAME } from "../../../../../ade-cli/src/serviceManager/common"; +import { RUNTIME_SERVICE_START_WAIT_MS } from "../../../../../ade-cli/src/serviceManager/runtimeServiceBudgets"; import { buildPackagedRuntimeNodePath, type PackagedRuntimeNodePathOptions } from "../runtime/packagedNodePath"; import { readLastFailure } from "../runtime/lastFailureStore"; import type { AdeRecoveryErrorCode } from "../../../shared/types/recovery"; @@ -84,6 +85,10 @@ type RuntimeServiceManagerOutput = { ok: boolean | null; path: string | null; message: string | null; + /** The installer left a live brain that has not answered yet; see `ServiceManagerResult.starting`. */ + starting: boolean; + /** The installer actually (re)started the service child; see `ServiceManagerResult.restarted`. */ + restarted: boolean; }; /** @@ -127,7 +132,23 @@ const LOCAL_RUNTIME_SERVICE_UNINSTALL_TIMEOUT_MS = 20_000; // `serve --install-service` does an unload → reap → load handover, so it is // allowed longer than the uninstall — but never forever: a wedged installer // used to pin `serviceInstallPromise` and block every later install. -const LOCAL_RUNTIME_SERVICE_INSTALL_TIMEOUT_MS = 60_000; +// Must cover the installer's own waits: the shared handover budget +// (RUNTIME_SERVICE_HANDOVER_TIMEOUT_MS, 30 s) plus one blocking probe overrun, +// predecessor termination and the launchctl round-trips. A child killed at +// this deadline reads as a failed install even when launchd's replacement is +// coming up, so the budget errs long. +const LOCAL_RUNTIME_SERVICE_INSTALL_TIMEOUT_MS = RUNTIME_SERVICE_START_WAIT_MS; +/** + * How long a freshly (re)installed service gets to answer on the socket before + * the desktop gives up on it. + * + * The same size as the install budget, and spent *after* it rather than + * instead of it: `tryRepairServiceConnection` runs the install first and then + * this connect loop, so a brain the installer returned as `starting` gets a + * second full window to come up (and one `createConnection` can therefore stay + * pending for up to about twice RUNTIME_SERVICE_START_WAIT_MS). + */ +const LOCAL_RUNTIME_SERVICE_REPAIR_CONNECT_TIMEOUT_MS = RUNTIME_SERVICE_START_WAIT_MS; const LOCAL_RUNTIME_STATUS_REFRESH_TIMEOUT_MS = 2_000; // The Windows service probe itself costs ~2.1s (two PowerShell spawns), so the // off-thread child needs generous headroom before it is treated as unanswerable. @@ -943,6 +964,8 @@ export function parseRuntimeServiceManagerOutput(output: string): RuntimeService ok: typeof record.ok === "boolean" ? record.ok : null, path: typeof record.path === "string" && record.path.trim() ? record.path.trim() : null, message: typeof record.message === "string" && record.message.trim() ? record.message.trim() : null, + starting: record.starting === true, + restarted: record.restarted === true, }; } @@ -976,6 +999,8 @@ export class LocalRuntimeConnectionPool { intent: ProjectRegistrationIntent; promise: Promise; }>(); + /** ISO time of the first install attempt since the last successful connect; see `runServiceInstallBestEffort`. */ + private serviceStartupStreakStartedAt: string | null = null; private serviceInstallStatus: LocalRuntimeStatus["serviceInstall"] = { state: "not_attempted", attempted: false, @@ -1430,13 +1455,24 @@ export class LocalRuntimeConnectionPool { }); return; } + // The streak start, not this attempt's start: installs recur (connect + // failures re-run them, isolated recovery re-runs them every 60s), and a + // brain that has not answered since the FIRST of those attempts is what + // recovery needs to age. The streak resets when a connection succeeds. + if (this.serviceStartupStreakStartedAt == null) { + this.serviceStartupStreakStartedAt = new Date().toISOString(); + } + const attemptStartedAt = this.serviceStartupStreakStartedAt; this.serviceInstallStatus = { state: "installing", attempted: true, path: cliPath, message: "Installing the ADE service login item.", exitCode: null, + // This transition is happening now; `attemptStartedAt` is the streak + // marker and the only field that reaches back to the first attempt. updatedAt: new Date().toISOString(), + attemptStartedAt, }; let result: ServiceManagerCommandResult; try { @@ -1462,6 +1498,7 @@ export class LocalRuntimeConnectionPool { message, exitCode: null, updatedAt: new Date().toISOString(), + attemptStartedAt, }; this.logger.warn("local_runtime.service_install_failed", { error: message }); return; @@ -1475,6 +1512,7 @@ export class LocalRuntimeConnectionPool { message, exitCode: null, updatedAt: new Date().toISOString(), + attemptStartedAt, }; this.logger.warn("local_runtime.service_install_failed", { cliPath, reason: "timeout", message }); return; @@ -1497,8 +1535,14 @@ export class LocalRuntimeConnectionPool { message: parsed?.message || output || "ADE service login item is installed.", exitCode: code, updatedAt: new Date().toISOString(), + starting: parsed?.starting === true, + restarted: parsed?.restarted === true, + attemptStartedAt, }; - this.logger.info("local_runtime.service_install_succeeded", payload); + this.logger.info( + parsed?.starting ? "local_runtime.service_install_starting" : "local_runtime.service_install_succeeded", + payload, + ); } else { this.serviceInstallStatus = { state: "failed", @@ -1507,6 +1551,7 @@ export class LocalRuntimeConnectionPool { message: parsed?.message || errorOutput || output || "ADE service login item installation failed.", exitCode: code, updatedAt: new Date().toISOString(), + attemptStartedAt, }; this.logger.warn("local_runtime.service_install_failed", payload); } @@ -2285,7 +2330,7 @@ export class LocalRuntimeConnectionPool { private async tryConnect(socketPath: string): Promise { try { - const client = await this.connectClient(socketPath); + const client = await this.connectClient(socketPath, { isMachineService: true }); this.ownedRuntimeChild = null; return { client, child: null, socketPath }; } catch (error) { @@ -2355,13 +2400,15 @@ export class LocalRuntimeConnectionPool { // attempt lands in that churn window and strands this desktop on an // isolated no-sync runtime, so keep retrying — through connect failures // AND through compatibility errors from the not-yet-replaced old brain — - // until the repaired service is actually reachable. - const deadline = Date.now() + 20_000; + // until the repaired service is actually reachable. The budget covers a + // brain the installer reported as `starting`: launchd/the supervisor owns + // it and it will answer, so waiting is right and restarting it is not. + const deadline = Date.now() + LOCAL_RUNTIME_SERVICE_REPAIR_CONNECT_TIMEOUT_MS; let lastError: unknown = null; for (;;) { try { await waitForSocket(socketPath, 2_000); - const client = await this.connectClient(socketPath); + const client = await this.connectClient(socketPath, { isMachineService: true }); this.ownedRuntimeChild = null; return { client, child: null, socketPath }; } catch (error) { @@ -2557,10 +2604,30 @@ export class LocalRuntimeConnectionPool { return localReleaseBuildOutputRuntimeBlock(resolveCliScriptPath()); } + /** + * Clears the service-install streak marker, but only when the connection + * that just succeeded is the machine background service. + * + * The streak answers "how long has the machine service been failing to + * answer", and recovery ages a `brain_starting` verdict off it. Connecting + * to an app-owned runtime — the isolated one, or a spawned primary — proves + * nothing about the service: it is what the desktop falls back to *because* + * the service did not answer. Clearing the marker there made the next + * isolated-recovery install (every 60s) restart the streak from now, so a + * service broken for an hour kept reporting a brand-new attempt and kept + * reading as a brain that is merely starting. + */ + private noteConnectedRuntime(isMachineService: boolean): void { + if (!isMachineService) return; + this.serviceStartupStreakStartedAt = null; + } + private async connectClient( socketPath: string, options: { preserveVersionSkew?: boolean; + /** True only for the machine service endpoint; gates the streak reset. */ + isMachineService?: boolean; expectedPid?: number | null; connectTimeoutMs?: number; initializeTimeoutMs?: number; @@ -2595,6 +2662,7 @@ export class LocalRuntimeConnectionPool { this.clearVersionSkewStatus(); } this.activeClient = client; + this.noteConnectedRuntime(options.isMachineService === true); this.activeRuntimePid = runtimeInfo.pid; this.activeRuntimeSyncPort = runtimeInfo.syncPort; this.activeRuntimePublishHealth = runtimeInfo.publishHealth; diff --git a/apps/desktop/src/main/services/runtime/lastFailureStore.ts b/apps/desktop/src/main/services/runtime/lastFailureStore.ts index 0ee8bd264..0099dc4d4 100644 --- a/apps/desktop/src/main/services/runtime/lastFailureStore.ts +++ b/apps/desktop/src/main/services/runtime/lastFailureStore.ts @@ -10,7 +10,7 @@ import { readValidJson, writeFileAtomic } from "../state/durableFile"; const MESSAGE_MAX_BYTES = 2 * 1024; const DETAIL_MAX_BYTES = 8 * 1024; -const CRASH_LOOP_WINDOW_MS = 5 * 60 * 1_000; +export const LAST_FAILURE_CRASH_LOOP_WINDOW_MS = 5 * 60 * 1_000; export type LastFailureTarget = | { kind: "machine"; env?: NodeJS.ProcessEnv } @@ -160,7 +160,7 @@ export function computeStartupBackoffMs( if (!report || report.count < 3) return 0; const nowMs = now instanceof Date ? now.getTime() : now; const firstAtMs = Date.parse(report.firstAt); - if (!Number.isFinite(firstAtMs) || nowMs - firstAtMs < 0 || nowMs - firstAtMs > CRASH_LOOP_WINDOW_MS) { + if (!Number.isFinite(firstAtMs) || nowMs - firstAtMs < 0 || nowMs - firstAtMs > LAST_FAILURE_CRASH_LOOP_WINDOW_MS) { return 0; } return Math.min((report.count - 2) * 10_000, 60_000); diff --git a/apps/desktop/src/main/services/runtime/projectRecoveryService.test.ts b/apps/desktop/src/main/services/runtime/projectRecoveryService.test.ts index cf81d77ed..fc414fdea 100644 --- a/apps/desktop/src/main/services/runtime/projectRecoveryService.test.ts +++ b/apps/desktop/src/main/services/runtime/projectRecoveryService.test.ts @@ -22,6 +22,8 @@ function tempRoot(): string { return root; } +const NOW = Date.parse("2026-07-12T12:01:00.000Z"); + function logger(): Logger { return { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }; } @@ -98,7 +100,7 @@ function deps(overrides: Partial = {}): ProjectRecov readFailureReports: vi.fn(async () => ({ project: null, machine: null })), clearFailureReports: vi.fn(async () => {}), socketExists: vi.fn(() => false), - now: () => Date.parse("2026-07-12T12:01:00.000Z"), + now: () => NOW, ...overrides, }; } @@ -162,6 +164,36 @@ describe("ProjectRecoveryService.diagnose", () => { canAutoRepair: false, overrides: { probeSocket: vi.fn(async () => true), pingEndpoint: vi.fn(async () => false) }, }, + { + name: "starting brain (service running, install just began, socket not up yet)", + expected: "brain_starting", + canAutoRepair: false, + overrides: { + connectionPool: pool(status({ + serviceInstall: { + state: "installed", attempted: true, path: null, message: null, exitCode: null, + updatedAt: new Date(NOW).toISOString(), starting: true, + attemptStartedAt: new Date(NOW - 15_000).toISOString(), + }, + serviceHealth: { state: "running", installed: true, running: true, path: null, message: null, checkedAt: null }, + })), + }, + }, + { + name: "brain still quiet long after its install began (not starting any more)", + expected: "brain_crash_looping", + canAutoRepair: true, + overrides: { + connectionPool: pool(status({ + serviceInstall: { + state: "installed", attempted: true, path: null, message: null, exitCode: null, + updatedAt: new Date(NOW).toISOString(), starting: true, + attemptStartedAt: new Date(NOW - 10 * 60_000).toISOString(), + }, + serviceHealth: { state: "installed", installed: true, running: false, path: null, message: null, checkedAt: null }, + })), + }, + }, { name: "unknown", expected: "unknown_failure", @@ -176,6 +208,84 @@ describe("ProjectRecoveryService.diagnose", () => { expect(diagnosis.canAutoRepair).toBe(canAutoRepair); expectNoJargon(`${diagnosis.headline} ${diagnosis.body}`); }); + + it("calls a bound-but-not-yet-answering brain starting, not another program's socket", async () => { + // The brain binds its RPC socket before it can answer `ade/initialize`, so + // reachable-but-silent is its own boot window, not a stranger's socket. + const service = createProjectRecoveryService(deps({ + probeSocket: vi.fn(async () => true), + pingEndpoint: vi.fn(async () => false), + connectionPool: pool(status({ + serviceInstall: { + state: "installed", attempted: true, path: null, message: null, exitCode: null, + updatedAt: new Date(NOW).toISOString(), starting: true, + attemptStartedAt: new Date(NOW - 15_000).toISOString(), + }, + serviceHealth: { state: "running", installed: true, running: true, path: null, message: null, checkedAt: null }, + })), + })); + + const diagnosis = await service.diagnose(tempRoot()); + + expect(diagnosis.state).toBe("brain_starting"); + expect(diagnosis.canAutoRepair).toBe(false); + }); + + it("does not classify a future install timestamp as brain_starting", async () => { + // A clock that moved backwards after the attempt was recorded leaves a + // stamp in the future. Only the upper bound was checked, so it read as + // "always starting" and suppressed repair until the clock caught up. + const service = createProjectRecoveryService(deps({ + connectionPool: pool(status({ + serviceInstall: { + state: "installed", attempted: true, path: null, message: null, exitCode: null, + updatedAt: new Date(NOW).toISOString(), starting: true, + attemptStartedAt: new Date(NOW + 60 * 60_000).toISOString(), + }, + serviceHealth: { state: "running", installed: true, running: true, path: null, message: null, checkedAt: null }, + })), + })); + + const diagnosis = await service.diagnose(tempRoot()); + + // The concrete state matters: "not brain_starting" would pass for any + // wrong answer, and the point of the fix is that the stuck project falls + // through to a diagnosis Repair is allowed to act on. + expect(diagnosis.state).toBe("unknown_failure"); + expect(diagnosis.canAutoRepair).toBe(true); + }); + + it("does not report a healthy answering brain as broken over a fresh sync-host failure", async () => { + const syncHostFailure: AdeLastFailureReport = { + ...failure("socket_owned_by_other"), + component: "sync_host", + }; + const service = createProjectRecoveryService(deps({ + probeSocket: vi.fn(async () => true), + pingEndpoint: vi.fn(async () => true), + readFailureReports: vi.fn(async () => ({ project: null, machine: syncHostFailure })), + })); + + const diagnosis = await service.diagnose(tempRoot()); + + expect(diagnosis.state).toBe("healthy"); + // Still reported, just not treated as a reason to repair. + expect(diagnosis.lastFailure?.component).toBe("sync_host"); + }); + + it("still repairs on a fresh sync-host failure when the brain does not answer", async () => { + const syncHostFailure: AdeLastFailureReport = { + ...failure("socket_owned_by_other"), + component: "sync_host", + }; + const service = createProjectRecoveryService(deps({ + readFailureReports: vi.fn(async () => ({ project: null, machine: syncHostFailure })), + })); + + const diagnosis = await service.diagnose(tempRoot()); + + expect(diagnosis.state).toBe("socket_owned_by_other"); + }); }); describe("ProjectRecoveryService.repair", () => { @@ -330,7 +440,7 @@ describe("ProjectRecoveryService.restartBrain", () => { // The ping is explicitly bounded: the RPC client's default is 10 minutes, // which would park this call — and any repair waiting on it — on a brain // that binds the socket but never answers. - expect(connectionPool.callSync).toHaveBeenCalledWith("ping", {}, { timeoutMs: 20_000 }); + expect(connectionPool.callSync).toHaveBeenCalledWith("ping", {}, { timeoutMs: 90_000 }); }); const installStatusPool = ( diff --git a/apps/desktop/src/main/services/runtime/projectRecoveryService.ts b/apps/desktop/src/main/services/runtime/projectRecoveryService.ts index 60fd3f31f..312852d31 100644 --- a/apps/desktop/src/main/services/runtime/projectRecoveryService.ts +++ b/apps/desktop/src/main/services/runtime/projectRecoveryService.ts @@ -1,15 +1,22 @@ import fs from "node:fs"; import net from "node:net"; import path from "node:path"; -import type { - AdeLastFailureReport, - AdeRecoveryErrorCode, - ProjectRecoveryDiagnosis, - ProjectRepairReport, - RepairStepId, - RepairStepResult, +import { + REPAIR_STEP_LABELS, + REPAIR_STEP_ORDER, + stateForCode, + type AdeLastFailureReport, + type AdeRecoveryErrorCode, + type ProjectRecoveryDiagnosis, + type ProjectRepairReport, + type RepairStepId, + type RepairStepResult, } from "../../../shared/types/recovery"; import { resolveMachineAdeLayout } from "../../../../../ade-cli/src/services/projects/machineLayout"; +import { + RUNTIME_SERVICE_START_WAIT_MS, + RUNTIME_SERVICE_YOUNG_BRAIN_MS, +} from "../../../../../ade-cli/src/serviceManager/runtimeServiceBudgets"; import type { Logger } from "../logging/logger"; import type { LocalRuntimeConnectionPool } from "../localRuntime/localRuntimeConnectionPool"; import { RuntimeRpcClient, type RuntimeRpcTransport } from "../remoteRuntime/runtimeRpcClient"; @@ -28,30 +35,19 @@ const MIB = 1024 * 1024; const GIB = 1024 * MIB; const FRESH_FAILURE_MS = 5 * 60 * 1_000; // How long a restarted brain gets to rebind the machine endpoint, shared by -// `repair()`'s restart_service step and `restartBrain()`. -const BRAIN_RESTART_TIMEOUT_MS = 20_000; - -const STEP_LABELS: Record = { - check_space: "Checking storage space", - stop_service: "Stopping ADE's background service", - validate_database: "Checking project data", - resolve_migrations: "Finishing interrupted saves", - restart_service: "Restarting ADE's background service", - verify_endpoint: "Checking the background service", - verify_project_rpc: "Checking this project", - reconcile_chats: "Checking chats", -}; +// `repair()`'s restart_service step and `restartBrain()`. Generous on purpose: +// the installer reports a live-but-slow brain as `starting`, and this wait is +// where such a brain gets the rest of its time. It used to be 20s, which on a +// cold or slow machine expired against a healthy brain and turned into +// "didn't restart — try again". +const BRAIN_RESTART_TIMEOUT_MS = RUNTIME_SERVICE_START_WAIT_MS; +// A brain whose install/restart began less than this long ago and that is not +// answering yet is presumed to still be starting, not stuck. +const BRAIN_STARTING_WINDOW_MS = RUNTIME_SERVICE_YOUNG_BRAIN_MS; + +const STEP_LABELS = REPAIR_STEP_LABELS; -const STEP_ORDER: readonly RepairStepId[] = [ - "check_space", - "stop_service", - "validate_database", - "resolve_migrations", - "restart_service", - "verify_endpoint", - "verify_project_rpc", - "reconcile_chats", -]; +const STEP_ORDER = REPAIR_STEP_ORDER; const REPAIR_MIN_FREE_BYTES = (dbSize: number): number => Math.max(GIB, dbSize + 512 * MIB); // Advice = repair gate + margin, so following the advice always satisfies repair. @@ -189,30 +185,21 @@ function diagnosisCopy(state: ProjectRecoveryDiagnosis["state"]): Pick< body: "Close other copies of ADE, then try again.", canAutoRepair: false, }; + case "brain_starting": + return { + headline: "ADE's background service is starting.", + body: "This can take a minute the first time or right after an update. ADE will open the project as soon as it's ready — nothing to do.", + canAutoRepair: false, + }; default: return { headline: "ADE couldn't open this project.", - body: "You can try a repair, or send the technical details to support.", + body: "Something stopped ADE's background service from answering. A repair restarts it and checks the project's data — your files and chats aren't touched.", canAutoRepair: true, }; } } -function stateForCode(code: AdeRecoveryErrorCode): ProjectRecoveryDiagnosis["state"] { - switch (code) { - case "disk_full": return "disk_full"; - case "insufficient_headroom": return "insufficient_headroom"; - case "db_integrity": - case "migration_incomplete": - case "migration_unknown_state": return "db_repair_needed"; - case "brain_crash_looping": return "brain_crash_looping"; - case "brain_not_installed": return "brain_not_installed"; - case "socket_stale_no_owner": return "socket_stale_no_owner"; - case "socket_owned_by_other": return "socket_owned_by_other"; - default: return "unknown_failure"; - } -} - function humanGb(bytes: number): string { const gb = Math.max(1, Math.ceil(bytes / GIB)); return `${gb} GB`; @@ -504,16 +491,43 @@ export class ProjectRecoveryService { const socketReachable = await this.probeSocket(this.socketPath, 750); const endpointHealthy = socketReachable && await this.pingEndpoint(this.socketPath, 1_500); const serviceStatus = this.deps.connectionPool.getStatus(); + const installStartedAt = Date.parse(serviceStatus.serviceInstall.attemptStartedAt ?? ""); + // Time-bounded on purpose: the installer's `starting` flag alone would keep + // a brain that wedged during boot reading as "starting" forever. + // + // Deliberately not gated on `!socketReachable`: the brain binds its RPC + // socket before it can answer `ade/initialize`, so there is a real window + // where the socket accepts connections and the ping still fails. Requiring + // an unreachable socket made that window impossible to classify as + // starting, and it fell through to "another program owns this" instead. + // A negative age means the recorded attempt is in the future -- a clock + // that moved backwards after it was written. Only the upper bound was + // checked, so such a stamp read as "always inside the window" and + // suppressed repair until the clock caught up with it. + const startupAgeMs = this.now() - installStartedAt; + const brainStarting = + serviceStatus.serviceHealth.running === true + && Number.isFinite(installStartedAt) + && startupAgeMs >= 0 + && startupAgeMs < BRAIN_STARTING_WINDOW_MS; const dbCheck = endpointHealthy ? { healthy: null, detail: "Project data check skipped because the background service is using it." } : await this.quickCheck(dbPath); + // A recorded sync-host failure says phone sync could not start. It says + // nothing about whether the desktop can reach this brain, and the brain + // deliberately keeps that record until sync really comes up — so a brain + // that is bound, answering, and healthy routinely carries a fresh + // `sync_host` failure. Repairing on it would kill a brain doing its job. + // It stays in `lastFailure` and the technical detail either way. + const actionableFailure = + freshFailure && endpointHealthy && freshFailure.component === "sync_host" ? null : freshFailure; const technicalParts = [ `freeBytes=${free}`, `dbSize=${dbSize}`, `socketPath=${this.socketPath}`, `socketReachable=${socketReachable}`, `endpointHealthy=${endpointHealthy}`, - `serviceInstall=${serviceStatus.serviceInstall.state}`, + `serviceInstall=${serviceStatus.serviceInstall.state}${serviceStatus.serviceInstall.starting ? " (starting)" : ""}`, `serviceHealth=${serviceStatus.serviceHealth.state}`, `database=${dbCheck.detail}`, ...(latestFailure ? [`lastFailure=${latestFailure.code}: ${latestFailure.message}${latestFailure.detail ? ` (${latestFailure.detail})` : ""}`] : []), @@ -524,15 +538,23 @@ export class ProjectRecoveryService { if (free < GIB) { state = "disk_full"; code = "disk_full"; - } else if (freshFailure) { - state = stateForCode(freshFailure.code); - code = freshFailure.code; + } else if (actionableFailure) { + state = stateForCode(actionableFailure.code); + code = actionableFailure.code; } else if (dbCheck.healthy === false) { state = "db_repair_needed"; code = "db_integrity"; } else if (endpointHealthy) { state = "healthy"; code = "unknown"; + } else if (brainStarting) { + // Ahead of the owner, crash-loop and stale-socket branches: a brain that + // the installer just started (or reported as still starting) and that + // launchd/the supervisor shows running is booting, not broken. Repair + // here would only kill it and start its clock over, and a socket it has + // bound but cannot answer on yet is this brain's, not a stranger's. + state = "brain_starting"; + code = "unknown"; } else if (socketReachable) { state = "socket_owned_by_other"; code = "socket_owned_by_other"; diff --git a/apps/desktop/src/main/services/shared/externalLinks.ts b/apps/desktop/src/main/services/shared/externalLinks.ts index 721622a98..2ca89f093 100644 --- a/apps/desktop/src/main/services/shared/externalLinks.ts +++ b/apps/desktop/src/main/services/shared/externalLinks.ts @@ -1,87 +1,8 @@ -import { execFile } from "node:child_process"; -import { - resolveTrustedWindowsTool, - trustedWindowsToolKernelPath, -} from "../../../../../ade-cli/src/lib/trustedWindowsTools"; - -const ALLOWED_EXTERNAL_URL_SCHEMES = new Set(["http:", "https:", "mailto:"]); -const OPEN_TIMEOUT_MS = 5_000; - -export function normalizeExternalUrl(url: string | undefined | null): string | null { - const raw = typeof url === "string" ? url.trim() : ""; - if (!raw) return null; - - let parsed: URL; - try { - parsed = new URL(raw); - } catch { - throw new Error("Invalid URL"); - } - - if (!ALLOWED_EXTERNAL_URL_SCHEMES.has(parsed.protocol)) { - throw new Error("Only http(s) and mailto: URLs are allowed."); - } - - return parsed.toString(); -} - -function execFileOpen(file: string, args: string[]): Promise { - return new Promise((resolve, reject) => { - execFile(file, args, { timeout: OPEN_TIMEOUT_MS, windowsHide: true }, (error) => { - if (error) { - reject(error); - return; - } - resolve(); - }); - }); -} - -function windowsRundll32Path(): string { - try { - return resolveTrustedWindowsTool("rundll32"); - } catch { - // Cross-platform unit tests mock `process.platform` to win32 on macOS/Linux, - // where the kernel SystemRoot alias cannot be canonicalized. The kernel - // path is still the command we would spawn; execFile is mocked in those tests. - return trustedWindowsToolKernelPath("rundll32"); - } -} - -function openWithPlatformHelper(url: string): Promise { - if (process.platform === "darwin") { - return execFileOpen("/usr/bin/open", [url]); - } - if (process.platform === "win32") { - return execFileOpen(windowsRundll32Path(), ["url.dll,FileProtocolHandler", url]); - } - return execFileOpen("/usr/bin/xdg-open", [url]); -} - -async function openWithElectronShell(url: string): Promise { - try { - // Dual-runtime: ADE CLI bundles this module, and a static - // `import { shell } from "electron"` crashes headless startup with - // "does not provide an export named 'shell'". Load Electron only when the - // OS opener failed and we are actually in the desktop process. - const electron = await import("electron"); - if (electron.shell?.openExternal) { - await electron.shell.openExternal(url); - return; - } - } catch { - // Not running inside Electron. - } - throw new Error("No external URL opener is available."); -} - -export async function openExternalUrl(url: string | undefined | null): Promise { - const normalized = normalizeExternalUrl(url); - if (!normalized) return; - - try { - await openWithPlatformHelper(normalized); - } catch { - await openWithElectronShell(normalized); - } -} +/** + * Re-export only. `openExternalUrl` has no Electron dependency of its own — it + * shells out to the OS opener and reaches for `electron.shell` lazily, exactly + * so the CLI can bundle it — so it lives in `apps/ade-cli/src/lib` with the + * rest of the dual-runtime helpers. This file keeps the desktop's existing + * import path working. + */ +export { normalizeExternalUrl, openExternalUrl } from "../../../../../ade-cli/src/lib/externalLinks"; diff --git a/apps/desktop/src/preload/global.d.ts b/apps/desktop/src/preload/global.d.ts index 4a152812c..738ae96c4 100644 --- a/apps/desktop/src/preload/global.d.ts +++ b/apps/desktop/src/preload/global.d.ts @@ -719,7 +719,8 @@ import type { StorageCompressionResult, StorageSnapshot, } from "../shared/types/storage"; -import type { ProjectRecoveryDiagnosis, ProjectRepairReport } from "../shared/types/recovery"; +import type { ProjectRecoveryDiagnosis, ProjectRepairReport, RepairStepResult } from "../shared/types/recovery"; +import type { DiagnosticReportPayload, DiagnosticReportRequestPayload } from "../shared/types/diagnostics"; import type { AppPackageChannel } from "../shared/packageChannel"; import type { ProductAnalyticsCapture, @@ -892,9 +893,24 @@ declare global { onMissing: (cb: (data: { rootPath: string }) => void) => () => void; onStateEvent: (cb: (event: AdeProjectEvent) => void) => () => void; }; + /** + * Absent on older preloads: every call site must tolerate `undefined` + * and simply not offer the button. + */ + diagnostics?: { + openIssue: (context: DiagnosticReportRequestPayload) => Promise; + }; recovery: { diagnose: (projectRoot: string) => Promise; repair: (projectRoot: string) => Promise; + /** + * Live repair steps for the window that started the repair. Optional + * for the same reason `diagnostics` is: an older preload does not have + * it, and every call site already guards before calling. + */ + onRepairStep?: ( + cb: (payload: { projectRoot: string; step: RepairStepResult }) => void, + ) => () => void; }; remoteRuntime: { listTargets: () => Promise; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index da9d18b64..d37b5df10 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -26,7 +26,8 @@ import { REMOTE_RUNTIME_EVENT_IDLE_POLL_MS, } from "./pinnedRuntimeEvents"; import type { OrchestrationEventPayload } from "../shared/types/orchestration"; -import type { ProjectRecoveryDiagnosis, ProjectRepairReport } from "../shared/types/recovery"; +import type { ProjectRecoveryDiagnosis, ProjectRepairReport, RepairStepResult } from "../shared/types/recovery"; +import type { DiagnosticReportPayload, DiagnosticReportRequestPayload } from "../shared/types/diagnostics"; import type { ProductAnalyticsCapture, ProductAnalyticsCaptureResult, @@ -3969,11 +3970,27 @@ contextBridge.exposeInMainWorld("ade", { }; }, }, + diagnostics: { + openIssue: ( + context: DiagnosticReportRequestPayload, + ): Promise => + ipcRenderer.invoke(IPC.diagnosticsOpenIssue, context), + }, recovery: { diagnose: (projectRoot: string): Promise => ipcRenderer.invoke(IPC.recoveryDiagnose, { projectRoot }), repair: (projectRoot: string): Promise => ipcRenderer.invoke(IPC.recoveryRepair, { projectRoot }), + onRepairStep: ( + cb: (payload: { projectRoot: string; step: RepairStepResult }) => void, + ): (() => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: { projectRoot: string; step: RepairStepResult }, + ) => cb(payload); + ipcRenderer.on(IPC.recoveryRepairStep, listener); + return () => ipcRenderer.removeListener(IPC.recoveryRepairStep, listener); + }, }, remoteRuntime: { listTargets: async (): Promise => diff --git a/apps/desktop/src/renderer/components/app/App.tsx b/apps/desktop/src/renderer/components/app/App.tsx index 8fa312ba5..da4ecb2bf 100644 --- a/apps/desktop/src/renderer/components/app/App.tsx +++ b/apps/desktop/src/renderer/components/app/App.tsx @@ -21,11 +21,11 @@ import { WindowsBetaNoticeHost } from "./WindowsBetaNoticeModal"; import { ClipboardDeeplinkBanner } from "./ClipboardDeeplinkBanner"; import { CrossRepoPrBanner } from "./CrossRepoPrBanner"; import { ProjectRecoveryScreen } from "./ProjectRecoveryScreen"; +import { PageErrorBoundary } from "./PageErrorBoundary"; import { ProjectWelcomePage } from "../projects/ProjectWelcomePage"; import { OnboardingBootstrap } from "../onboarding/OnboardingBootstrap"; import { LaunchGate } from "../onboarding/LaunchGate"; import { GlossaryPage } from "../onboarding/GlossaryPage"; -import { logRendererDebugEvent } from "../../lib/debugLog"; import { readStoredProjectRoute, writeStoredProjectRoute } from "./projectRouteStorage"; import { requestLinearIssueQuickView } from "../../lib/linearIssueQuickViewNavigation"; import { isWebClientMode } from "../../lib/webClientMode"; @@ -173,73 +173,6 @@ const StartupSplashScreen = ( /** Used by React.lazy Suspense boundaries while route chunks load. */ const GuardLoadingFallback = StartupSplashScreen; -/* ---------- Per-route error boundary ---------- */ - -type PageErrorBoundaryState = { hasError: boolean; message: string }; - -class PageErrorBoundaryInner extends React.Component< - { children: React.ReactNode; onGoHome: () => void }, - PageErrorBoundaryState -> { - state: PageErrorBoundaryState = { hasError: false, message: "" }; - - static getDerivedStateFromError(error: unknown): PageErrorBoundaryState { - return { hasError: true, message: error instanceof Error ? error.message : String(error) }; - } - - componentDidCatch(error: Error, errorInfo: React.ErrorInfo): void { - console.error("page.crash", error, errorInfo, error?.stack); - logRendererDebugEvent("renderer.page_boundary_crash", { - message: error?.message ?? String(error), - route: window.location.hash || window.location.pathname, - componentStack: errorInfo.componentStack ?? null, - causeStack: error?.stack ?? null, - }); - } - - render() { - if (this.state.hasError) { - return ( -
-
-
This page crashed
-
{this.state.message || "Unknown error"}
-
-
- - -
-
- ); - } - return this.props.children; - } -} - -function PageErrorBoundary({ children }: { children: React.ReactNode }) { - const navigate = useNavigate(); - return ( - navigate("/work")}> - {children} - - ); -} - const RouteLoadingFallback = (
{showBanner && banner ? ( -
-
diff --git a/apps/desktop/src/renderer/components/settings/BrainRepairButton.tsx b/apps/desktop/src/renderer/components/settings/BrainRepairButton.tsx index 443a63e16..fd1d32993 100644 --- a/apps/desktop/src/renderer/components/settings/BrainRepairButton.tsx +++ b/apps/desktop/src/renderer/components/settings/BrainRepairButton.tsx @@ -1,5 +1,6 @@ import type { BrainRepair } from "../../hooks/useBrainRepair"; import { COLORS, SANS_FONT, outlineButton } from "../lanes/laneDesignTokens"; +import { ReportIssueButton } from "../app/ReportIssueButton"; /** * The Repair control, shared by every surface that renders a brain-side @@ -31,12 +32,37 @@ export function BrainRepairButton({ {repair.pending ? "Repairing…" : "Repair"} {repair.error ? ( + <> - Repair failed — quit and reopen ADE. + {/* The main process already phrases restart failures for people + ("A newer ADE runtime is already running — quit and reopen ADE + instead."); hiding that behind a generic line and a tooltip + left users with an instruction and no reason. */} + {/* Two sentences, not an em dash: the main process hands back its own + capitalised sentence, and joining it with a dash read as + "Repair didn't finish — A newer ADE runtime is…". */} + {`Repair didn't finish. ${repair.error.replace(/\.?\s*$/, ".")}`} + {/* One import, one element: the surrounding surfaces own their own + layout, so this only ever appends after the failure line. */} + + ) : repair.notice ? ( { render(); fireEvent.click(screen.getByRole("button", { name: "Repair" })); - const failure = await screen.findByText("Repair failed — quit and reopen ADE."); - // Terse copy on screen; the technical detail rides along as the tooltip. + // The reason the main process gave is the message, not a hidden tooltip: + // "Repair failed" alone left people with an instruction and no reason. + const failure = await screen.findByText("Repair didn't finish. launchctl load failed."); expect(failure.getAttribute("title")).toBe("launchctl load failed."); expect(screen.getByRole("button", { name: "Repair" })).toBeTruthy(); }); diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.test.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.test.tsx index 9d3d19b61..a8047ce31 100644 --- a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.test.tsx +++ b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.test.tsx @@ -1,10 +1,13 @@ /* @vitest-environment jsdom */ import { afterEach, describe, expect, it, vi } from "vitest"; -import { cleanup, fireEvent, render } from "@testing-library/react"; -import { StorageDialogFrame } from "./StorageCleanupDialog"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { StorageCleanupDialog, StorageDialogFrame } from "./StorageCleanupDialog"; -afterEach(cleanup); +afterEach(() => { + cleanup(); + delete (window as unknown as { ade?: unknown }).ade; +}); describe("StorageDialogFrame", () => { it("lets only the topmost dialog close on Escape", () => { @@ -56,3 +59,141 @@ describe("StorageDialogFrame", () => { expect(closeBack).not.toHaveBeenCalled(); }); }); + +describe("StorageCleanupDialog failures", () => { + it("says which half of the job failed and offers a way to run it again", async () => { + const cleanupPreview = vi + .fn() + .mockRejectedValueOnce(new Error("EPERM: operation not permitted, scandir")) + .mockResolvedValueOnce({ items: [], blocked: [], totalBytes: 0 }); + (window as unknown as { ade?: unknown }).ade = { + storage: { cleanupPreview, cleanup: vi.fn() }, + }; + + render( + , + ); + + // Naming the failed half is the point: nothing was removed, so the person + // should not go hunting for half-deleted files. + expect(await screen.findByText("ADE couldn't check what's safe to remove.")).toBeTruthy(); + expect(screen.getByText(/Nothing was removed/)).toBeTruthy(); + // The raw errno stays behind the fold rather than on the main line. + const fold = screen.getByText("Show technical details").closest("details"); + expect(fold?.textContent).toContain("EPERM"); + + fireEvent.click(screen.getByRole("button", { name: "Try again" })); + await waitFor(() => expect(cleanupPreview).toHaveBeenCalledTimes(2)); + await waitFor(() => + expect(screen.queryByText("ADE couldn't check what's safe to remove.")).toBeNull(), + ); + }); + + it("shows the reopened dialog's preview when the abandoned read lands last", async () => { + const deferred: Array<(value: unknown) => void> = []; + const cleanupPreview = vi.fn( + () => new Promise((resolve) => { deferred.push(resolve); }), + ); + (window as unknown as { ade?: unknown }).ade = { + storage: { cleanupPreview, cleanup: vi.fn() }, + }; + + const props = { + title: "Free up space", + targets: [] as never[], + onClose: vi.fn(), + onCleaned: vi.fn(), + }; + const { rerender } = render(); + await waitFor(() => expect(cleanupPreview).toHaveBeenCalledTimes(1)); + + // Close before the first read answers, then reopen: a second read starts. + rerender(); + rerender(); + await waitFor(() => expect(cleanupPreview).toHaveBeenCalledTimes(2)); + + // The reopened dialog's read answers first, the abandoned one answers last. + deferred[1]({ + items: [{ path: "/tmp/fresh.log", label: "fresh.log", bytes: 10 }], + blocked: [], + totalBytes: 10, + }); + await screen.findByText("fresh.log"); + + deferred[0]({ + items: [{ path: "/tmp/stale.log", label: "stale.log", bytes: 99 }], + blocked: [], + totalBytes: 99, + }); + + // The stale answer must not repaint the dialog — it stays on the fresh + // review rather than falling back to a spinner or the abandoned list. + await waitFor(() => + expect(screen.getByRole("button", { name: /Remove 1 item/ })).toBeTruthy(), + ); + expect(screen.queryByText("stale.log")).toBeNull(); + expect(screen.getByText("fresh.log")).toBeTruthy(); + }); + + it("ignores a cleanup completion after close and reopen", async () => { + const previews: Array<(value: unknown) => void> = []; + const cleanupPreview = vi.fn(() => new Promise((resolve) => { previews.push(resolve); })); + const cleanups: Array<(value: unknown) => void> = []; + const cleanupCall = vi.fn(() => new Promise((resolve) => { cleanups.push(resolve); })); + (window as unknown as { ade?: unknown }).ade = { + storage: { cleanupPreview, cleanup: cleanupCall }, + }; + + const onCleaned = vi.fn(); + const props = { + title: "Free up space", + targets: [] as never[], + onClose: vi.fn(), + onCleaned, + }; + const { rerender } = render(); + await waitFor(() => expect(cleanupPreview).toHaveBeenCalledTimes(1)); + previews[0]({ + items: [{ path: "/tmp/first.log", label: "first.log", bytes: 10 }], + blocked: [], + totalBytes: 10, + }); + await screen.findByText("first.log"); + + // Start the removal, then close and reopen before it answers. + fireEvent.click(screen.getByRole("button", { name: /Remove 1 item/ })); + await waitFor(() => expect(cleanupCall).toHaveBeenCalledTimes(1)); + rerender(); + rerender(); + await waitFor(() => expect(cleanupPreview).toHaveBeenCalledTimes(2)); + previews[1]({ + items: [{ path: "/tmp/second.log", label: "second.log", bytes: 20 }], + blocked: [], + totalBytes: 20, + }); + await screen.findByText("second.log"); + + // The abandoned removal answers last: it must not settle the reopened + // dialog to "done", and must not report a result for a job nobody is + // looking at any more. + cleanups[0]({ removed: ["/tmp/first.log"], failed: [], freedBytes: 10 }); + // Flush the abandoned resolution's own continuation. Polling the cleanup + // call count would pass on the first tick -- that call happened before the + // reopen -- so the assertions below could run before the stale completion + // was even delivered, and would hold with or without the guard. + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(onCleaned).not.toHaveBeenCalled(); + expect(screen.getByRole("button", { name: /Remove 1 item/ })).toBeTruthy(); + expect(screen.getByText("second.log")).toBeTruthy(); + }); +}); diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx index 8647555e1..4564ce27d 100644 --- a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx +++ b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx @@ -251,6 +251,10 @@ export function StorageCleanupDialog({ const [result, setResult] = React.useState(null); const [report, setReport] = React.useState(null); const [error, setError] = React.useState(null); + // Which half of the job failed. "Something went wrong" told people nothing; + // whether ADE failed to *look* or failed to *remove* changes both what is + // true about their disk and what they should do next. + const [errorPhase, setErrorPhase] = React.useState<"checking" | "removing">("checking"); const maintenanceMode = Boolean(plan); @@ -261,39 +265,57 @@ export function StorageCleanupDialog({ const initRef = React.useRef({ targets }); initRef.current = { targets }; - React.useEffect(() => { - if (!open) return; - let active = true; + // Which read the dialog is currently showing. A close/reopen (or a Try again + // pressed twice) leaves the earlier `cleanupPreview` in flight, and those can + // land out of order — the stale one must not overwrite the fresh preview, nor + // drag a settled dialog back to "error". + const requestRef = React.useRef(0); + + const loadPreview = React.useCallback((): Promise => { const { targets: openTargets } = initRef.current; + const requestId = ++requestRef.current; setResult(null); setReport(null); setError(null); setStage("loading"); setPreview(null); - void window.ade.storage + return window.ade.storage .cleanupPreview(openTargets) .then((next) => { - if (!active) return; + if (requestRef.current !== requestId) return; setPreview(next); setStage("review"); }) .catch((err: unknown) => { - if (!active) return; + if (requestRef.current !== requestId) return; setError(err instanceof Error ? err.message : String(err)); + setErrorPhase("checking"); setStage("error"); }); + }, []); + + React.useEffect(() => { + if (!open) return; + void loadPreview(); return () => { - active = false; + // A dialog closed mid-read has nothing to show; retire the request so a + // late answer cannot paint over whatever the next open reads. + requestRef.current += 1; }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [open]); const confirm = React.useCallback(async () => { + // Same generation guard the preview uses, for the same reason: a removal + // (and the maintenance run after it) outlives a close, and a completion + // that lands after the dialog reopened would push the fresh dialog to + // "done" and hand the parent a result for a job it is no longer showing. + const requestId = requestRef.current; setStage("removing"); setError(null); try { if (!preview) { - setStage("review"); + if (requestRef.current === requestId) setStage("review"); return; } const filesystemResult = preview.items.length > 0 @@ -307,13 +329,16 @@ export function StorageCleanupDialog({ ...filesystemResult, freedBytes: filesystemResult.freedBytes + maintenanceBytes, }; + if (requestRef.current !== requestId) return; setReport(nextReport); setResult(next); setStage("done"); onCleaned(next); if (nextReport) plan?.onMaintenanceDone?.(nextReport); } catch (err) { + if (requestRef.current !== requestId) return; setError(err instanceof Error ? err.message : String(err)); + setErrorPhase("removing"); setStage("error"); } }, [plan, preview, targets, onCleaned]); @@ -357,17 +382,39 @@ export function StorageCleanupDialog({ {stage === "error" ? (
- {error ?? "Something went wrong."} + + {errorPhase === "checking" + ? "ADE couldn't check what's safe to remove." + : "The cleanup didn't finish."} + + + {errorPhase === "checking" + ? "Nothing was removed. Try again — if it keeps failing, close and reopen ADE, then come back here." + : "Some items may still be on disk. Nothing outside this list was touched, and you can run it again."} + + {error ? ( +
+ + Show technical details + +
+ {error} +
+
+ ) : null}
) : null} @@ -562,6 +609,22 @@ export function StorageCleanupDialog({ + ) : stage === "error" ? ( + <> + + + ) : ( <>