diff --git a/src/cli.ts b/src/cli.ts index b1c8fb8c6..e49b16794 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -32,6 +32,7 @@ import { randomBytes } from 'node:crypto'; import { validateWorkingDir } from './core/working-dir.js'; import { resolveSessionContext } from './core/session-marker.js'; import { resolveBotmuxDataDir } from './core/data-dir.js'; +import { clearSpawnFreeze, readActiveSpawnFreeze, spawnFreezePath, writeSpawnFreeze, SpawnFreezeConflictError, SPAWN_FREEZE_HARD_CAP_MS } from './core/spawn-freeze.js'; import { dashboardSecretPath } from './core/dashboard-secret.js'; import { acceptedDispatchBotAppIds, appendDispatchReportProtocol, appendLegacyDispatchReportProtocol, parseDispatchBotSpec, buildDispatchMessages, buildRepoPrimeText, buildReportContent, eligibleAutoMentionAliases, findDispatchRegistryEntry, offTopicSubBotTopic, resolveReportTarget, resolveSendTarget } from './core/dispatch.js'; import { pickTurnReplyTarget } from './core/reply-target.js'; @@ -4082,6 +4083,195 @@ async function cmdSuspend(): Promise { if (failed > 0) process.exitCode = 1; } +/** `--for 90s` / `5m` / 裸数字(秒)。返回毫秒;非法返回 null。 */ +function parseFreezeDuration(raw: string | undefined): number | null { + if (!raw) return null; + const m = /^(\d+)(s|m)?$/.exec(raw.trim()); + if (!m) return null; + const value = Number(m[1]); + if (!Number.isSafeInteger(value) || value <= 0) return null; + return m[2] === 'm' ? value * 60_000 : value * 1_000; +} + +function formatFreezeClock(epochMs: number): string { + const d = new Date(epochMs); + const p = (n: number) => String(n).padStart(2, '0'); + return `${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`; +} + +/** + * `botmux freeze` —— 声明「这台机器暂时不要起新的 CLI 会话」。 + * + * 用途:任何会让「新起的 CLI」踩到半成品状态的维护窗口——刷 Claude 凭证(伪过期 + * 期间冷启动的 CLI 会自己去刷、轮换掉 refresh token,把整队投毒)、升级 claude + * 二进制、重建某个 bot 的工作区。冻结期间 daemon 把该会话的这次 spawn 记下来, + * 解冻后自动重放——用户的消息只是晚几秒被处理,不会丢。 + * + * 只拦「新起」,不动已在跑的 CLI:那是 `botmux suspend` 的职责。典型顺序是 + * freeze → suspend → 干活 → 验证 → release。 + * + * 三重兜底保证一次维护不会把机器冻死:声明的 deadline、写文件进程(--pid)退出、 + * 以及 daemon 侧按文件 mtime 算的 10 分钟硬上限。daemon 读不到/读坏一律放行。 + */ +async function cmdFreeze(): Promise { + const argv = process.argv.slice(3); + const usage = (): never => { + console.error('用法: botmux freeze --reason <说明> [--for 120s] [--pid ] [--notify] [--bot ]... [--force]'); + console.error(' botmux freeze --status | --release [--pid ]'); + console.error(' 维护期间禁止本机起新的 CLI 会话;每个会话的第一条消息解冻后自动重放'); + console.error(' ⚠️ 同一会话在窗口内的后续消息不会排队(会打日志/发提示,需恢复后重发)'); + console.error(' --for 默认 120s,支持 90s / 5m / 裸数字(秒);上限 10m(daemon 侧硬截断)'); + console.error(' --pid 声明进程 pid(脚本里传 $$):进程一死立即解冻;--release 带上它只删自己的声明'); + console.error(' --notify 冻结期收到消息时回一条「维护中」;建议 deadline > 30s 才开'); + console.error(' --bot 只冻某个 bot(可重复);不给 = 全队'); + console.error(' --force 覆盖仍生效的其它声明(默认拒绝,避免两个维护脚本互相解除保护)'); + process.exit(1); + }; + + // 所有 value flag 走同一个取值器:缺值、或下一个 token 又是 flag,一律 fail-fast。 + // 静默容忍在这里是危险的 —— `--bot` 缺值曾会退化成「冻结全队」,`--reason --notify` + // 会把 reason 存成 "--notify"。 + const KNOWN_VALUE_FLAGS = new Set(['--reason', '--for', '--pid', '--bot']); + const KNOWN_BOOL_FLAGS = new Set(['--status', '--release', '--notify', '--force']); + const values = new Map(); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith('--')) { + console.error(`❌ 多余的参数: ${arg}`); + usage(); + } + if (KNOWN_BOOL_FLAGS.has(arg)) continue; + if (!KNOWN_VALUE_FLAGS.has(arg)) { + console.error(`❌ 未知参数: ${arg}`); + usage(); + } + const value = argv[i + 1]; + if (value === undefined || value.startsWith('--')) { + console.error(`❌ ${arg} 缺少取值`); + usage(); + } + const bucket = values.get(arg); + if (bucket) bucket.push(value); + else values.set(arg, [value]); + i++; + } + const single = (flag: string): string | undefined => { + const bucket = values.get(flag); + if (!bucket) return undefined; + if (bucket.length > 1) { + console.error(`❌ ${flag} 只能给一次`); + usage(); + } + return bucket[0]; + }; + + const release = argv.includes('--release'); + const status = argv.includes('--status'); + if (release && status) { + console.error('❌ --release 与 --status 不能同时用'); + process.exit(1); + } + // 静默忽略不相容的参数会让人以为自己冻结/解冻成了别的样子。 + if (status || release) { + const irrelevant = ['--reason', '--for', '--bot', '--notify', '--force'] + .concat(status ? ['--pid'] : []) // --pid 只对 acquire 与 --release 有意义 + .filter(flag => argv.includes(flag)); + if (irrelevant.length) { + console.error(`❌ ${status ? '--status' : '--release'} 不接受 ${irrelevant.join(' / ')}`); + usage(); + } + } + + const pidRaw = single('--pid'); + let pid: number | undefined; + if (pidRaw !== undefined) { + const parsed = Number(pidRaw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + console.error(`❌ --pid 非法: ${pidRaw}`); + process.exit(1); + } + pid = parsed; + } + + if (status) { + const freeze = readActiveSpawnFreeze(); + if (!freeze) { + console.log('当前无冻结:新 CLI 会话可正常启动。'); + return; + } + const remaining = Math.max(0, Math.ceil((freeze.effectiveUntil - Date.now()) / 1000)); + console.log('当前处于冻结中(新 CLI 会话不会启动):'); + console.log(` 原因 ${freeze.reason}`); + console.log(` 生效至 ${formatFreezeClock(freeze.effectiveUntil)}(剩 ${remaining}s)`); + if (freeze.effectiveUntil < freeze.deadline) { + console.log(` ⚠️ 已被 10 分钟硬上限截断(声明的 deadline 是 ${formatFreezeClock(freeze.deadline)})`); + } + console.log(` 范围 ${freeze.larkAppIds ? freeze.larkAppIds.join(', ') : '全部 bot'}`); + console.log(` 声明进程 ${freeze.declaredByPid ?? '(未记录 pid,仅靠 deadline 兜底)'}`); + console.log(` 维护提示 ${freeze.notify ? '开(冻结期每个会话回一条)' : '关(静默延迟)'}`); + console.log(` 文件 ${spawnFreezePath()}`); + return; + } + + if (release) { + const result = clearSpawnFreeze({}, pid !== undefined ? { ownerPid: pid } : {}); + if (result === 'cleared') { + console.log('✓ 已解冻:被暂存的 spawn 会在各 daemon 的下一次轮询(≤1s)自动重放。'); + return; + } + if (result === 'absent') { + console.log('· 本就没有冻结声明(目标态已达成)。'); + return; + } + // not_owner:声明是别人写的。解冻别人的窗口比不解冻更危险,所以拒绝并明说。 + console.log(`· 当前声明不是 pid ${pid} 写的,未解冻(不删别人的冻结)。要强制解除:botmux freeze --release`); + process.exitCode = 1; + return; + } + + const reason = single('--reason'); + if (!reason) usage(); + + const durationRaw = single('--for'); + if (durationRaw !== undefined && parseFreezeDuration(durationRaw) === null) { + console.error(`❌ --for 无法解析: ${durationRaw}(示例:90s / 5m / 120)`); + process.exit(1); + } + let durationMs = parseFreezeDuration(durationRaw) ?? 120_000; + if (durationMs > SPAWN_FREEZE_HARD_CAP_MS) { + console.log(`⚠️ --for 超过硬上限,按 ${SPAWN_FREEZE_HARD_CAP_MS / 60_000} 分钟处理。`); + durationMs = SPAWN_FREEZE_HARD_CAP_MS; + } + + const larkAppIds = values.get('--bot') ?? []; + const deadline = Date.now() + durationMs; + let path: string; + try { + path = writeSpawnFreeze({ + reason: reason!, + deadline, + ...(pid !== undefined ? { pid } : {}), + ...(argv.includes('--notify') ? { notify: true } : {}), + ...(larkAppIds.length ? { scope: { larkAppIds } } : {}), + }, {}, { force: argv.includes('--force') }); + } catch (err) { + if (err instanceof SpawnFreezeConflictError) { + const remaining = Math.max(0, Math.ceil((err.active.effectiveUntil - Date.now()) / 1000)); + console.error(`❌ 已有生效中的冻结声明(${err.active.reason},剩 ${remaining}s,` + + `pid ${err.active.declaredByPid ?? '未记录'})——未覆盖它。`); + console.error(' 两个维护窗口重叠时互相解除保护,所以默认拒绝。确实要接管:加 --force'); + process.exit(1); + } + throw err; + } + console.log(`✓ 已冻结(${reason}):${larkAppIds.length ? larkAppIds.join(', ') : '全部 bot'} 在 ${formatFreezeClock(deadline)} 前不会起新 CLI。`); + console.log(` 声明文件 ${path}`); + if (pid === undefined) { + console.log(' ⚠️ 未传 --pid:脚本中途被杀时只能等 deadline / 10 分钟硬上限自动解冻。建议加 --pid $$。'); + } + console.log(' 记得在结束时(含 trap)执行 botmux freeze --release;已在跑的 CLI 不受影响,需要的话另跑 botmux suspend。'); +} + /** 会话级 CLI IPC(slash/cd/close)的 POST:与 postAsk 同款双路径——能读 host secret * (非隔离进程)走 trusted-host HMAC 签名;读不到(沙箱 BOTMUX_SEND_RELAY / * macOS 读隔离 carve-out)改带本会话当前轮换的 origin capability,由 daemon @@ -4806,6 +4996,15 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 --bot 挂起该 bot 的全部活跃会话 --isolated 挂起所有读隔离 bot(凭证轮换后用;下次冷启动自动同步最新凭证) --dry-run 只列出目标,不执行 + freeze --reason <说明> 维护窗口内禁止本机起新 CLI 会话(每会话第一条消息解冻后自动重放; + 同会话后续消息不排队,会发提示并需重发) + --for 120s 冻结时长(默认 120s,上限 10m) + --pid 声明进程(脚本传 $$):进程一死立即解冻 + --notify 冻结期回一条「维护中」(deadline > 30s 建议开) + --bot 只冻某个 bot(可重复);不给 = 全队 + --force 覆盖仍生效的其它声明(默认拒绝) + --status 查看当前是否冻结、为什么、还剩多久 + --release 解冻(幂等;带 --pid 只删自己的声明,可放脚本 trap 里) slash "<斜杠命令>" 会话空闲后向本会话 CLI 注入一条原生斜杠命令(需 bots.json 配 tuiSlashAllow;/cd 恒被拒) role switch <目录> (会话内)切换本话题到角色库内的角色目录——角色切换用; 目录必须位于 ~/botmux-roles 之下 @@ -9922,6 +10121,7 @@ switch (command) { case 'rm': await cmdDelete(); break; case 'resume': await cmdResume(); break; case 'suspend': await cmdSuspend(); break; + case 'freeze': await cmdFreeze(); break; case 'slash': await cmdSlash(); break; case 'cd': { // Tombstone for the removed `botmux cd`(改名为 `botmux role switch`)。**必须 diff --git a/src/core/command-handler.ts b/src/core/command-handler.ts index edc23c3e3..16b2968a9 100644 --- a/src/core/command-handler.ts +++ b/src/core/command-handler.ts @@ -25,6 +25,7 @@ import { claimPairing } from '../services/pairing-store.js'; import { logger } from '../utils/logger.js'; import { scheduleTimeZone } from '../utils/timezone.js'; import { killWorker, suspendWorker, forkWorker, forkAdoptWorker, adoptSandboxBlocked, getCurrentCliVersion, postFreshStreamingCard, postPrivateSnapshotCard, resolvePrivateCardAudience, deliverEphemeralOrReply, deliverWritableTerminalCardTo, requestSessionRestart } from './worker-pool.js'; +import { forgetDeferredSpawn } from './spawn-freeze.js'; import { expandHome, getSessionWorkingDir, @@ -1650,6 +1651,9 @@ export async function handleCommand( ds!.scope, ); ds!.session = session; + // 这个 ds 从此代表新 session:冻结期为旧 session 暂存的 spawn 必须就地丢弃, + // 否则解冻重放会为同一个 ds 起第二个 worker,把刚起来的那个顶掉。 + forgetDeferredSpawn(oldSession.sessionId); ds!.lastUserPrompt = undefined; ds!.lastCliInput = undefined; ds!.workingDir = selectedPath; diff --git a/src/core/device-isolation-activation.ts b/src/core/device-isolation-activation.ts index 8de105761..b55f678ff 100644 --- a/src/core/device-isolation-activation.ts +++ b/src/core/device-isolation-activation.ts @@ -132,6 +132,11 @@ export function releaseDeviceIsolationFreeze(input: { * Queue only the first spawn request for a logical session. Later turns are * already retained by the session's normal pending-input machinery; replaying * multiple fork requests would instead kill and replace the first new worker. + * + * Callers must key on an IMMUTABLE session id captured at queue time and + * re-check it before forking: a repo switch replaces `ds.session` wholesale on + * the same session object, so a replay that re-reads it would accept an entry + * queued for the previous session. */ export function deferWorkerSpawnDuringDeviceIsolation( sessionId: string, diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts new file mode 100644 index 000000000..78c33b43d --- /dev/null +++ b/src/core/spawn-freeze.ts @@ -0,0 +1,518 @@ +/** + * Ops-declared CLI spawn barrier. + * + * A maintenance script — credential refresh, `claude` upgrade, workspace + * rebuild — declares a freeze by writing `/spawn-freeze.json`, and + * every daemon reads that declaration immediately before it would start a CLI. + * While the declaration is effective `forkWorker` parks at most one cold spawn + * per logical session and replays it on release. + * + * What that does and does NOT guarantee, precisely — this is a delay, not a + * queue: + * + * - the FIRST message that would have started a CLI in a session is replayed + * after release, so it is delayed rather than dropped; + * - a promptless spawn (restore re-attach, warm-up, a start that only exists so + * a queued raw command can be written once the CLI is ready) parks like any + * other, but a payload-bearing turn SUPERSEDES it, so a warm-up cannot + * consume the slot the first real turn needs; + * - a SECOND message to that same session in the same window is NOT parked + * (replaying two forks would kill and replace the first new worker), and + * nothing else is holding it either: the worker never started, so the + * worker-side pre-ready input buffer that normally absorbs follow-up turns + * does not exist yet. Such a turn is reported — log, plus a chat notice when + * `notify` is on — and has to be re-sent; + * - a daemon restart inside the window loses the parked spawns. The + * declaration is on disk; the parked closures are not. + * + * Buffering whole turns durably (message id, attachments, reaction settlement, + * structured CLI input) would make this a delivery queue rather than a gate; + * that is deliberately out of scope. The limits above are surfaced instead of + * papered over — keep maintenance windows short. + * + * `pid` is a cooperative identity, not a security boundary: it is only checked + * for liveness, so a same-user caller could name somebody else's pid (and one + * that can write this file could equally pass `--force`). It exists to keep an + * operator's own scripts from disarming each other, nothing more. + * + * Out of scope by definition: `/adopt` sessions. Botmux never started those + * CLIs (the user did, outside botmux) and re-attaching to one starts no new CLI, + * so a maintenance window has nothing to protect there. + * + * The file is data, never code: a declaration can only ask for "no new CLI + * until T", it can never make the daemon execute anything. That keeps the + * barrier usable by plain shell scripts and keeps its blast radius shaped like + * a timeout instead of an extension point. + * + * Three independent expiries bound it, and every read failure is fail-open: + * + * 1. the declared `deadline`, + * 2. the declaring process (`pid`) exiting, and + * 3. a hard cap measured from the file's mtime. + * + * The cap deserves a note on what it can and cannot do. mtime is not content, + * but it is still writable by whoever owns the file (`touch -t`), and a symlink + * would let the declaration borrow some other file's timestamps — so the reader + * refuses symlinks and refuses a future mtime outright instead of clamping it + * (clamping to `now` would slide the window forward on every read, i.e. exactly + * the permanent freeze the cap exists to prevent). What remains is that a live + * process can keep re-declaring; that is not a hole, it is the same authority + * as holding a legitimate freeze. The cap's job is bounding an ABANDONED + * declaration — `kill -9` before the cleanup trap, power loss — and it does. + * + * A freeze must never be able to wedge the fleet. Not starting CLIs for a few + * seconds is cheap; never starting them again is an outage — so anything + * ambiguous (missing file, unreadable, unparseable, out-of-range) resolves to + * "no freeze". + * + * Unlike the device-isolation lease (`device-isolation-activation.ts`), which + * is an in-memory lease acquired over authenticated IPC from every ONLINE + * daemon, this barrier lives on disk. That is deliberate: a daemon that boots + * *during* the window reads the same file and freezes itself, which a lease + * handed out before the boot can never cover. The two compose — `forkWorker` + * defers when either one applies. + */ +import { linkSync, lstatSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { resolveBotmuxDataDir } from './data-dir.js'; + +export const SPAWN_FREEZE_FILENAME = 'spawn-freeze.json'; + +/** Upper bound on how long one declaration may hold the barrier, measured from + * the file's mtime. A script that dies between writing the file and its own + * cleanup trap — `kill -9`, power loss — cannot freeze the fleet for longer + * than this even if it declared a deadline years out. */ +export const SPAWN_FREEZE_HARD_CAP_MS = 10 * 60_000; + +/** How far ahead of `now` an mtime may sit before we stop believing it. Covers + * ordinary clock skew / filesystem timestamp granularity; anything beyond it is + * a timestamp nobody legitimately produces, and honoring it would hand the + * writer an unbounded freeze. */ +export const SPAWN_FREEZE_MTIME_SKEW_MS = 60_000; + +/** How often a deferred spawn re-checks the declaration. Polling (rather than + * fs.watch) keeps this portable and cheap: the timer only runs while at least + * one spawn is actually parked. */ +export const SPAWN_FREEZE_POLL_MS = 1_000; + +/** Longest `reason` we accept. It lands in logs, cards and `--status`. */ +const MAX_REASON_LENGTH = 120; +const MAX_SCOPE_ENTRIES = 64; +const MAX_FILE_BYTES = 8 * 1024; + +/** On-disk shape. Written by `botmux freeze`, read by every daemon. */ +export interface SpawnFreezeDeclaration { + /** Why new CLIs are blocked. Shown to operators; never parsed for meaning. */ + reason: string; + /** Epoch ms after which the declaration stops applying. */ + deadline: number; + /** Declaring process. When it is gone the barrier lifts immediately, so a + * crashed script self-heals long before `deadline`. */ + pid?: number; + /** Reply once per chat while deferring, instead of silently delaying. Worth + * it for multi-minute windows, noise for a five-second one. */ + notify?: boolean; + /** Absent / empty `larkAppIds` = every bot on this host. */ + scope?: { larkAppIds?: string[] }; +} + +/** A declaration that has been read, validated and found to still apply. */ +export interface ActiveSpawnFreeze { + reason: string; + /** As declared. */ + deadline: number; + /** `min(deadline, mtime + hard cap)` — when the barrier really lifts. */ + effectiveUntil: number; + notify: boolean; + /** Undefined = fleet-wide. */ + larkAppIds?: string[]; + declaredByPid?: number; + /** Identity of this declaration, used to scope one-shot notices. */ + freezeId: string; +} + +export interface SpawnFreezeDeps { + dataDir?: () => string; + now?: () => number; + /** EPERM means "exists but not ours to signal" — that is still alive. */ + processAlive?: (pid: number) => boolean; + /** Release-poll interval. Only tests override it (production always uses + * `SPAWN_FREEZE_POLL_MS`); a tiny value keeps them on real timers, which is + * what the replay path — `setInterval` handing off to `setImmediate` — + * actually runs on. */ + pollMs?: number; +} + +function defaultProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException)?.code === 'EPERM'; + } +} + +function resolveDeps(deps: SpawnFreezeDeps = {}): Required { + return { + dataDir: deps.dataDir ?? (() => resolveBotmuxDataDir()), + now: deps.now ?? (() => Date.now()), + processAlive: deps.processAlive ?? defaultProcessAlive, + pollMs: deps.pollMs && deps.pollMs > 0 ? deps.pollMs : SPAWN_FREEZE_POLL_MS, + }; +} + +export function spawnFreezePath(deps: SpawnFreezeDeps = {}): string { + return join(resolveDeps(deps).dataDir(), SPAWN_FREEZE_FILENAME); +} + +function normalizeScope(value: unknown): string[] | undefined { + if (value === undefined || value === null) return undefined; + if (!Array.isArray(value)) throw new Error('scope.larkAppIds must be an array'); + const ids = value + .filter((id): id is string => typeof id === 'string') + .map(id => id.trim()) + .filter(id => id.length > 0 && id.length <= 64); + if (ids.length !== value.length) throw new Error('scope.larkAppIds has invalid entries'); + if (ids.length > MAX_SCOPE_ENTRIES) throw new Error('scope.larkAppIds too long'); + // An explicitly empty array means the same thing as no scope at all + // (fleet-wide); collapsing it here keeps `spawnFreezeApplies` trivial. + return ids.length > 0 ? ids : undefined; +} + +/** + * Validate a parsed declaration. Throws on anything malformed so the single + * catch in `readActiveSpawnFreeze` can turn every failure into fail-open. + */ +export function parseSpawnFreezeDeclaration(value: unknown): SpawnFreezeDeclaration { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('declaration must be an object'); + } + const record = value as Record; + const reason = typeof record.reason === 'string' ? record.reason.trim() : ''; + if (!reason || reason.length > MAX_REASON_LENGTH) throw new Error('invalid reason'); + const deadline = record.deadline; + if (typeof deadline !== 'number' || !Number.isSafeInteger(deadline) || deadline <= 0) { + throw new Error('invalid deadline'); + } + if (record.notify !== undefined && typeof record.notify !== 'boolean') { + throw new Error('invalid notify'); + } + let pid: number | undefined; + if (record.pid !== undefined) { + if (typeof record.pid !== 'number' || !Number.isSafeInteger(record.pid) || record.pid <= 0) { + throw new Error('invalid pid'); + } + pid = record.pid; + } + const scopeValue = record.scope; + let larkAppIds: string[] | undefined; + if (scopeValue !== undefined && scopeValue !== null) { + if (typeof scopeValue !== 'object' || Array.isArray(scopeValue)) throw new Error('invalid scope'); + larkAppIds = normalizeScope((scopeValue as Record).larkAppIds); + } + return { + reason, + deadline, + ...(pid !== undefined ? { pid } : {}), + ...(record.notify === true ? { notify: true } : {}), + ...(larkAppIds ? { scope: { larkAppIds } } : {}), + }; +} + +/** + * Read the declaration and decide whether it still applies. Returns null — "no + * freeze" — for a missing file and for every possible failure: an unreadable + * or oversized file, malformed JSON, a passed deadline, a dead declarer, or a + * declaration held past the hard cap. + */ +export function readActiveSpawnFreeze(deps: SpawnFreezeDeps = {}): ActiveSpawnFreeze | null { + const { dataDir, now, processAlive } = resolveDeps(deps); + const path = join(dataDir(), SPAWN_FREEZE_FILENAME); + try { + // lstat, not stat: a symlink would let the declaration inherit an arbitrary + // file's mtime (and thus escape the hard cap), so it is refused outright. + const stat = lstatSync(path); + if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null; + const at = now(); + // A future mtime cannot be honored and must not be clamped to `now` either: + // clamping would re-anchor the cap on every read and never expire. + if (stat.mtimeMs > at + SPAWN_FREEZE_MTIME_SKEW_MS) return null; + const declaration = parseSpawnFreezeDeclaration(JSON.parse(readFileSync(path, 'utf-8'))); + const effectiveUntil = Math.min(declaration.deadline, stat.mtimeMs + SPAWN_FREEZE_HARD_CAP_MS); + if (effectiveUntil <= at) return null; + if (declaration.pid !== undefined && !processAlive(declaration.pid)) return null; + return { + reason: declaration.reason, + deadline: declaration.deadline, + effectiveUntil, + notify: declaration.notify === true, + ...(declaration.scope?.larkAppIds ? { larkAppIds: declaration.scope.larkAppIds } : {}), + ...(declaration.pid !== undefined ? { declaredByPid: declaration.pid } : {}), + freezeId: `${declaration.reason}:${declaration.deadline}:${Math.round(stat.mtimeMs)}`, + }; + } catch { + // Fail-open, deliberately silent: this runs on every spawn, and a missing + // file is the overwhelmingly common case. + return null; + } +} + +/** True when a freeze covers this bot (no scope = the whole host). */ +export function spawnFreezeApplies(freeze: ActiveSpawnFreeze, larkAppId?: string): boolean { + if (!freeze.larkAppIds) return true; + if (!larkAppId) return false; + return freeze.larkAppIds.includes(larkAppId); +} + +/** The freeze in effect for one bot, or null. */ +export function activeSpawnFreezeFor( + larkAppId?: string, + deps: SpawnFreezeDeps = {}, +): ActiveSpawnFreeze | null { + const freeze = readActiveSpawnFreeze(deps); + if (!freeze) return null; + return spawnFreezeApplies(freeze, larkAppId) ? freeze : null; +} + +export interface SpawnFreezeDeferral { + freeze: ActiveSpawnFreeze; + /** False when this session already had a payload-bearing spawn parked — i.e. + * THIS request will not be replayed and the turn behind it is lost unless + * re-sent. */ + parked: boolean; + /** True when parking this request displaced a promptless one (warm-up / + * re-attach). Nothing is lost — that spawn carried no turn — but it is worth + * a log line. */ + superseded?: boolean; +} + +interface DeferredSpawn { + larkAppId?: string; + /** A promptless spawn (empty prompt, no structured input) carries no user turn + * and may be displaced by one that does. */ + hasPayload: boolean; + replay: () => void; +} + +const deferredSpawns = new Map(); +let pollTimer: NodeJS.Timeout | null = null; +let noticeFreezeId: string | null = null; +const noticedChats = new Set(); + +function stopPoll(): void { + if (!pollTimer) return; + clearInterval(pollTimer); + pollTimer = null; +} + +function flushReleasedSpawns(deps: SpawnFreezeDeps = {}): void { + for (const [sessionId, entry] of [...deferredSpawns.entries()]) { + // Per-entry, because a scoped freeze can lift for one bot while another + // declaration still covers a different one. + if (activeSpawnFreezeFor(entry.larkAppId, deps)) continue; + deferredSpawns.delete(sessionId); + setImmediate(entry.replay); + } + if (deferredSpawns.size === 0) stopPoll(); +} + +function ensurePoll(deps: SpawnFreezeDeps = {}): void { + if (pollTimer) return; + pollTimer = setInterval(() => flushReleasedSpawns(deps), resolveDeps(deps).pollMs); + pollTimer.unref?.(); +} + +/** + * Park a cold spawn while a freeze applies. Returns null when not frozen (the + * caller proceeds to spawn), otherwise the freeze plus whether THIS request is + * the one that got parked. + * + * Only the first request per logical session can be parked: replaying several + * fork requests would kill and replace the first new worker. Same invariant as + * `deferWorkerSpawnDuringDeviceIsolation` — and, as the module header spells + * out, a later turn is genuinely dropped rather than queued elsewhere, so + * `parked: false` is something the caller must surface, not swallow. + */ +export function deferSpawnDuringFreeze( + input: { sessionId: string; larkAppId?: string; hasPayload: boolean; replay: () => void }, + deps: SpawnFreezeDeps = {}, +): SpawnFreezeDeferral | null { + const freeze = activeSpawnFreezeFor(input.larkAppId, deps); + if (!freeze) return null; + const existing = deferredSpawns.get(input.sessionId); + // A promptless spawn still has to be parked — some of them exist precisely so + // a queued raw command or a dashboard wake gets a CLI — but it must not shut + // out the first real turn, so a payload-bearing request displaces it. + const superseded = existing !== undefined && !existing.hasPayload && input.hasPayload; + const parked = existing === undefined || superseded; + if (parked) { + deferredSpawns.set(input.sessionId, { + larkAppId: input.larkAppId, + hasPayload: input.hasPayload, + replay: input.replay, + }); + } + ensurePoll(deps); + return { freeze, parked, ...(superseded ? { superseded: true } : {}) }; +} + +/** Drop a parked spawn without replaying it (session closed while frozen). */ +export function forgetDeferredSpawn(sessionId: string): void { + deferredSpawns.delete(sessionId); + if (deferredSpawns.size === 0) stopPoll(); +} + +export function deferredSpawnCount(): number { + return deferredSpawns.size; +} + +/** + * One-shot gate for a "maintenance in progress" reply: true the first time a + * given anchor asks during a given declaration, per `kind`. Without it a busy + * chat gets one notice per message; with it it is told once and then waits. + * State resets when the declaration changes, so the next window speaks again. + * + * `anchor` must be the same key the notice is delivered to (the session's reply + * anchor, NOT its chat id): keyed by chat, a second topic in the same group + * would be silenced even though a different person is waiting in it. + * + * `kind` separates the two things worth saying once each: "parked, it continues + * by itself" and "this one was dropped, re-send it". + */ +export function shouldAnnounceSpawnFreeze( + freeze: ActiveSpawnFreeze, + anchor: string, + kind: 'parked' | 'dropped' = 'parked', +): boolean { + if (!freeze.notify) return false; + if (noticeFreezeId !== freeze.freezeId) { + noticeFreezeId = freeze.freezeId; + noticedChats.clear(); + } + const key = `${kind}:${anchor}`; + if (noticedChats.has(key)) return false; + noticedChats.add(key); + return true; +} + +export class SpawnFreezeConflictError extends Error { + constructor(public readonly active: ActiveSpawnFreeze) { + super(`another spawn freeze is active: ${active.reason}`); + this.name = 'SpawnFreezeConflictError'; + } +} + +/** + * Publish a declaration. A reader never observes a half-written file: the + * content is written to a same-directory temp and then linked/renamed into + * place. + * + * Refuses to clobber a declaration that is still in effect and belongs to + * somebody else. There is exactly one declaration file, so an unconditional + * write let two overlapping maintenance scripts silently disarm each other — + * the later writer erasing the earlier one's scope, then the earlier one's + * cleanup deleting the later one's freeze. Making that a visible failure (so the + * second script decides what to do) is worth the code; a multi-record lease + * protocol is not, so overlapping windows from DIFFERENT owners stay + * unsupported. + * + * Three cases are allowed through: + * - nothing in effect (missing / expired / dead declarer / unparseable); + * - same `pid` — an owner extending or amending its own window (both sides must + * name a pid; two anonymous declarations are two owners, not one); + * - `force` — an explicit operator takeover. + * + * The create path uses `link()`, which fails if the destination exists, so the + * check and the publish are one atomic step rather than a TOCTOU pair: two + * scripts racing from scratch cannot both win. + */ +export function writeSpawnFreeze( + declaration: SpawnFreezeDeclaration, + deps: SpawnFreezeDeps = {}, + opts: { force?: boolean } = {}, +): string { + const parsed = parseSpawnFreezeDeclaration(declaration); + const path = spawnFreezePath(deps); + const body = `${JSON.stringify(parsed, null, 2)}\n`; + const tmp = `${path}.tmp-${process.pid}`; + try { + // Inside the try: a partial write (disk full, EIO) must not leave the temp + // behind either. + writeFileSync(tmp, body, { mode: 0o600 }); + if (opts.force) { + renameSync(tmp, path); + return path; + } + // Bounded retry: each round either wins the atomic create, hands back a + // real conflict, or removes exactly one superseded declaration. + for (let attempt = 0; attempt < 3; attempt++) { + try { + linkSync(tmp, path); + return path; + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== 'EEXIST') throw error; + } + const active = readActiveSpawnFreeze(deps); + // Same owner requires BOTH sides to actually name one: two declarations + // that simply omit `pid` are not "the same owner", they are two anonymous + // writers, and treating undefined === undefined as a match let every racer + // replace every other one (verified: 10 concurrent writers all succeeded). + const sameOwner = active !== null + && parsed.pid !== undefined + && active.declaredByPid === parsed.pid; + if (active && !sameOwner) throw new SpawnFreezeConflictError(active); + // Ours to replace (same owner), or nothing in effect: drop the stale file + // and try the atomic create again. + try { unlinkSync(path); } catch { /* somebody else removed it first */ } + } + throw new Error('spawn-freeze declaration is being rewritten concurrently'); + } finally { + try { unlinkSync(tmp); } catch { /* already linked away or gone */ } + } +} + +export type ClearSpawnFreezeResult = 'cleared' | 'absent' | 'not_owner'; + +/** + * Remove the declaration. Idempotent — releasing a freeze that already expired + * must not fail a maintenance script's cleanup trap. + * + * With `ownerPid`, only a declaration made by that pid is removed: a script's + * EXIT trap must never delete somebody else's freeze. A declaration written + * without a pid has no owner to check and stays removable by anyone. + */ +export function clearSpawnFreeze( + deps: SpawnFreezeDeps = {}, + opts: { ownerPid?: number } = {}, +): ClearSpawnFreezeResult { + const path = spawnFreezePath(deps); + try { + // lstat for the same reason the reader uses it: "exists" must include a + // dangling symlink, which statSync would report as absent and leave behind. + lstatSync(path); + } catch { + return 'absent'; + } + if (opts.ownerPid !== undefined) { + // Read the raw declaration rather than `readActiveSpawnFreeze`: an EXPIRED + // declaration must still be removable by its owner, and one we cannot parse + // carries no ownership claim to respect. + let declaredPid: number | undefined; + try { + declaredPid = parseSpawnFreezeDeclaration(JSON.parse(readFileSync(path, 'utf-8'))).pid; + } catch { + declaredPid = undefined; + } + if (declaredPid !== undefined && declaredPid !== opts.ownerPid) return 'not_owner'; + } + rmSync(path, { force: true }); + return 'cleared'; +} + +/** Test-only: production state is driven entirely by the file and the timer. */ +export function resetSpawnFreezeStateForTest(): void { + deferredSpawns.clear(); + stopPoll(); + noticeFreezeId = null; + noticedChats.clear(); +} diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 70631f38d..883449547 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -106,6 +106,7 @@ import { isSilentScheduledTurn } from './silent-schedule-turns.js'; import { isTriggerFinalSuppressed } from './trigger-final-suppression.js'; import { writeDeferredTopicBinding } from './deferred-topic-binding.js'; import { deferWorkerSpawnDuringDeviceIsolation } from './device-isolation-activation.js'; +import { deferSpawnDuringFreeze, forgetDeferredSpawn, shouldAnnounceSpawnFreeze } from './spawn-freeze.js'; import { buildBotmuxLarkNativeSessionTitle, extractBotmuxLarkNativeSessionTitlePrompt, @@ -1640,6 +1641,10 @@ export async function closeSession( // 会话关闭即可回收其崩溃重启计数;否则每个曾崩溃过的 session 会在 daemon // 生命周期内永久占位(restartCounts 此前无任何 delete)。 restartCounts.delete(sessionId); + // A spawn parked by an ops freeze must not outlive its session. The replay + // callback re-checks the registry anyway, so this is about not holding the + // closure (and the poll timer) alive for a session that is already gone. + forgetDeferredSpawn(sessionId); if (ds) { // Usage ledger: flush the final delta before the worker goes away (a // crash/limited turn may never have reached an idle edge). @@ -2090,14 +2095,93 @@ export function forkWorker( // ANY session mutation or child fork. One deferred spawn per logical session // is replayed after the exact freeze lease is released; a session closed by // the activation transaction is deliberately not revived. - if (deferWorkerSpawnDuringDeviceIsolation(ds.session.sessionId, () => { - if (findActiveBySessionId(ds.session.sessionId) === ds) { + // Same immutable-identity rule as the ops freeze below: `ds.session` is + // replaced wholesale by a repo switch, so both the queue key and the replay + // guard must use the id captured here, never a re-read of `ds.session`. + const deferredSessionId = ds.session.sessionId; + if (deferWorkerSpawnDuringDeviceIsolation(deferredSessionId, () => { + if ( + ds.session.sessionId === deferredSessionId + && findActiveBySessionId(deferredSessionId) === ds + ) { forkWorker(ds, promptInput, resumeOrTurnId); } })) { logger.info(`[${tag(ds)}] worker spawn deferred during device credential activation`); return; } + // An ops maintenance window (credential refresh, `claude` upgrade, workspace + // rebuild) can declare that no new CLI may start on this host. Same defer + + // replay shape as the device-isolation lease above, but driven by an on-disk + // declaration so a daemon that BOOTS during the window freezes itself too. + // Deliberately only blocks new spawns: tearing down live CLIs is + // `botmux suspend`'s job, and conflating the two would make every freeze a + // fleet-wide cold restart. + // Whether this spawn carries a user turn. A promptless one (restore re-attach, + // warm-up, a start whose only job is to let a queued raw command reach a ready + // CLI) is still parked — skipping it outright would silently strand dashboard + // wakes and `pendingRawInput` — but it must not shut out the first real turn, + // so the gate lets a payload-bearing request displace it. + const gateHasPayload = typeof promptInput === 'string' + ? promptInput.trim() !== '' + : promptInput.content.trim() !== '' || !!promptInput.codexAppInput; + const opsFreeze = deferSpawnDuringFreeze({ + sessionId: deferredSessionId, + larkAppId: ds.larkAppId, + hasPayload: gateHasPayload, + replay: () => { + // Replay only when this ds is STILL the live session it was queued for: + // closed, replaced or transferred sessions must never be revived. + if ( + ds.session.sessionId === deferredSessionId + && findActiveBySessionId(deferredSessionId) === ds + ) { + forkWorker(ds, promptInput, resumeOrTurnId); + } + }, + }); + if (opsFreeze) { + const { freeze, parked } = opsFreeze; + const remainingSec = Math.max(1, Math.ceil((freeze.effectiveUntil - Date.now()) / 1000)); + const anchor = sessionAnchorId(ds); + // `parked: false` means this session already has a spawn waiting, so THIS + // turn will not be replayed — it is dropped. Say so loudly: the reaction + // bookkeeping settles pending turns when the replayed one finishes, so an + // unreported drop would show up to the user as "handled". + if (parked) { + logger.info( + `[${tag(ds)}] worker spawn parked by spawn-freeze (reason=${freeze.reason}, ` + + `~${remainingSec}s left${opsFreeze.superseded ? ', superseded a promptless spawn' : ''})`, + ); + } else { + logger.warn( + `[${tag(ds)}] spawn-freeze (reason=${freeze.reason}) already holds a parked spawn for this ` + + `session — THIS turn will not be replayed and must be re-sent after release ` + + `(~${remainingSec}s left)`, + ); + } + if (shouldAnnounceSpawnFreeze(freeze, anchor, parked ? 'parked' : 'dropped')) { + const gateTurnId = typeof resumeOrTurnId === 'string' + ? resumeOrTurnId + : (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null ? resumeOrTurnId.turnId : undefined); + const notice = parked + ? `🔧 维护中(${freeze.reason}),暂不启动新的 CLI 会话;约 ${remainingSec} 秒后自动继续处理这条消息。` + : `🔧 维护中(${freeze.reason}),这条消息不会被自动处理——本会话已有一条在排队。请在约 ${remainingSec} 秒后重发。`; + try { + void requireCallbacks().sessionReply( + anchor, + notice, + 'text', + ds.larkAppId, + fallbackTurnId(ds, gateTurnId), + ).catch(err => logger.warn(`[${tag(ds)}] spawn-freeze notice failed: ${err?.message ?? err}`)); + } catch { + // No callbacks wired (unit tests / early boot) — the log above is the + // only report, which is strictly better than failing the gate itself. + } + } + return; + } const cb = requireCallbacks(); const bot = getBot(ds.larkAppId); const botCfg = bot.config; diff --git a/src/worker.ts b/src/worker.ts index 911909e8b..1154b0853 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -118,6 +118,7 @@ import { setCodexAppThreadName, } from './services/codex-app-threads.js'; import { buildBotmuxLarkNativeSessionTitle } from './core/session-title.js'; +import { activeSpawnFreezeFor } from './core/spawn-freeze.js'; import { drainCodexRollout, findCodexRolloutBySessionId, findCodexRolloutByPid, splitCodexEventsByCutoff, extractLastCodexTurn, codexSessionIdFromRolloutPath, type CodexBridgeEvent } from './services/codex-transcript.js'; import { drainTraexRollout, findTraexRolloutBySessionId, findTraexRolloutByPid } from './services/traex-transcript.js'; import { parseTraexUserInputQuestions } from './services/traex-user-input.js'; @@ -940,6 +941,7 @@ function provisionIsolatedBotHome( cliId: string, hookInstall: HookInstallConfig | undefined, log: (m: string) => void, + larkAppId: string, ): void { try { if (isClaude) { @@ -967,12 +969,41 @@ function provisionIsolatedBotHome( // on EVERY spawn (verified: Claude logs in from that file). Refreshing here (not // just seeding once) means a re-login elsewhere self-heals on the next cold // spawn — no separate sync step needed. Same shared account for every bot. - const fresh = freshestClaudeCred(); - if (fresh) writeCredIfChanged(join(cdir, '.credentials.json'), fresh); - else if ( - !existsSync(join(cdir, '.credentials.json')) - && !claudeSettingsHasProviderAuth(isolatedSettingsPath) - ) { + // …EXCEPT while an ops freeze is in effect. A credential-refresh script + // deliberately fake-expires the shared file before asking claude to + // rotate, so a copy taken mid-window hands this bot a token that is + // already dead — and a bot that then refreshes on its own rotates the + // shared refresh token out from under the script, which is the whole + // poisoning chain the freeze exists to prevent. + // + // The daemon's spawn gate does not make this check redundant: the freeze + // can begin AFTER forkWorker cleared it and before this line runs, so the + // decision has to be made here too. + // + // The rule while frozen is simply "do not touch credential copies" — not + // "copy something safer". Trying to be clever here means deciding whether + // an expired source is the script's fake-expire step (copying it feeds the + // poisoning chain) or a genuinely idle host (copying it is the self-heal + // path), and that is not decidable from the file. So: a bot that already + // has a copy keeps it, and a bot that has none stays unseeded until the + // window closes. + // + // The cost is explicit: a brand-new isolated bot whose FIRST spawn lands + // inside a maintenance window can reach a login screen, and recovers on + // the next spawn after release (≤ the freeze's own bound). That is one bot + // briefly unusable versus a shared-account rotation that takes the whole + // fleet offline — the trade is not close. + const credPath = join(cdir, '.credentials.json'); + const hasCredCopy = existsSync(credPath); + const credFreeze = activeSpawnFreezeFor(larkAppId); + if (credFreeze) { + log(hasCredCopy + ? `[read-isolation] spawn-freeze active (${credFreeze.reason}) — keeping existing per-bot credential copy` + : `[read-isolation] WARN spawn-freeze active (${credFreeze.reason}) and this bot has no credential copy yet — NOT seeding one mid-window; it may hit a login screen and will sync on the first spawn after release`); + } + const fresh = credFreeze ? null : freshestClaudeCred(); + if (fresh) writeCredIfChanged(credPath, fresh); + else if (!credFreeze && !hasCredCopy && !claudeSettingsHasProviderAuth(isolatedSettingsPath)) { log(`[read-isolation] WARN no Claude provider auth found (global settings env, keychain, or ~/.claude/.credentials.json) — bot may hit login screen`); } // State: seed /.claude.json from the GLOBAL one MINUS `projects` (keeps the @@ -983,9 +1014,20 @@ function provisionIsolatedBotHome( const cdir = join(botHome, 'codex'); mkdirSync(cdir, { recursive: true }); // auth.json: keep synced to the shared account's copy on EVERY spawn (a re-login - // elsewhere rotates the refresh token, which would strand a stale per-bot copy). + // elsewhere rotates the refresh token, which would strand a stale per-bot copy) — + // skipped mid-freeze for the same reason as the Claude branch above. const authSrc = join(homedir(), '.codex', 'auth.json'); - if (existsSync(authSrc)) writeCredIfChanged(join(cdir, 'auth.json'), readFileSync(authSrc, 'utf-8')); + const authDst = join(cdir, 'auth.json'); + const codexFreeze = activeSpawnFreezeFor(larkAppId); + // Same rule as the Claude branch, same reasoning: mid-window we do not + // write credential copies at all. + if (codexFreeze) { + log(existsSync(authDst) + ? `[read-isolation] spawn-freeze active (${codexFreeze.reason}) — keeping existing per-bot codex auth copy` + : `[read-isolation] WARN spawn-freeze active (${codexFreeze.reason}) and this bot has no codex auth copy yet — NOT seeding one mid-window; it will sync on the first spawn after release`); + } else if (existsSync(authSrc)) { + writeCredIfChanged(authDst, readFileSync(authSrc, 'utf-8')); + } // config.toml: seed ONCE (it may carry per-bot customizations afterwards). const cfgDst = join(cdir, 'config.toml'); const cfgSrc = join(homedir(), '.codex', 'config.toml'); @@ -1022,17 +1064,30 @@ function freshestClaudeCred(): string | null { const expOf = (raw: string): number => { try { return Number(JSON.parse(raw)?.claudeAiOauth?.expiresAt) || 0; } catch { return 0; } }; + // "Non-empty" is not the same as "a credential". A source mid-rewrite (or an + // OAuth block stripped down to a logged-out shell) parses fine yet carries no + // usable token; copying it into a per-bot file only guarantees a login screen + // on the next spawn. Note this checks STRUCTURE, not expiry: an expired token + // with a live refresh token is normal and must still be copied, otherwise a + // bot can never self-heal after the host has been idle past a token lifetime. + const usable = (raw: string): boolean => { + try { + const oauth = JSON.parse(raw)?.claudeAiOauth; + return typeof oauth?.accessToken === 'string' && oauth.accessToken.length > 0 + && typeof oauth?.refreshToken === 'string' && oauth.refreshToken.length > 0; + } catch { return false; } + }; try { const p = join(homedir(), '.claude', '.credentials.json'); if (existsSync(p)) { const raw = readFileSync(p, 'utf-8').trim(); - if (raw) cands.push({ raw, exp: expOf(raw) }); + if (raw && usable(raw)) cands.push({ raw, exp: expOf(raw) }); } } catch { /* unreadable file → skip candidate */ } try { const r = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials', '-w'], { encoding: 'utf-8' }); const raw = (r.stdout ?? '').trim(); - if (raw) cands.push({ raw, exp: expOf(raw) }); + if (raw && usable(raw)) cands.push({ raw, exp: expOf(raw) }); } catch { /* no keychain (non-mac) → skip candidate */ } if (!cands.length) return null; cands.sort((a, b) => b.exp - a.exp); @@ -6796,7 +6851,7 @@ async function spawnCli( if (isClaudeFam) claudeDataDir = join(isolationBotHome, 'claude'); // Provision the per-bot config dir (auth + onboarding/trust seed + hooks for claude; // auth/config copy for codex) so the CLI starts fully set up under the Seatbelt wrapper. - provisionIsolatedBotHome(isolationBotHome, cfg.workingDir, isClaudeFam, cfg.cliId, cliAdapter.hookInstall, log); + provisionIsolatedBotHome(isolationBotHome, cfg.workingDir, isClaudeFam, cfg.cliId, cliAdapter.hookInstall, log, cfg.larkAppId); if (isClaudeFam && effectiveReadyHookInstall) { effectiveReadyHookInstall = { ...effectiveReadyHookInstall, diff --git a/test/spawn-freeze.test.ts b/test/spawn-freeze.test.ts new file mode 100644 index 000000000..89d36be93 --- /dev/null +++ b/test/spawn-freeze.test.ts @@ -0,0 +1,369 @@ +import { mkdtempSync, rmSync, symlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + activeSpawnFreezeFor, + clearSpawnFreeze, + deferSpawnDuringFreeze, + SpawnFreezeConflictError, + deferredSpawnCount, + forgetDeferredSpawn, + readActiveSpawnFreeze, + resetSpawnFreezeStateForTest, + shouldAnnounceSpawnFreeze, + SPAWN_FREEZE_FILENAME, + SPAWN_FREEZE_HARD_CAP_MS, + SPAWN_FREEZE_MTIME_SKEW_MS, + spawnFreezeApplies, + writeSpawnFreeze, + type SpawnFreezeDeclaration, +} from '../src/core/spawn-freeze.js'; + +const NOW = 1_000_000_000_000; +const dirs: string[] = []; + +function freshDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'spawn-freeze-')); + dirs.push(dir); + return dir; +} + +function deps(dir: string, now = NOW, processAlive: (pid: number) => boolean = () => true) { + return { dataDir: () => dir, now: () => now, processAlive, pollMs: POLL_MS }; +} + +/** The replay path is `setInterval` → `setImmediate`; both are real timers here + * (a fake clock does not drive the immediate), so tests just wait a few of the + * injected poll intervals. */ +const POLL_MS = 5; +const waitForPolls = (n = 3): Promise => new Promise(r => setTimeout(r, POLL_MS * n)); + +/** Write raw content (possibly invalid) plus an explicit mtime. */ +function writeRaw(dir: string, body: string, mtimeMs = NOW): void { + const path = join(dir, SPAWN_FREEZE_FILENAME); + writeFileSync(path, body); + utimesSync(path, new Date(mtimeMs), new Date(mtimeMs)); +} + +function writeDeclaration(dir: string, declaration: SpawnFreezeDeclaration, mtimeMs = NOW): void { + // force: several tests deliberately replace an in-effect declaration. + writeSpawnFreeze(declaration, deps(dir), { force: true }); + stampMtime(dir, mtimeMs); +} + +/** Write WITHOUT force (the conflict check runs), then normalize mtime: the + * harness clock (`NOW`) is far in the past, so a real write time would read as + * a future mtime and be refused. */ +function writeUnforced(dir: string, declaration: SpawnFreezeDeclaration, mtimeMs = NOW): void { + writeSpawnFreeze(declaration, deps(dir)); + stampMtime(dir, mtimeMs); +} + +function stampMtime(dir: string, mtimeMs: number): void { + const path = join(dir, SPAWN_FREEZE_FILENAME); + utimesSync(path, new Date(mtimeMs), new Date(mtimeMs)); +} + +afterEach(() => { + resetSpawnFreezeStateForTest(); + while (dirs.length) rmSync(dirs.pop()!, { recursive: true, force: true }); +}); + +describe('spawn freeze declaration', () => { + it('applies while the deadline is in the future and lifts once it passes', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + expect(readActiveSpawnFreeze(deps(dir))).toMatchObject({ + reason: 'cred-refresh', + effectiveUntil: NOW + 60_000, + notify: false, + }); + expect(readActiveSpawnFreeze(deps(dir, NOW + 60_001))).toBeNull(); + }); + + it('lifts as soon as the declaring process is gone, long before the deadline', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 300_000, pid: 4242 }); + expect(readActiveSpawnFreeze(deps(dir))).toMatchObject({ declaredByPid: 4242 }); + expect(readActiveSpawnFreeze(deps(dir, NOW, () => false))).toBeNull(); + }); + + it('treats EPERM from the liveness probe as alive, not as a dead declarer', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000, pid: 1 }); + const eperm = (pid: number): boolean => { + // Mirrors the production probe: process.kill(pid, 0) throwing EPERM means + // the process exists but belongs to somebody else. + expect(pid).toBe(1); + const error = Object.assign(new Error('EPERM'), { code: 'EPERM' }); + try { throw error; } catch (e) { return (e as NodeJS.ErrnoException).code === 'EPERM'; } + }; + expect(readActiveSpawnFreeze(deps(dir, NOW, eperm))).not.toBeNull(); + }); + + it('caps an over-long declaration at the hard cap measured from mtime', () => { + const dir = freshDir(); + // A declaration asking for a year, written 11 minutes ago: the cap has + // already elapsed even though the deadline is far away. + writeDeclaration(dir, { reason: 'oops', deadline: NOW + 365 * 24 * 3_600_000 }, NOW - 11 * 60_000); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + // Same declaration written just now: capped, but still in effect. + writeDeclaration(dir, { reason: 'oops', deadline: NOW + 365 * 24 * 3_600_000 }, NOW); + expect(readActiveSpawnFreeze(deps(dir))).toMatchObject({ + effectiveUntil: NOW + SPAWN_FREEZE_HARD_CAP_MS, + }); + }); + + it.each([ + ['missing file', null], + ['not json', 'definitely not json'], + ['array', '[]'], + ['no reason', JSON.stringify({ deadline: NOW + 60_000 })], + ['blank reason', JSON.stringify({ reason: ' ', deadline: NOW + 60_000 })], + ['reason too long', JSON.stringify({ reason: 'x'.repeat(121), deadline: NOW + 60_000 })], + ['no deadline', JSON.stringify({ reason: 'x' })], + ['deadline not a number', JSON.stringify({ reason: 'x', deadline: '999' })], + ['fractional deadline', JSON.stringify({ reason: 'x', deadline: NOW + 0.5 })], + ['negative pid', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, pid: -1 })], + ['notify not boolean', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, notify: 'yes' })], + ['scope not object', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, scope: [] })], + ['scope entry not string', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, scope: { larkAppIds: [1] } })], + ['oversized file', JSON.stringify({ reason: 'x'.repeat(100), deadline: NOW + 60_000, pad: 'p'.repeat(9000) })], + ])('fails open on %s', (_label, body) => { + const dir = freshDir(); + if (body !== null) writeRaw(dir, body); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + }); + + it('scopes to the declared bots, and an empty scope means fleet-wide', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'rebuild', deadline: NOW + 60_000, scope: { larkAppIds: ['cli_a'] } }); + const scoped = readActiveSpawnFreeze(deps(dir))!; + expect(spawnFreezeApplies(scoped, 'cli_a')).toBe(true); + expect(spawnFreezeApplies(scoped, 'cli_b')).toBe(false); + // A session with no known bot is never covered by a scoped freeze — it + // cannot be proven in scope, and over-freezing is the worse failure. + expect(spawnFreezeApplies(scoped, undefined)).toBe(false); + expect(activeSpawnFreezeFor('cli_b', deps(dir))).toBeNull(); + + writeDeclaration(dir, { reason: 'fleet', deadline: NOW + 60_000, scope: { larkAppIds: [] } }); + const fleet = readActiveSpawnFreeze(deps(dir))!; + expect(fleet.larkAppIds).toBeUndefined(); + expect(spawnFreezeApplies(fleet, undefined)).toBe(true); + }); + + it('refuses a future mtime instead of clamping it (a clamp would never expire)', () => { + const dir = freshDir(); + // `touch -t 2099…` on the declaration would otherwise re-anchor the hard cap + // on every read — a permanent, un-expiring freeze. + writeDeclaration(dir, { reason: 'stuck', deadline: Number.MAX_SAFE_INTEGER, pid: 1 }, + NOW + SPAWN_FREEZE_MTIME_SKEW_MS + 1_000); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + // Inside the skew tolerance it is still honored (clock drift is not an attack). + writeDeclaration(dir, { reason: 'ok', deadline: NOW + 60_000 }, NOW + 1_000); + expect(readActiveSpawnFreeze(deps(dir))).not.toBeNull(); + }); + + it('refuses a symlinked declaration (it would borrow another file\'s mtime)', () => { + const dir = freshDir(); + const target = join(dir, 'freeze-target.json'); + writeFileSync(target, JSON.stringify({ reason: 'stuck', deadline: Number.MAX_SAFE_INTEGER, pid: 1 })); + utimesSync(target, new Date(NOW + 10 * 365 * 24 * 3_600_000), new Date(NOW + 10 * 365 * 24 * 3_600_000)); + symlinkSync(target, join(dir, SPAWN_FREEZE_FILENAME)); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + }); + + it('clears idempotently', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + expect(clearSpawnFreeze(deps(dir))).toBe('cleared'); + expect(clearSpawnFreeze(deps(dir))).toBe('absent'); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + }); + + it('refuses to clobber an in-effect declaration unless forced', () => { + const dir = freshDir(); + // Two overlapping maintenance scripts would otherwise disarm each other: + // the later write erases the earlier scope, the earlier cleanup deletes the + // later freeze. + writeDeclaration(dir, { reason: 'first', deadline: NOW + 60_000, pid: 111 }); + expect(() => writeSpawnFreeze({ reason: 'second', deadline: NOW + 60_000 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('first'); + // --force is the explicit takeover… + writeDeclaration(dir, { reason: 'second', deadline: NOW + 60_000 }); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('second'); + // …and an EXPIRED declaration is not a conflict at all: an abandoned file + // must never block the next maintenance window. + writeDeclaration(dir, { reason: 'stale', deadline: NOW - 1 }); + writeUnforced(dir, { reason: 'third', deadline: NOW + 60_000 }); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('third'); + }); + + it('lets the same owner update its own window, and stays atomic against a racer', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000, pid: 111 }); + // Extending / amending your own declaration must not require --force. + writeUnforced(dir, { reason: 'cred-refresh', deadline: NOW + 120_000, pid: 111 }); + expect(readActiveSpawnFreeze(deps(dir))!.deadline).toBe(NOW + 120_000); + // A different owner is still refused. + expect(() => writeSpawnFreeze({ reason: 'other', deadline: NOW + 60_000, pid: 222 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + // …and an ownerless declaration is not the same owner as a pid-bearing one. + expect(() => writeSpawnFreeze({ reason: 'other', deadline: NOW + 60_000 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + // Two ownerless declarations are two owners, not one — otherwise every + // anonymous writer replaces every other one (10 concurrent writers all won). + clearSpawnFreeze(deps(dir)); + writeDeclaration(dir, { reason: 'anon-1', deadline: NOW + 60_000 }); + expect(() => writeSpawnFreeze({ reason: 'anon-2', deadline: NOW + 60_000 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('anon-1'); + }); + + it('releases only its own declaration when an owner pid is given', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'theirs', deadline: NOW + 60_000, pid: 4242 }); + // A script's EXIT trap must not delete somebody else's freeze. + expect(clearSpawnFreeze(deps(dir), { ownerPid: 777 })).toBe('not_owner'); + expect(readActiveSpawnFreeze(deps(dir))).not.toBeNull(); + expect(clearSpawnFreeze(deps(dir), { ownerPid: 4242 })).toBe('cleared'); + + // An owner must still be able to clean up its own EXPIRED declaration. + writeDeclaration(dir, { reason: 'mine', deadline: NOW - 1, pid: 4242 }); + expect(clearSpawnFreeze(deps(dir), { ownerPid: 4242 })).toBe('cleared'); + + // No pid recorded → no ownership claim to respect. + writeDeclaration(dir, { reason: 'ownerless', deadline: NOW + 60_000 }); + expect(clearSpawnFreeze(deps(dir), { ownerPid: 999 })).toBe('cleared'); + }); +}); + +describe('deferred spawns', () => { + it('queues only the first spawn per session and replays it on release', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + const d = deps(dir); + + const replays: string[] = []; + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('first') }, d)) + .toMatchObject({ parked: true }); + // A second turn for the same session must NOT queue a second fork (replaying + // two forks would kill and replace the first new worker). `parked: false` is + // how the caller learns THIS turn is dropped and has to be reported — the + // barrier is a delay, not a queue. + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('second') }, d)) + .toMatchObject({ parked: false }); + expect(deferSpawnDuringFreeze({ sessionId: 's2', hasPayload: true, replay: () => replays.push('other') }, d)) + .toMatchObject({ parked: true }); + expect(deferredSpawnCount()).toBe(2); + expect(replays).toEqual([]); + + // Still frozen → the poll must not release anything. + await waitForPolls(); + expect(replays).toEqual([]); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays.sort()).toEqual(['first', 'other']); + expect(deferredSpawnCount()).toBe(0); + }); + + it('releases each session against its own scope', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'rebuild', deadline: NOW + 60_000, scope: { larkAppIds: ['cli_a'] } }); + const d = deps(dir); + + const replays: string[] = []; + expect(deferSpawnDuringFreeze({ sessionId: 's1', larkAppId: 'cli_a', hasPayload: true, replay: () => replays.push('a') }, d)) + .toMatchObject({ parked: true }); + // Out of scope: never deferred in the first place. + expect(deferSpawnDuringFreeze({ sessionId: 's2', larkAppId: 'cli_b', hasPayload: true, replay: () => replays.push('b') }, d)).toBeNull(); + expect(deferredSpawnCount()).toBe(1); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays).toEqual(['a']); + }); + + it('lets a real turn supersede a parked promptless spawn, but never the reverse', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 60_000 }); + const d = deps(dir); + + const replays: string[] = []; + // A warm-up / re-attach parks (some of those exist so a queued raw command + // reaches a ready CLI — skipping them outright would strand it)… + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: false, replay: () => replays.push('warmup') }, d)) + .toMatchObject({ parked: true }); + // …but the first real turn takes the slot from it. + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('real') }, d)) + .toMatchObject({ parked: true, superseded: true }); + // A later promptless spawn must NOT displace the real turn. + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: false, replay: () => replays.push('warmup2') }, d)) + .toMatchObject({ parked: false }); + expect(deferredSpawnCount()).toBe(1); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays).toEqual(['real']); + }); + + it('drops a parked spawn when its session is closed', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + const d = deps(dir); + + const replays: string[] = []; + deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('s1') }, d); + forgetDeferredSpawn('s1'); + expect(deferredSpawnCount()).toBe(0); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays).toEqual([]); + }); + + it('replays when the deadline passes even if nobody releases the freeze', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'crashed-script', deadline: NOW + 2_000 }); + let now = NOW; + const d = { dataDir: () => dir, now: () => now, processAlive: () => true, pollMs: POLL_MS }; + + const replays: string[] = []; + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('s1') }, d)) + .toMatchObject({ parked: true }); + + now = NOW + 2_001; + await waitForPolls(); + expect(replays).toEqual(['s1']); + }); +}); + +describe('freeze announcements', () => { + it('announces once per chat per declaration, and stays silent without notify', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 300_000, notify: true }); + const first = readActiveSpawnFreeze(deps(dir))!; + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1')).toBe(true); + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1')).toBe(false); + // A different session in the same group has its own anchor and must still be + // told — keying by chat id would silence whoever is waiting in it. + expect(shouldAnnounceSpawnFreeze(first, 'anchor-2')).toBe(true); + // "parked" and "dropped" are different messages, one each. + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1', 'dropped')).toBe(true); + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1', 'dropped')).toBe(false); + + // A NEW declaration is a new window: the same chat is told again. + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 400_000, notify: true }); + const second = readActiveSpawnFreeze(deps(dir))!; + expect(second.freezeId).not.toBe(first.freezeId); + expect(shouldAnnounceSpawnFreeze(second, 'anchor-1')).toBe(true); + + writeDeclaration(dir, { reason: 'quiet', deadline: NOW + 60_000 }); + const quiet = readActiveSpawnFreeze(deps(dir))!; + expect(shouldAnnounceSpawnFreeze(quiet, 'anchor-1')).toBe(false); + expect(shouldAnnounceSpawnFreeze(quiet, 'anchor-1', 'dropped')).toBe(false); + }); +});