From cf0aba3d49d91100dcf2f0239f258d33a2b0a1a0 Mon Sep 17 00:00:00 2001 From: xu4wang Date: Wed, 29 Jul 2026 01:53:25 +0700 Subject: [PATCH 1/6] =?UTF-8?q?feat(spawn-freeze):=20=E8=BF=90=E7=BB=B4?= =?UTF-8?q?=E5=A3=B0=E6=98=8E=E5=BC=8F=20CLI=20spawn=20=E9=97=B8=E9=97=A8?= =?UTF-8?q?=20=E2=80=94=E2=80=94=20=E7=BB=B4=E6=8A=A4=E7=AA=97=E5=8F=A3?= =?UTF-8?q?=E5=86=85=E4=B8=8D=E8=B5=B7=E6=96=B0=20CLI=EF=BC=8C=E8=A7=A3?= =?UTF-8?q?=E5=86=BB=E5=90=8E=E8=87=AA=E5=8A=A8=E9=87=8D=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 刷 Claude 凭证时脚本会先把 live 凭证伪过期再逼 claude 刷新。这个窗口里任何冷启动的 CLI 都会看到「过期 token」并自己去刷，轮换掉共享账号的 refresh token，让脚本这次刷新 用的旧 RT 当场作废（失败回滚 → 全队投毒）。读隔离 bot 更糟：它每次冷启动都把「最新 凭证」复制进自己那份副本，伪过期的文件会被原样拷走。 新增 core/spawn-freeze.ts：维护脚本写一份 /spawn-freeze.json 声明「T 之前 不要起新 CLI」，daemon 在 forkWorker 里读它，命中就把这次 spawn 暂存（每会话一个， 与 device-isolation 同不变式），解冻后自动重放 —— 用户消息只是晚几秒，不会丢。 设计要点： - 是数据不是代码：声明只能要求「T 前别起」，不能让 daemon 执行任何东西。既能被纯 shell 脚本使用，影响面也被限制成一个超时而不是扩展点。 - 三重失效 + 全路径 fail-open：deadline / 声明进程退出 / 按文件 mtime 算的 10 分钟 硬上限；文件缺失、读坏、解析失败、越界一律当「无冻结」。冻几秒很便宜，永远起不来 是事故。 - 与 device-isolation 的内存租约互补：租约只覆盖 acquire 那刻在线的 daemon，而窗口 里新启动的 daemon 会读到同一个文件并自我冻结。forkWorker 两个来源任一命中即 defer。 - 只拦新起，不动在跑的 CLI：拆掉现有 CLI 是 botmux suspend 的职责。 - worker 侧读隔离 provisioning 冻结期不覆盖 per-bot 凭证副本（claude + codex）， 堵住 worker 进程内部那两条不经过 forkWorker 的自重启路径。 - freshestClaudeCred() 补结构校验（accessToken/refreshToken 存在）。刻意不按过期时间 拒绝：token 过期而 RT 仍有效是正常状态，拒绝复制会让 bot 再也无法自愈。 CLI：botmux freeze --reason X [--for 120s] [--pid $$] [--notify] [--bot ] botmux freeze --status | --release 测试：test/spawn-freeze.test.ts 25 例（deadline/pid 死/EPERM 当活着/mtime 硬上限/ 13 种坏输入 fail-open/scope/幂等 clear/每会话只暂存一个/按 scope 分别释放/会话关闭 丢弃/无人释放也会因 deadline 重放/通知每话题一条）。 Co-Authored-By: Claude Opus 5 --- src/cli.ts | 137 +++++++++++++++ src/core/spawn-freeze.ts | 342 ++++++++++++++++++++++++++++++++++++++ src/core/worker-pool.ts | 48 ++++++ src/worker.ts | 47 +++++- test/spawn-freeze.test.ts | 240 ++++++++++++++++++++++++++ 5 files changed, 807 insertions(+), 7 deletions(-) create mode 100644 src/core/spawn-freeze.ts create mode 100644 test/spawn-freeze.test.ts diff --git a/src/cli.ts b/src/cli.ts index b1c8fb8c6..29b4a19c4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -32,6 +32,7 @@ import { randomBytes } from 'node:crypto'; import { validateWorkingDir } from './core/working-dir.js'; import { resolveSessionContext } from './core/session-marker.js'; import { resolveBotmuxDataDir } from './core/data-dir.js'; +import { clearSpawnFreeze, readActiveSpawnFreeze, spawnFreezePath, writeSpawnFreeze, SPAWN_FREEZE_HARD_CAP_MS } from './core/spawn-freeze.js'; import { dashboardSecretPath } from './core/dashboard-secret.js'; import { acceptedDispatchBotAppIds, appendDispatchReportProtocol, appendLegacyDispatchReportProtocol, parseDispatchBotSpec, buildDispatchMessages, buildRepoPrimeText, buildReportContent, eligibleAutoMentionAliases, findDispatchRegistryEntry, offTopicSubBotTopic, resolveReportTarget, resolveSendTarget } from './core/dispatch.js'; import { pickTurnReplyTarget } from './core/reply-target.js'; @@ -4082,6 +4083,134 @@ async function cmdSuspend(): Promise { if (failed > 0) process.exitCode = 1; } +/** `--for 90s` / `5m` / 裸数字（秒）。返回毫秒；非法返回 null。 */ +function parseFreezeDuration(raw: string | undefined): number | null { + if (!raw) return null; + const m = /^(\d+)(s|m)?$/.exec(raw.trim()); + if (!m) return null; + const value = Number(m[1]); + if (!Number.isSafeInteger(value) || value <= 0) return null; + return m[2] === 'm' ? value * 60_000 : value * 1_000; +} + +function formatFreezeClock(epochMs: number): string { + const d = new Date(epochMs); + const p = (n: number) => String(n).padStart(2, '0'); + return `${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`; +} + +/** + * `botmux freeze` —— 声明「这台机器暂时不要起新的 CLI 会话」。 + * + * 用途：任何会让「新起的 CLI」踩到半成品状态的维护窗口——刷 Claude 凭证（伪过期 + * 期间冷启动的 CLI 会自己去刷、轮换掉 refresh token，把整队投毒）、升级 claude + * 二进制、重建某个 bot 的工作区。冻结期间 daemon 把该会话的这次 spawn 记下来， + * 解冻后自动重放——用户的消息只是晚几秒被处理，不会丢。 + * + * 只拦「新起」，不动已在跑的 CLI：那是 `botmux suspend` 的职责。典型顺序是 + * freeze → suspend → 干活 → 验证 → release。 + * + * 三重兜底保证一次维护不会把机器冻死：声明的 deadline、写文件进程（--pid）退出、 + * 以及 daemon 侧按文件 mtime 算的 10 分钟硬上限。daemon 读不到/读坏一律放行。 + */ +async function cmdFreeze(): Promise { + const argv = process.argv.slice(3); + const flagValue = (name: string): string | undefined => { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : undefined; + }; + const release = argv.includes('--release'); + const status = argv.includes('--status'); + + if (release && status) { + console.error('❌ --release 与 --status 不能同时用'); + process.exit(1); + } + + if (status) { + const freeze = readActiveSpawnFreeze(); + if (!freeze) { + console.log('当前无冻结：新 CLI 会话可正常启动。'); + return; + } + const remaining = Math.max(0, Math.ceil((freeze.effectiveUntil - Date.now()) / 1000)); + console.log('当前处于冻结中（新 CLI 会话不会启动）：'); + console.log(` 原因 ${freeze.reason}`); + console.log(` 生效至 ${formatFreezeClock(freeze.effectiveUntil)}（剩 ${remaining}s）`); + if (freeze.effectiveUntil < freeze.deadline) { + console.log(` ⚠️ 已被 10 分钟硬上限截断（声明的 deadline 是 ${formatFreezeClock(freeze.deadline)}）`); + } + console.log(` 范围 ${freeze.larkAppIds ? freeze.larkAppIds.join(', ') : '全部 bot'}`); + console.log(` 声明进程 ${freeze.declaredByPid ?? '（未记录 pid，仅靠 deadline 兜底）'}`); + console.log(` 维护提示 ${freeze.notify ? '开（冻结期每个话题回一条）' : '关（静默延迟）'}`); + console.log(` 文件 ${spawnFreezePath()}`); + return; + } + + if (release) { + const removed = clearSpawnFreeze(); + console.log(removed + ? '✓ 已解冻：被暂存的 spawn 会在各 daemon 的下一次轮询（≤1s）自动重放。' + : '· 本就没有冻结声明（目标态已达成）。'); + return; + } + + const reason = flagValue('--reason'); + if (!reason) { + console.error('用法: botmux freeze --reason <说明> [--for 120s] [--pid ] [--notify] [--bot ]...'); + console.error(' botmux freeze --status | --release'); + console.error(' 维护期间禁止本机起新的 CLI 会话；被拦下的 spawn 解冻后自动重放（消息不丢）'); + console.error(' --for 默认 120s，支持 90s / 5m / 裸数字(秒)；上限 10m（daemon 侧硬截断）'); + console.error(' --pid 声明进程 pid（脚本里传 $$）：进程一死立即解冻，崩了也能自愈'); + console.error(' --notify 冻结期收到消息时回一条「维护中」；建议 deadline > 30s 才开'); + console.error(' --bot 只冻某个 bot（可重复）；不给 = 全队'); + process.exit(1); + } + + const durationRaw = flagValue('--for'); + if (durationRaw !== undefined && parseFreezeDuration(durationRaw) === null) { + console.error(`❌ --for 无法解析: ${durationRaw}（示例：90s / 5m / 120）`); + process.exit(1); + } + let durationMs = parseFreezeDuration(durationRaw) ?? 120_000; + if (durationMs > SPAWN_FREEZE_HARD_CAP_MS) { + console.log(`⚠️ --for 超过硬上限，按 ${SPAWN_FREEZE_HARD_CAP_MS / 60_000} 分钟处理。`); + durationMs = SPAWN_FREEZE_HARD_CAP_MS; + } + + const pidRaw = flagValue('--pid'); + let pid: number | undefined; + if (pidRaw !== undefined) { + const parsed = Number(pidRaw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + console.error(`❌ --pid 非法: ${pidRaw}`); + process.exit(1); + } + pid = parsed; + } + + // --bot 可重复 + const larkAppIds: string[] = []; + argv.forEach((arg, i) => { + if (arg === '--bot' && argv[i + 1] && !argv[i + 1].startsWith('--')) larkAppIds.push(argv[i + 1]); + }); + + const deadline = Date.now() + durationMs; + const path = writeSpawnFreeze({ + reason, + deadline, + ...(pid !== undefined ? { pid } : {}), + ...(argv.includes('--notify') ? { notify: true } : {}), + ...(larkAppIds.length ? { scope: { larkAppIds } } : {}), + }); + console.log(`✓ 已冻结（${reason}）：${larkAppIds.length ? larkAppIds.join(', ') : '全部 bot'} 在 ${formatFreezeClock(deadline)} 前不会起新 CLI。`); + console.log(` 声明文件 ${path}`); + if (pid === undefined) { + console.log(' ⚠️ 未传 --pid：脚本中途被杀时只能等 deadline / 10 分钟硬上限自动解冻。建议加 --pid $$。'); + } + console.log(' 记得在结束时（含 trap）执行 botmux freeze --release；已在跑的 CLI 不受影响，需要的话另跑 botmux suspend。'); +} + /** 会话级 CLI IPC（slash/cd/close）的 POST：与 postAsk 同款双路径——能读 host secret * （非隔离进程）走 trusted-host HMAC 签名；读不到（沙箱 BOTMUX_SEND_RELAY / * macOS 读隔离 carve-out）改带本会话当前轮换的 origin capability，由 daemon @@ -4806,6 +4935,13 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 --bot 挂起该 bot 的全部活跃会话 --isolated 挂起所有读隔离 bot（凭证轮换后用；下次冷启动自动同步最新凭证） --dry-run 只列出目标，不执行 + freeze --reason <说明> 维护窗口内禁止本机起新 CLI 会话（被拦下的 spawn 解冻后自动重放，消息不丢） + --for 120s 冻结时长（默认 120s，上限 10m） + --pid 声明进程（脚本传 $$）：进程一死立即解冻 + --notify 冻结期回一条「维护中」（deadline > 30s 建议开） + --bot 只冻某个 bot（可重复）；不给 = 全队 + --status 查看当前是否冻结、为什么、还剩多久 + --release 解冻（幂等，可放脚本 trap 里） slash "<斜杠命令>" 会话空闲后向本会话 CLI 注入一条原生斜杠命令（需 bots.json 配 tuiSlashAllow；/cd 恒被拒） role switch <目录> （会话内）切换本话题到角色库内的角色目录——角色切换用； 目录必须位于 ~/botmux-roles 之下 @@ -9922,6 +10058,7 @@ switch (command) { case 'rm': await cmdDelete(); break; case 'resume': await cmdResume(); break; case 'suspend': await cmdSuspend(); break; + case 'freeze': await cmdFreeze(); break; case 'slash': await cmdSlash(); break; case 'cd': { // Tombstone for the removed `botmux cd`（改名为 `botmux role switch`）。**必须 diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts new file mode 100644 index 000000000..4dea1887a --- /dev/null +++ b/src/core/spawn-freeze.ts @@ -0,0 +1,342 @@ +/** + * Ops-declared CLI spawn barrier. + * + * A maintenance script — credential refresh, `claude` upgrade, workspace + * rebuild — declares a freeze by writing `/spawn-freeze.json`, and + * every daemon reads that declaration immediately before it would start a CLI. + * While the declaration is effective `forkWorker` queues at most one cold spawn + * per logical session and replays it on release, so a user's message is delayed + * instead of lost. + * + * The file is data, never code: a declaration can only ask for "no new CLI + * until T", it can never make the daemon execute anything. That keeps the + * barrier usable by plain shell scripts and keeps its blast radius shaped like + * a timeout instead of an extension point. + * + * Three independent expiries bound it, and every read failure is fail-open: + * + * 1. the declared `deadline`, + * 2. the declaring process (`pid`) exiting, and + * 3. a hard cap measured from the file's own mtime, which the content cannot + * forge. + * + * A freeze must never be able to wedge the fleet. Not starting CLIs for a few + * seconds is cheap; never starting them again is an outage — so anything + * ambiguous (missing file, unreadable, unparseable, out-of-range) resolves to + * "no freeze". + * + * Unlike the device-isolation lease (`device-isolation-activation.ts`), which + * is an in-memory lease acquired over authenticated IPC from every ONLINE + * daemon, this barrier lives on disk. That is deliberate: a daemon that boots + * *during* the window reads the same file and freezes itself, which a lease + * handed out before the boot can never cover. The two compose — `forkWorker` + * defers when either one applies. + */ +import { readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { resolveBotmuxDataDir } from './data-dir.js'; + +export const SPAWN_FREEZE_FILENAME = 'spawn-freeze.json'; + +/** Upper bound on how long one declaration may hold the barrier, measured from + * the file's mtime. A script that dies between writing the file and its own + * cleanup trap — `kill -9`, power loss — cannot freeze the fleet for longer + * than this even if it declared a deadline years out. */ +export const SPAWN_FREEZE_HARD_CAP_MS = 10 * 60_000; + +/** How often a deferred spawn re-checks the declaration. Polling (rather than + * fs.watch) keeps this portable and cheap: the timer only runs while at least + * one spawn is actually parked. */ +export const SPAWN_FREEZE_POLL_MS = 1_000; + +/** Longest `reason` we accept. It lands in logs, cards and `--status`. */ +const MAX_REASON_LENGTH = 120; +const MAX_SCOPE_ENTRIES = 64; +const MAX_FILE_BYTES = 8 * 1024; + +/** On-disk shape. Written by `botmux freeze`, read by every daemon. */ +export interface SpawnFreezeDeclaration { + /** Why new CLIs are blocked. Shown to operators; never parsed for meaning. */ + reason: string; + /** Epoch ms after which the declaration stops applying. */ + deadline: number; + /** Declaring process. When it is gone the barrier lifts immediately, so a + * crashed script self-heals long before `deadline`. */ + pid?: number; + /** Reply once per chat while deferring, instead of silently delaying. Worth + * it for multi-minute windows, noise for a five-second one. */ + notify?: boolean; + /** Absent / empty `larkAppIds` = every bot on this host. */ + scope?: { larkAppIds?: string[] }; +} + +/** A declaration that has been read, validated and found to still apply. */ +export interface ActiveSpawnFreeze { + reason: string; + /** As declared. */ + deadline: number; + /** `min(deadline, mtime + hard cap)` — when the barrier really lifts. */ + effectiveUntil: number; + notify: boolean; + /** Undefined = fleet-wide. */ + larkAppIds?: string[]; + declaredByPid?: number; + /** Identity of this declaration, used to scope one-shot notices. */ + freezeId: string; +} + +export interface SpawnFreezeDeps { + dataDir?: () => string; + now?: () => number; + /** EPERM means "exists but not ours to signal" — that is still alive. */ + processAlive?: (pid: number) => boolean; + /** Release-poll interval. Only tests override it (production always uses + * `SPAWN_FREEZE_POLL_MS`); a tiny value keeps them on real timers, which is + * what the replay path — `setInterval` handing off to `setImmediate` — + * actually runs on. */ + pollMs?: number; +} + +function defaultProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException)?.code === 'EPERM'; + } +} + +function resolveDeps(deps: SpawnFreezeDeps = {}): Required { + return { + dataDir: deps.dataDir ?? (() => resolveBotmuxDataDir()), + now: deps.now ?? (() => Date.now()), + processAlive: deps.processAlive ?? defaultProcessAlive, + pollMs: deps.pollMs && deps.pollMs > 0 ? deps.pollMs : SPAWN_FREEZE_POLL_MS, + }; +} + +export function spawnFreezePath(deps: SpawnFreezeDeps = {}): string { + return join(resolveDeps(deps).dataDir(), SPAWN_FREEZE_FILENAME); +} + +function normalizeScope(value: unknown): string[] | undefined { + if (value === undefined || value === null) return undefined; + if (!Array.isArray(value)) throw new Error('scope.larkAppIds must be an array'); + const ids = value + .filter((id): id is string => typeof id === 'string') + .map(id => id.trim()) + .filter(id => id.length > 0 && id.length <= 64); + if (ids.length !== value.length) throw new Error('scope.larkAppIds has invalid entries'); + if (ids.length > MAX_SCOPE_ENTRIES) throw new Error('scope.larkAppIds too long'); + // An explicitly empty array means the same thing as no scope at all + // (fleet-wide); collapsing it here keeps `spawnFreezeApplies` trivial. + return ids.length > 0 ? ids : undefined; +} + +/** + * Validate a parsed declaration. Throws on anything malformed so the single + * catch in `readActiveSpawnFreeze` can turn every failure into fail-open. + */ +export function parseSpawnFreezeDeclaration(value: unknown): SpawnFreezeDeclaration { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('declaration must be an object'); + } + const record = value as Record; + const reason = typeof record.reason === 'string' ? record.reason.trim() : ''; + if (!reason || reason.length > MAX_REASON_LENGTH) throw new Error('invalid reason'); + const deadline = record.deadline; + if (typeof deadline !== 'number' || !Number.isSafeInteger(deadline) || deadline <= 0) { + throw new Error('invalid deadline'); + } + if (record.notify !== undefined && typeof record.notify !== 'boolean') { + throw new Error('invalid notify'); + } + let pid: number | undefined; + if (record.pid !== undefined) { + if (typeof record.pid !== 'number' || !Number.isSafeInteger(record.pid) || record.pid <= 0) { + throw new Error('invalid pid'); + } + pid = record.pid; + } + const scopeValue = record.scope; + let larkAppIds: string[] | undefined; + if (scopeValue !== undefined && scopeValue !== null) { + if (typeof scopeValue !== 'object' || Array.isArray(scopeValue)) throw new Error('invalid scope'); + larkAppIds = normalizeScope((scopeValue as Record).larkAppIds); + } + return { + reason, + deadline, + ...(pid !== undefined ? { pid } : {}), + ...(record.notify === true ? { notify: true } : {}), + ...(larkAppIds ? { scope: { larkAppIds } } : {}), + }; +} + +/** + * Read the declaration and decide whether it still applies. Returns null — "no + * freeze" — for a missing file and for every possible failure: an unreadable + * or oversized file, malformed JSON, a passed deadline, a dead declarer, or a + * declaration held past the hard cap. + */ +export function readActiveSpawnFreeze(deps: SpawnFreezeDeps = {}): ActiveSpawnFreeze | null { + const { dataDir, now, processAlive } = resolveDeps(deps); + const path = join(dataDir(), SPAWN_FREEZE_FILENAME); + try { + const stat = statSync(path); + if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null; + const declaration = parseSpawnFreezeDeclaration(JSON.parse(readFileSync(path, 'utf-8'))); + // The hard cap is anchored to mtime, not to a timestamp inside the file: + // content is whatever the writer says, mtime is what actually happened. + const effectiveUntil = Math.min(declaration.deadline, stat.mtimeMs + SPAWN_FREEZE_HARD_CAP_MS); + if (effectiveUntil <= now()) return null; + if (declaration.pid !== undefined && !processAlive(declaration.pid)) return null; + return { + reason: declaration.reason, + deadline: declaration.deadline, + effectiveUntil, + notify: declaration.notify === true, + ...(declaration.scope?.larkAppIds ? { larkAppIds: declaration.scope.larkAppIds } : {}), + ...(declaration.pid !== undefined ? { declaredByPid: declaration.pid } : {}), + freezeId: `${declaration.reason}:${declaration.deadline}:${Math.round(stat.mtimeMs)}`, + }; + } catch { + // Fail-open, deliberately silent: this runs on every spawn, and a missing + // file is the overwhelmingly common case. + return null; + } +} + +/** True when a freeze covers this bot (no scope = the whole host). */ +export function spawnFreezeApplies(freeze: ActiveSpawnFreeze, larkAppId?: string): boolean { + if (!freeze.larkAppIds) return true; + if (!larkAppId) return false; + return freeze.larkAppIds.includes(larkAppId); +} + +/** The freeze in effect for one bot, or null. */ +export function activeSpawnFreezeFor( + larkAppId?: string, + deps: SpawnFreezeDeps = {}, +): ActiveSpawnFreeze | null { + const freeze = readActiveSpawnFreeze(deps); + if (!freeze) return null; + return spawnFreezeApplies(freeze, larkAppId) ? freeze : null; +} + +interface DeferredSpawn { + larkAppId?: string; + replay: () => void; +} + +const deferredSpawns = new Map(); +let pollTimer: NodeJS.Timeout | null = null; +let noticeFreezeId: string | null = null; +const noticedChats = new Set(); + +function stopPoll(): void { + if (!pollTimer) return; + clearInterval(pollTimer); + pollTimer = null; +} + +function flushReleasedSpawns(deps: SpawnFreezeDeps = {}): void { + for (const [sessionId, entry] of [...deferredSpawns.entries()]) { + // Per-entry, because a scoped freeze can lift for one bot while another + // declaration still covers a different one. + if (activeSpawnFreezeFor(entry.larkAppId, deps)) continue; + deferredSpawns.delete(sessionId); + setImmediate(entry.replay); + } + if (deferredSpawns.size === 0) stopPoll(); +} + +function ensurePoll(deps: SpawnFreezeDeps = {}): void { + if (pollTimer) return; + pollTimer = setInterval(() => flushReleasedSpawns(deps), resolveDeps(deps).pollMs); + pollTimer.unref?.(); +} + +/** + * Park a cold spawn while a freeze applies, and return the freeze that caused + * it (null = not frozen, caller proceeds). + * + * Only the FIRST request per logical session is queued: later turns are + * already retained by the session's own pending-input machinery, and replaying + * several fork requests would kill and replace the first new worker. Same + * invariant as `deferWorkerSpawnDuringDeviceIsolation`. + */ +export function deferSpawnDuringFreeze( + input: { sessionId: string; larkAppId?: string; replay: () => void }, + deps: SpawnFreezeDeps = {}, +): ActiveSpawnFreeze | null { + const freeze = activeSpawnFreezeFor(input.larkAppId, deps); + if (!freeze) return null; + if (!deferredSpawns.has(input.sessionId)) { + deferredSpawns.set(input.sessionId, { larkAppId: input.larkAppId, replay: input.replay }); + } + ensurePoll(deps); + return freeze; +} + +/** Drop a parked spawn without replaying it (session closed while frozen). */ +export function forgetDeferredSpawn(sessionId: string): void { + deferredSpawns.delete(sessionId); + if (deferredSpawns.size === 0) stopPoll(); +} + +export function deferredSpawnCount(): number { + return deferredSpawns.size; +} + +/** + * One-shot gate for the "maintenance in progress" reply: true the first time a + * given chat asks during a given declaration. Without this a busy group gets + * one notice per message; with it the group is told once and then simply waits. + * State resets when the declaration changes, so the next window speaks again. + */ +export function shouldAnnounceSpawnFreeze(freeze: ActiveSpawnFreeze, chatKey: string): boolean { + if (!freeze.notify) return false; + if (noticeFreezeId !== freeze.freezeId) { + noticeFreezeId = freeze.freezeId; + noticedChats.clear(); + } + if (noticedChats.has(chatKey)) return false; + noticedChats.add(chatKey); + return true; +} + +/** Atomically publish a declaration. Same-directory temp + rename so a reader + * never observes a half-written file. */ +export function writeSpawnFreeze( + declaration: SpawnFreezeDeclaration, + deps: SpawnFreezeDeps = {}, +): string { + const parsed = parseSpawnFreezeDeclaration(declaration); + const path = spawnFreezePath(deps); + const tmp = `${path}.tmp-${process.pid}`; + writeFileSync(tmp, `${JSON.stringify(parsed, null, 2)}\n`, { mode: 0o600 }); + renameSync(tmp, path); + return path; +} + +/** Remove the declaration. Idempotent — releasing a freeze that already + * expired must not fail a maintenance script's cleanup trap. */ +export function clearSpawnFreeze(deps: SpawnFreezeDeps = {}): boolean { + const path = spawnFreezePath(deps); + try { + statSync(path); + } catch { + return false; + } + rmSync(path, { force: true }); + return true; +} + +/** Test-only: production state is driven entirely by the file and the timer. */ +export function resetSpawnFreezeStateForTest(): void { + deferredSpawns.clear(); + stopPoll(); + noticeFreezeId = null; + noticedChats.clear(); +} diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 70631f38d..203278b79 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -106,6 +106,7 @@ import { isSilentScheduledTurn } from './silent-schedule-turns.js'; import { isTriggerFinalSuppressed } from './trigger-final-suppression.js'; import { writeDeferredTopicBinding } from './deferred-topic-binding.js'; import { deferWorkerSpawnDuringDeviceIsolation } from './device-isolation-activation.js'; +import { deferSpawnDuringFreeze, forgetDeferredSpawn, shouldAnnounceSpawnFreeze } from './spawn-freeze.js'; import { buildBotmuxLarkNativeSessionTitle, extractBotmuxLarkNativeSessionTitlePrompt, @@ -1640,6 +1641,10 @@ export async function closeSession( // 会话关闭即可回收其崩溃重启计数；否则每个曾崩溃过的 session 会在 daemon // 生命周期内永久占位（restartCounts 此前无任何 delete）。 restartCounts.delete(sessionId); + // A spawn parked by an ops freeze must not outlive its session. The replay + // callback re-checks the registry anyway, so this is about not holding the + // closure (and the poll timer) alive for a session that is already gone. + forgetDeferredSpawn(sessionId); if (ds) { // Usage ledger: flush the final delta before the worker goes away (a // crash/limited turn may never have reached an idle edge). @@ -2098,6 +2103,49 @@ export function forkWorker( logger.info(`[${tag(ds)}] worker spawn deferred during device credential activation`); return; } + // An ops maintenance window (credential refresh, `claude` upgrade, workspace + // rebuild) can declare that no new CLI may start on this host. Same defer + + // replay shape as the device-isolation lease above, but driven by an on-disk + // declaration so a daemon that BOOTS during the window freezes itself too. + // Deliberately only blocks new spawns: tearing down live CLIs is + // `botmux suspend`'s job, and conflating the two would make every freeze a + // fleet-wide cold restart. + const opsFreeze = deferSpawnDuringFreeze({ + sessionId: ds.session.sessionId, + larkAppId: ds.larkAppId, + replay: () => { + // The session may have been closed while parked; never revive it. + if (findActiveBySessionId(ds.session.sessionId) === ds) { + forkWorker(ds, promptInput, resumeOrTurnId); + } + }, + }); + if (opsFreeze) { + const remainingSec = Math.max(1, Math.ceil((opsFreeze.effectiveUntil - Date.now()) / 1000)); + logger.info( + `[${tag(ds)}] worker spawn deferred by spawn-freeze (reason=${opsFreeze.reason}, ` + + `~${remainingSec}s left)`, + ); + const chatKey = ds.session.chatId ?? ds.session.sessionId; + if (shouldAnnounceSpawnFreeze(opsFreeze, chatKey)) { + const gateTurnId = typeof resumeOrTurnId === 'string' + ? resumeOrTurnId + : (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null ? resumeOrTurnId.turnId : undefined); + try { + void requireCallbacks().sessionReply( + sessionAnchorId(ds), + `🔧 维护中（${opsFreeze.reason}），暂不启动新的 CLI 会话；约 ${remainingSec} 秒后自动继续处理这条消息。`, + 'text', + ds.larkAppId, + fallbackTurnId(ds, gateTurnId), + ).catch(err => logger.warn(`[${tag(ds)}] spawn-freeze notice failed: ${err?.message ?? err}`)); + } catch { + // No callbacks wired (unit tests / early boot) — the delay is silent, + // which is strictly better than failing the gate itself. + } + } + return; + } const cb = requireCallbacks(); const bot = getBot(ds.larkAppId); const botCfg = bot.config; diff --git a/src/worker.ts b/src/worker.ts index 911909e8b..041da6340 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -118,6 +118,7 @@ import { setCodexAppThreadName, } from './services/codex-app-threads.js'; import { buildBotmuxLarkNativeSessionTitle } from './core/session-title.js'; +import { activeSpawnFreezeFor } from './core/spawn-freeze.js'; import { drainCodexRollout, findCodexRolloutBySessionId, findCodexRolloutByPid, splitCodexEventsByCutoff, extractLastCodexTurn, codexSessionIdFromRolloutPath, type CodexBridgeEvent } from './services/codex-transcript.js'; import { drainTraexRollout, findTraexRolloutBySessionId, findTraexRolloutByPid } from './services/traex-transcript.js'; import { parseTraexUserInputQuestions } from './services/traex-user-input.js'; @@ -940,6 +941,7 @@ function provisionIsolatedBotHome( cliId: string, hookInstall: HookInstallConfig | undefined, log: (m: string) => void, + larkAppId: string, ): void { try { if (isClaude) { @@ -967,10 +969,22 @@ function provisionIsolatedBotHome( // on EVERY spawn (verified: Claude logs in from that file). Refreshing here (not // just seeding once) means a re-login elsewhere self-heals on the next cold // spawn — no separate sync step needed. Same shared account for every bot. - const fresh = freshestClaudeCred(); + // …EXCEPT while an ops freeze is in effect. A credential-refresh script + // deliberately fake-expires the shared file before asking claude to + // rotate, so a copy taken mid-window hands this bot a token that is + // already dead — and a bot that then refreshes on its own rotates the + // shared refresh token out from under the script, which is the whole + // poisoning chain the freeze exists to prevent. Keep whatever copy this + // bot already has; the next cold spawn after release re-syncs. + const credFreeze = activeSpawnFreezeFor(larkAppId); + if (credFreeze) { + log(`[read-isolation] spawn-freeze active (${credFreeze.reason}) — keeping existing per-bot credential copy`); + } + const fresh = credFreeze ? null : freshestClaudeCred(); if (fresh) writeCredIfChanged(join(cdir, '.credentials.json'), fresh); else if ( - !existsSync(join(cdir, '.credentials.json')) + !credFreeze + && !existsSync(join(cdir, '.credentials.json')) && !claudeSettingsHasProviderAuth(isolatedSettingsPath) ) { log(`[read-isolation] WARN no Claude provider auth found (global settings env, keychain, or ~/.claude/.credentials.json) — bot may hit login screen`); @@ -983,9 +997,15 @@ function provisionIsolatedBotHome( const cdir = join(botHome, 'codex'); mkdirSync(cdir, { recursive: true }); // auth.json: keep synced to the shared account's copy on EVERY spawn (a re-login - // elsewhere rotates the refresh token, which would strand a stale per-bot copy). + // elsewhere rotates the refresh token, which would strand a stale per-bot copy) — + // skipped mid-freeze for the same reason as the Claude branch above. const authSrc = join(homedir(), '.codex', 'auth.json'); - if (existsSync(authSrc)) writeCredIfChanged(join(cdir, 'auth.json'), readFileSync(authSrc, 'utf-8')); + const codexFreeze = activeSpawnFreezeFor(larkAppId); + if (codexFreeze) { + log(`[read-isolation] spawn-freeze active (${codexFreeze.reason}) — keeping existing per-bot codex auth copy`); + } else if (existsSync(authSrc)) { + writeCredIfChanged(join(cdir, 'auth.json'), readFileSync(authSrc, 'utf-8')); + } // config.toml: seed ONCE (it may carry per-bot customizations afterwards). const cfgDst = join(cdir, 'config.toml'); const cfgSrc = join(homedir(), '.codex', 'config.toml'); @@ -1022,17 +1042,30 @@ function freshestClaudeCred(): string | null { const expOf = (raw: string): number => { try { return Number(JSON.parse(raw)?.claudeAiOauth?.expiresAt) || 0; } catch { return 0; } }; + // "Non-empty" is not the same as "a credential". A source mid-rewrite (or an + // OAuth block stripped down to a logged-out shell) parses fine yet carries no + // usable token; copying it into a per-bot file only guarantees a login screen + // on the next spawn. Note this checks STRUCTURE, not expiry: an expired token + // with a live refresh token is normal and must still be copied, otherwise a + // bot can never self-heal after the host has been idle past a token lifetime. + const usable = (raw: string): boolean => { + try { + const oauth = JSON.parse(raw)?.claudeAiOauth; + return typeof oauth?.accessToken === 'string' && oauth.accessToken.length > 0 + && typeof oauth?.refreshToken === 'string' && oauth.refreshToken.length > 0; + } catch { return false; } + }; try { const p = join(homedir(), '.claude', '.credentials.json'); if (existsSync(p)) { const raw = readFileSync(p, 'utf-8').trim(); - if (raw) cands.push({ raw, exp: expOf(raw) }); + if (raw && usable(raw)) cands.push({ raw, exp: expOf(raw) }); } } catch { /* unreadable file → skip candidate */ } try { const r = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials', '-w'], { encoding: 'utf-8' }); const raw = (r.stdout ?? '').trim(); - if (raw) cands.push({ raw, exp: expOf(raw) }); + if (raw && usable(raw)) cands.push({ raw, exp: expOf(raw) }); } catch { /* no keychain (non-mac) → skip candidate */ } if (!cands.length) return null; cands.sort((a, b) => b.exp - a.exp); @@ -6796,7 +6829,7 @@ async function spawnCli( if (isClaudeFam) claudeDataDir = join(isolationBotHome, 'claude'); // Provision the per-bot config dir (auth + onboarding/trust seed + hooks for claude; // auth/config copy for codex) so the CLI starts fully set up under the Seatbelt wrapper. - provisionIsolatedBotHome(isolationBotHome, cfg.workingDir, isClaudeFam, cfg.cliId, cliAdapter.hookInstall, log); + provisionIsolatedBotHome(isolationBotHome, cfg.workingDir, isClaudeFam, cfg.cliId, cliAdapter.hookInstall, log, cfg.larkAppId); if (isClaudeFam && effectiveReadyHookInstall) { effectiveReadyHookInstall = { ...effectiveReadyHookInstall, diff --git a/test/spawn-freeze.test.ts b/test/spawn-freeze.test.ts new file mode 100644 index 000000000..5ef8b0993 --- /dev/null +++ b/test/spawn-freeze.test.ts @@ -0,0 +1,240 @@ +import { mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + activeSpawnFreezeFor, + clearSpawnFreeze, + deferSpawnDuringFreeze, + deferredSpawnCount, + forgetDeferredSpawn, + readActiveSpawnFreeze, + resetSpawnFreezeStateForTest, + shouldAnnounceSpawnFreeze, + SPAWN_FREEZE_FILENAME, + SPAWN_FREEZE_HARD_CAP_MS, + spawnFreezeApplies, + writeSpawnFreeze, + type SpawnFreezeDeclaration, +} from '../src/core/spawn-freeze.js'; + +const NOW = 1_000_000_000_000; +const dirs: string[] = []; + +function freshDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'spawn-freeze-')); + dirs.push(dir); + return dir; +} + +function deps(dir: string, now = NOW, processAlive: (pid: number) => boolean = () => true) { + return { dataDir: () => dir, now: () => now, processAlive, pollMs: POLL_MS }; +} + +/** The replay path is `setInterval` → `setImmediate`; both are real timers here + * (a fake clock does not drive the immediate), so tests just wait a few of the + * injected poll intervals. */ +const POLL_MS = 5; +const waitForPolls = (n = 3): Promise => new Promise(r => setTimeout(r, POLL_MS * n)); + +/** Write raw content (possibly invalid) plus an explicit mtime. */ +function writeRaw(dir: string, body: string, mtimeMs = NOW): void { + const path = join(dir, SPAWN_FREEZE_FILENAME); + writeFileSync(path, body); + utimesSync(path, new Date(mtimeMs), new Date(mtimeMs)); +} + +function writeDeclaration(dir: string, declaration: SpawnFreezeDeclaration, mtimeMs = NOW): void { + writeSpawnFreeze(declaration, deps(dir)); + const path = join(dir, SPAWN_FREEZE_FILENAME); + utimesSync(path, new Date(mtimeMs), new Date(mtimeMs)); +} + +afterEach(() => { + resetSpawnFreezeStateForTest(); + while (dirs.length) rmSync(dirs.pop()!, { recursive: true, force: true }); +}); + +describe('spawn freeze declaration', () => { + it('applies while the deadline is in the future and lifts once it passes', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + expect(readActiveSpawnFreeze(deps(dir))).toMatchObject({ + reason: 'cred-refresh', + effectiveUntil: NOW + 60_000, + notify: false, + }); + expect(readActiveSpawnFreeze(deps(dir, NOW + 60_001))).toBeNull(); + }); + + it('lifts as soon as the declaring process is gone, long before the deadline', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 300_000, pid: 4242 }); + expect(readActiveSpawnFreeze(deps(dir))).toMatchObject({ declaredByPid: 4242 }); + expect(readActiveSpawnFreeze(deps(dir, NOW, () => false))).toBeNull(); + }); + + it('treats EPERM from the liveness probe as alive, not as a dead declarer', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000, pid: 1 }); + const eperm = (pid: number): boolean => { + // Mirrors the production probe: process.kill(pid, 0) throwing EPERM means + // the process exists but belongs to somebody else. + expect(pid).toBe(1); + const error = Object.assign(new Error('EPERM'), { code: 'EPERM' }); + try { throw error; } catch (e) { return (e as NodeJS.ErrnoException).code === 'EPERM'; } + }; + expect(readActiveSpawnFreeze(deps(dir, NOW, eperm))).not.toBeNull(); + }); + + it('caps an over-long declaration at the hard cap measured from mtime', () => { + const dir = freshDir(); + // A declaration asking for a year, written 11 minutes ago: the cap has + // already elapsed even though the deadline is far away. + writeDeclaration(dir, { reason: 'oops', deadline: NOW + 365 * 24 * 3_600_000 }, NOW - 11 * 60_000); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + // Same declaration written just now: capped, but still in effect. + writeDeclaration(dir, { reason: 'oops', deadline: NOW + 365 * 24 * 3_600_000 }, NOW); + expect(readActiveSpawnFreeze(deps(dir))).toMatchObject({ + effectiveUntil: NOW + SPAWN_FREEZE_HARD_CAP_MS, + }); + }); + + it.each([ + ['missing file', null], + ['not json', 'definitely not json'], + ['array', '[]'], + ['no reason', JSON.stringify({ deadline: NOW + 60_000 })], + ['blank reason', JSON.stringify({ reason: ' ', deadline: NOW + 60_000 })], + ['reason too long', JSON.stringify({ reason: 'x'.repeat(121), deadline: NOW + 60_000 })], + ['no deadline', JSON.stringify({ reason: 'x' })], + ['deadline not a number', JSON.stringify({ reason: 'x', deadline: '999' })], + ['fractional deadline', JSON.stringify({ reason: 'x', deadline: NOW + 0.5 })], + ['negative pid', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, pid: -1 })], + ['notify not boolean', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, notify: 'yes' })], + ['scope not object', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, scope: [] })], + ['scope entry not string', JSON.stringify({ reason: 'x', deadline: NOW + 60_000, scope: { larkAppIds: [1] } })], + ['oversized file', JSON.stringify({ reason: 'x'.repeat(100), deadline: NOW + 60_000, pad: 'p'.repeat(9000) })], + ])('fails open on %s', (_label, body) => { + const dir = freshDir(); + if (body !== null) writeRaw(dir, body); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + }); + + it('scopes to the declared bots, and an empty scope means fleet-wide', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'rebuild', deadline: NOW + 60_000, scope: { larkAppIds: ['cli_a'] } }); + const scoped = readActiveSpawnFreeze(deps(dir))!; + expect(spawnFreezeApplies(scoped, 'cli_a')).toBe(true); + expect(spawnFreezeApplies(scoped, 'cli_b')).toBe(false); + // A session with no known bot is never covered by a scoped freeze — it + // cannot be proven in scope, and over-freezing is the worse failure. + expect(spawnFreezeApplies(scoped, undefined)).toBe(false); + expect(activeSpawnFreezeFor('cli_b', deps(dir))).toBeNull(); + + writeDeclaration(dir, { reason: 'fleet', deadline: NOW + 60_000, scope: { larkAppIds: [] } }); + const fleet = readActiveSpawnFreeze(deps(dir))!; + expect(fleet.larkAppIds).toBeUndefined(); + expect(spawnFreezeApplies(fleet, undefined)).toBe(true); + }); + + it('clears idempotently', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + expect(clearSpawnFreeze(deps(dir))).toBe(true); + expect(clearSpawnFreeze(deps(dir))).toBe(false); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + }); +}); + +describe('deferred spawns', () => { + it('queues only the first spawn per session and replays it on release', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + const d = deps(dir); + + const replays: string[] = []; + expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('first') }, d)).not.toBeNull(); + // A second turn for the same session must NOT queue a second fork: replaying + // two forks would kill and replace the first new worker. + expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('second') }, d)).not.toBeNull(); + expect(deferSpawnDuringFreeze({ sessionId: 's2', replay: () => replays.push('other') }, d)).not.toBeNull(); + expect(deferredSpawnCount()).toBe(2); + expect(replays).toEqual([]); + + // Still frozen → the poll must not release anything. + await waitForPolls(); + expect(replays).toEqual([]); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays.sort()).toEqual(['first', 'other']); + expect(deferredSpawnCount()).toBe(0); + }); + + it('releases each session against its own scope', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'rebuild', deadline: NOW + 60_000, scope: { larkAppIds: ['cli_a'] } }); + const d = deps(dir); + + const replays: string[] = []; + expect(deferSpawnDuringFreeze({ sessionId: 's1', larkAppId: 'cli_a', replay: () => replays.push('a') }, d)).not.toBeNull(); + // Out of scope: never deferred in the first place. + expect(deferSpawnDuringFreeze({ sessionId: 's2', larkAppId: 'cli_b', replay: () => replays.push('b') }, d)).toBeNull(); + expect(deferredSpawnCount()).toBe(1); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays).toEqual(['a']); + }); + + it('drops a parked spawn when its session is closed', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); + const d = deps(dir); + + const replays: string[] = []; + deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('s1') }, d); + forgetDeferredSpawn('s1'); + expect(deferredSpawnCount()).toBe(0); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays).toEqual([]); + }); + + it('replays when the deadline passes even if nobody releases the freeze', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'crashed-script', deadline: NOW + 2_000 }); + let now = NOW; + const d = { dataDir: () => dir, now: () => now, processAlive: () => true, pollMs: POLL_MS }; + + const replays: string[] = []; + expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('s1') }, d)).not.toBeNull(); + + now = NOW + 2_001; + await waitForPolls(); + expect(replays).toEqual(['s1']); + }); +}); + +describe('freeze announcements', () => { + it('announces once per chat per declaration, and stays silent without notify', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 300_000, notify: true }); + const first = readActiveSpawnFreeze(deps(dir))!; + expect(shouldAnnounceSpawnFreeze(first, 'chat-1')).toBe(true); + expect(shouldAnnounceSpawnFreeze(first, 'chat-1')).toBe(false); + expect(shouldAnnounceSpawnFreeze(first, 'chat-2')).toBe(true); + + // A NEW declaration is a new window: the same chat is told again. + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 400_000, notify: true }); + const second = readActiveSpawnFreeze(deps(dir))!; + expect(second.freezeId).not.toBe(first.freezeId); + expect(shouldAnnounceSpawnFreeze(second, 'chat-1')).toBe(true); + + writeDeclaration(dir, { reason: 'quiet', deadline: NOW + 60_000 }); + const quiet = readActiveSpawnFreeze(deps(dir))!; + expect(shouldAnnounceSpawnFreeze(quiet, 'chat-1')).toBe(false); + }); +}); From 8f42bc13d0f9721d6474a37a0f4cb11110be4cd1 Mon Sep 17 00:00:00 2001 From: xu4wang Date: Wed, 29 Jul 2026 01:58:55 +0700 Subject: [PATCH 2/6] =?UTF-8?q?fix(spawn-freeze):=20codex=20=E5=A4=8D?= =?UTF-8?q?=E5=AE=A1=E4=B8=89=E5=A4=84=20=E2=80=94=E2=80=94=20=E7=A1=AC?= =?UTF-8?q?=E4=B8=8A=E9=99=90=E5=8F=AF=E8=A2=AB=E4=BC=AA=E9=80=A0=20mtime?= =?UTF-8?q?=20=E7=BB=95=E8=BF=87=20/=20=E9=87=8D=E6=94=BE=E8=AE=A4?= =?UTF-8?q?=E9=94=99=E4=BC=9A=E8=AF=9D=20/=20=E5=86=BB=E7=BB=93=E6=9C=9F?= =?UTF-8?q?=E6=96=B0=20bot=20=E6=97=A0=E5=87=AD=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. [P0] 10 分钟硬上限可被绕过 → 永久自锁。statSync 跟随符号链接，mtime 又能被 `touch -t` 改到未来，两者任一都能让 effectiveUntil 落在很远的将来，配上 pid=1（恒存活）与合法 deadline，三重失效同时失守。改为 lstatSync 拒绝符号链接， 并【拒绝】未来 mtime 而不是把它 clamp 到 now —— clamp 会在每次读取时重新锚定 窗口，正是要防的那种永不过期。容忍 60s 时钟漂移。 2. [P1] 重放可能认错会话。deferred map 以入队时的 sessionId 为 key，但回调在执行时 读的是可变的 ds.session.sessionId；切 repo 会在同一个 ds 上整体替换 session （command-handler.ts:1652），于是旧 entry 的守卫也会通过 → 为新会话起第二个 worker，把刚起来的顶掉。改为入队时捕获不可变 id，重放时同时校验；切 repo 处 显式 forgetDeferredSpawn(旧 id)。 3. [P1] 冻结期首次 provisioning 会让 bot 停在登录界面。daemon 的闸门只保证它读取 那一刻没冻结，worker 真正 provisioning 时会重新读一次——冻结若在两者之间开始， 凭证复制被跳过；若该 bot 还没有任何副本，就等于用空凭证启动。「跳过」只能表示 「保留已有副本」，没有副本时改为播种（优先未过期的那份，避免propagate 脚本的 伪过期文件），claude / codex 两支同此。 测试：+2 例（未来 mtime、符号链接），共 27 例。 Co-Authored-By: Claude Opus 5 --- src/core/command-handler.ts | 4 +++ src/core/spawn-freeze.ts | 33 +++++++++++++++++----- src/core/worker-pool.ts | 16 +++++++++-- src/worker.ts | 55 ++++++++++++++++++++++++++----------- test/spawn-freeze.test.ts | 24 +++++++++++++++- 5 files changed, 105 insertions(+), 27 deletions(-) diff --git a/src/core/command-handler.ts b/src/core/command-handler.ts index edc23c3e3..16b2968a9 100644 --- a/src/core/command-handler.ts +++ b/src/core/command-handler.ts @@ -25,6 +25,7 @@ import { claimPairing } from '../services/pairing-store.js'; import { logger } from '../utils/logger.js'; import { scheduleTimeZone } from '../utils/timezone.js'; import { killWorker, suspendWorker, forkWorker, forkAdoptWorker, adoptSandboxBlocked, getCurrentCliVersion, postFreshStreamingCard, postPrivateSnapshotCard, resolvePrivateCardAudience, deliverEphemeralOrReply, deliverWritableTerminalCardTo, requestSessionRestart } from './worker-pool.js'; +import { forgetDeferredSpawn } from './spawn-freeze.js'; import { expandHome, getSessionWorkingDir, @@ -1650,6 +1651,9 @@ export async function handleCommand( ds!.scope, ); ds!.session = session; + // 这个 ds 从此代表新 session：冻结期为旧 session 暂存的 spawn 必须就地丢弃， + // 否则解冻重放会为同一个 ds 起第二个 worker，把刚起来的那个顶掉。 + forgetDeferredSpawn(oldSession.sessionId); ds!.lastUserPrompt = undefined; ds!.lastCliInput = undefined; ds!.workingDir = selectedPath; diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts index 4dea1887a..45a8c4ac7 100644 --- a/src/core/spawn-freeze.ts +++ b/src/core/spawn-freeze.ts @@ -17,8 +17,17 @@ * * 1. the declared `deadline`, * 2. the declaring process (`pid`) exiting, and - * 3. a hard cap measured from the file's own mtime, which the content cannot - * forge. + * 3. a hard cap measured from the file's mtime. + * + * The cap deserves a note on what it can and cannot do. mtime is not content, + * but it is still writable by whoever owns the file (`touch -t`), and a symlink + * would let the declaration borrow some other file's timestamps — so the reader + * refuses symlinks and refuses a future mtime outright instead of clamping it + * (clamping to `now` would slide the window forward on every read, i.e. exactly + * the permanent freeze the cap exists to prevent). What remains is that a live + * process can keep re-declaring; that is not a hole, it is the same authority + * as holding a legitimate freeze. The cap's job is bounding an ABANDONED + * declaration — `kill -9` before the cleanup trap, power loss — and it does. * * A freeze must never be able to wedge the fleet. Not starting CLIs for a few * seconds is cheap; never starting them again is an outage — so anything @@ -32,7 +41,7 @@ * handed out before the boot can never cover. The two compose — `forkWorker` * defers when either one applies. */ -import { readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { lstatSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { resolveBotmuxDataDir } from './data-dir.js'; @@ -44,6 +53,12 @@ export const SPAWN_FREEZE_FILENAME = 'spawn-freeze.json'; * than this even if it declared a deadline years out. */ export const SPAWN_FREEZE_HARD_CAP_MS = 10 * 60_000; +/** How far ahead of `now` an mtime may sit before we stop believing it. Covers + * ordinary clock skew / filesystem timestamp granularity; anything beyond it is + * a timestamp nobody legitimately produces, and honoring it would hand the + * writer an unbounded freeze. */ +export const SPAWN_FREEZE_MTIME_SKEW_MS = 60_000; + /** How often a deferred spawn re-checks the declaration. Polling (rather than * fs.watch) keeps this portable and cheap: the timer only runs while at least * one spawn is actually parked. */ @@ -183,13 +198,17 @@ export function readActiveSpawnFreeze(deps: SpawnFreezeDeps = {}): ActiveSpawnFr const { dataDir, now, processAlive } = resolveDeps(deps); const path = join(dataDir(), SPAWN_FREEZE_FILENAME); try { - const stat = statSync(path); + // lstat, not stat: a symlink would let the declaration inherit an arbitrary + // file's mtime (and thus escape the hard cap), so it is refused outright. + const stat = lstatSync(path); if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null; + const at = now(); + // A future mtime cannot be honored and must not be clamped to `now` either: + // clamping would re-anchor the cap on every read and never expire. + if (stat.mtimeMs > at + SPAWN_FREEZE_MTIME_SKEW_MS) return null; const declaration = parseSpawnFreezeDeclaration(JSON.parse(readFileSync(path, 'utf-8'))); - // The hard cap is anchored to mtime, not to a timestamp inside the file: - // content is whatever the writer says, mtime is what actually happened. const effectiveUntil = Math.min(declaration.deadline, stat.mtimeMs + SPAWN_FREEZE_HARD_CAP_MS); - if (effectiveUntil <= now()) return null; + if (effectiveUntil <= at) return null; if (declaration.pid !== undefined && !processAlive(declaration.pid)) return null; return { reason: declaration.reason, diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 203278b79..cc3263d61 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -2110,12 +2110,22 @@ export function forkWorker( // Deliberately only blocks new spawns: tearing down live CLIs is // `botmux suspend`'s job, and conflating the two would make every freeze a // fleet-wide cold restart. + // Capture the session id NOW: `ds.session` is replaced wholesale on a repo + // switch (command-handler swaps in the new Session on the SAME ds), so a + // closure that read `ds.session.sessionId` at replay time would accept an + // entry queued for the previous session and fork a second worker for the new + // one — killing and replacing the first. + const deferredSessionId = ds.session.sessionId; const opsFreeze = deferSpawnDuringFreeze({ - sessionId: ds.session.sessionId, + sessionId: deferredSessionId, larkAppId: ds.larkAppId, replay: () => { - // The session may have been closed while parked; never revive it. - if (findActiveBySessionId(ds.session.sessionId) === ds) { + // Replay only when this ds is STILL the live session it was queued for: + // closed, replaced or transferred sessions must never be revived. + if ( + ds.session.sessionId === deferredSessionId + && findActiveBySessionId(deferredSessionId) === ds + ) { forkWorker(ds, promptInput, resumeOrTurnId); } }, diff --git a/src/worker.ts b/src/worker.ts index 041da6340..e4606af14 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -974,19 +974,32 @@ function provisionIsolatedBotHome( // rotate, so a copy taken mid-window hands this bot a token that is // already dead — and a bot that then refreshes on its own rotates the // shared refresh token out from under the script, which is the whole - // poisoning chain the freeze exists to prevent. Keep whatever copy this - // bot already has; the next cold spawn after release re-syncs. + // poisoning chain the freeze exists to prevent. + // + // The daemon's spawn gate does not make this check redundant: the freeze + // can begin AFTER forkWorker cleared it and before this line runs, so the + // decision has to be made here too. + // + // "Skip" can only mean "keep the copy this bot already has". With no copy + // at all — a brand-new isolated bot, or one whose file was removed — there + // is nothing to protect and skipping would start the CLI straight into a + // login screen. Seed it, preferring an unexpired credential so the + // script's fake-expired file is not what gets propagated. + const credPath = join(cdir, '.credentials.json'); + const hasCredCopy = existsSync(credPath); const credFreeze = activeSpawnFreezeFor(larkAppId); - if (credFreeze) { + let fresh: string | null; + if (!credFreeze) { + fresh = freshestClaudeCred(); + } else if (hasCredCopy) { log(`[read-isolation] spawn-freeze active (${credFreeze.reason}) — keeping existing per-bot credential copy`); + fresh = null; + } else { + fresh = freshestClaudeCred({ requireUnexpired: true }) ?? freshestClaudeCred(); + log(`[read-isolation] spawn-freeze active (${credFreeze.reason}) but this bot has no credential copy yet — seeding ${fresh ? 'the freshest unexpired credential' : 'nothing (none found)'}`); } - const fresh = credFreeze ? null : freshestClaudeCred(); - if (fresh) writeCredIfChanged(join(cdir, '.credentials.json'), fresh); - else if ( - !credFreeze - && !existsSync(join(cdir, '.credentials.json')) - && !claudeSettingsHasProviderAuth(isolatedSettingsPath) - ) { + if (fresh) writeCredIfChanged(credPath, fresh); + else if (!hasCredCopy && !claudeSettingsHasProviderAuth(isolatedSettingsPath)) { log(`[read-isolation] WARN no Claude provider auth found (global settings env, keychain, or ~/.claude/.credentials.json) — bot may hit login screen`); } // State: seed /.claude.json from the GLOBAL one MINUS `projects` (keeps the @@ -1000,11 +1013,15 @@ function provisionIsolatedBotHome( // elsewhere rotates the refresh token, which would strand a stale per-bot copy) — // skipped mid-freeze for the same reason as the Claude branch above. const authSrc = join(homedir(), '.codex', 'auth.json'); + const authDst = join(cdir, 'auth.json'); const codexFreeze = activeSpawnFreezeFor(larkAppId); - if (codexFreeze) { + // Same rule as the Claude branch: a freeze protects an EXISTING copy; with + // none yet there is nothing to protect and skipping would only produce a + // logged-out CLI. + if (codexFreeze && existsSync(authDst)) { log(`[read-isolation] spawn-freeze active (${codexFreeze.reason}) — keeping existing per-bot codex auth copy`); } else if (existsSync(authSrc)) { - writeCredIfChanged(join(cdir, 'auth.json'), readFileSync(authSrc, 'utf-8')); + writeCredIfChanged(authDst, readFileSync(authSrc, 'utf-8')); } // config.toml: seed ONCE (it may carry per-bot customizations afterwards). const cfgDst = join(cdir, 'config.toml'); @@ -1037,7 +1054,7 @@ function claudeSettingsHasProviderAuth(settingsPath: string): boolean { * `~/.claude/.credentials.json`, by `claudeAiOauth.expiresAt` (longest runway * wins — a re-login updates one of the two, and this picks whichever is newer). * Returns the raw credential JSON string, or null when neither source exists. */ -function freshestClaudeCred(): string | null { +function freshestClaudeCred(opts: { requireUnexpired?: boolean } = {}): string | null { const cands: { raw: string; exp: number }[] = []; const expOf = (raw: string): number => { try { return Number(JSON.parse(raw)?.claudeAiOauth?.expiresAt) || 0; } catch { return 0; } @@ -1067,9 +1084,15 @@ function freshestClaudeCred(): string | null { const raw = (r.stdout ?? '').trim(); if (raw && usable(raw)) cands.push({ raw, exp: expOf(raw) }); } catch { /* no keychain (non-mac) → skip candidate */ } - if (!cands.length) return null; - cands.sort((a, b) => b.exp - a.exp); - return cands[0].raw; + // `requireUnexpired` is only for the mid-freeze seed path: an expired token + // whose refresh token still works is a perfectly normal credential (and the + // default path must keep copying it, or a bot can never self-heal), but it is + // also exactly what the refresh script's fake-expire step produces — so the + // one case that must not copy it is seeding a fresh bot during a freeze. + const usableCands = opts.requireUnexpired ? cands.filter(c => c.exp > Date.now()) : cands; + if (!usableCands.length) return null; + usableCands.sort((a, b) => b.exp - a.exp); + return usableCands[0].raw; } /** Write a credential file (mode 0600) only when its content actually changed — diff --git a/test/spawn-freeze.test.ts b/test/spawn-freeze.test.ts index 5ef8b0993..93e78a19b 100644 --- a/test/spawn-freeze.test.ts +++ b/test/spawn-freeze.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, rmSync, symlinkSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; @@ -13,6 +13,7 @@ import { shouldAnnounceSpawnFreeze, SPAWN_FREEZE_FILENAME, SPAWN_FREEZE_HARD_CAP_MS, + SPAWN_FREEZE_MTIME_SKEW_MS, spawnFreezeApplies, writeSpawnFreeze, type SpawnFreezeDeclaration, @@ -138,6 +139,27 @@ describe('spawn freeze declaration', () => { expect(spawnFreezeApplies(fleet, undefined)).toBe(true); }); + it('refuses a future mtime instead of clamping it (a clamp would never expire)', () => { + const dir = freshDir(); + // `touch -t 2099…` on the declaration would otherwise re-anchor the hard cap + // on every read — a permanent, un-expiring freeze. + writeDeclaration(dir, { reason: 'stuck', deadline: Number.MAX_SAFE_INTEGER, pid: 1 }, + NOW + SPAWN_FREEZE_MTIME_SKEW_MS + 1_000); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + // Inside the skew tolerance it is still honored (clock drift is not an attack). + writeDeclaration(dir, { reason: 'ok', deadline: NOW + 60_000 }, NOW + 1_000); + expect(readActiveSpawnFreeze(deps(dir))).not.toBeNull(); + }); + + it('refuses a symlinked declaration (it would borrow another file\'s mtime)', () => { + const dir = freshDir(); + const target = join(dir, 'freeze-target.json'); + writeFileSync(target, JSON.stringify({ reason: 'stuck', deadline: Number.MAX_SAFE_INTEGER, pid: 1 })); + utimesSync(target, new Date(NOW + 10 * 365 * 24 * 3_600_000), new Date(NOW + 10 * 365 * 24 * 3_600_000)); + symlinkSync(target, join(dir, SPAWN_FREEZE_FILENAME)); + expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); + }); + it('clears idempotently', () => { const dir = freshDir(); writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); From 6f16c549943d10416dd78b8be8b7ba5083524705 Mon Sep 17 00:00:00 2001 From: xu4wang Date: Wed, 29 Jul 2026 02:02:12 +0700 Subject: [PATCH 3/6] =?UTF-8?q?fix(spawn-freeze):=20codex=20=E7=AC=AC?= =?UTF-8?q?=E4=BA=8C=E8=BD=AE=20=E2=80=94=E2=80=94=20=E5=86=BB=E7=BB=93?= =?UTF-8?q?=E6=9C=9F=E4=B8=80=E5=BE=8B=E4=B8=8D=E5=86=99=E5=87=AD=E8=AF=81?= =?UTF-8?q?=E5=89=AF=E6=9C=AC=EF=BC=9Bclear=20=E7=94=A8=20lstat=EF=BC=9Bde?= =?UTF-8?q?vice-isolation=20=E5=90=8C=E7=B1=BB=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 上一轮的 `requireUnexpired ?? 回退` 是假修复：所有候选都过期时回退分支会把 伪过期凭证照样种进去，日志还宣称种的是「未过期的那份」。改成规则更硬的一条: 【冻结期一律不写凭证副本】。因为「过期的源」到底是脚本的伪过期步骤（拷了就投毒） 还是机器闲置（拷了才自愈），从文件上无法判定 —— 不猜。 代价写进注释:首次 spawn 恰好撞进维护窗口的全新隔离 bot 可能撞登录页，解冻后 第一次 spawn 自动同步（上限就是冻结自身的时限）。一个 bot 短暂不可用 vs 共享账号 被轮换导致全队掉线,取舍不接近。requireUnexpired 参数随之删除。 2. clearSpawnFreeze 仍用 statSync,悬空符号链接会被判「不存在」而删不掉,与 reader 的 lstat 语义不一致 → 一并改 lstatSync。 3. device-isolation 那个更早的 deferred gate 有同一类缺陷（重放时重读可变 ds.session.sessionId）。它不是本次引入的,但就在本 PR 扩展的同一行上方,顺手用 同一个不可变 id 修掉,并把这条不变式写进 deferWorkerSpawnDuringDeviceIsolation 的文档注释。 Co-Authored-By: Claude Opus 5 --- src/core/device-isolation-activation.ts | 5 +++ src/core/spawn-freeze.ts | 6 ++- src/core/worker-pool.ts | 17 +++---- src/worker.ts | 59 ++++++++++++------------- 4 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/core/device-isolation-activation.ts b/src/core/device-isolation-activation.ts index 8de105761..b55f678ff 100644 --- a/src/core/device-isolation-activation.ts +++ b/src/core/device-isolation-activation.ts @@ -132,6 +132,11 @@ export function releaseDeviceIsolationFreeze(input: { * Queue only the first spawn request for a logical session. Later turns are * already retained by the session's normal pending-input machinery; replaying * multiple fork requests would instead kill and replace the first new worker. + * + * Callers must key on an IMMUTABLE session id captured at queue time and + * re-check it before forking: a repo switch replaces `ds.session` wholesale on + * the same session object, so a replay that re-reads it would accept an entry + * queued for the previous session. */ export function deferWorkerSpawnDuringDeviceIsolation( sessionId: string, diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts index 45a8c4ac7..01c67c295 100644 --- a/src/core/spawn-freeze.ts +++ b/src/core/spawn-freeze.ts @@ -41,7 +41,7 @@ * handed out before the boot can never cover. The two compose — `forkWorker` * defers when either one applies. */ -import { lstatSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { lstatSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { resolveBotmuxDataDir } from './data-dir.js'; @@ -344,7 +344,9 @@ export function writeSpawnFreeze( export function clearSpawnFreeze(deps: SpawnFreezeDeps = {}): boolean { const path = spawnFreezePath(deps); try { - statSync(path); + // lstat for the same reason the reader uses it: "exists" must include a + // dangling symlink, which statSync would report as absent and leave behind. + lstatSync(path); } catch { return false; } diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index cc3263d61..34468e920 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -2095,8 +2095,15 @@ export function forkWorker( // ANY session mutation or child fork. One deferred spawn per logical session // is replayed after the exact freeze lease is released; a session closed by // the activation transaction is deliberately not revived. - if (deferWorkerSpawnDuringDeviceIsolation(ds.session.sessionId, () => { - if (findActiveBySessionId(ds.session.sessionId) === ds) { + // Same immutable-identity rule as the ops freeze below: `ds.session` is + // replaced wholesale by a repo switch, so both the queue key and the replay + // guard must use the id captured here, never a re-read of `ds.session`. + const deferredSessionId = ds.session.sessionId; + if (deferWorkerSpawnDuringDeviceIsolation(deferredSessionId, () => { + if ( + ds.session.sessionId === deferredSessionId + && findActiveBySessionId(deferredSessionId) === ds + ) { forkWorker(ds, promptInput, resumeOrTurnId); } })) { @@ -2110,12 +2117,6 @@ export function forkWorker( // Deliberately only blocks new spawns: tearing down live CLIs is // `botmux suspend`'s job, and conflating the two would make every freeze a // fleet-wide cold restart. - // Capture the session id NOW: `ds.session` is replaced wholesale on a repo - // switch (command-handler swaps in the new Session on the SAME ds), so a - // closure that read `ds.session.sessionId` at replay time would accept an - // entry queued for the previous session and fork a second worker for the new - // one — killing and replacing the first. - const deferredSessionId = ds.session.sessionId; const opsFreeze = deferSpawnDuringFreeze({ sessionId: deferredSessionId, larkAppId: ds.larkAppId, diff --git a/src/worker.ts b/src/worker.ts index e4606af14..1154b0853 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -980,26 +980,30 @@ function provisionIsolatedBotHome( // can begin AFTER forkWorker cleared it and before this line runs, so the // decision has to be made here too. // - // "Skip" can only mean "keep the copy this bot already has". With no copy - // at all — a brand-new isolated bot, or one whose file was removed — there - // is nothing to protect and skipping would start the CLI straight into a - // login screen. Seed it, preferring an unexpired credential so the - // script's fake-expired file is not what gets propagated. + // The rule while frozen is simply "do not touch credential copies" — not + // "copy something safer". Trying to be clever here means deciding whether + // an expired source is the script's fake-expire step (copying it feeds the + // poisoning chain) or a genuinely idle host (copying it is the self-heal + // path), and that is not decidable from the file. So: a bot that already + // has a copy keeps it, and a bot that has none stays unseeded until the + // window closes. + // + // The cost is explicit: a brand-new isolated bot whose FIRST spawn lands + // inside a maintenance window can reach a login screen, and recovers on + // the next spawn after release (≤ the freeze's own bound). That is one bot + // briefly unusable versus a shared-account rotation that takes the whole + // fleet offline — the trade is not close. const credPath = join(cdir, '.credentials.json'); const hasCredCopy = existsSync(credPath); const credFreeze = activeSpawnFreezeFor(larkAppId); - let fresh: string | null; - if (!credFreeze) { - fresh = freshestClaudeCred(); - } else if (hasCredCopy) { - log(`[read-isolation] spawn-freeze active (${credFreeze.reason}) — keeping existing per-bot credential copy`); - fresh = null; - } else { - fresh = freshestClaudeCred({ requireUnexpired: true }) ?? freshestClaudeCred(); - log(`[read-isolation] spawn-freeze active (${credFreeze.reason}) but this bot has no credential copy yet — seeding ${fresh ? 'the freshest unexpired credential' : 'nothing (none found)'}`); + if (credFreeze) { + log(hasCredCopy + ? `[read-isolation] spawn-freeze active (${credFreeze.reason}) — keeping existing per-bot credential copy` + : `[read-isolation] WARN spawn-freeze active (${credFreeze.reason}) and this bot has no credential copy yet — NOT seeding one mid-window; it may hit a login screen and will sync on the first spawn after release`); } + const fresh = credFreeze ? null : freshestClaudeCred(); if (fresh) writeCredIfChanged(credPath, fresh); - else if (!hasCredCopy && !claudeSettingsHasProviderAuth(isolatedSettingsPath)) { + else if (!credFreeze && !hasCredCopy && !claudeSettingsHasProviderAuth(isolatedSettingsPath)) { log(`[read-isolation] WARN no Claude provider auth found (global settings env, keychain, or ~/.claude/.credentials.json) — bot may hit login screen`); } // State: seed /.claude.json from the GLOBAL one MINUS `projects` (keeps the @@ -1015,11 +1019,12 @@ function provisionIsolatedBotHome( const authSrc = join(homedir(), '.codex', 'auth.json'); const authDst = join(cdir, 'auth.json'); const codexFreeze = activeSpawnFreezeFor(larkAppId); - // Same rule as the Claude branch: a freeze protects an EXISTING copy; with - // none yet there is nothing to protect and skipping would only produce a - // logged-out CLI. - if (codexFreeze && existsSync(authDst)) { - log(`[read-isolation] spawn-freeze active (${codexFreeze.reason}) — keeping existing per-bot codex auth copy`); + // Same rule as the Claude branch, same reasoning: mid-window we do not + // write credential copies at all. + if (codexFreeze) { + log(existsSync(authDst) + ? `[read-isolation] spawn-freeze active (${codexFreeze.reason}) — keeping existing per-bot codex auth copy` + : `[read-isolation] WARN spawn-freeze active (${codexFreeze.reason}) and this bot has no codex auth copy yet — NOT seeding one mid-window; it will sync on the first spawn after release`); } else if (existsSync(authSrc)) { writeCredIfChanged(authDst, readFileSync(authSrc, 'utf-8')); } @@ -1054,7 +1059,7 @@ function claudeSettingsHasProviderAuth(settingsPath: string): boolean { * `~/.claude/.credentials.json`, by `claudeAiOauth.expiresAt` (longest runway * wins — a re-login updates one of the two, and this picks whichever is newer). * Returns the raw credential JSON string, or null when neither source exists. */ -function freshestClaudeCred(opts: { requireUnexpired?: boolean } = {}): string | null { +function freshestClaudeCred(): string | null { const cands: { raw: string; exp: number }[] = []; const expOf = (raw: string): number => { try { return Number(JSON.parse(raw)?.claudeAiOauth?.expiresAt) || 0; } catch { return 0; } @@ -1084,15 +1089,9 @@ function freshestClaudeCred(opts: { requireUnexpired?: boolean } = {}): string | const raw = (r.stdout ?? '').trim(); if (raw && usable(raw)) cands.push({ raw, exp: expOf(raw) }); } catch { /* no keychain (non-mac) → skip candidate */ } - // `requireUnexpired` is only for the mid-freeze seed path: an expired token - // whose refresh token still works is a perfectly normal credential (and the - // default path must keep copying it, or a bot can never self-heal), but it is - // also exactly what the refresh script's fake-expire step produces — so the - // one case that must not copy it is seeding a fresh bot during a freeze. - const usableCands = opts.requireUnexpired ? cands.filter(c => c.exp > Date.now()) : cands; - if (!usableCands.length) return null; - usableCands.sort((a, b) => b.exp - a.exp); - return usableCands[0].raw; + if (!cands.length) return null; + cands.sort((a, b) => b.exp - a.exp); + return cands[0].raw; } /** Write a credential file (mode 0600) only when its content actually changed — From 28e30f66a948aef9f41cfa1803065d65b07ab9b2 Mon Sep 17 00:00:00 2001 From: xu4wang Date: Wed, 29 Jul 2026 09:46:47 +0700 Subject: [PATCH 4/6] =?UTF-8?q?fix(spawn-freeze):=20=E9=87=87=E7=BA=B3=20r?= =?UTF-8?q?eview=20=E7=9A=84=20A=20=E6=96=B9=E6=A1=88=20=E2=80=94=E2=80=94?= =?UTF-8?q?=20=E4=B8=8D=E5=86=8D=E6=89=BF=E8=AF=BA=E3=80=8C=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E4=B8=8D=E4=B8=A2=E3=80=8D=EF=BC=8C=E5=B9=B6=E6=8A=8A?= =?UTF-8?q?=E9=9D=99=E9=BB=98=E4=B8=A2=E5=A4=B1=E5=8F=98=E6=88=90=E5=8F=AF?= =?UTF-8?q?=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两位 reviewer 的 P1 判断成立，我核过调用链：冻结期 forkWorker 提前 return，从不设 ds.worker，所以同会话第 2 条消息会走 daemon.ts 的 worker-null re-fork 分支，再被 sessionId 去重丢掉。我原来那句「later turns are already retained by the session's pending-input machinery」在冻结路径下是错的 —— 那套缓冲活在 worker 进程里，而 worker 从没起来。这是我写错的注释，不是既有行为的延续。 按定策走 A（不做 turn 缓冲/持久化，那会把闸门变成投递队列），但把限制说清、把静默变可见： - 模块头改写成精确的「保证什么、不保证什么」：每会话第一条延后重放；同会话后续消息 不排队、需重发；冻结期 daemon 重启会丢掉已暂存的 spawn（声明在盘上，闭包不在）。 - deferSpawnDuringFreeze 返回 { freeze, parked }。parked=false（本会话已有一条在排队） 时 forkWorker 打 WARN，--notify 开着就在该会话回一条「这条不会被自动处理，请重发」。 必须出声:后续 finishTurnReactions 会把 pending ✋ 批量翻成 ✅，不报就等于骗用户。 - 通知去重键从 chatId 改成 sessionAnchorId（投递锚点）:按群去重会让同群第二个话题里 等着的人一条提示都收不到。parked / dropped 两类各说一次。 顺带修 review 指出的输入与并发问题（纯校验，非新机制）： - CLI 所有 value flag 缺值一律 fail-fast，未知参数也拒绝。此前 `--reason --notify` 会把 reason 存成 "--notify"；更糟的是 `--bot` 缺值被静默忽略 → 退化成【冻结全队】。 - writeSpawnFreeze 拒绝覆盖仍生效的声明（--force 显式接管），clearSpawnFreeze 支持 ownerPid 只删自己那份。此前两个维护脚本重叠会互相解除保护:后写者顶掉前者 scope， 先结束者又把后者的删掉。刻意不做多记录 lease / scope union —— 重叠窗口不支持，报错就好。 过期声明不算冲突（否则一份被遗弃的文件会挡住下一次维护）。 测试 29 例（+并发拒绝覆盖、+按 owner pid 解冻、+parked/dropped 语义与通知分类）。 CLI 六种情况实机验证过:三种缺值/未知参数报用法、正常冻结、覆盖被拒、非 owner 解冻被拒。 Co-Authored-By: Claude Opus 5 --- src/cli.ts | 145 ++++++++++++++++++++++++++------------ src/core/spawn-freeze.ts | 135 ++++++++++++++++++++++++++++------- src/core/worker-pool.ts | 38 +++++++--- test/spawn-freeze.test.ts | 91 ++++++++++++++++++++---- 4 files changed, 313 insertions(+), 96 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 29b4a19c4..ce53c8f99 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -32,7 +32,7 @@ import { randomBytes } from 'node:crypto'; import { validateWorkingDir } from './core/working-dir.js'; import { resolveSessionContext } from './core/session-marker.js'; import { resolveBotmuxDataDir } from './core/data-dir.js'; -import { clearSpawnFreeze, readActiveSpawnFreeze, spawnFreezePath, writeSpawnFreeze, SPAWN_FREEZE_HARD_CAP_MS } from './core/spawn-freeze.js'; +import { clearSpawnFreeze, readActiveSpawnFreeze, spawnFreezePath, writeSpawnFreeze, SpawnFreezeConflictError, SPAWN_FREEZE_HARD_CAP_MS } from './core/spawn-freeze.js'; import { dashboardSecretPath } from './core/dashboard-secret.js'; import { acceptedDispatchBotAppIds, appendDispatchReportProtocol, appendLegacyDispatchReportProtocol, parseDispatchBotSpec, buildDispatchMessages, buildRepoPrimeText, buildReportContent, eligibleAutoMentionAliases, findDispatchRegistryEntry, offTopicSubBotTopic, resolveReportTarget, resolveSendTarget } from './core/dispatch.js'; import { pickTurnReplyTarget } from './core/reply-target.js'; @@ -4115,18 +4115,74 @@ function formatFreezeClock(epochMs: number): string { */ async function cmdFreeze(): Promise { const argv = process.argv.slice(3); - const flagValue = (name: string): string | undefined => { - const i = argv.indexOf(name); - return i >= 0 ? argv[i + 1] : undefined; + const usage = (): never => { + console.error('用法: botmux freeze --reason <说明> [--for 120s] [--pid ] [--notify] [--bot ]... [--force]'); + console.error(' botmux freeze --status | --release [--pid ]'); + console.error(' 维护期间禁止本机起新的 CLI 会话；每个会话的第一条消息解冻后自动重放'); + console.error(' ⚠️ 同一会话在窗口内的后续消息不会排队（会打日志/发提示，需恢复后重发）'); + console.error(' --for 默认 120s，支持 90s / 5m / 裸数字(秒)；上限 10m（daemon 侧硬截断）'); + console.error(' --pid 声明进程 pid（脚本里传 $$）：进程一死立即解冻；--release 带上它只删自己的声明'); + console.error(' --notify 冻结期收到消息时回一条「维护中」；建议 deadline > 30s 才开'); + console.error(' --bot 只冻某个 bot（可重复）；不给 = 全队'); + console.error(' --force 覆盖仍生效的其它声明（默认拒绝，避免两个维护脚本互相解除保护）'); + process.exit(1); + }; + + // 所有 value flag 走同一个取值器：缺值、或下一个 token 又是 flag，一律 fail-fast。 + // 静默容忍在这里是危险的 —— `--bot` 缺值曾会退化成「冻结全队」，`--reason --notify` + // 会把 reason 存成 "--notify"。 + const KNOWN_VALUE_FLAGS = new Set(['--reason', '--for', '--pid', '--bot']); + const KNOWN_BOOL_FLAGS = new Set(['--status', '--release', '--notify', '--force']); + const values = new Map(); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith('--')) { + console.error(`❌ 多余的参数: ${arg}`); + usage(); + } + if (KNOWN_BOOL_FLAGS.has(arg)) continue; + if (!KNOWN_VALUE_FLAGS.has(arg)) { + console.error(`❌ 未知参数: ${arg}`); + usage(); + } + const value = argv[i + 1]; + if (value === undefined || value.startsWith('--')) { + console.error(`❌ ${arg} 缺少取值`); + usage(); + } + const bucket = values.get(arg); + if (bucket) bucket.push(value); + else values.set(arg, [value]); + i++; + } + const single = (flag: string): string | undefined => { + const bucket = values.get(flag); + if (!bucket) return undefined; + if (bucket.length > 1) { + console.error(`❌ ${flag} 只能给一次`); + usage(); + } + return bucket[0]; }; + const release = argv.includes('--release'); const status = argv.includes('--status'); - if (release && status) { console.error('❌ --release 与 --status 不能同时用'); process.exit(1); } + const pidRaw = single('--pid'); + let pid: number | undefined; + if (pidRaw !== undefined) { + const parsed = Number(pidRaw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + console.error(`❌ --pid 非法: ${pidRaw}`); + process.exit(1); + } + pid = parsed; + } + if (status) { const freeze = readActiveSpawnFreeze(); if (!freeze) { @@ -4142,32 +4198,31 @@ async function cmdFreeze(): Promise { } console.log(` 范围 ${freeze.larkAppIds ? freeze.larkAppIds.join(', ') : '全部 bot'}`); console.log(` 声明进程 ${freeze.declaredByPid ?? '（未记录 pid，仅靠 deadline 兜底）'}`); - console.log(` 维护提示 ${freeze.notify ? '开（冻结期每个话题回一条）' : '关（静默延迟）'}`); + console.log(` 维护提示 ${freeze.notify ? '开（冻结期每个会话回一条）' : '关（静默延迟）'}`); console.log(` 文件 ${spawnFreezePath()}`); return; } if (release) { - const removed = clearSpawnFreeze(); - console.log(removed - ? '✓ 已解冻：被暂存的 spawn 会在各 daemon 的下一次轮询（≤1s）自动重放。' - : '· 本就没有冻结声明（目标态已达成）。'); + const result = clearSpawnFreeze({}, pid !== undefined ? { ownerPid: pid } : {}); + if (result === 'cleared') { + console.log('✓ 已解冻：被暂存的 spawn 会在各 daemon 的下一次轮询（≤1s）自动重放。'); + return; + } + if (result === 'absent') { + console.log('· 本就没有冻结声明（目标态已达成）。'); + return; + } + // not_owner：声明是别人写的。解冻别人的窗口比不解冻更危险，所以拒绝并明说。 + console.log(`· 当前声明不是 pid ${pid} 写的，未解冻（不删别人的冻结）。要强制解除：botmux freeze --release`); + process.exitCode = 1; return; } - const reason = flagValue('--reason'); - if (!reason) { - console.error('用法: botmux freeze --reason <说明> [--for 120s] [--pid ] [--notify] [--bot ]...'); - console.error(' botmux freeze --status | --release'); - console.error(' 维护期间禁止本机起新的 CLI 会话；被拦下的 spawn 解冻后自动重放（消息不丢）'); - console.error(' --for 默认 120s，支持 90s / 5m / 裸数字(秒)；上限 10m（daemon 侧硬截断）'); - console.error(' --pid 声明进程 pid（脚本里传 $$）：进程一死立即解冻，崩了也能自愈'); - console.error(' --notify 冻结期收到消息时回一条「维护中」；建议 deadline > 30s 才开'); - console.error(' --bot 只冻某个 bot（可重复）；不给 = 全队'); - process.exit(1); - } + const reason = single('--reason'); + if (!reason) usage(); - const durationRaw = flagValue('--for'); + const durationRaw = single('--for'); if (durationRaw !== undefined && parseFreezeDuration(durationRaw) === null) { console.error(`❌ --for 无法解析: ${durationRaw}（示例：90s / 5m / 120）`); process.exit(1); @@ -4178,31 +4233,27 @@ async function cmdFreeze(): Promise { durationMs = SPAWN_FREEZE_HARD_CAP_MS; } - const pidRaw = flagValue('--pid'); - let pid: number | undefined; - if (pidRaw !== undefined) { - const parsed = Number(pidRaw); - if (!Number.isSafeInteger(parsed) || parsed <= 0) { - console.error(`❌ --pid 非法: ${pidRaw}`); + const larkAppIds = values.get('--bot') ?? []; + const deadline = Date.now() + durationMs; + let path: string; + try { + path = writeSpawnFreeze({ + reason: reason!, + deadline, + ...(pid !== undefined ? { pid } : {}), + ...(argv.includes('--notify') ? { notify: true } : {}), + ...(larkAppIds.length ? { scope: { larkAppIds } } : {}), + }, {}, { force: argv.includes('--force') }); + } catch (err) { + if (err instanceof SpawnFreezeConflictError) { + const remaining = Math.max(0, Math.ceil((err.active.effectiveUntil - Date.now()) / 1000)); + console.error(`❌ 已有生效中的冻结声明（${err.active.reason}，剩 ${remaining}s，` + + `pid ${err.active.declaredByPid ?? '未记录'}）——未覆盖它。`); + console.error(' 两个维护窗口重叠时互相解除保护，所以默认拒绝。确实要接管：加 --force'); process.exit(1); } - pid = parsed; + throw err; } - - // --bot 可重复 - const larkAppIds: string[] = []; - argv.forEach((arg, i) => { - if (arg === '--bot' && argv[i + 1] && !argv[i + 1].startsWith('--')) larkAppIds.push(argv[i + 1]); - }); - - const deadline = Date.now() + durationMs; - const path = writeSpawnFreeze({ - reason, - deadline, - ...(pid !== undefined ? { pid } : {}), - ...(argv.includes('--notify') ? { notify: true } : {}), - ...(larkAppIds.length ? { scope: { larkAppIds } } : {}), - }); console.log(`✓ 已冻结（${reason}）：${larkAppIds.length ? larkAppIds.join(', ') : '全部 bot'} 在 ${formatFreezeClock(deadline)} 前不会起新 CLI。`); console.log(` 声明文件 ${path}`); if (pid === undefined) { @@ -4935,13 +4986,15 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 --bot 挂起该 bot 的全部活跃会话 --isolated 挂起所有读隔离 bot（凭证轮换后用；下次冷启动自动同步最新凭证） --dry-run 只列出目标，不执行 - freeze --reason <说明> 维护窗口内禁止本机起新 CLI 会话（被拦下的 spawn 解冻后自动重放，消息不丢） + freeze --reason <说明> 维护窗口内禁止本机起新 CLI 会话（每会话第一条消息解冻后自动重放； + 同会话后续消息不排队，会发提示并需重发） --for 120s 冻结时长（默认 120s，上限 10m） --pid 声明进程（脚本传 $$）：进程一死立即解冻 --notify 冻结期回一条「维护中」（deadline > 30s 建议开） --bot 只冻某个 bot（可重复）；不给 = 全队 + --force 覆盖仍生效的其它声明（默认拒绝） --status 查看当前是否冻结、为什么、还剩多久 - --release 解冻（幂等，可放脚本 trap 里） + --release 解冻（幂等；带 --pid 只删自己的声明，可放脚本 trap 里） slash "<斜杠命令>" 会话空闲后向本会话 CLI 注入一条原生斜杠命令（需 bots.json 配 tuiSlashAllow；/cd 恒被拒） role switch <目录> （会话内）切换本话题到角色库内的角色目录——角色切换用； 目录必须位于 ~/botmux-roles 之下 diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts index 01c67c295..10a673269 100644 --- a/src/core/spawn-freeze.ts +++ b/src/core/spawn-freeze.ts @@ -4,9 +4,27 @@ * A maintenance script — credential refresh, `claude` upgrade, workspace * rebuild — declares a freeze by writing `/spawn-freeze.json`, and * every daemon reads that declaration immediately before it would start a CLI. - * While the declaration is effective `forkWorker` queues at most one cold spawn - * per logical session and replays it on release, so a user's message is delayed - * instead of lost. + * While the declaration is effective `forkWorker` parks at most one cold spawn + * per logical session and replays it on release. + * + * What that does and does NOT guarantee, precisely — this is a delay, not a + * queue: + * + * - the FIRST message that would have started a CLI in a session is replayed + * after release, so it is delayed rather than dropped; + * - a SECOND message to that same session in the same window is NOT parked + * (replaying two forks would kill and replace the first new worker), and + * nothing else is holding it either: the worker never started, so the + * worker-side pre-ready input buffer that normally absorbs follow-up turns + * does not exist yet. Such a turn is reported — log, plus a chat notice when + * `notify` is on — and has to be re-sent; + * - a daemon restart inside the window loses the parked spawns. The + * declaration is on disk; the parked closures are not. + * + * Buffering whole turns durably (message id, attachments, reaction settlement, + * structured CLI input) would make this a delivery queue rather than a gate; + * that is deliberately out of scope. The limits above are surfaced instead of + * papered over — keep maintenance windows short. * * The file is data, never code: a declaration can only ask for "no new CLI * until T", it can never make the daemon execute anything. That keeps the @@ -243,6 +261,13 @@ export function activeSpawnFreezeFor( return spawnFreezeApplies(freeze, larkAppId) ? freeze : null; } +export interface SpawnFreezeDeferral { + freeze: ActiveSpawnFreeze; + /** False when this session already had a spawn parked — i.e. THIS request will + * not be replayed and the turn behind it is lost unless re-sent. */ + parked: boolean; +} + interface DeferredSpawn { larkAppId?: string; replay: () => void; @@ -277,25 +302,28 @@ function ensurePoll(deps: SpawnFreezeDeps = {}): void { } /** - * Park a cold spawn while a freeze applies, and return the freeze that caused - * it (null = not frozen, caller proceeds). + * Park a cold spawn while a freeze applies. Returns null when not frozen (the + * caller proceeds to spawn), otherwise the freeze plus whether THIS request is + * the one that got parked. * - * Only the FIRST request per logical session is queued: later turns are - * already retained by the session's own pending-input machinery, and replaying - * several fork requests would kill and replace the first new worker. Same - * invariant as `deferWorkerSpawnDuringDeviceIsolation`. + * Only the first request per logical session can be parked: replaying several + * fork requests would kill and replace the first new worker. Same invariant as + * `deferWorkerSpawnDuringDeviceIsolation` — and, as the module header spells + * out, a later turn is genuinely dropped rather than queued elsewhere, so + * `parked: false` is something the caller must surface, not swallow. */ export function deferSpawnDuringFreeze( input: { sessionId: string; larkAppId?: string; replay: () => void }, deps: SpawnFreezeDeps = {}, -): ActiveSpawnFreeze | null { +): SpawnFreezeDeferral | null { const freeze = activeSpawnFreezeFor(input.larkAppId, deps); if (!freeze) return null; - if (!deferredSpawns.has(input.sessionId)) { + const parked = !deferredSpawns.has(input.sessionId); + if (parked) { deferredSpawns.set(input.sessionId, { larkAppId: input.larkAppId, replay: input.replay }); } ensurePoll(deps); - return freeze; + return { freeze, parked }; } /** Drop a parked spawn without replaying it (session closed while frozen). */ @@ -309,29 +337,63 @@ export function deferredSpawnCount(): number { } /** - * One-shot gate for the "maintenance in progress" reply: true the first time a - * given chat asks during a given declaration. Without this a busy group gets - * one notice per message; with it the group is told once and then simply waits. + * One-shot gate for a "maintenance in progress" reply: true the first time a + * given anchor asks during a given declaration, per `kind`. Without it a busy + * chat gets one notice per message; with it it is told once and then waits. * State resets when the declaration changes, so the next window speaks again. + * + * `anchor` must be the same key the notice is delivered to (the session's reply + * anchor, NOT its chat id): keyed by chat, a second topic in the same group + * would be silenced even though a different person is waiting in it. + * + * `kind` separates the two things worth saying once each: "parked, it continues + * by itself" and "this one was dropped, re-send it". */ -export function shouldAnnounceSpawnFreeze(freeze: ActiveSpawnFreeze, chatKey: string): boolean { +export function shouldAnnounceSpawnFreeze( + freeze: ActiveSpawnFreeze, + anchor: string, + kind: 'parked' | 'dropped' = 'parked', +): boolean { if (!freeze.notify) return false; if (noticeFreezeId !== freeze.freezeId) { noticeFreezeId = freeze.freezeId; noticedChats.clear(); } - if (noticedChats.has(chatKey)) return false; - noticedChats.add(chatKey); + const key = `${kind}:${anchor}`; + if (noticedChats.has(key)) return false; + noticedChats.add(key); return true; } -/** Atomically publish a declaration. Same-directory temp + rename so a reader - * never observes a half-written file. */ +export class SpawnFreezeConflictError extends Error { + constructor(public readonly active: ActiveSpawnFreeze) { + super(`another spawn freeze is active: ${active.reason}`); + this.name = 'SpawnFreezeConflictError'; + } +} + +/** + * Atomically publish a declaration. Same-directory temp + rename so a reader + * never observes a half-written file. + * + * Refuses to clobber a declaration that is still in effect. There is exactly one + * declaration file, so an unconditional write let two overlapping maintenance + * scripts silently disarm each other — the later writer erasing the earlier + * one's scope, then the earlier one's cleanup deleting the later one's freeze. + * Making that a visible failure (so the second script decides what to do) is + * worth ~10 lines; a multi-record lease protocol is not, so overlapping windows + * are simply unsupported. + */ export function writeSpawnFreeze( declaration: SpawnFreezeDeclaration, deps: SpawnFreezeDeps = {}, + opts: { force?: boolean } = {}, ): string { const parsed = parseSpawnFreezeDeclaration(declaration); + if (!opts.force) { + const active = readActiveSpawnFreeze(deps); + if (active) throw new SpawnFreezeConflictError(active); + } const path = spawnFreezePath(deps); const tmp = `${path}.tmp-${process.pid}`; writeFileSync(tmp, `${JSON.stringify(parsed, null, 2)}\n`, { mode: 0o600 }); @@ -339,19 +401,42 @@ export function writeSpawnFreeze( return path; } -/** Remove the declaration. Idempotent — releasing a freeze that already - * expired must not fail a maintenance script's cleanup trap. */ -export function clearSpawnFreeze(deps: SpawnFreezeDeps = {}): boolean { +export type ClearSpawnFreezeResult = 'cleared' | 'absent' | 'not_owner'; + +/** + * Remove the declaration. Idempotent — releasing a freeze that already expired + * must not fail a maintenance script's cleanup trap. + * + * With `ownerPid`, only a declaration made by that pid is removed: a script's + * EXIT trap must never delete somebody else's freeze. A declaration written + * without a pid has no owner to check and stays removable by anyone. + */ +export function clearSpawnFreeze( + deps: SpawnFreezeDeps = {}, + opts: { ownerPid?: number } = {}, +): ClearSpawnFreezeResult { const path = spawnFreezePath(deps); try { // lstat for the same reason the reader uses it: "exists" must include a // dangling symlink, which statSync would report as absent and leave behind. lstatSync(path); } catch { - return false; + return 'absent'; + } + if (opts.ownerPid !== undefined) { + // Read the raw declaration rather than `readActiveSpawnFreeze`: an EXPIRED + // declaration must still be removable by its owner, and one we cannot parse + // carries no ownership claim to respect. + let declaredPid: number | undefined; + try { + declaredPid = parseSpawnFreezeDeclaration(JSON.parse(readFileSync(path, 'utf-8'))).pid; + } catch { + declaredPid = undefined; + } + if (declaredPid !== undefined && declaredPid !== opts.ownerPid) return 'not_owner'; } rmSync(path, { force: true }); - return true; + return 'cleared'; } /** Test-only: production state is driven entirely by the file and the timer. */ diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 34468e920..907e787bc 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -2132,27 +2132,43 @@ export function forkWorker( }, }); if (opsFreeze) { - const remainingSec = Math.max(1, Math.ceil((opsFreeze.effectiveUntil - Date.now()) / 1000)); - logger.info( - `[${tag(ds)}] worker spawn deferred by spawn-freeze (reason=${opsFreeze.reason}, ` - + `~${remainingSec}s left)`, - ); - const chatKey = ds.session.chatId ?? ds.session.sessionId; - if (shouldAnnounceSpawnFreeze(opsFreeze, chatKey)) { + const { freeze, parked } = opsFreeze; + const remainingSec = Math.max(1, Math.ceil((freeze.effectiveUntil - Date.now()) / 1000)); + const anchor = sessionAnchorId(ds); + // `parked: false` means this session already has a spawn waiting, so THIS + // turn will not be replayed — it is dropped. Say so loudly: the reaction + // bookkeeping settles pending turns when the replayed one finishes, so an + // unreported drop would show up to the user as "handled". + if (parked) { + logger.info( + `[${tag(ds)}] worker spawn parked by spawn-freeze (reason=${freeze.reason}, ` + + `~${remainingSec}s left)`, + ); + } else { + logger.warn( + `[${tag(ds)}] spawn-freeze (reason=${freeze.reason}) already holds a parked spawn for this ` + + `session — THIS turn will not be replayed and must be re-sent after release ` + + `(~${remainingSec}s left)`, + ); + } + if (shouldAnnounceSpawnFreeze(freeze, anchor, parked ? 'parked' : 'dropped')) { const gateTurnId = typeof resumeOrTurnId === 'string' ? resumeOrTurnId : (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null ? resumeOrTurnId.turnId : undefined); + const notice = parked + ? `🔧 维护中（${freeze.reason}），暂不启动新的 CLI 会话；约 ${remainingSec} 秒后自动继续处理这条消息。` + : `🔧 维护中（${freeze.reason}），这条消息不会被自动处理——本会话已有一条在排队。请在约 ${remainingSec} 秒后重发。`; try { void requireCallbacks().sessionReply( - sessionAnchorId(ds), - `🔧 维护中（${opsFreeze.reason}），暂不启动新的 CLI 会话；约 ${remainingSec} 秒后自动继续处理这条消息。`, + anchor, + notice, 'text', ds.larkAppId, fallbackTurnId(ds, gateTurnId), ).catch(err => logger.warn(`[${tag(ds)}] spawn-freeze notice failed: ${err?.message ?? err}`)); } catch { - // No callbacks wired (unit tests / early boot) — the delay is silent, - // which is strictly better than failing the gate itself. + // No callbacks wired (unit tests / early boot) — the log above is the + // only report, which is strictly better than failing the gate itself. } } return; diff --git a/test/spawn-freeze.test.ts b/test/spawn-freeze.test.ts index 93e78a19b..f3b613083 100644 --- a/test/spawn-freeze.test.ts +++ b/test/spawn-freeze.test.ts @@ -6,6 +6,7 @@ import { activeSpawnFreezeFor, clearSpawnFreeze, deferSpawnDuringFreeze, + SpawnFreezeConflictError, deferredSpawnCount, forgetDeferredSpawn, readActiveSpawnFreeze, @@ -46,7 +47,20 @@ function writeRaw(dir: string, body: string, mtimeMs = NOW): void { } function writeDeclaration(dir: string, declaration: SpawnFreezeDeclaration, mtimeMs = NOW): void { + // force: several tests deliberately replace an in-effect declaration. + writeSpawnFreeze(declaration, deps(dir), { force: true }); + stampMtime(dir, mtimeMs); +} + +/** Write WITHOUT force (the conflict check runs), then normalize mtime: the + * harness clock (`NOW`) is far in the past, so a real write time would read as + * a future mtime and be refused. */ +function writeUnforced(dir: string, declaration: SpawnFreezeDeclaration, mtimeMs = NOW): void { writeSpawnFreeze(declaration, deps(dir)); + stampMtime(dir, mtimeMs); +} + +function stampMtime(dir: string, mtimeMs: number): void { const path = join(dir, SPAWN_FREEZE_FILENAME); utimesSync(path, new Date(mtimeMs), new Date(mtimeMs)); } @@ -163,10 +177,46 @@ describe('spawn freeze declaration', () => { it('clears idempotently', () => { const dir = freshDir(); writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); - expect(clearSpawnFreeze(deps(dir))).toBe(true); - expect(clearSpawnFreeze(deps(dir))).toBe(false); + expect(clearSpawnFreeze(deps(dir))).toBe('cleared'); + expect(clearSpawnFreeze(deps(dir))).toBe('absent'); expect(readActiveSpawnFreeze(deps(dir))).toBeNull(); }); + + it('refuses to clobber an in-effect declaration unless forced', () => { + const dir = freshDir(); + // Two overlapping maintenance scripts would otherwise disarm each other: + // the later write erases the earlier scope, the earlier cleanup deletes the + // later freeze. + writeDeclaration(dir, { reason: 'first', deadline: NOW + 60_000, pid: 111 }); + expect(() => writeSpawnFreeze({ reason: 'second', deadline: NOW + 60_000 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('first'); + // --force is the explicit takeover… + writeDeclaration(dir, { reason: 'second', deadline: NOW + 60_000 }); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('second'); + // …and an EXPIRED declaration is not a conflict at all: an abandoned file + // must never block the next maintenance window. + writeDeclaration(dir, { reason: 'stale', deadline: NOW - 1 }); + writeUnforced(dir, { reason: 'third', deadline: NOW + 60_000 }); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('third'); + }); + + it('releases only its own declaration when an owner pid is given', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'theirs', deadline: NOW + 60_000, pid: 4242 }); + // A script's EXIT trap must not delete somebody else's freeze. + expect(clearSpawnFreeze(deps(dir), { ownerPid: 777 })).toBe('not_owner'); + expect(readActiveSpawnFreeze(deps(dir))).not.toBeNull(); + expect(clearSpawnFreeze(deps(dir), { ownerPid: 4242 })).toBe('cleared'); + + // An owner must still be able to clean up its own EXPIRED declaration. + writeDeclaration(dir, { reason: 'mine', deadline: NOW - 1, pid: 4242 }); + expect(clearSpawnFreeze(deps(dir), { ownerPid: 4242 })).toBe('cleared'); + + // No pid recorded → no ownership claim to respect. + writeDeclaration(dir, { reason: 'ownerless', deadline: NOW + 60_000 }); + expect(clearSpawnFreeze(deps(dir), { ownerPid: 999 })).toBe('cleared'); + }); }); describe('deferred spawns', () => { @@ -176,11 +226,16 @@ describe('deferred spawns', () => { const d = deps(dir); const replays: string[] = []; - expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('first') }, d)).not.toBeNull(); - // A second turn for the same session must NOT queue a second fork: replaying - // two forks would kill and replace the first new worker. - expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('second') }, d)).not.toBeNull(); - expect(deferSpawnDuringFreeze({ sessionId: 's2', replay: () => replays.push('other') }, d)).not.toBeNull(); + expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('first') }, d)) + .toMatchObject({ parked: true }); + // A second turn for the same session must NOT queue a second fork (replaying + // two forks would kill and replace the first new worker). `parked: false` is + // how the caller learns THIS turn is dropped and has to be reported — the + // barrier is a delay, not a queue. + expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('second') }, d)) + .toMatchObject({ parked: false }); + expect(deferSpawnDuringFreeze({ sessionId: 's2', replay: () => replays.push('other') }, d)) + .toMatchObject({ parked: true }); expect(deferredSpawnCount()).toBe(2); expect(replays).toEqual([]); @@ -200,7 +255,8 @@ describe('deferred spawns', () => { const d = deps(dir); const replays: string[] = []; - expect(deferSpawnDuringFreeze({ sessionId: 's1', larkAppId: 'cli_a', replay: () => replays.push('a') }, d)).not.toBeNull(); + expect(deferSpawnDuringFreeze({ sessionId: 's1', larkAppId: 'cli_a', replay: () => replays.push('a') }, d)) + .toMatchObject({ parked: true }); // Out of scope: never deferred in the first place. expect(deferSpawnDuringFreeze({ sessionId: 's2', larkAppId: 'cli_b', replay: () => replays.push('b') }, d)).toBeNull(); expect(deferredSpawnCount()).toBe(1); @@ -232,7 +288,8 @@ describe('deferred spawns', () => { const d = { dataDir: () => dir, now: () => now, processAlive: () => true, pollMs: POLL_MS }; const replays: string[] = []; - expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('s1') }, d)).not.toBeNull(); + expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('s1') }, d)) + .toMatchObject({ parked: true }); now = NOW + 2_001; await waitForPolls(); @@ -245,18 +302,24 @@ describe('freeze announcements', () => { const dir = freshDir(); writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 300_000, notify: true }); const first = readActiveSpawnFreeze(deps(dir))!; - expect(shouldAnnounceSpawnFreeze(first, 'chat-1')).toBe(true); - expect(shouldAnnounceSpawnFreeze(first, 'chat-1')).toBe(false); - expect(shouldAnnounceSpawnFreeze(first, 'chat-2')).toBe(true); + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1')).toBe(true); + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1')).toBe(false); + // A different session in the same group has its own anchor and must still be + // told — keying by chat id would silence whoever is waiting in it. + expect(shouldAnnounceSpawnFreeze(first, 'anchor-2')).toBe(true); + // "parked" and "dropped" are different messages, one each. + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1', 'dropped')).toBe(true); + expect(shouldAnnounceSpawnFreeze(first, 'anchor-1', 'dropped')).toBe(false); // A NEW declaration is a new window: the same chat is told again. writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 400_000, notify: true }); const second = readActiveSpawnFreeze(deps(dir))!; expect(second.freezeId).not.toBe(first.freezeId); - expect(shouldAnnounceSpawnFreeze(second, 'chat-1')).toBe(true); + expect(shouldAnnounceSpawnFreeze(second, 'anchor-1')).toBe(true); writeDeclaration(dir, { reason: 'quiet', deadline: NOW + 60_000 }); const quiet = readActiveSpawnFreeze(deps(dir))!; - expect(shouldAnnounceSpawnFreeze(quiet, 'chat-1')).toBe(false); + expect(shouldAnnounceSpawnFreeze(quiet, 'anchor-1')).toBe(false); + expect(shouldAnnounceSpawnFreeze(quiet, 'anchor-1', 'dropped')).toBe(false); }); }); From c4e69cb072f8112ce90dd6c0c7aaf1d0b1d7a930 Mon Sep 17 00:00:00 2001 From: xu4wang Date: Wed, 29 Jul 2026 09:51:17 +0700 Subject: [PATCH 5/6] =?UTF-8?q?fix(spawn-freeze):=20codex=20=E7=AC=AC?= =?UTF-8?q?=E5=9B=9B=E8=BD=AE=20=E2=80=94=E2=80=94=20=E5=86=99=E5=85=A5?= =?UTF-8?q?=E6=94=B9=E5=8E=9F=E5=AD=90=20CAS=E3=80=81=E5=90=8C=20owner=20?= =?UTF-8?q?=E5=8F=AF=E7=BB=AD=E6=9C=9F=E3=80=81=E7=A9=BA=20prompt=20?= =?UTF-8?q?=E4=B8=8D=E5=8D=A0=E6=8E=92=E9=98=9F=E4=BD=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 并发保护原来是 TOCTOU（先 read 再 rename），两个脚本同时通过检查仍会互相覆盖。 改成 link() 原子创建 + 有界重试:link 在目标存在时失败,所以「检查 + 发布」是一步。 实测 10 个并发写入者:修前 10 个全成功,修后恰好 1 个成功。 ⚠️ 同 owner 判定必须两边都写了 pid —— undefined === undefined 会让每个匿名写入者都 能替换别人（正是上面 10 个全赢的原因）。 2. 同一个脚本续期/修改自己的窗口原来会被自己挡住（只能 --force）。现在同 pid 直接放行, 不同 owner 或任一方匿名一律冲突。过期声明照旧不算冲突。 3. 空 prompt 的 spawn（restore 重连 / 预热）会占掉该会话唯一的排队位,导致紧随其后的 【第一条真实用户消息】被丢 —— 文档声称的「每会话第一条会重放」因此不恒真。现在冻结期 直接跳过这类 spawn:会话保持 worker-less,下条消息冷启动,与 suspend 后的行为一致。 4. --status / --release 不再静默接受 --reason/--for/--bot/--notify/--force。 另外把「/adopt 会话不在闸门范围内」写进模块头:那些 CLI 是用户自己起的,重连不产生新 CLI, 维护窗口没有要保护的东西。ownerPid 遇到「合法但无 pid」的声明仍会删除 —— 没写 pid 就没有 所有权主张,这是明写的取舍,不是遗漏。 测试 30 例（+同 owner 续期 / 匿名写入者互不相认 / 不同 owner 冲突）。 Co-Authored-By: Claude Opus 5 --- src/cli.ts | 9 +++++ src/core/spawn-freeze.ts | 78 ++++++++++++++++++++++++++++++--------- src/core/worker-pool.ts | 20 +++++++++- test/spawn-freeze.test.ts | 21 +++++++++++ 4 files changed, 110 insertions(+), 18 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index ce53c8f99..da6474550 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -4171,6 +4171,15 @@ async function cmdFreeze(): Promise { console.error('❌ --release 与 --status 不能同时用'); process.exit(1); } + // 静默忽略不相容的参数会让人以为自己冻结/解冻成了别的样子。 + if (status || release) { + const irrelevant = ['--reason', '--for', '--bot', '--notify', '--force'] + .filter(flag => argv.includes(flag)); + if (irrelevant.length) { + console.error(`❌ ${status ? '--status' : '--release'} 不接受 ${irrelevant.join(' / ')}`); + usage(); + } + } const pidRaw = single('--pid'); let pid: number | undefined; diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts index 10a673269..480d2bb18 100644 --- a/src/core/spawn-freeze.ts +++ b/src/core/spawn-freeze.ts @@ -12,6 +12,8 @@ * * - the FIRST message that would have started a CLI in a session is replayed * after release, so it is delayed rather than dropped; + * - a promptless spawn (restore re-attach / warm-up) is skipped rather than + * parked, so it cannot consume the slot that the first real turn needs; * - a SECOND message to that same session in the same window is NOT parked * (replaying two forks would kill and replace the first new worker), and * nothing else is holding it either: the worker never started, so the @@ -26,6 +28,10 @@ * that is deliberately out of scope. The limits above are surfaced instead of * papered over — keep maintenance windows short. * + * Out of scope by definition: `/adopt` sessions. Botmux never started those + * CLIs (the user did, outside botmux) and re-attaching to one starts no new CLI, + * so a maintenance window has nothing to protect there. + * * The file is data, never code: a declaration can only ask for "no new CLI * until T", it can never make the daemon execute anything. That keeps the * barrier usable by plain shell scripts and keeps its blast radius shaped like @@ -59,7 +65,7 @@ * handed out before the boot can never cover. The two compose — `forkWorker` * defers when either one applies. */ -import { lstatSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { linkSync, lstatSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { resolveBotmuxDataDir } from './data-dir.js'; @@ -373,16 +379,28 @@ export class SpawnFreezeConflictError extends Error { } /** - * Atomically publish a declaration. Same-directory temp + rename so a reader - * never observes a half-written file. + * Publish a declaration. A reader never observes a half-written file: the + * content is written to a same-directory temp and then linked/renamed into + * place. + * + * Refuses to clobber a declaration that is still in effect and belongs to + * somebody else. There is exactly one declaration file, so an unconditional + * write let two overlapping maintenance scripts silently disarm each other — + * the later writer erasing the earlier one's scope, then the earlier one's + * cleanup deleting the later one's freeze. Making that a visible failure (so the + * second script decides what to do) is worth the code; a multi-record lease + * protocol is not, so overlapping windows from DIFFERENT owners stay + * unsupported. * - * Refuses to clobber a declaration that is still in effect. There is exactly one - * declaration file, so an unconditional write let two overlapping maintenance - * scripts silently disarm each other — the later writer erasing the earlier - * one's scope, then the earlier one's cleanup deleting the later one's freeze. - * Making that a visible failure (so the second script decides what to do) is - * worth ~10 lines; a multi-record lease protocol is not, so overlapping windows - * are simply unsupported. + * Three cases are allowed through: + * - nothing in effect (missing / expired / dead declarer / unparseable); + * - same `pid` — an owner extending or amending its own window (both sides must + * name a pid; two anonymous declarations are two owners, not one); + * - `force` — an explicit operator takeover. + * + * The create path uses `link()`, which fails if the destination exists, so the + * check and the publish are one atomic step rather than a TOCTOU pair: two + * scripts racing from scratch cannot both win. */ export function writeSpawnFreeze( declaration: SpawnFreezeDeclaration, @@ -390,15 +408,41 @@ export function writeSpawnFreeze( opts: { force?: boolean } = {}, ): string { const parsed = parseSpawnFreezeDeclaration(declaration); - if (!opts.force) { - const active = readActiveSpawnFreeze(deps); - if (active) throw new SpawnFreezeConflictError(active); - } const path = spawnFreezePath(deps); + const body = `${JSON.stringify(parsed, null, 2)}\n`; const tmp = `${path}.tmp-${process.pid}`; - writeFileSync(tmp, `${JSON.stringify(parsed, null, 2)}\n`, { mode: 0o600 }); - renameSync(tmp, path); - return path; + writeFileSync(tmp, body, { mode: 0o600 }); + try { + if (opts.force) { + renameSync(tmp, path); + return path; + } + // Bounded retry: each round either wins the atomic create, hands back a + // real conflict, or removes exactly one superseded declaration. + for (let attempt = 0; attempt < 3; attempt++) { + try { + linkSync(tmp, path); + return path; + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== 'EEXIST') throw error; + } + const active = readActiveSpawnFreeze(deps); + // Same owner requires BOTH sides to actually name one: two declarations + // that simply omit `pid` are not "the same owner", they are two anonymous + // writers, and treating undefined === undefined as a match let every racer + // replace every other one (verified: 10 concurrent writers all succeeded). + const sameOwner = active !== null + && parsed.pid !== undefined + && active.declaredByPid === parsed.pid; + if (active && !sameOwner) throw new SpawnFreezeConflictError(active); + // Ours to replace (same owner), or nothing in effect: drop the stale file + // and try the atomic create again. + try { unlinkSync(path); } catch { /* somebody else removed it first */ } + } + throw new Error('spawn-freeze declaration is being rewritten concurrently'); + } finally { + try { unlinkSync(tmp); } catch { /* already linked away or gone */ } + } } export type ClearSpawnFreezeResult = 'cleared' | 'absent' | 'not_owner'; diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 907e787bc..8d9cc564a 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -106,7 +106,7 @@ import { isSilentScheduledTurn } from './silent-schedule-turns.js'; import { isTriggerFinalSuppressed } from './trigger-final-suppression.js'; import { writeDeferredTopicBinding } from './deferred-topic-binding.js'; import { deferWorkerSpawnDuringDeviceIsolation } from './device-isolation-activation.js'; -import { deferSpawnDuringFreeze, forgetDeferredSpawn, shouldAnnounceSpawnFreeze } from './spawn-freeze.js'; +import { activeSpawnFreezeFor, deferSpawnDuringFreeze, forgetDeferredSpawn, shouldAnnounceSpawnFreeze } from './spawn-freeze.js'; import { buildBotmuxLarkNativeSessionTitle, extractBotmuxLarkNativeSessionTitlePrompt, @@ -2117,6 +2117,24 @@ export function forkWorker( // Deliberately only blocks new spawns: tearing down live CLIs is // `botmux suspend`'s job, and conflating the two would make every freeze a // fleet-wide cold restart. + // A promptless spawn (restore re-attach, warm-up) carries nothing to deliver, + // so it must NOT occupy this session's single parked slot: doing so would make + // the freeze drop the FIRST real user turn that arrives behind it. During a + // window such a spawn is simply skipped — the session stays worker-less and + // cold-resumes on its next message, exactly as after `botmux suspend`. + const gatePayloadEmpty = typeof promptInput === 'string' + ? promptInput.trim() === '' + : promptInput.content.trim() === '' && !promptInput.codexAppInput; + if (gatePayloadEmpty) { + const skipFreeze = activeSpawnFreezeFor(ds.larkAppId); + if (skipFreeze) { + logger.info( + `[${tag(ds)}] promptless worker spawn skipped during spawn-freeze ` + + `(reason=${skipFreeze.reason}) — session stays worker-less and cold-resumes later`, + ); + return; + } + } const opsFreeze = deferSpawnDuringFreeze({ sessionId: deferredSessionId, larkAppId: ds.larkAppId, diff --git a/test/spawn-freeze.test.ts b/test/spawn-freeze.test.ts index f3b613083..af2052033 100644 --- a/test/spawn-freeze.test.ts +++ b/test/spawn-freeze.test.ts @@ -201,6 +201,27 @@ describe('spawn freeze declaration', () => { expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('third'); }); + it('lets the same owner update its own window, and stays atomic against a racer', () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000, pid: 111 }); + // Extending / amending your own declaration must not require --force. + writeUnforced(dir, { reason: 'cred-refresh', deadline: NOW + 120_000, pid: 111 }); + expect(readActiveSpawnFreeze(deps(dir))!.deadline).toBe(NOW + 120_000); + // A different owner is still refused. + expect(() => writeSpawnFreeze({ reason: 'other', deadline: NOW + 60_000, pid: 222 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + // …and an ownerless declaration is not the same owner as a pid-bearing one. + expect(() => writeSpawnFreeze({ reason: 'other', deadline: NOW + 60_000 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + // Two ownerless declarations are two owners, not one — otherwise every + // anonymous writer replaces every other one (10 concurrent writers all won). + clearSpawnFreeze(deps(dir)); + writeDeclaration(dir, { reason: 'anon-1', deadline: NOW + 60_000 }); + expect(() => writeSpawnFreeze({ reason: 'anon-2', deadline: NOW + 60_000 }, deps(dir))) + .toThrow(SpawnFreezeConflictError); + expect(readActiveSpawnFreeze(deps(dir))!.reason).toBe('anon-1'); + }); + it('releases only its own declaration when an owner pid is given', () => { const dir = freshDir(); writeDeclaration(dir, { reason: 'theirs', deadline: NOW + 60_000, pid: 4242 }); From d23f5d1600be6a7ad957f884f8644365fddd9c25 Mon Sep 17 00:00:00 2001 From: xu4wang Date: Wed, 29 Jul 2026 09:54:43 +0700 Subject: [PATCH 6/6] =?UTF-8?q?fix(spawn-freeze):=20codex=20=E7=AC=AC?= =?UTF-8?q?=E4=BA=94=E8=BD=AE=20=E2=80=94=E2=80=94=20=E7=A9=BA=20prompt=20?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E3=80=8C=E5=8F=AF=E8=A2=AB=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=20turn=20=E9=A1=B6=E6=8E=89=E3=80=8D=E8=80=8C=E4=B8=8D?= =?UTF-8?q?=E6=98=AF=E8=B7=B3=E8=BF=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一轮为了不让预热 spawn 占掉排队位，直接在冻结期跳过空 prompt 的 spawn。codex 指出这会 悄悄搞坏真实路径:command-handler 的 pendingRawInput 正是先 forkWorker(ds,'',false)、 等 prompt_ready 再把命令写进 PTY;dashboard 唤醒/web 终端也走空 prompt，跳过后它们会报 成功却没有 worker，终端等 10 秒超时。 改成排队位有优先级:空 prompt 照常入队（它们确实需要 CLI），但携带用户 turn 的请求会 【顶掉】它（superseded，日志记一笔）；反之不成立，后来的空 prompt 不会顶掉真实 turn。 这样既不丢唤醒/raw 命令，也保住「窗口内第一条真实消息会被重放」。 另外两处： - writeFileSync 移进 try:半途写失败（磁盘满/EIO）也不会留下 tmp。 - --status 不再静默接受 --pid（此前被忽略）。 pid 的定位在模块头写清:它只是协作标识、不是安全边界（只校验存活;能写这个文件的人本来 也能 --force）。它的作用是让运维自己的多个脚本别互相解除保护，仅此而已。 测试 31 例（+顶替语义:预热入队 → 真实 turn 顶掉 → 后续空 prompt 顶不掉，解冻只重放真实那条）。 Co-Authored-By: Claude Opus 5 --- src/cli.ts | 1 + src/core/spawn-freeze.ts | 44 +++++++++++++++++++++++++++++++-------- src/core/worker-pool.ts | 31 ++++++++++----------------- test/spawn-freeze.test.ts | 37 +++++++++++++++++++++++++------- 4 files changed, 77 insertions(+), 36 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index da6474550..e49b16794 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -4174,6 +4174,7 @@ async function cmdFreeze(): Promise { // 静默忽略不相容的参数会让人以为自己冻结/解冻成了别的样子。 if (status || release) { const irrelevant = ['--reason', '--for', '--bot', '--notify', '--force'] + .concat(status ? ['--pid'] : []) // --pid 只对 acquire 与 --release 有意义 .filter(flag => argv.includes(flag)); if (irrelevant.length) { console.error(`❌ ${status ? '--status' : '--release'} 不接受 ${irrelevant.join(' / ')}`); diff --git a/src/core/spawn-freeze.ts b/src/core/spawn-freeze.ts index 480d2bb18..78c33b43d 100644 --- a/src/core/spawn-freeze.ts +++ b/src/core/spawn-freeze.ts @@ -12,8 +12,10 @@ * * - the FIRST message that would have started a CLI in a session is replayed * after release, so it is delayed rather than dropped; - * - a promptless spawn (restore re-attach / warm-up) is skipped rather than - * parked, so it cannot consume the slot that the first real turn needs; + * - a promptless spawn (restore re-attach, warm-up, a start that only exists so + * a queued raw command can be written once the CLI is ready) parks like any + * other, but a payload-bearing turn SUPERSEDES it, so a warm-up cannot + * consume the slot the first real turn needs; * - a SECOND message to that same session in the same window is NOT parked * (replaying two forks would kill and replace the first new worker), and * nothing else is holding it either: the worker never started, so the @@ -28,6 +30,11 @@ * that is deliberately out of scope. The limits above are surfaced instead of * papered over — keep maintenance windows short. * + * `pid` is a cooperative identity, not a security boundary: it is only checked + * for liveness, so a same-user caller could name somebody else's pid (and one + * that can write this file could equally pass `--force`). It exists to keep an + * operator's own scripts from disarming each other, nothing more. + * * Out of scope by definition: `/adopt` sessions. Botmux never started those * CLIs (the user did, outside botmux) and re-attaching to one starts no new CLI, * so a maintenance window has nothing to protect there. @@ -269,13 +276,21 @@ export function activeSpawnFreezeFor( export interface SpawnFreezeDeferral { freeze: ActiveSpawnFreeze; - /** False when this session already had a spawn parked — i.e. THIS request will - * not be replayed and the turn behind it is lost unless re-sent. */ + /** False when this session already had a payload-bearing spawn parked — i.e. + * THIS request will not be replayed and the turn behind it is lost unless + * re-sent. */ parked: boolean; + /** True when parking this request displaced a promptless one (warm-up / + * re-attach). Nothing is lost — that spawn carried no turn — but it is worth + * a log line. */ + superseded?: boolean; } interface DeferredSpawn { larkAppId?: string; + /** A promptless spawn (empty prompt, no structured input) carries no user turn + * and may be displaced by one that does. */ + hasPayload: boolean; replay: () => void; } @@ -319,17 +334,26 @@ function ensurePoll(deps: SpawnFreezeDeps = {}): void { * `parked: false` is something the caller must surface, not swallow. */ export function deferSpawnDuringFreeze( - input: { sessionId: string; larkAppId?: string; replay: () => void }, + input: { sessionId: string; larkAppId?: string; hasPayload: boolean; replay: () => void }, deps: SpawnFreezeDeps = {}, ): SpawnFreezeDeferral | null { const freeze = activeSpawnFreezeFor(input.larkAppId, deps); if (!freeze) return null; - const parked = !deferredSpawns.has(input.sessionId); + const existing = deferredSpawns.get(input.sessionId); + // A promptless spawn still has to be parked — some of them exist precisely so + // a queued raw command or a dashboard wake gets a CLI — but it must not shut + // out the first real turn, so a payload-bearing request displaces it. + const superseded = existing !== undefined && !existing.hasPayload && input.hasPayload; + const parked = existing === undefined || superseded; if (parked) { - deferredSpawns.set(input.sessionId, { larkAppId: input.larkAppId, replay: input.replay }); + deferredSpawns.set(input.sessionId, { + larkAppId: input.larkAppId, + hasPayload: input.hasPayload, + replay: input.replay, + }); } ensurePoll(deps); - return { freeze, parked }; + return { freeze, parked, ...(superseded ? { superseded: true } : {}) }; } /** Drop a parked spawn without replaying it (session closed while frozen). */ @@ -411,8 +435,10 @@ export function writeSpawnFreeze( const path = spawnFreezePath(deps); const body = `${JSON.stringify(parsed, null, 2)}\n`; const tmp = `${path}.tmp-${process.pid}`; - writeFileSync(tmp, body, { mode: 0o600 }); try { + // Inside the try: a partial write (disk full, EIO) must not leave the temp + // behind either. + writeFileSync(tmp, body, { mode: 0o600 }); if (opts.force) { renameSync(tmp, path); return path; diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 8d9cc564a..883449547 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -106,7 +106,7 @@ import { isSilentScheduledTurn } from './silent-schedule-turns.js'; import { isTriggerFinalSuppressed } from './trigger-final-suppression.js'; import { writeDeferredTopicBinding } from './deferred-topic-binding.js'; import { deferWorkerSpawnDuringDeviceIsolation } from './device-isolation-activation.js'; -import { activeSpawnFreezeFor, deferSpawnDuringFreeze, forgetDeferredSpawn, shouldAnnounceSpawnFreeze } from './spawn-freeze.js'; +import { deferSpawnDuringFreeze, forgetDeferredSpawn, shouldAnnounceSpawnFreeze } from './spawn-freeze.js'; import { buildBotmuxLarkNativeSessionTitle, extractBotmuxLarkNativeSessionTitlePrompt, @@ -2117,27 +2117,18 @@ export function forkWorker( // Deliberately only blocks new spawns: tearing down live CLIs is // `botmux suspend`'s job, and conflating the two would make every freeze a // fleet-wide cold restart. - // A promptless spawn (restore re-attach, warm-up) carries nothing to deliver, - // so it must NOT occupy this session's single parked slot: doing so would make - // the freeze drop the FIRST real user turn that arrives behind it. During a - // window such a spawn is simply skipped — the session stays worker-less and - // cold-resumes on its next message, exactly as after `botmux suspend`. - const gatePayloadEmpty = typeof promptInput === 'string' - ? promptInput.trim() === '' - : promptInput.content.trim() === '' && !promptInput.codexAppInput; - if (gatePayloadEmpty) { - const skipFreeze = activeSpawnFreezeFor(ds.larkAppId); - if (skipFreeze) { - logger.info( - `[${tag(ds)}] promptless worker spawn skipped during spawn-freeze ` - + `(reason=${skipFreeze.reason}) — session stays worker-less and cold-resumes later`, - ); - return; - } - } + // Whether this spawn carries a user turn. A promptless one (restore re-attach, + // warm-up, a start whose only job is to let a queued raw command reach a ready + // CLI) is still parked — skipping it outright would silently strand dashboard + // wakes and `pendingRawInput` — but it must not shut out the first real turn, + // so the gate lets a payload-bearing request displace it. + const gateHasPayload = typeof promptInput === 'string' + ? promptInput.trim() !== '' + : promptInput.content.trim() !== '' || !!promptInput.codexAppInput; const opsFreeze = deferSpawnDuringFreeze({ sessionId: deferredSessionId, larkAppId: ds.larkAppId, + hasPayload: gateHasPayload, replay: () => { // Replay only when this ds is STILL the live session it was queued for: // closed, replaced or transferred sessions must never be revived. @@ -2160,7 +2151,7 @@ export function forkWorker( if (parked) { logger.info( `[${tag(ds)}] worker spawn parked by spawn-freeze (reason=${freeze.reason}, ` - + `~${remainingSec}s left)`, + + `~${remainingSec}s left${opsFreeze.superseded ? ', superseded a promptless spawn' : ''})`, ); } else { logger.warn( diff --git a/test/spawn-freeze.test.ts b/test/spawn-freeze.test.ts index af2052033..89d36be93 100644 --- a/test/spawn-freeze.test.ts +++ b/test/spawn-freeze.test.ts @@ -247,15 +247,15 @@ describe('deferred spawns', () => { const d = deps(dir); const replays: string[] = []; - expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('first') }, d)) + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('first') }, d)) .toMatchObject({ parked: true }); // A second turn for the same session must NOT queue a second fork (replaying // two forks would kill and replace the first new worker). `parked: false` is // how the caller learns THIS turn is dropped and has to be reported — the // barrier is a delay, not a queue. - expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('second') }, d)) + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('second') }, d)) .toMatchObject({ parked: false }); - expect(deferSpawnDuringFreeze({ sessionId: 's2', replay: () => replays.push('other') }, d)) + expect(deferSpawnDuringFreeze({ sessionId: 's2', hasPayload: true, replay: () => replays.push('other') }, d)) .toMatchObject({ parked: true }); expect(deferredSpawnCount()).toBe(2); expect(replays).toEqual([]); @@ -276,10 +276,10 @@ describe('deferred spawns', () => { const d = deps(dir); const replays: string[] = []; - expect(deferSpawnDuringFreeze({ sessionId: 's1', larkAppId: 'cli_a', replay: () => replays.push('a') }, d)) + expect(deferSpawnDuringFreeze({ sessionId: 's1', larkAppId: 'cli_a', hasPayload: true, replay: () => replays.push('a') }, d)) .toMatchObject({ parked: true }); // Out of scope: never deferred in the first place. - expect(deferSpawnDuringFreeze({ sessionId: 's2', larkAppId: 'cli_b', replay: () => replays.push('b') }, d)).toBeNull(); + expect(deferSpawnDuringFreeze({ sessionId: 's2', larkAppId: 'cli_b', hasPayload: true, replay: () => replays.push('b') }, d)).toBeNull(); expect(deferredSpawnCount()).toBe(1); clearSpawnFreeze(d); @@ -287,13 +287,36 @@ describe('deferred spawns', () => { expect(replays).toEqual(['a']); }); + it('lets a real turn supersede a parked promptless spawn, but never the reverse', async () => { + const dir = freshDir(); + writeDeclaration(dir, { reason: 'claude-update', deadline: NOW + 60_000 }); + const d = deps(dir); + + const replays: string[] = []; + // A warm-up / re-attach parks (some of those exist so a queued raw command + // reaches a ready CLI — skipping them outright would strand it)… + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: false, replay: () => replays.push('warmup') }, d)) + .toMatchObject({ parked: true }); + // …but the first real turn takes the slot from it. + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('real') }, d)) + .toMatchObject({ parked: true, superseded: true }); + // A later promptless spawn must NOT displace the real turn. + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: false, replay: () => replays.push('warmup2') }, d)) + .toMatchObject({ parked: false }); + expect(deferredSpawnCount()).toBe(1); + + clearSpawnFreeze(d); + await waitForPolls(); + expect(replays).toEqual(['real']); + }); + it('drops a parked spawn when its session is closed', async () => { const dir = freshDir(); writeDeclaration(dir, { reason: 'cred-refresh', deadline: NOW + 60_000 }); const d = deps(dir); const replays: string[] = []; - deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('s1') }, d); + deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('s1') }, d); forgetDeferredSpawn('s1'); expect(deferredSpawnCount()).toBe(0); @@ -309,7 +332,7 @@ describe('deferred spawns', () => { const d = { dataDir: () => dir, now: () => now, processAlive: () => true, pollMs: POLL_MS }; const replays: string[] = []; - expect(deferSpawnDuringFreeze({ sessionId: 's1', replay: () => replays.push('s1') }, d)) + expect(deferSpawnDuringFreeze({ sessionId: 's1', hasPayload: true, replay: () => replays.push('s1') }, d)) .toMatchObject({ parked: true }); now = NOW + 2_001;