From dcf37d4562653443d8692f20116fb4591d357bba Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:26:25 +0800 Subject: [PATCH 1/3] =?UTF-8?q?refactor(core):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=9C=BA=E5=99=A8=E4=BA=BA=E7=BA=A7=20turn=20admission/mutatio?= =?UTF-8?q?n=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/bot-turn-mutation-gate.ts | 296 ++++++++++++++++++++ test/bot-turn-mutation-gate.test.ts | 400 ++++++++++++++++++++++++++++ 2 files changed, 696 insertions(+) create mode 100644 src/core/bot-turn-mutation-gate.ts create mode 100644 test/bot-turn-mutation-gate.test.ts diff --git a/src/core/bot-turn-mutation-gate.ts b/src/core/bot-turn-mutation-gate.ts new file mode 100644 index 000000000..317b706b5 --- /dev/null +++ b/src/core/bot-turn-mutation-gate.ts @@ -0,0 +1,296 @@ +/** + * Per-bot admission/drain gate for mutations that replace every live worker + * generation (currently read-isolation). JavaScript's single thread makes the + * check+increment and close+publish edges atomic, while the waiters cover all + * intervening awaits in inbound/HTTP turn preparation. + */ +import { AsyncLocalStorage } from 'node:async_hooks'; + +type GateState = { + activeAdmissions: number; + mutating: boolean; + openWaiters: Array; + drainWaiters: Array; +}; + +type GateWaiter = { wake: () => void }; + +const gates = new Map(); +type AdmissionLease = { active: boolean; ownerFinished: boolean }; +type MutationLease = { active: boolean }; +const admissionContext = new AsyncLocalStorage>(); +const mutationContext = new AsyncLocalStorage>(); + +function stateFor(larkAppId: string): GateState { + let state = gates.get(larkAppId); + if (!state) { + state = { activeAdmissions: 0, mutating: false, openWaiters: [], drainWaiters: [] }; + gates.set(larkAppId, state); + } + return state; +} + +function waitUntilOpen(state: GateState): Promise { + return state.mutating + ? new Promise(resolve => state.openWaiters.push({ wake: resolve })) + : Promise.resolve(); +} + +function waitUntilDrained(state: GateState): Promise { + return state.activeAdmissions > 0 + ? new Promise(resolve => state.drainWaiters.push({ wake: resolve })) + : Promise.resolve(); +} + +function wakeAll(waiters: GateWaiter[]): void { + const pending = waiters.splice(0); + for (const waiter of pending) waiter.wake(); +} + +/** Cancellable condition wait used only by bounded acquisition. Timeout + * removes the exact waiter, so it cannot resume later and run a stale shutdown + * after its caller already reported refusal. */ +function waitUntilBefore( + waiters: GateWaiter[], + ready: () => boolean, + deadlineMs: number, +): Promise { + if (Date.now() >= deadlineMs) return Promise.resolve(false); + if (ready()) return Promise.resolve(true); + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) return Promise.resolve(false); + return new Promise(resolve => { + let settled = false; + let timer: NodeJS.Timeout; + const waiter: GateWaiter = { + wake: () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(Date.now() < deadlineMs); + }, + }; + waiters.push(waiter); + timer = setTimeout(() => { + if (settled) return; + settled = true; + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + resolve(false); + }, remaining); + }); +} + +export async function withBotTurnAdmission( + larkAppId: string, + action: () => Promise | T, +): Promise { + // A mutation already owns the bot exclusively. Re-entering an admission + // boundary from that mutation must not wait for the gate it owns. + const inheritedMutation = mutationContext.getStore()?.get(larkAppId); + if (inheritedMutation?.active) return action(); + const inherited = admissionContext.getStore(); + // Some admitted event paths delegate to triggerSessionTurn, which is also a + // public admission boundary. Treat that nested call as part of the outer + // lease; otherwise a mutation that closed the gate while the outer handler + // awaited would make the handler wait on itself and deadlock the drain. + const inheritedLease = inherited?.get(larkAppId); + if (inheritedLease?.active) return action(); + const state = stateFor(larkAppId); + // A mutation may finish and another queued mutation may close the gate again + // before this continuation runs, so re-check after every wake. + while (state.mutating) await waitUntilOpen(state); + state.activeAdmissions++; + const lease: AdmissionLease = { active: true, ownerFinished: false }; + try { + const next = new Map(inherited ?? []); + next.set(larkAppId, lease); + return await admissionContext.run(next, action); + } finally { + // AsyncLocalStorage descendants can outlive the awaited action when code + // starts fire-and-forget work. They retain this object, so revoke it before + // decrementing the drain count; a detached continuation must acquire a new + // admission instead of impersonating a still-live nested call. + // An admitted handler may upgrade itself to a mutation. The upgrade + // temporarily releases and later reacquires this exact lease; only the + // still-active owner decrements it here. + lease.ownerFinished = true; + if (lease.active) { + lease.active = false; + state.activeAdmissions--; + if (state.activeAdmissions === 0) { + wakeAll(state.drainWaiters); + } + } + } +} + +export async function withBotTurnMutation( + larkAppId: string, + action: () => Promise | T, +): Promise { + const inheritedMutation = mutationContext.getStore()?.get(larkAppId); + if (inheritedMutation?.active) return action(); + const state = stateFor(larkAppId); + const inheritedAdmission = admissionContext.getStore()?.get(larkAppId); + + // Explicit abandon actions are discovered inside already-admitted message + // and card handlers. Upgrade that admission instead of nesting and + // deadlocking: release only this handler's lease, drain every other turn, + // perform the mutation, then atomically reacquire the outer lease before the + // gate is reopened. The caller may safely finish delivery/logging under its + // original admission after the exclusive state change. + const upgrading = inheritedAdmission?.active === true; + if (upgrading) { + inheritedAdmission.active = false; + state.activeAdmissions--; + if (state.activeAdmissions === 0) { + wakeAll(state.drainWaiters); + } + } + + while (state.mutating) await waitUntilOpen(state); + state.mutating = true; + await waitUntilDrained(state); + const mutationLease: MutationLease = { active: true }; + try { + const next = new Map(mutationContext.getStore() ?? []); + next.set(larkAppId, mutationLease); + return await mutationContext.run(next, action); + } finally { + // Detached descendants retain the AsyncLocalStorage map. Revoke the + // mutable lease before reopening the gate so they cannot impersonate the + // completed mutation later. + mutationLease.active = false; + // Reacquire before waking queued admissions/mutations. This keeps the + // remainder of an upgraded outer handler inside the admission lifetime and + // prevents its finally block from double-decrementing the gate. + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + state.mutating = false; + wakeAll(state.openWaiters); + // Keep the state object stable. Resolved admission waiters resume in later + // promise jobs and still hold this object; deleting it here would let a new + // caller create a second gate and bypass those about-to-reacquire waiters. + // The key space is bounded by configured bot ids. + } +} + +export type BoundedBotTurnMutationResult = + | { acquired: true; value: T } + | { acquired: false; reason: 'timeout' | 'upgrade_conflict' }; + +/** Acquire one exact mutation lease within a single absolute deadline. + * + * Unlike Promise.race around `withBotTurnMutation`, this removes a timed-out + * waiter and rolls back a gate already closed while admissions were draining. + * Therefore `action` can never run after `{ acquired:false }` was returned. + * + * A same-bot admission may upgrade only when no other mutation already owns + * the gate. Returning `upgrade_conflict` leaves that admission untouched; this + * keeps the bounded path safe without resuming an admission concurrently with + * a mutation it had temporarily released for. */ +export async function tryWithBotTurnMutation( + larkAppId: string, + acquireTimeoutMs: number, + action: () => Promise | T, +): Promise> { + const inheritedMutation = mutationContext.getStore()?.get(larkAppId); + if (inheritedMutation?.active) { + return { acquired: true, value: await action() }; + } + const state = stateFor(larkAppId); + const inheritedAdmission = admissionContext.getStore()?.get(larkAppId); + const upgrading = inheritedAdmission?.active === true; + if (upgrading && state.mutating) { + return { acquired: false, reason: 'upgrade_conflict' }; + } + + const deadlineMs = Date.now() + Math.max(0, acquireTimeoutMs); + if (upgrading) { + inheritedAdmission.active = false; + state.activeAdmissions--; + if (state.activeAdmissions === 0) wakeAll(state.drainWaiters); + } else { + while (state.mutating) { + if (!await waitUntilBefore(state.openWaiters, () => !state.mutating, deadlineMs)) { + return { acquired: false, reason: 'timeout' }; + } + } + } + + if (Date.now() >= deadlineMs) { + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + return { acquired: false, reason: 'timeout' }; + } + + state.mutating = true; + const drained = await waitUntilBefore( + state.drainWaiters, + () => state.activeAdmissions === 0, + deadlineMs, + ); + if (!drained || Date.now() >= deadlineMs) { + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + state.mutating = false; + wakeAll(state.openWaiters); + return { acquired: false, reason: 'timeout' }; + } + + const mutationLease: MutationLease = { active: true }; + try { + const next = new Map(mutationContext.getStore() ?? []); + next.set(larkAppId, mutationLease); + if (Date.now() >= deadlineMs) { + return { acquired: false, reason: 'timeout' }; + } + const value = await mutationContext.run(next, action); + return { acquired: true, value }; + } finally { + mutationLease.active = false; + if (upgrading && !inheritedAdmission.ownerFinished) { + state.activeAdmissions++; + inheritedAdmission.active = true; + } + state.mutating = false; + wakeAll(state.openWaiters); + } +} + +/** + * Start an independent admission branch from fire-and-forget code. Normal + * same-bot nesting is intentionally reentrant and MUST be awaited by its + * caller; JavaScript cannot infer whether a returned Promise was floated. + * Detached callers clear both contexts and acquire a fresh counted root lease. + */ +export function runDetachedBotTurnAdmission( + larkAppId: string, + action: () => Promise | T, +): Promise { + return admissionContext.run(new Map(), () => + mutationContext.run(new Map(), () => withBotTurnAdmission(larkAppId, action))); +} + +/** Fire-and-forget counterpart for exclusive lifecycle mutations. A detached + * mutation inside an admission waits for that admission to drain; one inside a + * mutation waits for the parent mutation to reopen the gate. Do not await this + * helper from the parent scope whose lease it must wait on. */ +export function runDetachedBotTurnMutation( + larkAppId: string, + action: () => Promise | T, +): Promise { + return admissionContext.run(new Map(), () => + mutationContext.run(new Map(), () => withBotTurnMutation(larkAppId, action))); +} + +export function __testOnly_resetBotTurnMutationGates(): void { + gates.clear(); +} diff --git a/test/bot-turn-mutation-gate.test.ts b/test/bot-turn-mutation-gate.test.ts new file mode 100644 index 000000000..5dff838e9 --- /dev/null +++ b/test/bot-turn-mutation-gate.test.ts @@ -0,0 +1,400 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + __testOnly_resetBotTurnMutationGates, + runDetachedBotTurnAdmission, + runDetachedBotTurnMutation, + tryWithBotTurnMutation, + withBotTurnAdmission, + withBotTurnMutation, +} from '../src/core/bot-turn-mutation-gate.js'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +describe('per-bot turn mutation gate', () => { + afterEach(() => { + vi.useRealTimers(); + __testOnly_resetBotTurnMutationGates(); + }); + + it('drains an in-flight admission and blocks new turns through the mutation', async () => { + const finishFirst = deferred(); + const finishMutation = deferred(); + const firstEntered = vi.fn(); + const mutationEntered = vi.fn(); + const secondEntered = vi.fn(); + + const first = withBotTurnAdmission('app-a', async () => { + firstEntered(); + await finishFirst.promise; + }); + await vi.waitFor(() => expect(firstEntered).toHaveBeenCalledOnce()); + + const mutation = withBotTurnMutation('app-a', async () => { + mutationEntered(); + await finishMutation.promise; + }); + await Promise.resolve(); + expect(mutationEntered).not.toHaveBeenCalled(); + + const second = withBotTurnAdmission('app-a', () => { secondEntered(); }); + await Promise.resolve(); + expect(secondEntered).not.toHaveBeenCalled(); + + finishFirst.resolve(); + await vi.waitFor(() => expect(mutationEntered).toHaveBeenCalledOnce()); + expect(secondEntered).not.toHaveBeenCalled(); + + finishMutation.resolve(); + await Promise.all([first, mutation, second]); + expect(secondEntered).toHaveBeenCalledOnce(); + }); + + it('does not serialize an unrelated bot', async () => { + const finishMutation = deferred(); + const mutation = withBotTurnMutation('app-a', () => finishMutation.promise); + const other = vi.fn(); + await withBotTurnAdmission('app-b', () => other()); + expect(other).toHaveBeenCalledOnce(); + finishMutation.resolve(); + await mutation; + }); + + it('keeps woken waiters on the canonical state for the next mutation', async () => { + const finishFirstMutation = deferred(); + const finishQueuedAdmission = deferred(); + const firstMutation = withBotTurnMutation('app-a', () => finishFirstMutation.promise); + const queuedEntered = vi.fn(); + const queuedAdmission = withBotTurnAdmission('app-a', async () => { + queuedEntered(); + await finishQueuedAdmission.promise; + }); + await Promise.resolve(); + expect(queuedEntered).not.toHaveBeenCalled(); + + finishFirstMutation.resolve(); + await firstMutation; + await vi.waitFor(() => expect(queuedEntered).toHaveBeenCalledOnce()); + + const nextMutationEntered = vi.fn(); + const nextMutation = withBotTurnMutation('app-a', () => nextMutationEntered()); + await Promise.resolve(); + expect(nextMutationEntered).not.toHaveBeenCalled(); + + finishQueuedAdmission.resolve(); + await Promise.all([queuedAdmission, nextMutation]); + expect(nextMutationEntered).toHaveBeenCalledOnce(); + }); + + it('lets a draining admitted handler enter a nested same-bot boundary', async () => { + const enterNested = deferred(); + const outerEntered = vi.fn(); + const nestedEntered = vi.fn(); + const mutationEntered = vi.fn(); + const outer = withBotTurnAdmission('app-a', async () => { + outerEntered(); + await enterNested.promise; + await withBotTurnAdmission('app-a', () => nestedEntered()); + }); + await vi.waitFor(() => expect(outerEntered).toHaveBeenCalledOnce()); + + const mutation = withBotTurnMutation('app-a', () => mutationEntered()); + await Promise.resolve(); + expect(mutationEntered).not.toHaveBeenCalled(); + + enterNested.resolve(); + await Promise.all([outer, mutation]); + expect(nestedEntered).toHaveBeenCalledOnce(); + expect(mutationEntered).toHaveBeenCalledOnce(); + }); + + it('revokes inherited reentrancy for a detached descendant after the outer lease ends', async () => { + const releaseDetached = deferred(); + const finishMutation = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + + await withBotTurnAdmission('app-a', async () => { + detached = (async () => { + await releaseDetached.promise; + await withBotTurnAdmission('app-a', () => detachedEntered()); + })(); + }); + + const mutation = withBotTurnMutation('app-a', () => finishMutation.promise); + await Promise.resolve(); + releaseDetached.resolve(); + await Promise.resolve(); + await Promise.resolve(); + expect(detachedEntered).not.toHaveBeenCalled(); + + finishMutation.resolve(); + await Promise.all([mutation, detached]); + expect(detachedEntered).toHaveBeenCalledOnce(); + }); + + it('upgrades an admitted handler to a mutation without self-deadlock', async () => { + const order: string[] = []; + await withBotTurnAdmission('app-a', async () => { + order.push('admission'); + await withBotTurnMutation('app-a', async () => { + order.push('mutation'); + await withBotTurnAdmission('app-a', () => order.push('nested-admission')); + await withBotTurnMutation('app-a', () => order.push('nested-mutation')); + }); + order.push('after'); + }); + expect(order).toEqual([ + 'admission', + 'mutation', + 'nested-admission', + 'nested-mutation', + 'after', + ]); + }); + + it('upgrades through an awaited nested admission without retaining an ancestor count', async () => { + const events: string[] = []; + await withBotTurnAdmission('app-a', async () => { + await withBotTurnAdmission('app-a', async () => { + await withBotTurnMutation('app-a', () => events.push('mutated')); + }); + events.push('outer-finished'); + }); + expect(events).toEqual(['mutated', 'outer-finished']); + }); + + it('an upgraded mutation drains peer admissions and blocks later turns', async () => { + const letPeerFinish = deferred(); + const letCloseFinish = deferred(); + const startUpgrade = deferred(); + const events: string[] = []; + + const closer = withBotTurnAdmission('app-a', async () => { + events.push('close-admitted'); + await startUpgrade.promise; + await withBotTurnMutation('app-a', async () => { + events.push('close-mutating'); + await letCloseFinish.promise; + }); + events.push('close-after'); + }); + const peer = withBotTurnAdmission('app-a', async () => { + events.push('peer-admitted'); + await letPeerFinish.promise; + events.push('peer-finished'); + }); + await vi.waitFor(() => expect(events).toContain('peer-admitted')); + + startUpgrade.resolve(); + await Promise.resolve(); + expect(events).not.toContain('close-mutating'); + + letPeerFinish.resolve(); + await vi.waitFor(() => expect(events).toContain('close-mutating')); + + const later = withBotTurnAdmission('app-a', () => events.push('later-admitted')); + await Promise.resolve(); + expect(events).not.toContain('later-admitted'); + + letCloseFinish.resolve(); + await Promise.all([closer, peer, later]); + expect(events.indexOf('peer-finished')).toBeLessThan(events.indexOf('close-mutating')); + expect(events.indexOf('close-mutating')).toBeLessThan(events.indexOf('close-after')); + }); + + it('revokes inherited mutation ownership for detached descendants', async () => { + const releaseDetached = deferred(); + const holdNextMutation = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + + await withBotTurnMutation('app-a', async () => { + detached = (async () => { + await releaseDetached.promise; + await withBotTurnAdmission('app-a', () => detachedEntered()); + })(); + }); + + const nextMutation = withBotTurnMutation('app-a', () => holdNextMutation.promise); + await Promise.resolve(); + releaseDetached.resolve(); + await Promise.resolve(); + await Promise.resolve(); + expect(detachedEntered).not.toHaveBeenCalled(); + + holdNextMutation.resolve(); + await Promise.all([nextMutation, detached]); + expect(detachedEntered).toHaveBeenCalledOnce(); + }); + + it('runs a detached mutation only after its admission parent drains', async () => { + const releaseDetachedMutation = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + + await withBotTurnAdmission('app-a', () => { + detached = runDetachedBotTurnMutation('app-a', async () => { + detachedEntered(); + await releaseDetachedMutation.promise; + }); + expect(detachedEntered).not.toHaveBeenCalled(); + }); + await vi.waitFor(() => expect(detachedEntered).toHaveBeenCalledOnce()); + + releaseDetachedMutation.resolve(); + await detached; + + const laterMutation = vi.fn(); + const laterAdmission = vi.fn(); + await withBotTurnMutation('app-a', () => laterMutation()); + await withBotTurnAdmission('app-a', () => laterAdmission()); + expect(laterMutation).toHaveBeenCalledOnce(); + expect(laterAdmission).toHaveBeenCalledOnce(); + }); + + it('counts an explicit detached admission begun before its parent returns', async () => { + const releaseChild = deferred(); + const childEntered = vi.fn(); + let child!: Promise; + + await withBotTurnAdmission('app-a', () => { + child = runDetachedBotTurnAdmission('app-a', async () => { + childEntered(); + await releaseChild.promise; + }); + }); + expect(childEntered).toHaveBeenCalledOnce(); + + const mutationEntered = vi.fn(); + const mutation = withBotTurnMutation('app-a', () => mutationEntered()); + await Promise.resolve(); + expect(mutationEntered).not.toHaveBeenCalled(); + + releaseChild.resolve(); + await Promise.all([child, mutation]); + expect(mutationEntered).toHaveBeenCalledOnce(); + }); + + it('clears mutation context for an explicit detached mutation', async () => { + const releaseChild = deferred(); + const childEntered = vi.fn(); + let child!: Promise; + + const parent = withBotTurnMutation('app-a', () => { + child = runDetachedBotTurnMutation('app-a', async () => { + childEntered(); + await releaseChild.promise; + }); + }); + await parent; + await vi.waitFor(() => expect(childEntered).toHaveBeenCalledOnce()); + const admissionEntered = vi.fn(); + const admission = withBotTurnAdmission('app-a', () => admissionEntered()); + await Promise.resolve(); + expect(admissionEntered).not.toHaveBeenCalled(); + + releaseChild.resolve(); + await Promise.all([child, admission]); + expect(admissionEntered).toHaveBeenCalledOnce(); + }); + + it('keeps an explicit detached admission behind its mutation parent', async () => { + const holdParent = deferred(); + const detachedEntered = vi.fn(); + let detached!: Promise; + const parent = withBotTurnMutation('app-a', async () => { + detached = runDetachedBotTurnAdmission('app-a', () => detachedEntered()); + await Promise.resolve(); + expect(detachedEntered).not.toHaveBeenCalled(); + await holdParent.promise; + }); + await Promise.resolve(); + holdParent.resolve(); + await parent; + await detached; + expect(detachedEntered).toHaveBeenCalledOnce(); + }); + + it('removes a bounded waiter on timeout so its action can never run later', async () => { + const releaseOwner = deferred(); + const ownerEntered = vi.fn(); + const owner = withBotTurnMutation('app-a', async () => { + ownerEntered(); + await releaseOwner.promise; + }); + await vi.waitFor(() => expect(ownerEntered).toHaveBeenCalledOnce()); + + const staleAction = vi.fn(); + await expect(tryWithBotTurnMutation('app-a', 10, staleAction)).resolves.toEqual({ + acquired: false, + reason: 'timeout', + }); + releaseOwner.resolve(); + await owner; + await new Promise(resolve => setTimeout(resolve, 20)); + expect(staleAction).not.toHaveBeenCalled(); + + const later = vi.fn(); + await expect(tryWithBotTurnMutation('app-a', 50, later)).resolves.toEqual({ + acquired: true, + value: undefined, + }); + expect(later).toHaveBeenCalledOnce(); + }); + + it('rejects an overdue wake even when owner release runs before the timer callback', async () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + const releaseOwner = deferred(); + const ownerEntered = vi.fn(); + const owner = withBotTurnMutation('app-a', async () => { + ownerEntered(); + await releaseOwner.promise; + }); + await Promise.resolve(); + expect(ownerEntered).toHaveBeenCalledOnce(); + + const staleAction = vi.fn(); + const bounded = tryWithBotTurnMutation('app-a', 10, staleAction); + await Promise.resolve(); + // Advance wall time without running the overdue timer. Releasing the owner + // wakes the waiter first; the wake path itself must enforce the deadline. + vi.setSystemTime(1_020); + releaseOwner.resolve(); + await owner; + + await expect(bounded).resolves.toEqual({ acquired: false, reason: 'timeout' }); + expect(staleAction).not.toHaveBeenCalled(); + await vi.runAllTimersAsync(); + expect(staleAction).not.toHaveBeenCalled(); + }); + + it('rolls back a closed gate when admissions do not drain before the deadline', async () => { + const releaseAdmission = deferred(); + const heldEntered = vi.fn(); + const held = withBotTurnAdmission('app-a', async () => { + heldEntered(); + await releaseAdmission.promise; + }); + await vi.waitFor(() => expect(heldEntered).toHaveBeenCalledOnce()); + + const mutationAction = vi.fn(); + const bounded = tryWithBotTurnMutation('app-a', 10, mutationAction); + const admittedAfterRollback = vi.fn(); + const queuedAdmission = withBotTurnAdmission('app-a', admittedAfterRollback); + + await expect(bounded).resolves.toEqual({ acquired: false, reason: 'timeout' }); + await queuedAdmission; + expect(admittedAfterRollback).toHaveBeenCalledOnce(); + expect(mutationAction).not.toHaveBeenCalled(); + + releaseAdmission.resolve(); + await held; + await new Promise(resolve => setTimeout(resolve, 20)); + expect(mutationAction).not.toHaveBeenCalled(); + }); +}); From 09b96ef5cec0c13a68c72006c160ff1deaaa4a3e Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:49:21 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(riff):=20=E4=B8=BA=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E4=B8=8E=E5=AE=88=E6=8A=A4=E8=BF=9B=E7=A8=8B=E9=80=80=E5=87=BA?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=A4=E9=98=B6=E6=AE=B5=E5=85=B3=E5=81=9C?= =?UTF-8?q?=E5=9B=B4=E6=A0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/adapters/backend/riff-backend.ts | 334 ++++- src/adapters/backend/types.ts | 42 +- src/core/explicit-session-backing-cleanup.ts | 110 ++ src/core/persistent-backend.ts | 10 +- src/core/riff-shutdown-detach.ts | 1042 ++++++++++++++++ src/core/riff-worker-shutdown-readiness.ts | 31 + src/core/shutdown-budgets.ts | 33 + src/core/types.ts | 21 + src/core/worker-pool.ts | 603 ++++++++- src/daemon.ts | 199 ++- src/services/session-store.ts | 374 +++++- src/types.ts | 33 +- src/worker.ts | 271 +++- test/explicit-session-backing-cleanup.test.ts | 113 ++ test/persistent-backend-type.test.ts | 4 + test/riff-backend.test.ts | 241 +++- test/riff-explicit-close.test.ts | 283 +++++ test/riff-shutdown-detach.test.ts | 1093 +++++++++++++++++ test/riff-worker-shutdown-readiness.test.ts | 37 + test/session-store.test.ts | 38 + test/worker-riff-retirement-protocol.test.ts | 134 ++ 21 files changed, 4910 insertions(+), 136 deletions(-) create mode 100644 src/core/explicit-session-backing-cleanup.ts create mode 100644 src/core/riff-shutdown-detach.ts create mode 100644 src/core/riff-worker-shutdown-readiness.ts create mode 100644 src/core/shutdown-budgets.ts create mode 100644 test/explicit-session-backing-cleanup.test.ts create mode 100644 test/riff-explicit-close.test.ts create mode 100644 test/riff-shutdown-detach.test.ts create mode 100644 test/riff-worker-shutdown-readiness.test.ts create mode 100644 test/worker-riff-retirement-protocol.test.ts diff --git a/src/adapters/backend/riff-backend.ts b/src/adapters/backend/riff-backend.ts index 4eb793a65..56bd85b5e 100644 --- a/src/adapters/backend/riff-backend.ts +++ b/src/adapters/backend/riff-backend.ts @@ -2,7 +2,12 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { execFileSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import type { SessionBackend, SpawnOpts } from './types.js'; +import type { + SessionBackend, + SessionDestroyResult, + SessionShutdownDetachResult, + SpawnOpts, +} from './types.js'; import { logger } from '../../utils/logger.js'; /** @@ -339,6 +344,24 @@ export class RiffBackend implements SessionBackend { private cancelTimeoutMs = 4_000; private createTimeoutMs = 10_000; private destroyDeadlineMs = 20_000; + /** Exact late/current task whose close cancellation failed. Retained across + * the prepare-close handshake so the daemon can persist a retry handle. */ + private closeFailureTaskId: string | null = null; + private closeFailureError: string | null = null; + private closeLateTaskHandled = false; + private closePrepared = false; + private closeAttempt: symbol | null = null; + private destroyInFlight: Promise | null = null; + private cancelInFlight: Promise | null = null; + private abortInFlight: Promise | null = null; + /** Graceful daemon shutdown is a non-cancelling two-phase detach. It fences + * only writes arriving after prepare; writes already appended to writeChain + * still drain so a late child id can be durably handed to the daemon. */ + private shutdownDetaching = false; + private shutdownDetachPrepared = false; + private shutdownDetachAttempt: symbol | null = null; + private shutdownDetachInFlight: Promise | null = null; + private shutdownDetachAbortInFlight: Promise | null = null; /** Serializes write() → createTask/followUp. Without this, a second message * arriving before the first task-execute HTTP returns would see * currentTaskId === null and create a duplicate task. */ @@ -417,8 +440,16 @@ export class RiffBackend implements SessionBackend { // No actual process to spawn. Task creation happens on first write(). } - write(data: string): void { - if (this.killed || this.closing) return; + write(data: string): boolean { + if (this.killed) return false; + if (this.shutdownDetaching) { + logger.warn('[riff] write rejected while graceful shutdown detach is preparing/prepared'); + return false; + } + if (this.closing) { + logger.warn('[riff] write rejected while explicit close is preparing/prepared'); + return false; + } const { text, attachments } = this.extractAttachments(data); @@ -440,6 +471,7 @@ export class RiffBackend implements SessionBackend { .catch((err) => { logger.warn(`[riff] queued write failed: ${err}`); }); + return true; } resize(_cols: number, _rows: number): void { @@ -467,8 +499,8 @@ export class RiffBackend implements SessionBackend { this.exitCb?.(0, null); } - async destroySession(): Promise { - // /close（及 /restart 的替换路径）必须把远端任务真正取消掉——fire-and-forget + async destroySession(): Promise { + // /close 必须把远端任务真正取消掉——fire-and-forget // 在 worker 紧接 process.exit 时大概率发不出去，已关闭话题的远端 agent 会 // 继续拿着注入的凭证发消息。有界 await + 一次重试，失败也明确留痕。 // @@ -476,36 +508,249 @@ export class RiffBackend implements SessionBackend { // currentTaskId 还是 null/旧值，直接 cancel 会漏掉 late task。先立 closing // 门（拒新写 + 令 in-flight 完成后自取消），再有界等 writeChain 沉降，最后 // cancel 沉降后的 current task。 + if (this.shutdownDetaching) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'shutdown_detach_in_progress', + }; + } + if (this.destroyInFlight) return this.destroyInFlight; + if (this.closePrepared) { + return { + ok: true, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + }; + } + const attempt = Symbol('riff-close-prepare'); + this.closeAttempt = attempt; this.closing = true; + this.closeFailureTaskId = null; + this.closeFailureError = null; + this.closeLateTaskHandled = false; // 预算层级（单调覆盖，无内层 race——writeChain 本身有界）： // create/follow-up fetch 10s + late cancel 4s×2 = chain 最坏 18s // own cancel 4s×2 = 8s（与 late 情形互斥：closing 分支不登记 current） - // → destroySession 总 deadline 20s → worker close/restart race 22s + // → destroySession 总 deadline 20s → worker close handshake 22s // → daemon SIGTERM backstop 24s / SIGKILL 29s。 // 对 writeChain 只整体 await：单独给它小窗口会在窗口边缘掐掉链内的 // late cancel（create 于 t≈窗口末返回 → cancel 尚 pending → teardown 提前 // resolve → process.exit 掐断取消）。 - const teardown = (async () => { + const teardown = (async (): Promise => { try { await this.writeChain; } catch { /* writeChain never rejects (caught internally) */ } - if (this.currentTaskId && !this.taskDone) { + if (this.closeAttempt !== attempt) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + if (this.closeFailureTaskId) { + return { + ok: false, + taskId: this.closeFailureTaskId, + error: this.closeFailureError ?? 'late_task_cancel_failed', + }; + } + // A task materialized while closing was already cancelled inside the + // writeChain. Do not then cancel its stale parent lineage as if it were + // still the active execution. + if (!this.closeLateTaskHandled && this.currentTaskId && !this.taskDone) { const id = this.currentTaskId; - try { - await this.cancelTask(id); + const cancelled = await this.cancelTaskWithRetry(id, 'close'); + // abortDestroySession invalidates the exact attempt before waiting for + // an already-issued cancellation. A late successful HTTP response must + // not resurrect that aborted generation as a prepared close. + if (this.closeAttempt !== attempt || !this.closing) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + if (cancelled) { logger.info(`[riff] task ${id} cancelled on close`); - } catch (err) { - try { - await this.cancelTask(id); - logger.info(`[riff] task ${id} cancelled on close (retry)`); - } catch (err2) { - logger.warn(`[riff] task-cancel failed on close (task ${id} may keep running remotely): ${err2}`); - } + } else { + return { + ok: false, + taskId: id, + error: this.closeFailureError ?? 'task_cancel_failed', + }; } } + if (this.closeAttempt !== attempt || !this.closing) { + return { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + return { + ok: true, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + }; })(); - await Promise.race([teardown, new Promise((r) => setTimeout(r, this.destroyDeadlineMs))]); - this.kill(); + this.destroyInFlight = Promise.race([ + teardown, + new Promise(resolve => setTimeout(() => resolve({ + ok: false, + ...(this.closeFailureTaskId || this.currentTaskId + ? { taskId: this.closeFailureTaskId ?? this.currentTaskId! } + : {}), + error: 'close_timeout', + }), this.destroyDeadlineMs)), + ]).then(async (result) => { + // Promise.race and the teardown continuation each add a microtask + // boundary. Revalidate the generation immediately before publishing the + // prepared bit so a concurrent abort can never be overwritten. + if (result.ok && (this.closeAttempt !== attempt || !this.closing)) { + result = { + ok: false, + ...(this.currentTaskId ? { taskId: this.currentTaskId } : {}), + error: 'close_aborted', + }; + } + if (result.ok) { + this.closePrepared = true; + } else { + // A failed prepare is not a terminal close. Restore admission so the + // still-active durable owner can accept a follow-up or a close retry. + await this.abortDestroySession(); + } + return result; + }).finally(() => { + this.destroyInFlight = null; + }); + return this.destroyInFlight; + } + + async abortDestroySession(): Promise { + if (this.killed) return; + if (this.abortInFlight) return this.abortInFlight; + this.closeAttempt = null; + this.closePrepared = false; + const pendingCancel = this.cancelInFlight; + this.abortInFlight = (async () => { + // A close timeout can win Promise.race after task-cancel was already + // issued. Reopening admission before that request settles lets a new + // follow-up race a late successful cancellation of its parent. Keep the + // backend fenced until the exact cancellation attempt reaches terminal. + if (pendingCancel) { + try { await pendingCancel; } catch { /* cancel helper returns boolean */ } + } + if (this.killed || this.closeAttempt !== null || this.closePrepared) return; + this.closing = false; + this.closeFailureTaskId = null; + this.closeFailureError = null; + this.closeLateTaskHandled = false; + logger.info('[riff] explicit close aborted; write admission restored'); + })().finally(() => { + this.abortInFlight = null; + }); + return this.abortInFlight; + } + + commitDestroySession(): void { + // The daemon has durably published the closed row. Keep admission fenced + // until the worker immediately detaches/exits. + this.closePrepared = false; + this.closeAttempt = null; + this.closing = true; + } + + async prepareShutdownDetach(): Promise { + if (this.shutdownDetachInFlight) return this.shutdownDetachInFlight; + if (this.shutdownDetachPrepared) { + return { ok: true, taskId: this.currentTaskId }; + } + if (this.killed) { + return { ok: false, taskId: this.currentTaskId, error: 'backend_killed' }; + } + if (this.closing || this.destroyInFlight || this.closePrepared) { + return { ok: false, taskId: this.currentTaskId, error: 'explicit_close_in_progress' }; + } + + const attempt = Symbol('riff-shutdown-detach'); + this.shutdownDetachAttempt = attempt; + this.shutdownDetaching = true; + // Existing SSE delivery is presentation-only. Stop it now, but do not + // cancel the remote task. Any create/follow-up already accepted before the + // fence remains in writeChain and is allowed to materialize below. + this.abortController?.abort(); + + const drain = (async (): Promise => { + try { await this.writeChain; } + catch { /* writeChain catches its own failures */ } + if (this.killed || this.shutdownDetachAttempt !== attempt || !this.shutdownDetaching) { + return { ok: false, taskId: this.currentTaskId, error: 'shutdown_detach_aborted' }; + } + if (this.closing || this.closePrepared) { + return { ok: false, taskId: this.currentTaskId, error: 'explicit_close_in_progress' }; + } + this.shutdownDetachPrepared = true; + logger.info( + `[riff] graceful shutdown detach prepared` + + `${this.currentTaskId ? ` (task ${this.currentTaskId})` : ' (no task lineage)'}`, + ); + return { ok: true, taskId: this.currentTaskId }; + })(); + this.shutdownDetachInFlight = drain.finally(() => { + this.shutdownDetachInFlight = null; + }); + return this.shutdownDetachInFlight; + } + + async abortShutdownDetach(): Promise { + if (this.killed) { + return { ok: false, taskId: this.currentTaskId, error: 'backend_killed' }; + } + if (this.shutdownDetachAbortInFlight) return this.shutdownDetachAbortInFlight; + const pending = this.shutdownDetachInFlight; + const pendingCancel = this.cancelInFlight; + this.shutdownDetachAttempt = null; + this.shutdownDetachPrepared = false; + this.shutdownDetachAbortInFlight = (async (): Promise => { + // Normally shutdown detach never cancels a remote task. Still wait for + // any exact cancellation already issued by an overlapping explicit close + // before reopening admission, otherwise its late result could invalidate + // a newly accepted follow-up. + await Promise.all([ + pending ? pending.catch(() => undefined) : Promise.resolve(), + pendingCancel ? pendingCancel.catch(() => false) : Promise.resolve(), + ]); + if (this.killed) { + return { ok: false, taskId: this.currentTaskId, error: 'backend_killed' }; + } + if (this.closing || this.shutdownDetachAttempt !== null) { + return { + ok: false, + taskId: this.currentTaskId, + error: this.closing ? 'explicit_close_in_progress' : 'new_shutdown_detach_in_progress', + }; + } + this.shutdownDetaching = false; + // prepare stopped SSE before the persistence ACK. If shutdown is + // aborted, reconnect the exact current task so the still-live owner + // resumes normal output and completion tracking. + if (this.currentTaskId && !this.taskDone) { + this.reconnectAttempts = 0; + void this.streamTask(this.currentTaskId); + } + logger.info('[riff] graceful shutdown detach aborted; write admission restored'); + return { ok: true, taskId: this.currentTaskId }; + })().finally(() => { + this.shutdownDetachAbortInFlight = null; + }); + return this.shutdownDetachAbortInFlight; + } + + commitShutdownDetach(): void { + this.shutdownDetachPrepared = false; + this.shutdownDetachAttempt = null; + // Keep admission fenced until the worker exits immediately after commit. + this.shutdownDetaching = true; } getChildPid(): number | null { @@ -760,8 +1005,8 @@ export class RiffBackend implements SessionBackend { * Post-await adoption gate for a freshly created/followed-up task id. * - closing（/close 竞态窗口）：这个 late task 已经没有会话可服务——立即取消 * （有界+一次重试），绝不 stream/登记，防远端 orphan； - * - killed（detach：重启/休眠）：登记 id 让 daemon 持久化血缘，但不 stream - * （任务合法续跑，重启后 follow-up 接上）； + * - killed / shutdownDetaching（detach）：登记 id 让 daemon 持久化血缘， + * 但不 stream（任务合法续跑，重启后 follow-up 接上）； * - 正常：登记 + 由调用方启动 stream。 */ private async adoptLateTask(taskId: string): Promise { @@ -769,20 +1014,21 @@ export class RiffBackend implements SessionBackend { // 在 writeChain 内 await——destroySession 等 writeChain 沉降时就能把这次 // 取消一起等到（void 触发会在 worker exit 时被掐断）。 logger.info(`[riff] task ${taskId} created during close — cancelling late task`); - try { - await this.cancelTask(taskId); - } catch { - try { - await this.cancelTask(taskId); - } catch (err) { - logger.warn(`[riff] late-task cancel failed (task ${taskId} may keep running remotely): ${err}`); - } - } + this.closeLateTaskHandled = true; + // Preserve the exact newest lineage even when cancellation succeeds. + // If the daemon cannot durably commit the close and sends abort, the + // next follow-up must continue from this child rather than its stale + // parent. Publishing before the cancel also makes a failed cancel + // retryable by the daemon. + this.currentTaskId = taskId; + this.taskIdCb?.(taskId); + const cancelled = await this.cancelTaskWithRetry(taskId, 'late-task close'); + if (!cancelled) this.taskDone = false; return false; } this.currentTaskId = taskId; this.taskIdCb?.(taskId); - if (this.killed) return false; + if (this.killed || this.shutdownDetaching) return false; return true; } @@ -809,6 +1055,32 @@ export class RiffBackend implements SessionBackend { if (!resp.ok) throw new Error(`task-cancel HTTP ${resp.status}`); } + private async cancelTaskWithRetry(taskId: string, context: string): Promise { + const operation = (async (): Promise => { + try { + await this.cancelTask(taskId); + return true; + } catch { + try { + await this.cancelTask(taskId); + logger.info(`[riff] task ${taskId} cancelled on ${context} (retry)`); + return true; + } catch (err) { + this.closeFailureTaskId = taskId; + this.closeFailureError = err instanceof Error ? err.message : String(err); + logger.warn(`[riff] ${context} cancel failed (task ${taskId} may keep running remotely): ${err}`); + return false; + } + } + })(); + this.cancelInFlight = operation; + try { + return await operation; + } finally { + if (this.cancelInFlight === operation) this.cancelInFlight = null; + } + } + private async streamTask(taskId: string): Promise { const url = `${this.config.baseUrl}/api2/task-stream?id=${encodeURIComponent(taskId)}`; const headers: Record = {}; diff --git a/src/adapters/backend/types.ts b/src/adapters/backend/types.ts index c709e470c..996ad8d96 100644 --- a/src/adapters/backend/types.ts +++ b/src/adapters/backend/types.ts @@ -53,7 +53,9 @@ export interface SpawnOpts { export interface SessionBackend { spawn(bin: string, args: string[], opts: SpawnOpts): void; - write(data: string): void; + /** Returns false only when the backend can prove the write was not accepted. + * Legacy implementations may return void on success. */ + write(data: string): void | boolean; resize(cols: number, rows: number): void; onData(cb: (data: string) => void): void; onExit(cb: (code: number | null, signal: string | null) => void): void; @@ -87,8 +89,42 @@ export interface SessionBackend { * so the follow-up lineage survives daemon restarts. `null` clears the * persisted lineage (follow-up failed → next message starts fresh). */ onTaskId?(cb: (taskId: string | null) => void): void; - /** Async-capable teardown: riff awaits the remote task-cancel here. */ - destroySession?(): void | Promise; + /** Async-capable teardown: Riff returns a confirmed cancellation result so + * daemon-driven explicit close can fence late create/follow-up races before + * publishing the durable closed row. Local backends remain synchronous. */ + destroySession?(): void | Promise; + /** Roll back a successful remote prepare when the daemon could not commit + * the durable closed row. The backend must restore write admission without + * discarding the last task lineage. */ + abortDestroySession?(): void | Promise; + /** Finalize a successful prepare after the durable row is closed. */ + commitDestroySession?(): void; + /** Graceful daemon shutdown for a remote-task backend. Fence only NEW + * writes, drain every write accepted before the fence, and return the exact + * final lineage without cancelling it. The daemon persists that lineage + * before telling the worker it may exit. */ + prepareShutdownDetach?(): Promise; + /** Restore admission/streaming when the daemon cannot complete a prepared + * shutdown detach (for example, lineage persistence failed). */ + abortShutdownDetach?(): SessionShutdownDetachResult | Promise; + /** Finalize a shutdown detach after exact lineage persistence has been + * acknowledged by the daemon. Shutdown refusal never cancels remote work. */ + commitShutdownDetach?(): void; +} + +export interface SessionDestroyResult { + ok: boolean; + /** Exact remote task that failed cancellation and must remain retryable. */ + taskId?: string; + error?: string; +} + +export interface SessionShutdownDetachResult { + ok: boolean; + /** Exact final lineage after all pre-fence writes have drained. `null` is + * authoritative and clears any stale durable parent. */ + taskId: string | null; + error?: string; } /** diff --git a/src/core/explicit-session-backing-cleanup.ts b/src/core/explicit-session-backing-cleanup.ts new file mode 100644 index 000000000..d5b560194 --- /dev/null +++ b/src/core/explicit-session-backing-cleanup.ts @@ -0,0 +1,110 @@ +import type { BackendType } from '../adapters/backend/types.js'; +import type { RiffBackendConfig } from '../adapters/backend/riff-backend.js'; +import { cancelRiffTaskById } from '../adapters/backend/riff-backend.js'; +import { + isSuspendableBackendType, + killPersistentSession, + persistentSessionName, + probePersistentSession, +} from './persistent-backend.js'; + +export type ExplicitSessionBackingCleanupResult = + | { ok: true; kind: 'skipped_adopted' | 'no_backing' } + | { ok: true; kind: 'destroyed_persistent'; backendType: 'tmux' | 'herdr' | 'zellij'; name: string } + | { ok: true; kind: 'cancelled_riff'; taskId: string } + | { + ok: false; + kind: 'persistent_destroy_failed' | 'riff_config_missing' | 'riff_cancel_failed'; + backendType: BackendType; + taskId?: string; + error?: string; + }; + +export interface ExplicitSessionBackingCleanupInput { + sessionId: string; + backendType?: BackendType; + riffParentTaskId?: string; + /** Adopted panes/agents are user-owned. Explicit Botmux close only detaches + * its logical row and must never destroy the observed backing resource. */ + adopted?: boolean; + /** Current authoritative config for the row's bot. Required only when the + * row is frozen to Riff and still carries a remote task id. */ + riffConfig?: RiffBackendConfig; +} + +export interface ExplicitSessionBackingCleanupDeps { + cancelRiffTask?: typeof cancelRiffTaskById; + killPersistent?: typeof killPersistentSession; + probePersistent?: typeof probePersistentSession; +} + +/** + * Destroy the Botmux-owned backing resource before an offline/worker-less + * explicit close is published. + * + * This helper never mutates the session record. In particular, callers may + * erase `riffParentTaskId` only after `kind === 'cancelled_riff'`; a failed or + * unconfigurable cancellation therefore retains the durable retry handle. + */ +export async function cleanupExplicitSessionBacking( + input: ExplicitSessionBackingCleanupInput, + deps: ExplicitSessionBackingCleanupDeps = {}, +): Promise { + if (input.adopted) return { ok: true, kind: 'skipped_adopted' }; + + if (input.backendType === 'riff') { + const taskId = input.riffParentTaskId; + if (!taskId) return { ok: true, kind: 'no_backing' }; + if (!input.riffConfig?.baseUrl) { + return { ok: false, kind: 'riff_config_missing', backendType: 'riff', taskId }; + } + const cancel = deps.cancelRiffTask ?? cancelRiffTaskById; + try { + const cancelled = await cancel(input.riffConfig, taskId); + return cancelled + ? { ok: true, kind: 'cancelled_riff', taskId } + : { ok: false, kind: 'riff_cancel_failed', backendType: 'riff', taskId }; + } catch (err) { + return { + ok: false, + kind: 'riff_cancel_failed', + backendType: 'riff', + taskId, + error: err instanceof Error ? err.message : String(err), + }; + } + } + + if (!isSuspendableBackendType(input.backendType)) { + // Explicit pty and legacy unstamped rows have no deterministic persistent + // backing session to destroy. Never guess that an unstamped row was tmux. + return { ok: true, kind: 'no_backing' }; + } + + const name = persistentSessionName(input.backendType, input.sessionId); + const kill = deps.killPersistent ?? killPersistentSession; + const probe = deps.probePersistent ?? probePersistentSession; + try { + kill(input.backendType, name); + // Backend kill helpers are intentionally idempotent and historically + // swallow command errors. Offline explicit abandon needs a stronger + // contract: only publish closed after a post-kill probe confirms absence. + const after = probe(input.backendType, name); + if (after !== 'missing') { + return { + ok: false, + kind: 'persistent_destroy_failed', + backendType: input.backendType, + error: after === 'exists' ? 'backing_session_still_exists' : 'backing_session_state_unknown', + }; + } + return { ok: true, kind: 'destroyed_persistent', backendType: input.backendType, name }; + } catch (err) { + return { + ok: false, + kind: 'persistent_destroy_failed', + backendType: input.backendType, + error: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/src/core/persistent-backend.ts b/src/core/persistent-backend.ts index a91b89330..88c886489 100644 --- a/src/core/persistent-backend.ts +++ b/src/core/persistent-backend.ts @@ -119,12 +119,12 @@ export function resolvePairedSpawnBackendType( * Freezing here stops a live backendType edit from changing how a running session * tears down — e.g. detach-preserving a "herdr" session whose real pane is tmux. */ -export function shutdownBackendDisposition(ds: DaemonSession): 'detach' | 'close' { - // riff：远端任务独立于本地进程存活。daemon shutdown 走 'close' 会经 worker 的 - // destroySession() 取消远端任务——重启不该杀任务（血缘已持久化，重启后 - // follow-up 续上，agent 的 botmux send 照常送达）。detach = 仅 SIGTERM worker。 +export function shutdownBackendDisposition(ds: DaemonSession): 'riff-drain-detach' | 'detach' | 'close' { + // Riff 不能落进普通 detach 的直接 SIGTERM：create/follow-up 最长 10s 才返回 + // task id，而 worker SIGTERM 会立即 exit，丢掉唯一血缘。独立 disposition 迫使 + // daemon 先走 drain → durable ACK → commit 协议；类型检查防止未来回归。 const frozen = ds.initConfig?.backendType ?? ds.session.backendType; - if (frozen === 'riff') return 'detach'; + if (frozen === 'riff') return 'riff-drain-detach'; return getSessionPersistentBackendType(ds) ? 'detach' : 'close'; } diff --git a/src/core/riff-shutdown-detach.ts b/src/core/riff-shutdown-detach.ts new file mode 100644 index 000000000..79bbbb9ef --- /dev/null +++ b/src/core/riff-shutdown-detach.ts @@ -0,0 +1,1042 @@ +import { randomUUID } from 'node:crypto'; +import type { ChildProcess } from 'node:child_process'; +import type { DaemonToWorker, WorkerToDaemon } from '../types.js'; +import * as sessionStore from '../services/session-store.js'; +import { logger } from '../utils/logger.js'; +import type { DaemonSession } from './types.js'; +import { + RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS, + RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS, + RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS, +} from './shutdown-budgets.js'; + +type ShutdownPhase = 'prepare' | 'abort'; + +type ShutdownPhaseResult = { + ok: boolean; + taskId: string | null; + error?: string; +}; + +export type PreparedRiffShutdown = { + ok: true; + fence: 'prepared'; + requestId: string; + taskId: string | null; + /** Runtime lineage sampled before the worker fence. A workerless transaction + * must still own this exact value at persistence time; a live transaction may + * advance only to its drained `taskId`. */ + runtimeTaskIdAtPrepare: string | null; + /** Fresh durable lineage sampled before installing the fence. Phase 2 uses + * it as a lock-protected compare-and-set guard, while also accepting an + * already-idempotent target written by the ordinary task-id callback. */ + durableTaskIdAtPrepare: string | null; + durableOwnerAtPrepare: { + pid: number | null; + larkAppId: string | null; + backendType: string | null; + }; + /** Set only after the exact cross-process fresh read succeeds. Used when a + * prepared worker exits before an all-or-nothing rollback can reach it. */ + lineageVerified: boolean; + /** Exact generation fenced by `requestId`. Null means the active logical + * session was already workerless and only its runtime lineage needs an exact + * durable verification. */ + worker: ChildProcess | null; +}; + +export type RiffShutdownFailure = { + ok: false; + requestId?: string; + taskId: string | null; + error: string; + /** Phase-2 coordinator policy. Ownership ambiguity must stay fenced; a + * plain atomic-write I/O failure may restore the exact prepared worker. */ + rollbackDisposition?: 'abort_safe' | 'retain_fence'; +}; + +/** A prepare refusal that is proven to have happened before the worker/backend + * fence. It must never be sent an abort request. */ +export type UnfencedRiffShutdownRefusal = RiffShutdownFailure & { + fence: 'none'; +}; + +/** A prepare attempt whose exact worker may have installed its backend fence. + * Preparation deliberately does not restore admission inline; the fleet + * coordinator includes this handle in its one concurrent abort wave. */ +export type PossiblyFencedRiffShutdown = RiffShutdownFailure & { + fence: 'possible'; + requestId: string; + worker: ChildProcess; + expectedAbortTaskId?: string | null; +}; + +export type RiffShutdownPrepareResult = + | PreparedRiffShutdown + | UnfencedRiffShutdownRefusal + | PossiblyFencedRiffShutdown; + +export type RiffShutdownPrepareOptions = { + drainTimeoutMs?: number; + abortTimeoutMs?: number; + /** Absolute transaction deadline. A worker is never asked to fence unless + * phase-2 plus the configured admission-restore reserve remain after drain. */ + deadlineMs?: number; + now?: () => number; + /** One projection sampled by prepareRiffFleetForShutdown before any fence. */ + durableSnapshot?: sessionStore.ActiveRiffShutdownSnapshot; +}; + +export type RiffFleetPrepareEntry = { + ds: DaemonSession; + result: RiffShutdownPrepareResult; +}; + +export type FencedRiffShutdownParticipant = + | PreparedRiffShutdown + | PossiblyFencedRiffShutdown; + +export type UniqueDaemonShutdownSessions = + | { ok: true; sessions: DaemonSession[] } + | { ok: false; sessionId: string; error: string }; + +/** The active registry can retain multiple aliases to one exact runtime object + * after transfer/restore. Process that object once, but refuse two distinct + * objects claiming the same durable session id: there is no unique generation + * that shutdown can safely fence or retire. */ +export function collectUniqueDaemonShutdownSessions( + candidates: Iterable, +): UniqueDaemonShutdownSessions { + const seenObjects = new Set(); + const bySessionId = new Map(); + const sessions: DaemonSession[] = []; + for (const ds of candidates) { + if (seenObjects.has(ds)) continue; + seenObjects.add(ds); + const sessionId = ds.session.sessionId; + const existing = bySessionId.get(sessionId); + if (existing && existing !== ds) { + return { + ok: false, + sessionId, + error: `distinct daemon session generations share session id ${sessionId}`, + }; + } + bySessionId.set(sessionId, ds); + sessions.push(ds); + } + return { ok: true, sessions }; +} + +export type RiffShutdownDetachOutcome = + | { + ok: true; + requestId: string; + taskId: string | null; + disposition: 'lineage_persisted'; + worker?: ChildProcess; + } + | RiffShutdownFailure; + +function label(ds: DaemonSession): string { + return ds.session.sessionId.slice(0, 8); +} + +function workerHasExited(worker: ChildProcess): boolean { + return (worker.exitCode !== null && worker.exitCode !== undefined) + || (worker.signalCode !== null && worker.signalCode !== undefined); +} + +function send(worker: ChildProcess, message: DaemonToWorker): boolean { + try { + worker.send(message); + return true; + } catch { + return false; + } +} + +/** Attach the exact-worker response/exit listeners before publishing a phase + * request. Results from another generation/session/request are inert. */ +function requestPhase( + ds: DaemonSession, + worker: ChildProcess, + requestId: string, + phase: ShutdownPhase, + timeoutMs: number, +): Promise { + return new Promise(resolve => { + let settled = false; + const finish = (result: ShutdownPhaseResult): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.removeListener('message', onMessage); + worker.removeListener('exit', onExit); + resolve(result); + }; + const onMessage = (raw: unknown): void => { + const msg = raw as WorkerToDaemon; + if (msg?.type !== 'riff_shutdown_result' + || msg.requestId !== requestId + || msg.phase !== phase) return; + if (ds.worker !== worker) { + finish({ ok: false, taskId: msg.taskId, error: 'stale_worker_generation' }); + return; + } + finish({ + ok: msg.ok, + taskId: msg.taskId, + ...(msg.error ? { error: msg.error } : {}), + }); + }; + const onExit = (): void => finish({ + ok: false, + taskId: null, + error: `worker_exited_during_shutdown_${phase}`, + }); + const timer = setTimeout(() => finish({ + ok: false, + taskId: null, + error: `riff_shutdown_${phase}_timeout`, + }), timeoutMs); + timer.unref?.(); + worker.on('message', onMessage); + worker.once('exit', onExit); + const message: DaemonToWorker = phase === 'prepare' + ? { type: 'riff_shutdown_prepare', requestId } + : { type: 'riff_shutdown_abort', requestId }; + if (!send(worker, message)) { + finish({ ok: false, taskId: null, error: `riff_shutdown_${phase}_send_failed` }); + } + }); +} + +function clearWorkerOwnership(ds: DaemonSession, worker: ChildProcess): void { + if (ds.worker !== worker) return; + ds.worker = null; + ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; + ds.managedTurnOrigin = undefined; + ds.riffShutdownState = undefined; +} + +function clearWorkerlessShutdownState(ds: DaemonSession, requestId: string): void { + if (ds.riffShutdownState?.requestId !== requestId) return; + if (ds.worker) return; + ds.riffShutdownState = undefined; +} + +/** Daemon-owned accepted input has not crossed worker IPC yet. The bot-wide + * mutation lease blocks new admissions, but these older buffers must also be + * empty before a worker can be detached. */ +function daemonInputBlocker(ds: DaemonSession): string | null { + const parts: string[] = []; + if (ds.pendingRepoCommitInFlight || ds.worktreeCreating) parts.push('initial_start=1'); + if (ds.pendingPrompt) parts.push('prompt=1'); + if (ds.pendingRawInput) parts.push('raw=1'); + if (ds.pendingFollowUpInput) parts.push('raw_followup=1'); + if ((ds.pendingFollowUps?.length ?? 0) > 0) { + parts.push(`followups=${ds.pendingFollowUps!.length}`); + } + if (ds.session.queued) parts.push('queued=1'); + return parts.length > 0 ? parts.join(',') : null; +} + +async function abortWorkerPreparation( + ds: DaemonSession, + worker: ChildProcess, + requestId: string, + timeoutMs: number, + lineageVerified = false, + exactTask?: { taskId: string | null }, +): Promise { + if (workerHasExited(worker)) { + if (ds.worker && ds.worker !== worker) { + return { + ok: false, + taskId: ds.riffShutdownState?.taskId ?? ds.session.riffParentTaskId ?? null, + error: 'new_worker_generation', + }; + } + if (lineageVerified) { + if (ds.worker === worker) clearWorkerOwnership(ds, worker); + else clearWorkerlessShutdownState(ds, requestId); + return { ok: true, taskId: ds.session.riffParentTaskId ?? null }; + } + // No backend remains to ACK admission restoration and the drained lineage + // was not durably recovered. Retain a daemon-side fence instead of claiming + // rollback success and permitting a stale-lineage replacement. + if (!ds.riffShutdownState || ds.riffShutdownState.requestId === requestId) { + ds.riffShutdownState = { + phase: 'prepared', + requestId, + taskId: ds.session.riffParentTaskId ?? null, + }; + } + return { + ok: false, + taskId: ds.session.riffParentTaskId ?? null, + error: 'worker_exited_before_admission_restore', + }; + } + if (ds.worker !== worker) { + return { + ok: false, + taskId: ds.riffShutdownState?.taskId ?? ds.session.riffParentTaskId ?? null, + error: 'new_worker_generation', + }; + } + const result = await requestPhase(ds, worker, requestId, 'abort', timeoutMs); + if (result.ok && exactTask && result.taskId !== exactTask.taskId) { + return { + ok: false, + taskId: result.taskId, + error: `abort_task_lineage_mismatch:expected=${exactTask.taskId ?? 'none'},` + + `actual=${result.taskId ?? 'none'}`, + }; + } + if (result.ok && ds.riffShutdownState?.requestId === requestId) { + ds.riffShutdownState = undefined; + } + return result; +} + +function unfencedRefusal( + ds: DaemonSession, + error: string, + requestId?: string, +): UnfencedRiffShutdownRefusal { + return { + ok: false, + fence: 'none', + ...(requestId ? { requestId } : {}), + taskId: ds.session.riffParentTaskId ?? null, + error, + }; +} + +/** Phase 1: fence one exact Riff generation and drain task-id materialization. + * Nothing exits or restores admission here. A failure after the prepare send + * returns an exact possibly-fenced handle so all peers can be restored in one + * concurrent fleet wave rather than serial drain+abort chains. */ +export async function prepareRiffSessionForShutdown( + ds: DaemonSession, + options: RiffShutdownPrepareOptions = {}, +): Promise { + const frozenBackend = ds.initConfig?.backendType ?? ds.session.backendType; + if (frozenBackend !== 'riff') { + return unfencedRefusal(ds, 'not_riff_backend'); + } + if (ds.riffCloseState || ds.riffShutdownState) { + return unfencedRefusal( + ds, + ds.riffCloseState ? 'explicit_close_in_progress' : 'shutdown_detach_in_progress', + ); + } + const daemonBlockerBeforePrepare = daemonInputBlocker(ds); + if (daemonBlockerBeforePrepare) { + return unfencedRefusal(ds, `daemon_inputs_not_drained:${daemonBlockerBeforePrepare}`); + } + + const runtimeTaskIdAtPrepare = ds.session.riffParentTaskId ?? null; + let durableSnapshot = options.durableSnapshot; + if (!durableSnapshot) { + try { + [durableSnapshot] = sessionStore.getActiveRiffShutdownSnapshotsBatch([ + ds.session.sessionId, + ]); + } catch (err) { + return unfencedRefusal( + ds, + `durable_session_read_failed:${err instanceof Error ? err.message : String(err)}`, + ); + } + } + if (!durableSnapshot || durableSnapshot.sessionId !== ds.session.sessionId) { + return unfencedRefusal(ds, 'durable_session_snapshot_mismatch'); + } + const durableTaskIdAtPrepare = durableSnapshot.taskId; + const runtimeOwner = { + pid: ds.session.pid ?? null, + larkAppId: ds.session.larkAppId ?? null, + backendType: ds.session.backendType ?? null, + }; + const durableOwnerAtPrepare = durableSnapshot.owner; + if (durableOwnerAtPrepare.pid !== runtimeOwner.pid + || durableOwnerAtPrepare.larkAppId !== runtimeOwner.larkAppId + || durableOwnerAtPrepare.backendType !== runtimeOwner.backendType) { + return unfencedRefusal( + ds, + `durable_session_owner_mismatch:current=${JSON.stringify(durableOwnerAtPrepare)},` + + `runtime=${JSON.stringify(runtimeOwner)}`, + ); + } + + const now = options.now ?? Date.now; + const abortReserveMs = options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS; + let drainTimeoutMs = options.drainTimeoutMs ?? RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS; + if (options.deadlineMs !== undefined) { + const availableDrainMs = options.deadlineMs + - now() + - RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS + - abortReserveMs; + if (availableDrainMs <= 0) { + return unfencedRefusal(ds, 'insufficient_abort_budget_before_fence'); + } + drainTimeoutMs = Math.min(drainTimeoutMs, availableDrainMs); + } + + const requestId = randomUUID(); + const worker = ds.worker; + if (!worker || workerHasExited(worker)) { + // Retire a previously observed dead handle before installing the + // workerless fence. Any later non-null ds.worker is then unambiguously a + // new generation and cannot be blessed by this transaction. + if (worker) clearWorkerOwnership(ds, worker); + // Workerless rows can still carry a newer runtime-only lineage after a + // failed ordinary riff_task_id save. Fence daemon admission now; phase 2 + // performs an exact owner/lineage CAS and fresh disk verification. + ds.riffShutdownState = { + phase: 'prepared', + requestId, + taskId: ds.session.riffParentTaskId ?? null, + }; + return { + ok: true, + fence: 'prepared', + requestId, + taskId: ds.session.riffParentTaskId ?? null, + runtimeTaskIdAtPrepare, + durableTaskIdAtPrepare, + durableOwnerAtPrepare, + lineageVerified: false, + worker: null, + }; + } + + ds.riffShutdownState = { phase: 'preparing', requestId }; + const prepared = await requestPhase( + ds, + worker, + requestId, + 'prepare', + drainTimeoutMs, + ); + if (!prepared.ok) { + // This exact response proves the worker refused before installing a backend + // fence. Every other failure has ambiguous fence state and requires a + // positive admission-restored ACK. + const unfencedWorkerRefusal = prepared.error === 'explicit_close_in_progress' + || prepared.error === 'not_riff_backend' + || prepared.error === 'riff_shutdown_prepare_send_failed' + || prepared.error?.startsWith('worker_inputs_not_drained:') === true; + if (unfencedWorkerRefusal && ds.riffShutdownState?.requestId === requestId) { + ds.riffShutdownState = undefined; + } + if (unfencedWorkerRefusal) { + return { + ok: false, + fence: 'none', + requestId, + taskId: prepared.taskId, + error: prepared.error ?? 'riff_shutdown_prepare_failed', + }; + } + return { + ok: false, + fence: 'possible', + requestId, + taskId: prepared.taskId, + error: prepared.error ?? 'riff_shutdown_prepare_failed', + worker, + }; + } + if (ds.worker !== worker) { + return { + ok: false, + fence: 'possible', + requestId, + taskId: prepared.taskId, + error: 'stale_worker_generation', + worker, + }; + } + ds.riffShutdownState = { phase: 'prepared', requestId, taskId: prepared.taskId }; + + // Worker events can release a queued-activation journal/tail independently + // of bot-turn admission. Re-sample after drain so commit cannot strand input. + const daemonBlockerAfterPrepare = daemonInputBlocker(ds); + if (daemonBlockerAfterPrepare) { + return { + ok: false, + fence: 'possible', + requestId, + taskId: prepared.taskId, + error: `daemon_inputs_not_drained:${daemonBlockerAfterPrepare}`, + worker, + expectedAbortTaskId: prepared.taskId, + }; + } + + return { + ok: true, + fence: 'prepared', + requestId, + taskId: prepared.taskId, + runtimeTaskIdAtPrepare, + durableTaskIdAtPrepare, + durableOwnerAtPrepare, + lineageVerified: false, + worker, + }; +} + +export type RiffFleetPrepareOptions = Omit< + RiffShutdownPrepareOptions, + 'durableSnapshot' +> & { + snapshotTimeoutMs?: number; +}; + +/** Take one fresh projection for the complete candidate set, then (and only + * then) publish prepare requests concurrently. */ +export async function prepareRiffFleetForShutdown( + candidates: readonly DaemonSession[], + options: RiffFleetPrepareOptions = {}, +): Promise { + if (candidates.length === 0) return []; + const now = options.now ?? Date.now; + const configuredSnapshotTimeout = options.snapshotTimeoutMs + ?? RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS; + const remaining = options.deadlineMs === undefined + ? configuredSnapshotTimeout + : Math.max(0, options.deadlineMs - now()); + if (remaining <= 0) { + return candidates.map(ds => ({ + ds, + result: unfencedRefusal(ds, 'shutdown_deadline_elapsed_before_initial_snapshot'), + })); + } + + let snapshots: sessionStore.ActiveRiffShutdownSnapshot[]; + try { + snapshots = sessionStore.getActiveRiffShutdownSnapshotsBatch( + candidates.map(ds => ds.session.sessionId), + { maxWaitMs: Math.min(configuredSnapshotTimeout, remaining) }, + ); + } catch (error) { + const message = `initial_riff_snapshot_failed:${error instanceof Error + ? error.message + : String(error)}`; + return candidates.map(ds => ({ ds, result: unfencedRefusal(ds, message) })); + } + + const snapshotsBySession = new Map(snapshots.map(snapshot => [snapshot.sessionId, snapshot])); + return Promise.all(candidates.map(async ds => ({ + ds, + result: await prepareRiffSessionForShutdown(ds, { + ...options, + durableSnapshot: snapshotsBySession.get(ds.session.sessionId), + }), + }))); +} + +function validatePreparedRiffShutdownForPersistence( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): { ok: true } | RiffShutdownFailure { + if (ds.riffShutdownState?.requestId !== prepared.requestId + || ds.riffShutdownState.phase !== 'prepared') { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'shutdown_prepare_ownership_lost', + rollbackDisposition: 'retain_fence', + }; + } + const daemonBlocker = daemonInputBlocker(ds); + if (daemonBlocker) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `daemon_inputs_not_drained:${daemonBlocker}`, + rollbackDisposition: 'abort_safe', + }; + } + + if (prepared.worker) { + if (ds.worker !== prepared.worker || workerHasExited(prepared.worker)) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'stale_worker_generation', + rollbackDisposition: 'retain_fence', + }; + } + } else if (ds.worker) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'new_worker_generation', + rollbackDisposition: 'retain_fence', + }; + } + const runtimeOwner = { + pid: ds.session.pid ?? null, + larkAppId: ds.session.larkAppId ?? null, + backendType: ds.session.backendType ?? null, + }; + if (runtimeOwner.pid !== prepared.durableOwnerAtPrepare.pid + || runtimeOwner.larkAppId !== prepared.durableOwnerAtPrepare.larkAppId + || runtimeOwner.backendType !== prepared.durableOwnerAtPrepare.backendType) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `runtime_owner_changed:${JSON.stringify(runtimeOwner)}`, + rollbackDisposition: 'retain_fence', + }; + } + const runtimeTaskId = ds.session.riffParentTaskId ?? null; + const runtimeLineageExpected = prepared.worker + ? runtimeTaskId === prepared.runtimeTaskIdAtPrepare || runtimeTaskId === prepared.taskId + : runtimeTaskId === prepared.runtimeTaskIdAtPrepare; + if (!runtimeLineageExpected) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `runtime_lineage_changed:${runtimeTaskId ?? 'none'}`, + rollbackDisposition: 'retain_fence', + }; + } + return { ok: true }; +} + +export type PreparedRiffFleetEntry = { + ds: DaemonSession; + result: PreparedRiffShutdown; +}; + +export type RiffFleetPersistenceResult = { ok: true } +| (RiffShutdownFailure & { + sessionIds: readonly string[]; + retainFencedSessionIds: readonly string[]; +}); + +/** Phase 2 fleet transaction: validate every runtime generation, then compare + * and publish all durable lineage rows with one lock and one rename. */ +export function persistPreparedRiffShutdownFleet( + entries: readonly PreparedRiffFleetEntry[], + options: { + persistTimeoutMs?: number; + deadlineMs?: number; + now?: () => number; + } = {}, +): RiffFleetPersistenceResult { + if (entries.length === 0) return { ok: true }; + const now = options.now ?? Date.now; + + const validationFailures = entries + .map(({ ds, result }) => ({ + sessionId: ds.session.sessionId, + failure: validatePreparedRiffShutdownForPersistence(ds, result), + })) + .filter((entry): entry is { + sessionId: string; + failure: RiffShutdownFailure; + } => !entry.failure.ok); + if (validationFailures.length > 0) { + const retainFencedSessionIds = validationFailures + .filter(({ failure }) => failure.rollbackDisposition === 'retain_fence') + .map(({ sessionId }) => sessionId); + return { + ok: false, + taskId: null, + error: `Riff fleet persistence preflight failed: ${validationFailures + .map(({ sessionId, failure }) => `${sessionId}:${failure.error}`) + .join(';')}`, + rollbackDisposition: retainFencedSessionIds.length > 0 ? 'retain_fence' : 'abort_safe', + sessionIds: validationFailures.map(({ sessionId }) => sessionId), + retainFencedSessionIds, + }; + } + + const configuredPersistTimeout = options.persistTimeoutMs + ?? RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS; + const remaining = options.deadlineMs === undefined + ? configuredPersistTimeout + : Math.max(0, options.deadlineMs - now()); + if (remaining <= 0) { + return { + ok: false, + taskId: null, + error: 'Riff fleet lineage batch failed (prewrite_io): shutdown deadline elapsed', + rollbackDisposition: 'abort_safe', + sessionIds: entries.map(({ ds }) => ds.session.sessionId), + retainFencedSessionIds: [], + }; + } + + try { + sessionStore.persistActiveRiffLineagesExactBatch(entries.map(({ ds, result }) => ({ + sessionId: ds.session.sessionId, + taskId: result.durableTaskIdAtPrepare, + owner: result.durableOwnerAtPrepare, + targetTaskId: result.taskId, + expectedCurrentTaskIds: [result.durableTaskIdAtPrepare, result.taskId], + })), { + maxWaitMs: Math.min(configuredPersistTimeout, remaining), + }); + } catch (error) { + const batchError = error instanceof sessionStore.RiffLineageBatchError ? error : undefined; + const stage = batchError?.stage ?? 'prewrite_io'; + const retainFencedSessionIds = stage === 'postrename_ambiguity' + ? entries.map(({ ds }) => ds.session.sessionId) + : stage === 'prewrite_ownership' + ? [...(batchError?.sessionIds ?? [])] + : []; + return { + ok: false, + taskId: null, + error: `Riff fleet lineage batch failed (${stage}): ${error instanceof Error + ? error.message + : String(error)}`, + rollbackDisposition: retainFencedSessionIds.length > 0 ? 'retain_fence' : 'abort_safe', + sessionIds: batchError?.sessionIds ?? entries.map(({ ds }) => ds.session.sessionId), + retainFencedSessionIds, + }; + } + + for (const { ds, result } of entries) { + ds.session.riffParentTaskId = result.taskId ?? undefined; + result.lineageVerified = true; + } + if (options.deadlineMs !== undefined && now() >= options.deadlineMs) { + const sessionIds = entries.map(({ ds }) => ds.session.sessionId); + return { + ok: false, + taskId: null, + error: 'shutdown_deadline_elapsed_after_batch_persist', + rollbackDisposition: 'retain_fence', + sessionIds, + retainFencedSessionIds: sessionIds, + }; + } + const lost = entries + .filter(({ ds, result }) => !isPreparedRiffSessionCurrent(ds, result)) + .map(({ ds }) => ds.session.sessionId); + if (lost.length > 0) { + return { + ok: false, + taskId: null, + error: `shutdown_prepare_ownership_lost_after_batch_persist:${lost.join(',')}`, + rollbackDisposition: 'retain_fence', + sessionIds: lost, + retainFencedSessionIds: lost, + }; + } + return { ok: true }; +} + +/** Single-session compatibility path. Fleet shutdown uses the batch function + * above; this API remains for explicit focused operations and older callers. */ +export function persistPreparedRiffShutdown( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): { ok: true } | RiffShutdownFailure { + const validation = validatePreparedRiffShutdownForPersistence(ds, prepared); + if (!validation.ok) return validation; + + try { + sessionStore.persistActiveRiffLineageExact(ds.session.sessionId, prepared.taskId, { + expectedCurrentTaskIds: [prepared.durableTaskIdAtPrepare, prepared.taskId], + expectedOwner: prepared.durableOwnerAtPrepare, + }); + } catch (err) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `lineage_persist_failed:${err instanceof Error ? err.message : String(err)}`, + rollbackDisposition: err instanceof sessionStore.RiffLineageOwnershipError + ? 'retain_fence' + : 'abort_safe', + }; + } + + let fresh: ReturnType; + try { + fresh = sessionStore.getSessionFresh(ds.session.sessionId); + } catch (err) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `fresh_lineage_verification_failed:${err instanceof Error ? err.message : String(err)}`, + rollbackDisposition: 'retain_fence', + }; + } + const freshTaskId = fresh?.riffParentTaskId ?? null; + const freshOwner = fresh + ? { + pid: fresh.pid ?? null, + larkAppId: fresh.larkAppId ?? null, + backendType: fresh.backendType ?? null, + } + : null; + const freshOwnerMatches = freshOwner !== null + && freshOwner.pid === prepared.durableOwnerAtPrepare.pid + && freshOwner.larkAppId === prepared.durableOwnerAtPrepare.larkAppId + && freshOwner.backendType === prepared.durableOwnerAtPrepare.backendType; + if (!fresh + || fresh.status !== 'active' + || freshTaskId !== prepared.taskId + || !freshOwnerMatches) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: `fresh_lineage_verification_failed:status=${fresh?.status ?? 'missing'},` + + `task=${freshTaskId ?? 'none'},expected=${prepared.taskId ?? 'none'},` + + `owner=${JSON.stringify(freshOwner)},` + + `expectedOwner=${JSON.stringify(prepared.durableOwnerAtPrepare)}`, + rollbackDisposition: 'retain_fence', + }; + } + ds.session.riffParentTaskId = prepared.taskId ?? undefined; + prepared.lineageVerified = true; + + if (!isPreparedRiffSessionCurrent(ds, prepared)) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'shutdown_prepare_ownership_lost_after_persist', + rollbackDisposition: 'retain_fence', + }; + } + return { ok: true }; +} + +export function isPreparedRiffSessionCurrent( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): boolean { + if (ds.riffShutdownState?.requestId !== prepared.requestId + || ds.riffShutdownState.phase !== 'prepared') return false; + if (prepared.worker) { + return ds.worker === prepared.worker && !workerHasExited(prepared.worker); + } + return ds.worker === null; +} + +/** After verified lineage persistence, an exact prepared worker that exited + * and was cleared by worker-pool can be safely rolled back locally. A new + * worker generation or a different fence remains ownership ambiguity. */ +export function canAbortVerifiedExitedRiffPreparation( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): boolean { + return prepared.lineageVerified + && !!prepared.worker + && workerHasExited(prepared.worker) + && ds.worker === null + && ds.riffShutdownState?.requestId === prepared.requestId; +} + +/** Roll back one prepared participant. State clears only after the exact worker + * ACKs admission restoration; timeout/late ACK remains deliberately fail-closed. */ +export async function abortPreparedRiffShutdown( + ds: DaemonSession, + prepared: PreparedRiffShutdown, + options: { abortTimeoutMs?: number } = {}, +): Promise { + if (ds.riffShutdownState?.requestId !== prepared.requestId) { + return { ok: false, taskId: prepared.taskId, error: 'shutdown_prepare_ownership_lost' }; + } + if (!prepared.worker) { + if (ds.worker) { + return { ok: false, taskId: prepared.taskId, error: 'new_worker_generation' }; + } + clearWorkerlessShutdownState(ds, prepared.requestId); + return { ok: true, taskId: prepared.taskId }; + } + if (workerHasExited(prepared.worker)) { + return abortWorkerPreparation( + ds, + prepared.worker, + prepared.requestId, + options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + prepared.lineageVerified, + { taskId: prepared.taskId }, + ); + } + return abortWorkerPreparation( + ds, + prepared.worker, + prepared.requestId, + options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + prepared.lineageVerified, + { taskId: prepared.taskId }, + ); +} + +async function abortPossiblyFencedRiffShutdown( + ds: DaemonSession, + participant: PossiblyFencedRiffShutdown, + timeoutMs: number, +): Promise { + if (ds.riffShutdownState?.requestId !== participant.requestId) { + return { + ok: false, + taskId: participant.taskId, + error: 'shutdown_prepare_ownership_lost', + }; + } + return abortWorkerPreparation( + ds, + participant.worker, + participant.requestId, + timeoutMs, + false, + Object.prototype.hasOwnProperty.call(participant, 'expectedAbortTaskId') + ? { taskId: participant.expectedAbortTaskId ?? null } + : undefined, + ); +} + +export type RiffFleetAbortEntry = { + ds: DaemonSession; + result: FencedRiffShutdownParticipant; +}; + +export type RiffFleetAbortResult = { + ds: DaemonSession; + participant: FencedRiffShutdownParticipant; + result: ShutdownPhaseResult; +}; + +/** Restore every exact prepared/possibly-fenced generation concurrently. No + * participant can consume another's timeout budget. */ +export async function abortRiffShutdownFleet( + entries: readonly RiffFleetAbortEntry[], + options: { + abortTimeoutMs?: number; + deadlineMs?: number; + now?: () => number; + } = {}, +): Promise { + const now = options.now ?? Date.now; + const configuredTimeout = options.abortTimeoutMs ?? RIFF_ADMISSION_RESTORE_TIMEOUT_MS; + const remaining = options.deadlineMs === undefined + ? configuredTimeout + : Math.max(0, options.deadlineMs - now()); + const timeoutMs = Math.min(configuredTimeout, remaining); + + return Promise.all(entries.map(async ({ ds, result: participant }) => { + if (timeoutMs <= 0) { + return { + ds, + participant, + result: { + ok: false, + taskId: participant.taskId, + error: 'shutdown_deadline_elapsed_before_abort', + }, + }; + } + const result = participant.ok + ? await abortPreparedRiffShutdown(ds, participant, { abortTimeoutMs: timeoutMs }) + : await abortPossiblyFencedRiffShutdown(ds, participant, timeoutMs); + return { ds, participant, result }; + })); +} + +/** Phase 3: synchronous, infallible-after-validation commit. The coordinator + * validates every participant first, then calls this without an intervening + * await, so one session can never refuse after a peer was detached. */ +export function commitPreparedRiffShutdown( + ds: DaemonSession, + prepared: PreparedRiffShutdown, +): boolean { + if (!isPreparedRiffSessionCurrent(ds, prepared)) return false; + if (!prepared.worker) { + clearWorkerlessShutdownState(ds, prepared.requestId); + return true; + } + if (!send(prepared.worker, { type: 'riff_shutdown_commit', requestId: prepared.requestId })) { + // Lineage is already durably fresh-verified. Direct retirement is safe even + // if the final IPC channel closed between validation and send. + try { prepared.worker.kill('SIGTERM'); } catch { /* already exited */ } + } + clearWorkerOwnership(ds, prepared.worker); + return true; +} + +/** Single-session compatibility wrapper. Daemon shutdown intentionally uses + * the explicit three-phase API above so a multi-Riff fleet cannot half-commit. */ +export async function detachRiffWorkerForShutdown( + ds: DaemonSession, + options: { drainTimeoutMs?: number; abortTimeoutMs?: number } = {}, +): Promise { + const prepared = await prepareRiffSessionForShutdown(ds, options); + if (!prepared.ok) { + if (prepared.fence === 'none') return prepared; + const [aborted] = await abortRiffShutdownFleet([{ ds, result: prepared }], options); + const abortSuffix = aborted.result.ok + ? '' + : `;admission_restore_failed:${aborted.result.error ?? 'unknown'}`; + logger.error( + `[${label(ds)}] Riff shutdown drain failed (${prepared.error}${abortSuffix}); ` + + (aborted.result.ok + ? 'worker retained with admission restored' + : 'worker retained and admission fence kept fail-closed'), + ); + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: prepared.error + abortSuffix, + }; + } + const persisted = persistPreparedRiffShutdown(ds, prepared); + if (!persisted.ok) { + if (persisted.rollbackDisposition === 'retain_fence') return persisted; + const aborted = await abortPreparedRiffShutdown(ds, prepared, options); + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: persisted.error + + (aborted.ok ? '' : `;admission_restore_failed:${aborted.error ?? 'unknown'}`), + }; + } + if (!commitPreparedRiffShutdown(ds, prepared)) { + return { + ok: false, + requestId: prepared.requestId, + taskId: prepared.taskId, + error: 'shutdown_prepare_ownership_lost_before_commit', + rollbackDisposition: 'retain_fence', + }; + } + logger.info( + `[${label(ds)}] Riff shutdown detach committed after durable lineage ` + + `${prepared.taskId ?? 'none'}`, + ); + return { + ok: true, + requestId: prepared.requestId, + taskId: prepared.taskId, + disposition: 'lineage_persisted', + ...(prepared.worker ? { worker: prepared.worker } : {}), + }; +} diff --git a/src/core/riff-worker-shutdown-readiness.ts b/src/core/riff-worker-shutdown-readiness.ts new file mode 100644 index 000000000..f7101dd19 --- /dev/null +++ b/src/core/riff-worker-shutdown-readiness.ts @@ -0,0 +1,31 @@ +export interface RiffWorkerShutdownInputSnapshot { + /** False while async init can still materialize an opening prompt later. */ + initPromptMaterialized: boolean; + /** A normal prompt has been removed from pendingMessages but has not yet + * crossed the adapter/backend write boundary. */ + isFlushing: boolean; + pendingMessages: number; + pendingRawInputs: number; + pendingSessionRename: boolean; + sessionRenameInFlight: boolean; + /** Raw command text -> Enter sequences that can still append a backend + * write after the shutdown fence is sampled. */ + commandLineWritesPending: number; +} + +/** Describe worker-owned input not proven to be inside RiffBackend.writeChain. */ +export function riffWorkerShutdownInputBlocker( + snapshot: RiffWorkerShutdownInputSnapshot, +): string | null { + const parts: string[] = []; + if (!snapshot.initPromptMaterialized) parts.push('init=materializing'); + if (snapshot.isFlushing) parts.push('flushing=1'); + if (snapshot.pendingMessages > 0) parts.push(`messages=${snapshot.pendingMessages}`); + if (snapshot.pendingRawInputs > 0) parts.push(`raw=${snapshot.pendingRawInputs}`); + if (snapshot.pendingSessionRename) parts.push('rename=1'); + if (snapshot.sessionRenameInFlight) parts.push('rename_inflight=1'); + if (snapshot.commandLineWritesPending > 0) { + parts.push(`command_writes=${snapshot.commandLineWritesPending}`); + } + return parts.length > 0 ? parts.join(',') : null; +} diff --git a/src/core/shutdown-budgets.ts b/src/core/shutdown-budgets.ts new file mode 100644 index 000000000..3501ae0ef --- /dev/null +++ b/src/core/shutdown-budgets.ts @@ -0,0 +1,33 @@ +/** + * Graceful-shutdown budgets are ordered so a bounded Riff create/follow-up can + * drain before the daemon persists its final lineage and asks workers to exit. + * Shutdown never cancels accepted Riff work as a fallback. + */ +export const RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS = 12_000; + +/** Admission restoration can wait for the same bounded 10s create/follow-up + * that prepare was draining. The daemon keeps its retirement fence throughout. */ +export const RIFF_ADMISSION_RESTORE_TIMEOUT_MS = 11_000; + +/** Bounded acquisition of the bot-wide mutation lease. A timed-out waiter is + * removed and can never run after shutdown has already been refused. */ +export const BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS = 1_000; + +/** Initial all-owner snapshot and phase-2 batch CAS each use one short lock. */ +export const RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS = 1_000; +export const RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS = 1_000; + +/** Scheduling/logging slack inside the graceful daemon shutdown budget. */ +export const DAEMON_SHUTDOWN_OVERHEAD_MS = 2_000; +export const DAEMON_WORKER_EXIT_GRACE_MS = 3_000; +export const DAEMON_SHUTDOWN_MAX_MS = + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS + + RIFF_SHUTDOWN_INITIAL_SNAPSHOT_TIMEOUT_MS + + RIFF_SHUTDOWN_DRAIN_TIMEOUT_MS + + RIFF_SHUTDOWN_BATCH_PERSIST_TIMEOUT_MS + + Math.max(RIFF_ADMISSION_RESTORE_TIMEOUT_MS, DAEMON_WORKER_EXIT_GRACE_MS) + + DAEMON_SHUTDOWN_OVERHEAD_MS; + +if (DAEMON_SHUTDOWN_MAX_MS > 28_000) { + throw new Error('complete daemon shutdown budget must remain at or below 28 seconds'); +} diff --git a/src/core/types.ts b/src/core/types.ts index 9fcabbcf8..a08e604b7 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -187,6 +187,19 @@ export interface DaemonSession { * "Web终端" button opens the riff sandbox. In-memory only — re-sent by the * worker on each task. */ riffAccessUrl?: string; + /** Explicit-close transaction: while present, no new Riff input may be + * admitted until cancellation commits or admission restoration is ACKed. */ + riffCloseState?: { + phase: 'preparing' | 'prepared' | 'uncertain'; + requestId: string; + taskId?: string; + }; + /** Graceful-shutdown transaction for the exact Riff worker generation. */ + riffShutdownState?: { + phase: 'preparing' | 'prepared'; + requestId: string; + taskId?: string | null; + }; usageLimit?: CliUsageLimitState; usageLimitRetryTimer?: NodeJS.Timeout; lastUserPrompt?: string; @@ -316,6 +329,14 @@ export interface DaemonSession { }; } +/** A non-null value means this Riff generation is deliberately rejecting new + * input until a close/shutdown commit or an ACKed admission restore. */ +export function riffRetirementAdmissionPhase(ds: DaemonSession): string | null { + if (ds.riffShutdownState) return `shutdown-${ds.riffShutdownState.phase}`; + if (ds.riffCloseState) return `close-${ds.riffCloseState.phase}`; + return null; +} + /** Composite key for activeSessions — allows multiple bots to have independent * sessions anchored on the same id. The first arg is the **routing anchor**: * - thread-scope → rootMessageId diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 133c0878e..2629f2f02 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -22,7 +22,7 @@ import { fallbackTurnId, isSubstituteTurn } from './reply-target.js'; import { updateMessage, deleteMessage, sendEphemeralCard, sendUserMessage, addReaction, removeReaction, getMessageChatId, MessageWithdrawnError } from '../im/lark/client.js'; import { buildStreamingCard, buildPrivateSnapshotCard, buildSessionCard, buildTuiPromptCard, buildTuiPromptResolvedCard, buildRelayedFrozenCard, getCliDisplayName } from '../im/lark/card-builder.js'; import { loadFrozenCards, saveFrozenCards } from '../services/frozen-card-store.js'; -import { hashUrlForLog, cancelRiffTaskById } from '../adapters/backend/riff-backend.js'; +import { hashUrlForLog } from '../adapters/backend/riff-backend.js'; import { logger } from '../utils/logger.js'; import { createCliAdapterSync } from '../adapters/cli/registry.js'; import { traeHome } from '../services/traex-paths.js'; @@ -75,7 +75,14 @@ import { isStructuredBridgeAdoptCli } from '../services/structured-bridge-clis.j import { resolveEffectivePluginIds } from './plugins/effective.js'; import { ensureGatewayEntry } from './plugins/mcp/gateway-installer.js'; import type { CliTurnPayload, CodexAppTurnInput, DaemonToWorker, WorkerToDaemon, Session, DisplayMode } from '../types.js'; -import { activeSessionKey, sessionKey, sessionAnchorId, isDocNativeSession, type DaemonSession } from './types.js'; +import { + activeSessionKey, + sessionKey, + sessionAnchorId, + isDocNativeSession, + riffRetirementAdmissionPhase, + type DaemonSession, +} from './types.js'; import { DONE_REACTION_EMOJI_TYPE } from './pending-response.js'; import { buildTerminalUrl } from './terminal-url.js'; import { prependBotmuxBin } from './botmux-wrapper.js'; @@ -101,6 +108,8 @@ import { } from './session-title.js'; import { acknowledgeSessionReady } from './session-ready-handshake.js'; import { recordDispatchInputCommit } from './dispatch.js'; +import { cleanupExplicitSessionBacking } from './explicit-session-backing-cleanup.js'; +import { RIFF_ADMISSION_RESTORE_TIMEOUT_MS } from './shutdown-budgets.js'; type WindowsForkOptions = ForkOptions & { windowsHide?: boolean }; @@ -227,6 +236,18 @@ function requireCallbacks(): WorkerPoolCallbacks { // linear-scan by sessionId. let activeSessionsRegistry: Map | undefined; +type RiffWorkerCloseResult = { + ok: boolean; + taskId?: string; + error?: string; +}; + +const pendingRiffWorkerCloses = new Map void; +}>(); + export function setActiveSessionsRegistry(m: Map): void { activeSessionsRegistry = m; } @@ -1308,7 +1329,23 @@ export function ensureClaudeFolderTrust(workingDir: string, stateJsonPath: strin // ─── Kill worker ──────────────────────────────────────────────────────────── -export function killWorker(ds: DaemonSession): void { +export function killWorker( + ds: DaemonSession, + opts: { riffCloseCommitRequestId?: string } = {}, +): void { + const closeFrozenType = ds.initConfig?.backendType ?? ds.session.backendType; + if (ds.worker && !ds.worker.killed + && closeFrozenType === 'riff' + && !opts.riffCloseCommitRequestId) { + // A generic synchronous retirement cannot safely detach Riff. An accepted + // create/follow-up may still be waiting for the task id that becomes the + // only durable lineage anchor. + logger.error( + `[${tag(ds)}] Refused unprepared live Riff worker retirement; ` + + 'preserving worker and remote-task lineage', + ); + return; + } clearUsageLimitState(ds); ds.localProcessAttestation = undefined; // A managed-turn capability belongs to one concrete worker generation. @@ -1328,18 +1365,28 @@ export function killWorker(ds: DaemonSession): void { // session would leave an orphaned CLI running in tmux that still replies. // Destroy the backing session directly here so /close always terminates it. destroyOrphanedBackingSession(ds); + if (opts.riffCloseCommitRequestId + && ds.riffCloseState?.requestId === opts.riffCloseCommitRequestId) { + ds.riffCloseState = undefined; + } return; } try { - ds.worker.send({ type: 'close' } as DaemonToWorker); + if (opts.riffCloseCommitRequestId) { + ds.worker.send({ + type: 'close_commit', + requestId: opts.riffCloseCommitRequestId, + } as DaemonToWorker); + } else { + ds.worker.send({ type: 'close' } as DaemonToWorker); + } } catch { /* IPC already closed */ } + if (opts.riffCloseCommitRequestId + && ds.riffCloseState?.requestId === opts.riffCloseCommitRequestId) { + ds.riffCloseState = undefined; + } const w = ds.worker; - // riff：worker close 分支要有界 await 远端 task-cancel（destroySession 5s×2 重试， - // 外层 race 8s）。默认 2s SIGTERM backstop 会在取消发出前掐死进程，已关闭话题 - // 的远端任务照跑——冻结为 riff 的会话放宽到 24s（层级：destroy 20s < worker 22s - // < SIGTERM 24s < SIGKILL 29s；正常路径 worker 自行 exit，不会等满）。 - const closeFrozenType = ds.initConfig?.backendType ?? ds.session.backendType; - armWorkerKillBackstop(w, tag(ds), closeFrozenType === 'riff' ? 24_000 : WORKER_SIGTERM_BACKSTOP_MS); + armWorkerKillBackstop(w, tag(ds)); ds.worker = null; ds.workerPort = null; ds.workerToken = null; @@ -1359,22 +1406,17 @@ export function killWorker(ds: DaemonSession): void { function destroyOrphanedBackingSession(ds: DaemonSession): void { if (ds.initConfig?.adoptMode || ds.adoptedFrom) return; reclaimParkedCrashDiagnostic(ds); - // riff：worker 已死时 /close 仍要取消持久化血缘指向的远端任务——否则已关闭 - // 话题的远端 agent 继续拿着注入凭证发消息。fire-and-forget（内部有界+重试）。 + // Riff cancellation is asynchronous and cannot be made safe from this + // synchronous best-effort helper. The authoritative closeSession path awaits + // cancellation before publishing the closed row and retains lineage on + // failure. const frozenType = ds.initConfig?.backendType ?? ds.session.backendType; if (frozenType === 'riff') { - const taskId = ds.session.riffParentTaskId; - if (taskId) { - try { - const riffCfg = getBot(ds.larkAppId).config.riff; - if (riffCfg?.baseUrl) { - void cancelRiffTaskById(riffCfg, taskId).then((ok) => { - if (ok) logger.info(`[${tag(ds)}] killWorker: orphan riff task ${taskId} cancelled`); - }); - } - } catch { /* bot deregistered — nothing to cancel with */ } - ds.session.riffParentTaskId = undefined; - sessionStore.updateSession(ds.session); + if (ds.session.riffParentTaskId) { + logger.warn( + `[${tag(ds)}] worker-less Riff teardown requires awaited explicit close; ` + + `retaining task ${ds.session.riffParentTaskId} for retry`, + ); } return; } @@ -1388,6 +1430,305 @@ function destroyOrphanedBackingSession(ds: DaemonSession): void { } } +type RiffClosePreparation = + | { ok: true; taskId?: string } + | { + ok: false; + error: + | 'riff_cancel_failed' + | 'riff_config_missing' + | 'riff_task_changed' + | 'riff_worker_close_failed' + | 'riff_row_inconsistent' + | 'riff_durable_close_failed' + | 'riff_close_reconciliation_required' + | 'riff_shutdown_fence_in_progress'; + retryable: true; + taskId?: string; + }; + +async function abortLiveRiffWorkerClose( + ds: DaemonSession, + requestId: string, + opts: { allowAbsentAfterProvenRestore?: boolean } = {}, +): Promise { + if (ds.riffCloseState?.requestId !== requestId) return false; + const worker = ds.worker; + if (!worker || worker.killed) { + if (opts.allowAbsentAfterProvenRestore) { + ds.riffCloseState = undefined; + return true; + } + ds.riffCloseState = { ...ds.riffCloseState, phase: 'uncertain' }; + return false; + } + + const restored = await new Promise<{ ok: boolean; error?: string }>(resolve => { + let settled = false; + const finish = (result: { ok: boolean; error?: string }): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.removeListener?.('message', onMessage); + worker.removeListener?.('exit', onExit); + resolve(result); + }; + const onMessage = (raw: unknown): void => { + const msg = raw as WorkerToDaemon; + if (msg?.type !== 'close_abort_result' || msg.requestId !== requestId) return; + if (ds.worker !== worker) { + finish({ ok: false, error: 'stale_worker_generation' }); + return; + } + finish({ ok: msg.ok, ...(msg.error ? { error: msg.error } : {}) }); + }; + const onExit = (): void => finish({ + ok: false, + error: 'worker_exited_before_close_abort_result', + }); + const timer = setTimeout( + () => finish({ ok: false, error: 'close_abort_result_timeout' }), + RIFF_ADMISSION_RESTORE_TIMEOUT_MS, + ); + timer.unref?.(); + worker.on?.('message', onMessage); + worker.once?.('exit', onExit); + try { + worker.send({ type: 'close_abort', requestId } as DaemonToWorker); + } catch (err) { + finish({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }); + + if (restored.ok && ds.riffCloseState?.requestId === requestId) { + ds.riffCloseState = undefined; + return true; + } + if (opts.allowAbsentAfterProvenRestore + && ds.riffCloseState?.requestId === requestId) { + ds.riffCloseState = undefined; + return true; + } + if (ds.riffCloseState?.requestId === requestId) { + ds.riffCloseState = { ...ds.riffCloseState, phase: 'uncertain' }; + } + logger.warn( + `[${tag(ds)}] Riff close abort was not acknowledged (${restored.error ?? 'unknown'}); ` + + 'retaining admission fence pending explicit lineage reconciliation', + ); + return false; +} + +async function prepareLiveRiffWorkerClose(ds: DaemonSession): Promise { + const worker = ds.worker; + if (!worker || worker.killed) { + return { ok: false, error: 'riff_worker_close_failed', retryable: true }; + } + if (ds.riffCloseState || ds.riffShutdownState) { + return { + ok: false, + error: 'riff_worker_close_failed', + retryable: true, + ...((ds.riffCloseState?.taskId ?? ds.riffShutdownState?.taskId) + ? { taskId: (ds.riffCloseState?.taskId ?? ds.riffShutdownState?.taskId)! } + : {}), + }; + } + const requestId = randomUUID(); + ds.riffCloseState = { + phase: 'preparing', + requestId, + ...(ds.session.riffParentTaskId ? { taskId: ds.session.riffParentTaskId } : {}), + }; + let matchedCloseResult = false; + const result = await new Promise((resolve) => { + let settled = false; + const finish = (value: RiffWorkerCloseResult): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.removeListener?.('exit', onExit); + pendingRiffWorkerCloses.delete(requestId); + resolve(value); + }; + const onExit = (): void => finish({ + ok: false, + error: 'worker_exited_before_close_result', + }); + const timer = setTimeout( + () => finish({ ok: false, error: 'worker_close_result_timeout' }), + 23_000, + ); + timer.unref?.(); + worker.once?.('exit', onExit); + pendingRiffWorkerCloses.set(requestId, { + sessionId: ds.session.sessionId, + worker, + resolve: value => { + matchedCloseResult = true; + finish(value); + }, + }); + try { + worker.send({ type: 'close', requestId } as DaemonToWorker); + } catch (err) { + finish({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }); + + const taskId = result.taskId ?? ds.session.riffParentTaskId; + if (result.taskId) { + ds.session.riffParentTaskId = result.taskId; + try { + sessionStore.updateSession(ds.session); + } catch (err) { + await abortLiveRiffWorkerClose(ds, requestId); + logger.error( + `[${tag(ds)}] Riff close lineage persistence failed; close aborted: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return { + ok: false, + error: 'riff_durable_close_failed', + retryable: true, + taskId: result.taskId, + }; + } + } + + if (!result.ok) { + await abortLiveRiffWorkerClose(ds, requestId, { + allowAbsentAfterProvenRestore: matchedCloseResult, + }); + logger.warn( + `[${tag(ds)}] Riff worker close prepare failed: ${result.error ?? 'unknown'}; ` + + `session remains active${taskId ? ` (task ${taskId})` : ''}`, + ); + return { + ok: false, + error: 'riff_worker_close_failed', + retryable: true, + ...(taskId ? { taskId } : {}), + }; + } + + ds.riffCloseState = { + phase: 'prepared', + requestId, + ...(taskId ? { taskId } : {}), + }; + logger.info(`[${tag(ds)}] Riff worker close prepared and remote task cancellation confirmed`); + return { ok: true, ...(taskId ? { taskId } : {}) }; +} + +/** Await remote cancellation for any Riff owner before its durable row is + * closed. Live workers use prepare/commit; worker-less rows cancel their exact + * persisted task through current authoritative bot config. */ +async function prepareRiffExplicitClose( + ds: DaemonSession | undefined, + stored: Session | undefined, +): Promise { + const session = ds?.session ?? stored; + if (!session) return { ok: true }; + const backendType = ds?.initConfig?.backendType ?? session.backendType; + const taskId = session.riffParentTaskId; + if (ds?.riffShutdownState) { + const fencedTaskId = Object.prototype.hasOwnProperty.call(ds.riffShutdownState, 'taskId') + ? ds.riffShutdownState.taskId + : taskId; + return { + ok: false, + error: 'riff_shutdown_fence_in_progress', + retryable: true, + ...(typeof fencedTaskId === 'string' && fencedTaskId ? { taskId: fencedTaskId } : {}), + }; + } + if (ds?.riffCloseState) { + return { + ok: false, + error: 'riff_close_reconciliation_required', + retryable: true, + ...(ds.riffCloseState.taskId ? { taskId: ds.riffCloseState.taskId } : {}), + }; + } + if (backendType !== 'riff') return { ok: true }; + if (ds?.initConfig?.adoptMode || ds?.adoptedFrom || session.adoptedFrom) return { ok: true }; + + if (ds?.worker && !ds.worker.killed) { + if (!stored || stored.status !== 'active') { + return { + ok: false, + error: 'riff_row_inconsistent', + retryable: true, + ...(taskId ? { taskId } : {}), + }; + } + return prepareLiveRiffWorkerClose(ds); + } + + const durableBefore = sessionStore.getSession(session.sessionId); + if (ds && durableBefore + && ds.session.riffParentTaskId !== durableBefore.riffParentTaskId) { + const authoritativeTaskId = durableBefore.riffParentTaskId ?? ds.session.riffParentTaskId; + logger.warn( + `[${tag(ds)}] explicit close refused before cancellation: runtime/durable Riff lineage differs ` + + `(${ds.session.riffParentTaskId ?? 'none'}/${durableBefore.riffParentTaskId ?? 'none'})`, + ); + return { + ok: false, + error: 'riff_task_changed', + retryable: true, + ...(authoritativeTaskId ? { taskId: authoritativeTaskId } : {}), + }; + } + if (!taskId) return { ok: true }; + + let riffConfig; + const larkAppId = ds?.larkAppId ?? session.larkAppId; + try { if (larkAppId) riffConfig = getBot(larkAppId).config.riff; } catch { /* bot removed */ } + const cleanup = await cleanupExplicitSessionBacking({ + sessionId: session.sessionId, + backendType, + riffParentTaskId: taskId, + riffConfig, + }); + const closeLabel = ds ? tag(ds) : session.sessionId.slice(0, 8); + if (!cleanup.ok) { + logger.warn( + `[${closeLabel}] explicit close refused: ${cleanup.kind}; ` + + `Riff task ${taskId} remains active and retryable`, + ); + return { + ok: false, + error: cleanup.kind === 'riff_config_missing' ? 'riff_config_missing' : 'riff_cancel_failed', + retryable: true, + taskId, + }; + } + if (cleanup.kind !== 'cancelled_riff') return { ok: true }; + + const latest = sessionStore.getSession(session.sessionId); + const latestTaskId = latest?.riffParentTaskId; + const runtimeTaskId = ds?.session.riffParentTaskId; + if (!latest || latest.status !== 'active' || latestTaskId !== cleanup.taskId + || (ds && runtimeTaskId !== cleanup.taskId)) { + logger.warn( + `[${closeLabel}] explicit close cancelled stale Riff task ${cleanup.taskId}, ` + + `but runtime/durable lineage or status changed to ` + + `${runtimeTaskId ?? 'none'}/${latestTaskId ?? 'none'}/${latest?.status ?? 'missing'}; retry required`, + ); + return { + ok: false, + error: 'riff_task_changed', + retryable: true, + taskId: latestTaskId ?? runtimeTaskId ?? cleanup.taskId, + }; + } + + logger.info(`[${closeLabel}] Riff task ${cleanup.taskId} cancellation prepared for explicit close`); + return { ok: true, taskId: cleanup.taskId }; +} + /** * Reclaim a session's parked crash-diagnostic shell (`bmx-diag-`) and its * captured `.ansi` file. The worker normally tears these down itself (killCli / @@ -1493,10 +1834,23 @@ function armWorkerKillBackstop(w: ChildProcess, label: string, sigtermMs: number * Calling this on an unknown sessionId, an already-closed session, or a session * whose worker died asynchronously must still resolve with `{ ok: true }`. */ +export type CloseSessionResult = + | { ok: true; alreadyClosed: boolean } + | ({ + ok: false; + alreadyClosed: false; + } & Exclude); + export async function closeSession( sessionId: string, -): Promise<{ ok: true; alreadyClosed: boolean }> { +): Promise { const ds = findActiveBySessionId(sessionId); + const stored = sessionStore.getSession(sessionId); + const wasOpen = !!stored && stored.status !== 'closed'; + const isOwnedRiffClose = !ds?.initConfig?.adoptMode + && !ds?.adoptedFrom + && !stored?.adoptedFrom + && (ds?.initConfig?.backendType ?? ds?.session.backendType ?? stored?.backendType) === 'riff'; let killedLive = false; // 会话关闭即可回收其崩溃重启计数；否则每个曾崩溃过的 session 会在 daemon // 生命周期内永久占位（restartCounts 此前无任何 delete）。 @@ -1505,40 +1859,43 @@ export async function closeSession( // Usage ledger: flush the final delta before the worker goes away (a // crash/limited turn may never have reached an idle edge). recordUsageForDaemonSession(ds); - killWorker(ds); - // 文档入口清理：会话关闭即删除其绑定。只有旧 - // /subscribe-lark-doc 记录需要调飞书逐文件退订 API； - // /watch-comment 仅依赖应用级评论事件，删本地监听表即可。 - try { - const anchor = sessionAnchorId(ds); - const subs = listDocSubscriptionsForSession(config.session.dataDir, ds.larkAppId, anchor); - for (const sub of subs) { - if (sub.managedBy !== 'watch-comment') { - await unsubscribeDocFile(ds.larkAppId, { fileToken: sub.fileToken, fileType: sub.fileType }); - } - removeDocSubscription(config.session.dataDir, ds.larkAppId, sub.fileToken); - } - if (subs.length) logger.info(`[doc-comment] session ${sessionId.slice(0, 8)} closed → removed ${subs.length} doc binding(s)`); - } catch (err: any) { - logger.warn(`[doc-comment] cleanup on close failed for ${sessionId.slice(0, 8)}: ${err?.message ?? err}`); - } - activeSessionsRegistry?.delete(activeSessionKey(ds)); - killedLive = true; - if (!ds.exitEventEmitted) { - ds.exitEventEmitted = true; - dashboardEventBus.publish({ - type: 'session.exited', - body: { sessionId, reason: 'dashboard_close' }, - }); - emitSessionLifecycleHook(ds, 'session.exit', { reason: 'dashboard_close' }); - } } - // Persistence path — load → mark closed → save (delegated to sessionStore). - const stored = sessionStore.getSession(sessionId); - const wasOpen = !!stored && stored.status !== 'closed'; + // Riff is a remote credential-bearing process. Await cancellation before + // closing its durable row; failures preserve the route, worker, and lineage + // so the operation is retryable. + const prepared = await prepareRiffExplicitClose(ds, stored); + if (!prepared.ok) { + return { ...prepared, alreadyClosed: false }; + } + const preparedRiffRequestId = ds?.riffCloseState?.phase === 'prepared' + ? ds.riffCloseState.requestId + : undefined; + if (wasOpen) { - sessionStore.closeSession(sessionId); + try { + if (isOwnedRiffClose) { + sessionStore.closeSession(sessionId, { clearRiffParentTaskId: true }); + } else { + sessionStore.closeSession(sessionId); + } + } catch (err) { + if (!isOwnedRiffClose) throw err; + if (ds && preparedRiffRequestId) { + await abortLiveRiffWorkerClose(ds, preparedRiffRequestId); + } + logger.error( + `[${sessionId.slice(0, 8)}] Durable session close failed after Riff prepare; ` + + `worker admission restored: ${err instanceof Error ? err.message : String(err)}`, + ); + return { + ok: false, + alreadyClosed: false, + error: 'riff_durable_close_failed', + retryable: true, + ...(prepared.taskId ? { taskId: prepared.taskId } : {}), + }; + } const after = sessionStore.getSession(sessionId); dashboardEventBus.publish({ type: 'session.update', @@ -1553,6 +1910,59 @@ export async function closeSession( }); } + if (ds) { + killWorker(ds, { + ...(preparedRiffRequestId ? { riffCloseCommitRequestId: preparedRiffRequestId } : {}), + }); + // A transferred/restored exact object may remain under more than one alias. + // Remove only identity matches, never a same-key successor. + if (activeSessionsRegistry) { + for (const [registeredKey, candidate] of activeSessionsRegistry) { + if (candidate === ds) activeSessionsRegistry.delete(registeredKey); + } + } + killedLive = true; + if (!ds.exitEventEmitted) { + ds.exitEventEmitted = true; + dashboardEventBus.publish({ + type: 'session.exited', + body: { sessionId, reason: 'dashboard_close' }, + }); + emitSessionLifecycleHook(ds, 'session.exit', { reason: 'dashboard_close' }); + } + + // Remove local delivery bindings synchronously with the authoritative + // close. Provider unsubscribe remains best-effort after routing is gone. + const anchor = sessionAnchorId(ds); + const subs = listDocSubscriptionsForSession(config.session.dataDir, ds.larkAppId, anchor); + for (const sub of subs) { + removeDocSubscription(config.session.dataDir, ds.larkAppId, sub.fileToken); + } + if (subs.length) { + logger.info( + `[doc-comment] session ${sessionId.slice(0, 8)} closed → ` + + `removed ${subs.length} local doc binding(s)`, + ); + } + void (async () => { + try { + for (const sub of subs) { + if (sub.managedBy !== 'watch-comment') { + await unsubscribeDocFile(ds.larkAppId, { + fileToken: sub.fileToken, + fileType: sub.fileType, + }); + } + } + } catch (err: any) { + logger.warn( + `[doc-comment] cleanup on close failed for ${sessionId.slice(0, 8)}: ` + + `${err?.message ?? err}`, + ); + } + })(); + } + // alreadyClosed = nothing happened on either path. const alreadyClosed = !killedLive && !wasOpen; return { ok: true, alreadyClosed }; @@ -1878,6 +2288,25 @@ export function sendWorkerInput( dispatchAttempt?: number; } = {}, ): boolean { + const riffRetirementPhase = riffRetirementAdmissionPhase(ds); + if (riffRetirementPhase) { + logger.warn( + `[${tag(ds)}] Rejected turn ${turnId ?? '?'} while Riff retirement fence is ${riffRetirementPhase}`, + ); + void callbacks?.sessionReply( + sessionAnchorId(ds), + tr('worker.riff_close_in_progress', undefined, localeForBot(ds.larkAppId)), + 'text', + ds.larkAppId, + turnId, + ).catch(err => { + logger.warn( + `[${tag(ds)}] Failed to notify rejected Riff close-race turn: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + }); + return false; + } if (!ds.worker || ds.worker.killed) return false; const normalized = typeof payload === 'string' ? { content: payload } : payload; const codexAppInput = codexAppInputForSession(ds, normalized.codexAppInput, turnId); @@ -2155,6 +2584,22 @@ export function forkWorker( worker.on('error', (err) => { const reason = (err as Error)?.message ?? String(err); logger.error(`[${t}] Worker fork error: ${reason}`); + if (ds.worker === worker) { + const retainExactRetirementGeneration = ds.riffShutdownState !== undefined + || ds.riffCloseState !== undefined; + if (!retainExactRetirementGeneration) { + ds.worker = null; + ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; + ds.managedTurnOrigin = undefined; + ds.riffCloseState = undefined; + } + // The retirement coordinator owns a prepared generation. Keep the exact + // ChildProcess pointer until exit so it can decide whether abort/commit + // was durably verified. + try { worker.kill(); } catch { /* best-effort failed-child fence */ } + } if (startupState.failureNotified) return; startupState.failureNotified = true; emitSessionLifecycleHook(ds, 'session.requires_attention', { @@ -3485,8 +3930,16 @@ function setupWorkerHandlers( if (msg.taskId === null) { // follow-up 血缘断裂：清掉持久化锚点，否则 daemon 重启会复活已判坏的 parent。 if (ds.session.riffParentTaskId) { + const priorTaskId = ds.session.riffParentTaskId; ds.session.riffParentTaskId = undefined; - sessionStore.updateSession(ds.session); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + ds.session.riffParentTaskId = priorTaskId; + logger.error( + `[${t}] Failed to clear Riff lineage: ${err instanceof Error ? err.message : String(err)}`, + ); + } } break; } @@ -3496,7 +3949,34 @@ function setupWorkerHandlers( // next message continues the riff conversation in the warm sandbox // instead of cold-booting a context-less fresh task (4-5 min). ds.session.riffParentTaskId = msg.taskId; - sessionStore.updateSession(ds.session); + try { + sessionStore.updateSession(ds.session); + } catch (err) { + // Retain the newest runtime lineage. A close_result or later task-id + // event retries persistence; reverting here would make a follow-up + // target the stale parent while the worker owns the new child. + logger.error( + `[${t}] Failed to persist Riff lineage ${msg.taskId}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + break; + } + + case 'close_result': { + const pending = pendingRiffWorkerCloses.get(msg.requestId); + if (!pending + || pending.worker !== worker + || pending.sessionId !== ds.session.sessionId + || ds.worker !== worker) { + logger.warn(`[${t}] Ignored stale/unmatched Riff close result ${msg.requestId}`); + break; + } + pendingRiffWorkerCloses.delete(msg.requestId); + pending.resolve({ + ok: msg.ok, + ...(msg.taskId ? { taskId: msg.taskId } : {}), + ...(msg.error ? { error: msg.error } : {}), + }); break; } @@ -3760,7 +4240,14 @@ function setupWorkerHandlers( if (ds.worker === worker) { ds.worker = null; ds.workerPort = null; + ds.workerToken = null; + ds.workerViewToken = null; ds.managedTurnOrigin = undefined; + if (ds.riffCloseState) { + ds.riffCloseState = { ...ds.riffCloseState, phase: 'uncertain' }; + } + // Do not clear riffShutdownState here. Only the shutdown coordinator can + // release a generation after lineage persistence or admission restore. // This worker generation is gone. Invalidate any stuck-warning card it // posted so a late click cannot inject keys into a replacement worker. invalidateStuckWarning(ds, 'worker_exit'); diff --git a/src/daemon.ts b/src/daemon.ts index 9f990594d..40723e126 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -284,6 +284,23 @@ import { buildDocCommentTurnInput, buildDocWatchWarmupTurnInput } from './core/d import { advanceDocCommentCursor, docCommentRepliesAfterCursor, latestDocCommentPollCursor } from './core/doc-comment-poller.js'; import { renderBufferedSenderBlock } from './core/session-manager.js'; import { shutdownBackendDisposition } from './core/persistent-backend.js'; +import { + abortRiffShutdownFleet, + canAbortVerifiedExitedRiffPreparation, + collectUniqueDaemonShutdownSessions, + commitPreparedRiffShutdown, + isPreparedRiffSessionCurrent, + persistPreparedRiffShutdownFleet, + prepareRiffFleetForShutdown, + type FencedRiffShutdownParticipant, + type PreparedRiffShutdown, +} from './core/riff-shutdown-detach.js'; +import { + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS, + DAEMON_SHUTDOWN_MAX_MS, + DAEMON_WORKER_EXIT_GRACE_MS, +} from './core/shutdown-budgets.js'; +import { tryWithBotTurnMutation } from './core/bot-turn-mutation-gate.js'; import { evaluateVcMeetingConsumerIsolation } from './services/vc-meeting-consumer-isolation.js'; import { markSessionActivity, @@ -17521,21 +17538,154 @@ export async function startDaemon(botIndex?: number): Promise { }, 5_000).unref?.(); } - // Graceful shutdown. Sends SIGTERM (or `{type:'close'}` IPC via killWorker) - // to every worker, then waits up to SHUTDOWN_GRACE_MS for them to exit + // Graceful shutdown. Riff owners first run a three-phase non-cancelling drain + // that durably ACKs every exact final task lineage as one fleet transaction. + // No worker exits until every participant is prepared and fresh-verified. + // Ordinary workers then receive SIGTERM / close IPC and the daemon waits up + // to DAEMON_WORKER_EXIT_GRACE_MS for them to exit // before sending SIGKILL to stragglers. Without the wait, daemon // `process.exit(0)` races worker signal delivery — and any worker whose // main thread is in a sync code path (e.g. the bridge fingerprint scan // bug fixed in v2.9.2) loses the signal and survives as a ppid=1 orphan // forever (we'd accumulated 841 such orphans across daemon restarts, // consuming ~65 GB of RAM until manually SIGKILL'd). - const SHUTDOWN_GRACE_MS = 3000; let shuttingDown = false; const shutdown = async () => { if (shuttingDown) return; shuttingDown = true; + const shutdownDeadlineMs = Date.now() + DAEMON_SHUTDOWN_MAX_MS; + const mutationResult = await tryWithBotTurnMutation( + cfg.larkAppId, + BOT_TURN_MUTATION_SHUTDOWN_ACQUIRE_TIMEOUT_MS, + async () => { setSessionLifecycleShutdown(true); logger.info(`Daemon shutting down... (active: ${getActiveCount()})`); + + const initialShutdownFleet = collectUniqueDaemonShutdownSessions(activeSessions.values()); + if (!initialShutdownFleet.ok) { + setSessionLifecycleShutdown(false); + shuttingDown = false; + logger.error(`Daemon remains online: ${initialShutdownFleet.error}`); + return; + } + + // Preflight Riff before stopping services/removing descriptors. A failed + // drain or persistence write aborts every successfully prepared peer, so a + // refused shutdown returns to a live fleet instead of leaving an early + // participant workerless. + const riffCandidates = initialShutdownFleet.sessions + .filter(ds => shutdownBackendDisposition(ds) === 'riff-drain-detach'); + const riffPrepareResults = await prepareRiffFleetForShutdown(riffCandidates, { + deadlineMs: shutdownDeadlineMs, + }); + const riffPrepared = riffPrepareResults.filter( + (entry): entry is { ds: DaemonSession; result: PreparedRiffShutdown } => entry.result.ok, + ); + const riffFenced = riffPrepareResults.filter( + (entry): entry is { ds: DaemonSession; result: FencedRiffShutdownParticipant } => + entry.result.fence !== 'none', + ); + + const abortRiffFleet = async ( + reason: string, + retainFenced: ReadonlySet = new Set(), + ): Promise => { + for (const ds of retainFenced) { + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Riff shutdown participant remains fenced: ` + + 'durable/runtime ownership could not be reconciled safely', + ); + } + const aborts = await abortRiffShutdownFleet(riffFenced + .filter(({ ds }) => !retainFenced.has(ds)) + .map(({ ds, result }) => ({ ds, result })), { + deadlineMs: shutdownDeadlineMs, + }); + for (const { ds, result } of aborts) { + if (result.ok) continue; + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Riff shutdown rollback was not ACKed: ` + + `${result.error ?? 'unknown'}; admission remains fail-closed`, + ); + } + setSessionLifecycleShutdown(false); + shuttingDown = false; + logger.error(`Daemon remains online: ${reason}`); + }; + + const riffPrepareFailures = riffPrepareResults.filter(entry => !entry.result.ok); + if (riffPrepareFailures.length > 0) { + for (const { ds, result } of riffPrepareFailures) { + if (result.ok) continue; + logger.error( + `[${ds.session.sessionId.slice(0, 8)}] Daemon shutdown prepare refused: ` + + `${result.error}${result.taskId ? ` (task ${result.taskId})` : ''}`, + ); + } + await abortRiffFleet( + `${riffPrepareFailures.length} Riff owner(s) could not be safely drained`, + ); + return; + } + + const riffPersistence = persistPreparedRiffShutdownFleet(riffPrepared, { + deadlineMs: shutdownDeadlineMs, + }); + if (!riffPersistence.ok) { + const retainIds = new Set(riffPersistence.retainFencedSessionIds); + const retainFenced = new Set(riffPrepared + .filter(({ ds }) => retainIds.has(ds.session.sessionId)) + .map(({ ds }) => ds)); + logger.error(`Daemon shutdown lineage verification refused: ${riffPersistence.error}`); + await abortRiffFleet( + `${riffPersistence.sessionIds.length} Riff lineage row(s) could not be fresh-verified`, + retainFenced, + ); + return; + } + + // Recheck generation ownership while every participant is still fenced and + // daemon services remain live. + const currentShutdownFleet = collectUniqueDaemonShutdownSessions(activeSessions.values()); + if (!currentShutdownFleet.ok) { + const ambiguousSessionId = currentShutdownFleet.sessionId; + await abortRiffFleet( + `Riff ownership became ambiguous after shutdown preflight: ${currentShutdownFleet.error}`, + new Set(riffPrepared + .filter(({ ds }) => ds.session.sessionId === ambiguousSessionId) + .map(({ ds }) => ds)), + ); + return; + } + const currentRiffOwners = currentShutdownFleet.sessions + .filter(ds => shutdownBackendDisposition(ds) === 'riff-drain-detach'); + const riffPreparedOwners = new Set(riffPrepared.map(entry => entry.ds)); + const riffGenerationMismatch = currentRiffOwners.some(ds => !riffPreparedOwners.has(ds)) + || riffPrepared.some(({ ds, result }) => + !currentRiffOwners.includes(ds) || !isPreparedRiffSessionCurrent(ds, result)); + if (riffGenerationMismatch) { + const retainFenced = new Set(riffPrepared + .filter(({ ds, result }) => + (!currentRiffOwners.includes(ds) || !isPreparedRiffSessionCurrent(ds, result)) + && !canAbortVerifiedExitedRiffPreparation(ds, result)) + .map(({ ds }) => ds)); + await abortRiffFleet( + 'Riff ownership changed after shutdown preflight', + retainFenced, + ); + return; + } + + // Validate-all above, then commit-all synchronously with no await or + // callback boundary between peers. + const riffRetiredWorkers: ChildProcess[] = []; + for (const { ds, result } of riffPrepared) { + if (!commitPreparedRiffShutdown(ds, result)) { + throw new Error(`Riff shutdown commit invariant lost for ${ds.session.sessionId}`); + } + if (result.worker) riffRetiredWorkers.push(result.worker); + } + scheduler.stopScheduler(); stopMaintenance(); vcMeetingTerminalReconciler?.stop(); @@ -17569,23 +17719,33 @@ export async function startDaemon(botIndex?: number): Promise { const pendingExits: Array> = []; const survivors: ChildProcess[] = []; - for (const [, ds] of activeSessions) { + const trackWorkerExit = (w: ChildProcess): void => { + if (w.exitCode !== null || w.signalCode !== null) return; + pendingExits.push(new Promise(resolve => { + w.once('exit', () => resolve()); + // Close the check/listener race if exit landed between the first + // status read and once(). + if (w.exitCode !== null || w.signalCode !== null) resolve(); + })); + survivors.push(w); + }; + for (const worker of riffRetiredWorkers) trackWorkerExit(worker); + for (const ds of currentShutdownFleet.sessions) { if (ds.worker && !ds.worker.killed) { logger.info(`Shutting down worker for session ${ds.session.sessionId}`); const w = ds.worker; - // Capture the exit promise BEFORE killWorker nulls ds.worker. - if (w.exitCode === null && w.signalCode === null) { - pendingExits.push(new Promise(resolve => { - w.once('exit', () => resolve()); - })); - survivors.push(w); + const disposition = shutdownBackendDisposition(ds); + if (disposition === 'riff-drain-detach') { + throw new Error(`undrained Riff generation after atomic commit: ${ds.session.sessionId}`); } + // Capture the exit promise BEFORE killWorker nulls ds.worker. + trackWorkerExit(w); // Branch by the session's FROZEN backend (stamped on Session.backendType // at spawn), NOT the bot's live config — a dashboard backendType edit must // not change how a running session is torn down, or we'd e.g. try to // detach-preserve a "herdr" session whose real pane is tmux (freeze-once). // undefined (frozen pty, or unresolvable legacy) → non-persistent → killWorker. - if (shutdownBackendDisposition(ds) === 'detach') { + if (disposition === 'detach') { // Persistent backends (tmux / herdr / zellij): just kill the worker process — // the multiplexer session survives for re-attach. The worker's SIGTERM // handler calls backend.kill(), which only DETACHES. Going through @@ -17605,7 +17765,11 @@ export async function startDaemon(botIndex?: number): Promise { } if (pendingExits.length > 0) { - const timeout = new Promise(resolve => setTimeout(resolve, SHUTDOWN_GRACE_MS)); + const exitGraceMs = Math.min( + DAEMON_WORKER_EXIT_GRACE_MS, + Math.max(0, shutdownDeadlineMs - Date.now()), + ); + const timeout = new Promise(resolve => setTimeout(resolve, exitGraceMs)); await Promise.race([Promise.all(pendingExits), timeout]); let stragglers = 0; for (const w of survivors) { @@ -17615,7 +17779,7 @@ export async function startDaemon(botIndex?: number): Promise { } } if (stragglers > 0) { - logger.warn(`${stragglers}/${survivors.length} worker(s) didn't exit within ${SHUTDOWN_GRACE_MS}ms — SIGKILL'd to prevent ppid=1 orphans.`); + logger.warn(`${stragglers}/${survivors.length} worker(s) didn't exit within ${exitGraceMs}ms — SIGKILL'd to prevent ppid=1 orphans.`); } } @@ -17626,6 +17790,15 @@ export async function startDaemon(botIndex?: number): Promise { removePidFile(); process.exit(0); + }, + ); + if (!mutationResult.acquired) { + shuttingDown = false; + logger.error( + `Daemon remains online: could not acquire exclusive shutdown mutation lease ` + + `(${mutationResult.reason})`, + ); + } }; process.on('SIGTERM', () => { shutdown().catch(err => { logger.error(`shutdown failed: ${err?.message ?? err}`); process.exit(1); }); }); diff --git a/src/services/session-store.ts b/src/services/session-store.ts index e70c649b8..69407c22d 100644 --- a/src/services/session-store.ts +++ b/src/services/session-store.ts @@ -1,4 +1,12 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, readdirSync } from 'node:fs'; +import { + readFileSync, + writeFileSync, + mkdirSync, + existsSync, + renameSync, + readdirSync, + unlinkSync, +} from 'node:fs'; import { join, dirname } from 'node:path'; import { randomUUID } from 'node:crypto'; import { config } from '../config.js'; @@ -6,6 +14,7 @@ import { logger } from '../utils/logger.js'; import { cleanupMaterializedDashboardImages } from '../core/dashboard-images.js'; import { deleteFrozenCards } from './frozen-card-store.js'; import type { Session } from '../types.js'; +import { withFileLockSync } from '../utils/file-lock.js'; let sessions: Map = new Map(); let loaded = false; @@ -20,6 +29,64 @@ export function stripLegacyPendingCardFields(session: Record): for (const f of LEGACY_PENDING_CARD_FIELDS) delete session[f]; } +/** The active row no longer has the lineage/ownership sampled by the caller. */ +export class RiffLineageOwnershipError extends Error { + override readonly name = 'RiffLineageOwnershipError'; +} + +export type RiffDurableOwner = { + pid: number | null; + larkAppId: string | null; + backendType: string | null; +}; + +export type ActiveRiffShutdownSnapshot = { + sessionId: string; + taskId: string | null; + owner: RiffDurableOwner; +}; + +export type ActiveRiffLineageBatchUpdate = ActiveRiffShutdownSnapshot & { + targetTaskId: string | null; + expectedCurrentTaskIds: readonly (string | null)[]; +}; + +export type RiffLineageBatchFailureStage = + | 'prewrite_ownership' + | 'prewrite_io' + | 'postrename_ambiguity'; + +export class RiffLineageBatchError extends Error { + override readonly name = 'RiffLineageBatchError'; + + constructor( + readonly stage: RiffLineageBatchFailureStage, + readonly sessionIds: readonly string[], + message: string, + ) { + super(message); + } +} + +function riffDurableOwner(session: Session): RiffDurableOwner { + return { + pid: session.pid ?? null, + larkAppId: session.larkAppId ?? null, + backendType: session.backendType ?? null, + }; +} + +function riffOwnersEqual(left: RiffDurableOwner, right: RiffDurableOwner): boolean { + return left.pid === right.pid + && left.larkAppId === right.larkAppId + && left.backendType === right.backendType; +} + +let testOnlyAfterRiffBatchRename: (() => void) | undefined; +export function __testOnly_setAfterRiffBatchRename(hook: (() => void) | undefined): void { + testOnlyAfterRiffBatchRename = hook; +} + /** * Initialise session store for a specific bot (multi-daemon mode). * When appId is set, sessions are stored in `sessions-{appId}.json`. @@ -139,6 +206,198 @@ function readExistingSessionsFromDisk(fp: string): { raw: string; parsed: Record } } +function readSessionsProjectionStrict(fp: string): { raw: string; parsed: Record } { + if (!existsSync(fp)) return { raw: '', parsed: {} }; + const raw = readFileSync(fp, 'utf-8'); + const value = JSON.parse(raw) as unknown; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`invalid sessions projection at ${fp}`); + } + return { raw, parsed: value as Record }; +} + +function duplicateIds(ids: readonly string[]): string[] { + const seen = new Set(); + const duplicates = new Set(); + for (const id of ids) { + if (seen.has(id)) duplicates.add(id); + else seen.add(id); + } + return [...duplicates]; +} + +/** + * Sample every active Riff participant from one fresh sessions projection. + * Fleet shutdown takes this snapshot before fencing any worker. + */ +export function getActiveRiffShutdownSnapshotsBatch( + sessionIds: readonly string[], + options: { maxWaitMs?: number } = {}, +): ActiveRiffShutdownSnapshot[] { + if (sessionIds.length === 0) return []; + const duplicates = duplicateIds(sessionIds); + if (duplicates.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + duplicates, + `duplicate Riff shutdown session ids: ${duplicates.join(', ')}`, + ); + } + + ensureDir(); + const fp = getFilePath(); + try { + return withFileLockSync(fp, () => { + const { parsed } = readSessionsProjectionStrict(fp); + const invalid = sessionIds.filter((sessionId) => { + const session = parsed[sessionId]; + return !session || session.status !== 'active'; + }); + if (invalid.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + invalid, + `cannot snapshot non-active Riff sessions: ${invalid.join(', ')}`, + ); + } + return sessionIds.map((sessionId) => { + const session = parsed[sessionId]!; + return { + sessionId, + taskId: session.riffParentTaskId ?? null, + owner: riffDurableOwner(session), + }; + }); + }, { maxWaitMs: options.maxWaitMs }); + } catch (error) { + if (error instanceof RiffLineageBatchError) throw error; + throw new RiffLineageBatchError( + 'prewrite_io', + [...sessionIds], + `failed to snapshot active Riff sessions: ${String(error)}`, + ); + } +} + +/** + * Commit every prepared Riff lineage as one compare-and-set transaction. + * The published projection is read back under the same lock before workers + * are allowed to exit. + */ +export function persistActiveRiffLineagesExactBatch( + updates: readonly ActiveRiffLineageBatchUpdate[], + options: { maxWaitMs?: number } = {}, +): ActiveRiffShutdownSnapshot[] { + if (updates.length === 0) return []; + const sessionIds = updates.map(update => update.sessionId); + const duplicates = duplicateIds(sessionIds); + if (duplicates.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + duplicates, + `duplicate Riff lineage batch session ids: ${duplicates.join(', ')}`, + ); + } + + ensureDir(); + const fp = getFilePath(); + let published = false; + let tmpFp: string | undefined; + try { + return withFileLockSync(fp, () => { + const { raw, parsed } = readSessionsProjectionStrict(fp); + const conflicts: string[] = []; + for (const update of updates) { + const durable = parsed[update.sessionId]; + const durableTaskId = durable?.riffParentTaskId ?? null; + if (!durable + || durable.status !== 'active' + || !update.expectedCurrentTaskIds.some(candidate => candidate === durableTaskId) + || !riffOwnersEqual(riffDurableOwner(durable), update.owner)) { + conflicts.push(update.sessionId); + } + } + if (conflicts.length > 0) { + throw new RiffLineageBatchError( + 'prewrite_ownership', + conflicts, + `Riff lineage batch compare-and-set failed for: ${conflicts.join(', ')}`, + ); + } + + for (const update of updates) { + const durable = parsed[update.sessionId]!; + const next: Session = { + ...durable, + riffParentTaskId: update.targetTaskId ?? undefined, + }; + stripLegacyPendingCardFields(next as unknown as Record); + parsed[update.sessionId] = next; + } + + const json = JSON.stringify(parsed, null, 2); + if (json !== raw) { + tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, json, 'utf-8'); + renameSync(tmpFp, fp); + tmpFp = undefined; + published = true; + testOnlyAfterRiffBatchRename?.(); + } + + let verifiedProjection: Record; + try { + verifiedProjection = readSessionsProjectionStrict(fp).parsed; + } catch (error) { + throw new RiffLineageBatchError( + published ? 'postrename_ambiguity' : 'prewrite_io', + [...sessionIds], + `failed to read back Riff lineage batch: ${String(error)}`, + ); + } + + const ambiguous = updates.filter((update) => { + const durable = verifiedProjection[update.sessionId]; + return !durable + || durable.status !== 'active' + || (durable.riffParentTaskId ?? null) !== update.targetTaskId + || !riffOwnersEqual(riffDurableOwner(durable), update.owner); + }).map(update => update.sessionId); + if (ambiguous.length > 0) { + throw new RiffLineageBatchError( + published ? 'postrename_ambiguity' : 'prewrite_ownership', + ambiguous, + `Riff lineage batch readback mismatch for: ${ambiguous.join(', ')}`, + ); + } + + const verified = updates.map((update) => ({ + sessionId: update.sessionId, + taskId: update.targetTaskId, + owner: riffDurableOwner(verifiedProjection[update.sessionId]!), + })); + if (loaded) { + for (const update of updates) { + const cached = sessions.get(update.sessionId); + if (cached) cached.riffParentTaskId = update.targetTaskId ?? undefined; + } + } + return verified; + }, { maxWaitMs: options.maxWaitMs }); + } catch (error) { + if (error instanceof RiffLineageBatchError) throw error; + throw new RiffLineageBatchError( + published ? 'postrename_ambiguity' : 'prewrite_io', + [...sessionIds], + `failed to persist Riff lineage batch: ${String(error)}`, + ); + } finally { + if (tmpFp) { + try { unlinkSync(tmpFp); } catch { /* best-effort orphan cleanup */ } + } + } +} + function save(): void { ensureDir(); const fp = getFilePath(); @@ -190,6 +449,21 @@ export function getSession(sessionId: string): Session | undefined { return sessions.get(sessionId) ?? findInOtherFiles(sessionId); } +/** Cross-process fresh read ordered after daemon/CLI writes by the shared lock. */ +export function getSessionFresh(sessionId: string): Session | undefined { + ensureDir(); + const fp = getFilePath(); + return withFileLockSync(fp, () => { + if (!existsSync(fp)) return undefined; + try { + const data = JSON.parse(readFileSync(fp, 'utf-8')) as Record; + return data[sessionId]; + } catch { + return undefined; + } + }); +} + /** * Search all session files for a session not found in the current file. * @@ -215,22 +489,42 @@ function findInOtherFiles(sessionId: string): Session | undefined { return undefined; } -export function closeSession(sessionId: string): void { +export function closeSession( + sessionId: string, + opts: { clearRiffParentTaskId?: boolean } = {}, +): void { load(); const session = sessions.get(sessionId); if (session) { - if (session.larkAppId && session.dashboardAttachments?.length) { + const priorStatus = session.status; + const priorClosedAt = session.closedAt; + const priorRiffParentTaskId = session.riffParentTaskId; + const priorDashboardAttachments = session.dashboardAttachments; + const priorQueuedAttachments = session.queuedAttachments; + session.status = 'closed'; + session.closedAt = new Date().toISOString(); + session.dashboardAttachments = undefined; + session.queuedAttachments = undefined; + // Riff cancellation has already completed before this durable transition. + // Clear its retry handle in the same atomic save as status='closed'. + if (opts.clearRiffParentTaskId) session.riffParentTaskId = undefined; + try { + save(); + } catch (err) { + session.status = priorStatus; + session.closedAt = priorClosedAt; + session.riffParentTaskId = priorRiffParentTaskId; + session.dashboardAttachments = priorDashboardAttachments; + session.queuedAttachments = priorQueuedAttachments; + throw err; + } + if (session.larkAppId && priorDashboardAttachments?.length) { try { - cleanupMaterializedDashboardImages(session.larkAppId, session.dashboardAttachments); - session.dashboardAttachments = undefined; - session.queuedAttachments = undefined; + cleanupMaterializedDashboardImages(session.larkAppId, priorDashboardAttachments); } catch (error: any) { logger.warn(`Failed to clean Dashboard images for session ${sessionId}: ${error?.message ?? error}`); } } - session.status = 'closed'; - session.closedAt = new Date().toISOString(); - save(); deleteFrozenCards(sessionId); logger.info(`Closed session ${sessionId}`); } @@ -251,6 +545,68 @@ export function updateSession(session: Session): void { save(); } +/** + * Persist one exact Riff follow-up lineage for an active durable owner. + * The process cache changes only after the atomic file replacement succeeds. + */ +export function persistActiveRiffLineageExact( + sessionId: string, + taskId: string | null, + options: { + expectedCurrentTaskIds?: readonly (string | null)[]; + expectedOwner?: RiffDurableOwner; + } = {}, +): Session { + load(); + ensureDir(); + const fp = getFilePath(); + return withFileLockSync(fp, () => { + const { raw, parsed } = readExistingSessionsFromDisk(fp); + const durable = parsed[sessionId]; + if (!durable || durable.status !== 'active') { + throw new RiffLineageOwnershipError( + `cannot persist Riff lineage for non-active session ${sessionId}`, + ); + } + const durableTaskId = durable.riffParentTaskId ?? null; + const expected = options.expectedCurrentTaskIds; + if (expected && !expected.some(candidate => candidate === durableTaskId)) { + throw new RiffLineageOwnershipError( + `Riff lineage compare-and-set failed for ${sessionId} ` + + `(current=${durableTaskId ?? 'none'}, expected=${expected.map(id => id ?? 'none').join('|')})`, + ); + } + if (options.expectedOwner && !riffOwnersEqual(riffDurableOwner(durable), options.expectedOwner)) { + throw new RiffLineageOwnershipError( + `Riff owner compare-and-set failed for ${sessionId} ` + + `(current=${JSON.stringify(riffDurableOwner(durable))}, ` + + `expected=${JSON.stringify(options.expectedOwner)})`, + ); + } + + const next: Session = { + ...durable, + riffParentTaskId: taskId ?? undefined, + }; + stripLegacyPendingCardFields(next as unknown as Record); + parsed[sessionId] = next; + const json = JSON.stringify(parsed, null, 2); + if (json !== raw) { + const tmpFp = `${fp}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmpFp, json, 'utf-8'); + renameSync(tmpFp, fp); + } + + const cached = sessions.get(sessionId); + if (cached) { + cached.riffParentTaskId = taskId ?? undefined; + return cached; + } + sessions.set(sessionId, next); + return next; + }); +} + export function listSessions(): Session[] { load(); return [...sessions.values()]; diff --git a/src/types.ts b/src/types.ts index 074cd9551..89ff2c2c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -582,7 +582,15 @@ export type DaemonToWorker = * as a model turn. Only adapters declaring buildSessionRenameCommand handle * it; all other CLIs ignore it. */ | { type: 'rename_session'; title: string } - | { type: 'close' } + | { type: 'close'; requestId?: string } + | { type: 'close_commit'; requestId: string } + | { type: 'close_abort'; requestId: string } + /** Fence new Riff writes, drain accepted writes, and report exact lineage. */ + | { type: 'riff_shutdown_prepare'; requestId: string } + /** Final lineage is durable; detach the worker generation. */ + | { type: 'riff_shutdown_commit'; requestId: string } + /** Shutdown could not commit; restore Riff write admission. */ + | { type: 'riff_shutdown_abort'; requestId: string } | { type: 'suspend' } | { type: 'restart' } /** Lease watchdog fencing: only the exact still-running durable attempt may @@ -716,4 +724,25 @@ export type WorkerToDaemon = | { type: 'adopt_preamble'; userText: string; assistantText: string; turnId?: string } | { type: 'deferred_topic_materialized'; sessionId: string; turnId: string; rootMessageId: string } | { type: 'riff_access_url'; accessUrl: string; directAccessUrl?: string; turnId?: string; dispatchAttempt?: number } - | { type: 'riff_task_id'; taskId: string | null }; + | { type: 'riff_task_id'; taskId: string | null } + | { + type: 'riff_shutdown_result'; + requestId: string; + phase: 'prepare' | 'abort'; + ok: boolean; + taskId: string | null; + error?: string; + } + | { + type: 'close_abort_result'; + requestId: string; + ok: boolean; + error?: string; + } + | { + type: 'close_result'; + requestId: string; + ok: boolean; + taskId?: string; + error?: string; + }; diff --git a/src/worker.ts b/src/worker.ts index 21c31bcb7..33e8d796f 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -60,6 +60,7 @@ import { terminalReleasesDurableTurn, type PendingCliInput, } from './utils/pending-input-queue.js'; +import { riffWorkerShutdownInputBlocker } from './core/riff-worker-shutdown-readiness.js'; import { ReadyGate, shouldArmReadyGate } from './utils/ready-gate.js'; import { shouldRunStartupCommandsOnSpawn, shouldDeferInitialPromptForStartup } from './core/startup-commands.js'; import { sanitizePerBotEnv } from './core/per-bot-env.js'; @@ -201,7 +202,12 @@ import { DEVICE_AUTHORITY_DIRECTORY, DEVICE_CREDENTIAL_FILE, } from './platform/device-paths.js'; -import type { BackendType, SessionBackend } from './adapters/backend/types.js'; +import type { + BackendType, + SessionBackend, + SessionDestroyResult, + SessionShutdownDetachResult, +} from './adapters/backend/types.js'; import { tmuxEnv, probeTmuxFunctionalWithRetry } from './setup/ensure-tmux.js'; import { tmuxRestartJitterMs } from './core/tmux-recovery.js'; import { IdleDetector } from './utils/idle-detector.js'; @@ -1093,6 +1099,13 @@ let effectiveBackendType: BackendType = 'pty'; * generation. The daemon receives this over private IPC; child-writable PID * marker files remain diagnostics only. */ let currentCliCredentialIsolated = false; +/** Successful Riff close prepare awaiting durable daemon commit. */ +let preparedCloseRequestId: string | null = null; +let closeRequestInFlightId: string | null = null; +let lastAbortedCloseRequestId: string | null = null; +/** Graceful daemon-shutdown detach stays alive until lineage commit. */ +let shutdownDetachRequestId: string | null = null; +let shutdownDetachPhase: 'preparing' | 'prepared' | null = null; /** pty-under-zellij backend (BACKEND_TYPE=zellij). Behaves like the non-tmux * pty path for the worker (renderer screenshots, relay web terminal) but owns * a persistent zellij session that survives daemon restart. */ @@ -1448,6 +1461,8 @@ async function runStartupCommands(): Promise { } const pendingMessages: PendingCliInput[] = []; +/** Async init must materialize its opening input before shutdown may fence. */ +let initPromptMaterialized = false; /** Literal commands that arrived while native /rename owned the TUI or while * an owned CLI restart was fenced. Normal raw_input commands are still * delivered immediately (including while busy). */ @@ -9660,6 +9675,7 @@ process.on('message', async (raw: unknown) => { initialInputCommitted = true; } if (initialInputCommitted) acknowledgeTurnInputCommitted(msg.turnId); + initPromptMaterialized = true; // A backend may become prompt-ready before spawnCli() returns. The // initial prompt is queued only afterwards, so the earlier @@ -9901,6 +9917,10 @@ process.on('message', async (raw: unknown) => { } case 'restart': { + if (effectiveBackendType === 'riff') { + log('Refused Riff generation restart; the existing lineage-owning worker is retained'); + break; + } await restartCliProcess('daemon request', { preservePending: true }); break; } @@ -10136,16 +10156,69 @@ process.on('message', async (raw: unknown) => { case 'close': { log('Close requested'); + if (effectiveBackendType === 'riff') { + if (!msg.requestId) { + log('Refused unsafe request-less Riff close; explicit close requires prepare/commit'); + break; + } + + let result: SessionDestroyResult; + let attemptedPrepare = false; + if (shutdownDetachRequestId) { + result = { + ok: false, + error: `shutdown detach already ${shutdownDetachPhase ?? 'active'} as ${shutdownDetachRequestId}`, + }; + } else if (preparedCloseRequestId) { + result = { + ok: false, + error: `close already prepared as ${preparedCloseRequestId}`, + }; + } else if (closeRequestInFlightId) { + result = { + ok: false, + error: `close prepare already in flight as ${closeRequestInFlightId}`, + }; + } else { + attemptedPrepare = true; + lastAbortedCloseRequestId = null; + closeRequestInFlightId = msg.requestId; + try { + const raw = await backend?.destroySession?.(); + result = raw && typeof raw === 'object' && 'ok' in raw + ? raw as SessionDestroyResult + : { ok: true }; + } catch (err) { + result = { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + if (result.ok) { + preparedCloseRequestId = msg.requestId; + } else if (attemptedPrepare) { + await backend?.abortDestroySession?.(); + lastAbortedCloseRequestId = msg.requestId; + } + if (closeRequestInFlightId === msg.requestId) closeRequestInFlightId = null; + send({ + type: 'close_result', + requestId: msg.requestId, + ok: result.ok, + ...(result.taskId ? { taskId: result.taskId } : {}), + ...(result.error ? { error: result.error } : {}), + }); + if (!result.ok) { + log(`Riff close prepare failed (${result.error ?? 'cancel failed'}); session stays active for retry`); + } + break; + } + stopScreenshotLoop(); - // destroySession kills tmux session permanently; kill() only detaches. - // riff 的 destroySession 是异步远端取消——必须有界 await：紧跟着的 - // process.exit 会掐断未发出的 fetch，让已关闭话题的远端 agent 继续跑。 + // Local close: destroySession kills persistent owned sessions. Riff has + // already been handled above and cannot enter this request-less path. const closeTeardown = backend?.destroySession?.(); if (closeTeardown && typeof (closeTeardown as Promise).then === 'function') { - try { - // 预算层级见 RiffBackend.destroySession（总 deadline 20s）——这里 22s 只作兜底。 - await Promise.race([closeTeardown, new Promise((r) => setTimeout(r, 22_000))]); - } catch { /* logged inside destroySession */ } + try { await Promise.race([closeTeardown, new Promise((r) => setTimeout(r, 22_000))]); } + catch { /* logged by backend */ } } killCli(); // Bridge marker file outlives a single CLI process (we keep it across @@ -10157,7 +10230,184 @@ process.on('message', async (raw: unknown) => { process.exit(0); } + case 'riff_shutdown_prepare': { + if (effectiveBackendType !== 'riff') { + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'prepare', + ok: false, + taskId: null, + error: 'not_riff_backend', + }); + break; + } + let result: SessionShutdownDetachResult; + const inputBlocker = riffWorkerShutdownInputBlocker({ + initPromptMaterialized, + isFlushing, + pendingMessages: pendingMessages.length, + pendingRawInputs: pendingRawInputs.length, + pendingSessionRename: pendingSessionRename !== null, + sessionRenameInFlight, + commandLineWritesPending, + }); + if (preparedCloseRequestId || closeRequestInFlightId) { + result = { ok: false, taskId: null, error: 'explicit_close_in_progress' }; + } else if (shutdownDetachRequestId) { + result = { + ok: false, + taskId: null, + error: `shutdown detach already ${shutdownDetachPhase ?? 'active'} as ${shutdownDetachRequestId}`, + }; + } else if (inputBlocker) { + result = { + ok: false, + taskId: null, + error: `worker_inputs_not_drained:${inputBlocker}`, + }; + } else { + shutdownDetachRequestId = msg.requestId; + shutdownDetachPhase = 'preparing'; + try { + result = await backend?.prepareShutdownDetach?.() + ?? { ok: false, taskId: null, error: 'shutdown_detach_unsupported' }; + } catch (err) { + result = { + ok: false, + taskId: null, + error: err instanceof Error ? err.message : String(err), + }; + } + if (shutdownDetachRequestId === msg.requestId && result.ok) { + shutdownDetachPhase = 'prepared'; + } + } + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'prepare', + ok: result.ok, + taskId: result.taskId, + ...(result.error ? { error: result.error } : {}), + }); + break; + } + + case 'riff_shutdown_commit': { + if (effectiveBackendType !== 'riff' + || shutdownDetachRequestId !== msg.requestId + || shutdownDetachPhase !== 'prepared') { + log(`Ignoring stale Riff shutdown commit ${msg.requestId}`); + break; + } + log(`Riff shutdown detach committed (${msg.requestId})`); + shutdownDetachRequestId = null; + shutdownDetachPhase = null; + backend?.commitShutdownDetach?.(); + intentionalRestartBackend = backend; + stopScreenshotLoop(); + killCli(); + cleanup(); + process.exit(0); + } + + case 'riff_shutdown_abort': { + if (shutdownDetachRequestId !== msg.requestId) { + log(`Ignoring stale Riff shutdown abort ${msg.requestId}`); + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'abort', + ok: false, + taskId: null, + error: 'shutdown_detach_not_active', + }); + break; + } + log(`Riff shutdown detach aborted (${msg.requestId})`); + let result: SessionShutdownDetachResult; + try { + result = await backend?.abortShutdownDetach?.() + ?? { ok: false, taskId: null, error: 'shutdown_abort_unsupported' }; + } catch (err) { + result = { + ok: false, + taskId: null, + error: err instanceof Error ? err.message : String(err), + }; + } + if (result.ok && shutdownDetachRequestId === msg.requestId) { + shutdownDetachRequestId = null; + shutdownDetachPhase = null; + } + send({ + type: 'riff_shutdown_result', + requestId: msg.requestId, + phase: 'abort', + ok: result.ok, + taskId: result.taskId, + ...(result.error ? { error: result.error } : {}), + }); + break; + } + + case 'close_commit': { + if (!preparedCloseRequestId || preparedCloseRequestId !== msg.requestId) { + log(`Ignoring stale close_commit ${msg.requestId}`); + break; + } + log(`Close committed (${msg.requestId})`); + preparedCloseRequestId = null; + closeRequestInFlightId = null; + lastAbortedCloseRequestId = null; + backend?.commitDestroySession?.(); + stopScreenshotLoop(); + killCli(); + clearSendMarkers(); + cleanup(); + process.exit(0); + } + + case 'close_abort': { + const alreadyRestored = lastAbortedCloseRequestId === msg.requestId; + if (!alreadyRestored + && preparedCloseRequestId !== msg.requestId + && closeRequestInFlightId !== msg.requestId) { + log(`Ignoring stale close_abort ${msg.requestId}`); + send({ + type: 'close_abort_result', + requestId: msg.requestId, + ok: false, + error: 'close_abort_not_active', + }); + break; + } + log(`Close aborted (${msg.requestId}); Riff admission restored`); + let abortError: string | null = null; + if (!alreadyRestored) { + try { await backend?.abortDestroySession?.(); } + catch (err) { abortError = err instanceof Error ? err.message : String(err); } + } + if (!abortError) { + preparedCloseRequestId = null; + closeRequestInFlightId = null; + lastAbortedCloseRequestId = null; + } + send({ + type: 'close_abort_result', + requestId: msg.requestId, + ok: abortError === null, + ...(abortError ? { error: abortError } : {}), + }); + break; + } + case 'suspend': { + if (effectiveBackendType === 'riff') { + log('Refused unsafe Riff suspend; explicit close requires prepare/commit'); + break; + } log('Suspend requested'); stopScreenshotLoop(); stopBridgeWatcher(); @@ -10176,10 +10426,7 @@ process.on('message', async (raw: unknown) => { // uses to recover sessions after a reboot kills the tmux server). revokeManagedTurnOriginForRestart(); try { - // riff：suspend 语义是「休眠待续」——绝不能 cancel 远端任务（血缘已持久化， - // 恢复时 follow-up 续上）；只断流 detach。 - if (effectiveBackendType === 'riff') backend?.kill(); - else (backend?.destroySession ?? backend?.kill)?.call(backend); + (backend?.destroySession ?? backend?.kill)?.call(backend); } catch { /* best-effort */ } backend = null; isPromptReady = false; diff --git a/test/explicit-session-backing-cleanup.test.ts b/test/explicit-session-backing-cleanup.test.ts new file mode 100644 index 000000000..eb15f436a --- /dev/null +++ b/test/explicit-session-backing-cleanup.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from 'vitest'; +import { cleanupExplicitSessionBacking } from '../src/core/explicit-session-backing-cleanup.js'; + +const SID = 'abcd1234-1111-2222-3333-444444444444'; + +describe('offline explicit session backing cleanup', () => { + for (const backendType of ['tmux', 'herdr', 'zellij'] as const) { + it(`confirms ${backendType} is absent before reporting cleanup success`, async () => { + const killPersistent = vi.fn(); + const probePersistent = vi.fn(() => 'missing' as const); + const result = await cleanupExplicitSessionBacking( + { sessionId: SID, backendType }, + { killPersistent, probePersistent }, + ); + + expect(result).toEqual({ + ok: true, + kind: 'destroyed_persistent', + backendType, + name: 'bmx-abcd1234', + }); + expect(killPersistent).toHaveBeenCalledWith(backendType, 'bmx-abcd1234'); + expect(probePersistent).toHaveBeenCalledWith(backendType, 'bmx-abcd1234'); + }); + } + + it('does not report success when a persistent backend still exists after kill', async () => { + const result = await cleanupExplicitSessionBacking( + { sessionId: SID, backendType: 'tmux' }, + { + killPersistent: vi.fn(), + probePersistent: vi.fn(() => 'exists'), + }, + ); + expect(result).toMatchObject({ + ok: false, + kind: 'persistent_destroy_failed', + error: 'backing_session_still_exists', + }); + }); + + it('cancels Riff using the supplied authoritative bot config and returns the exact task id', async () => { + const cancelRiffTask = vi.fn(async () => true); + const result = await cleanupExplicitSessionBacking( + { + sessionId: SID, + backendType: 'riff', + riffParentTaskId: 'riff-task-1', + riffConfig: { baseUrl: 'https://riff.example', jwt: 'token' }, + }, + { cancelRiffTask }, + ); + expect(cancelRiffTask).toHaveBeenCalledWith( + { baseUrl: 'https://riff.example', jwt: 'token' }, + 'riff-task-1', + ); + expect(result).toEqual({ ok: true, kind: 'cancelled_riff', taskId: 'riff-task-1' }); + }); + + it('fails closed on Riff cancel failure so the caller can preserve its task id', async () => { + const record = { + sessionId: SID, + backendType: 'riff' as const, + riffParentTaskId: 'riff-task-retry', + riffConfig: { baseUrl: 'https://riff.example' }, + }; + const result = await cleanupExplicitSessionBacking(record, { + cancelRiffTask: vi.fn(async () => false), + }); + expect(result).toEqual({ + ok: false, + kind: 'riff_cancel_failed', + backendType: 'riff', + taskId: 'riff-task-retry', + }); + expect(record.riffParentTaskId).toBe('riff-task-retry'); + }); + + it('fails closed when current Riff config is unavailable', async () => { + expect(await cleanupExplicitSessionBacking({ + sessionId: SID, + backendType: 'riff', + riffParentTaskId: 'riff-task-no-config', + })).toEqual({ + ok: false, + kind: 'riff_config_missing', + backendType: 'riff', + taskId: 'riff-task-no-config', + }); + }); + + it('never destroys an adopted user-owned pane', async () => { + const killPersistent = vi.fn(); + const probePersistent = vi.fn(); + expect(await cleanupExplicitSessionBacking( + { sessionId: SID, backendType: 'tmux', adopted: true }, + { killPersistent, probePersistent: probePersistent as any }, + )).toEqual({ ok: true, kind: 'skipped_adopted' }); + expect(killPersistent).not.toHaveBeenCalled(); + expect(probePersistent).not.toHaveBeenCalled(); + }); + + it('does not guess a persistent backend for an unstamped legacy row', async () => { + const killPersistent = vi.fn(); + const probePersistent = vi.fn(); + expect(await cleanupExplicitSessionBacking( + { sessionId: SID }, + { killPersistent, probePersistent: probePersistent as any }, + )).toEqual({ ok: true, kind: 'no_backing' }); + expect(killPersistent).not.toHaveBeenCalled(); + expect(probePersistent).not.toHaveBeenCalled(); + }); +}); diff --git a/test/persistent-backend-type.test.ts b/test/persistent-backend-type.test.ts index 7535a95df..065b95b43 100644 --- a/test/persistent-backend-type.test.ts +++ b/test/persistent-backend-type.test.ts @@ -162,4 +162,8 @@ describe('shutdownBackendDisposition (shutdown freeze-once)', () => { bot.backendType = 'herdr'; expect(shutdownBackendDisposition(ds({ sessionBackend: 'pty' }))).toBe('close'); }); + it('routes a frozen Riff worker through drain + durable lineage ACK, never direct SIGTERM', () => { + bot.backendType = 'pty'; + expect(shutdownBackendDisposition(ds({ sessionBackend: 'riff' }))).toBe('riff-drain-detach'); + }); }); diff --git a/test/riff-backend.test.ts b/test/riff-backend.test.ts index 26494d483..d1ddec915 100644 --- a/test/riff-backend.test.ts +++ b/test/riff-backend.test.ts @@ -170,6 +170,105 @@ describe('RiffBackend', () => { }); }); + describe('graceful shutdown detach drain', () => { + it('fences new writes, waits for a slow create, returns its exact id, and never cancels or streams it', async () => { + const be = makeBackend({ injectStatusLines: false }); + const ids: Array = []; + be.onTaskId(id => ids.push(id)); + be.spawn('', [], {} as any); + be.write('opening turn'); + await flush(); + + let settled = false; + const prepare = be.prepareShutdownDetach().then(result => { + settled = true; + return result; + }); + await flush(); + expect(settled).toBe(false); + + // This write arrived after the shutdown fence and must not extend the + // drain or create a new remote task. + be.write('must be rejected'); + resolvers.shift()!(taskResponse('task-shutdown-late')); + const result = await prepare; + + expect(result).toEqual({ ok: true, taskId: 'task-shutdown-late' }); + expect(ids).toEqual(['task-shutdown-late']); + expect(calls.filter(c => c.url.includes('/api/task-execute')).length).toBe(1); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(0); + expect(calls.filter(c => c.url.includes('/api/task-cancel')).length).toBe(0); + expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); + }); + + it('drains two writes accepted before the fence and reports the newest child lineage', async () => { + const be = makeBackend({ injectStatusLines: false }); + const ids: Array = []; + be.onTaskId(id => ids.push(id)); + be.write('first accepted write'); + be.write('second accepted write'); + await flush(); + + const prepare = be.prepareShutdownDetach(); + resolvers.shift()!(taskResponse('task-drain-1')); + await flush(); + const follow = calls.find(c => c.url.includes('/api/task-follow-up')); + expect(follow).toBeDefined(); + expect(JSON.parse(String(follow!.init?.body)).parentTaskId).toBe('task-drain-1'); + resolvers.shift()!(taskResponse('task-drain-2')); + + await expect(prepare).resolves.toEqual({ ok: true, taskId: 'task-drain-2' }); + expect(ids).toEqual(['task-drain-1', 'task-drain-2']); + expect(calls.filter(c => c.url.includes('/api/task-cancel')).length).toBe(0); + expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); + }); + + it('abort restores admission and reconnects the exact drained task', async () => { + const be = makeBackend({ injectStatusLines: false }); + be.write('opening turn'); + await flush(); + const prepare = be.prepareShutdownDetach(); + resolvers.shift()!(taskResponse('task-abort-resume')); + await expect(prepare).resolves.toEqual({ ok: true, taskId: 'task-abort-resume' }); + expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); + + await be.abortShutdownDetach(); + await flush(); + expect(calls.filter(c => c.url.includes('/api2/task-stream?id=task-abort-resume')).length).toBe(1); + + be.write('follow-up after aborted shutdown'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(1); + }); + + it('abort invalidates a still-draining prepare and restores admission only after it settles', async () => { + const be = makeBackend({ injectStatusLines: false }); + be.write('accepted before shutdown'); + await flush(); + + const prepare = be.prepareShutdownDetach(); + let abortSettled = false; + const abort = be.abortShutdownDetach().then(result => { + abortSettled = true; + return result; + }); + await flush(); + expect(abortSettled).toBe(false); + expect((be as any).shutdownDetaching).toBe(true); + + resolvers.shift()!(taskResponse('task-abort-during-drain')); + await expect(prepare).resolves.toEqual({ + ok: false, + taskId: 'task-abort-during-drain', + error: 'shutdown_detach_aborted', + }); + await expect(abort).resolves.toEqual({ ok: true, taskId: 'task-abort-during-drain' }); + expect((be as any).shutdownDetachPrepared).toBe(false); + expect((be as any).shutdownDetaching).toBe(false); + expect(calls.filter(c => c.url.includes('/api/task-cancel'))).toHaveLength(0); + }); + }); + describe('SSE clean EOF without done (finding C)', () => { it('treats a clean EOF with no done event as a stream failure — session must not stay busy forever', async () => { const be = makeBackend({ injectStatusLines: false }); @@ -515,7 +614,10 @@ describe('RiffBackend', () => { expect(cancels.length).toBeGreaterThanOrEqual(1); expect(JSON.parse(String(cancels[cancels.length - 1]!.init?.body ?? '{}')).id ?? JSON.parse(String(cancels[0]!.init?.body)).id).toBe('task-late'); expect(calls.filter(c => c.url.includes('/api2/task-stream')).length).toBe(0); - expect((be as any).currentTaskId).not.toBe('task-late'); + // The cancelled child remains the exact lineage anchor until durable + // close commit. An abort can therefore continue from it, not its stale + // parent; it is still never streamed while prepare is fenced. + expect((be as any).currentTaskId).toBe('task-late'); }); it('close during follow-up: the late follow-up task is cancelled', async () => { @@ -536,8 +638,141 @@ describe('RiffBackend', () => { await destroyP; const cancelIds = calls.filter(c => c.url.includes('/api/task-cancel')).map(c => JSON.parse(String(c.init?.body)).id); expect(cancelIds).toContain('task-late-2'); - // late follow-up 不得成为 current，也不得开流 - expect((be as any).currentTaskId).not.toBe('task-late-2'); + // Preserve the cancelled child as retry lineage, but never stream it. + expect((be as any).currentTaskId).toBe('task-late-2'); + expect(calls.filter(c => c.url.includes('/api2/task-stream?id=task-late-2')).length).toBe(0); + }); + }); + + describe('close prepare / abort / retry state contract', () => { + it('restores write admission after cancel failure, then allows a close retry', async () => { + const be = makeBackend({ resumeParentTaskId: 'task-parent', injectStatusLines: false }); + let cancelCalls = 0; + fetchMock.mockImplementation(async (url: string | URL, init?: RequestInit) => { + const u = String(url); + calls.push({ url: u, init }); + if (u.includes('/api/task-cancel')) { + cancelCalls++; + return cancelCalls <= 2 + ? new Response('cancel failed', { status: 500 }) + : Response.json({ success: true, data: {} }); + } + if (u.includes('/api/task-follow-up')) return taskResponse('task-after-failure'); + if (u.includes('/api2/task-stream')) return pendingSseResponse(); + if (u.includes('/api/task-detail')) return Response.json({ success: true, data: { task: {} } }); + return taskResponse('unexpected-create'); + }); + + const failed = await be.destroySession(); + expect(failed).toMatchObject({ ok: false, taskId: 'task-parent' }); + expect((be as any).closing).toBe(false); + + be.write('continue after failed close'); + await flush(); await flush(); + const follow = calls.find(c => c.url.includes('/api/task-follow-up')); + expect(follow).toBeDefined(); + expect(JSON.parse(String(follow!.init?.body)).parentTaskId).toBe('task-parent'); + + const retried = await be.destroySession(); + expect(retried).toEqual({ ok: true, taskId: 'task-after-failure' }); + }); + + it('aborts a successful prepare without losing a late-created child lineage', async () => { + const be = makeBackend({ injectStatusLines: false }); + be.write('opening message'); + await flush(); + const preparedP = be.destroySession(); + resolvers.shift()!(taskResponse('task-late-prepared')); + const prepared = await preparedP; + expect(prepared).toEqual({ ok: true, taskId: 'task-late-prepared' }); + expect((be as any).closing).toBe(true); + + await be.abortDestroySession(); + expect((be as any).closing).toBe(false); + be.write('continue after durable commit failure'); + await flush(); + const follow = calls.find(c => c.url.includes('/api/task-follow-up')); + expect(follow).toBeDefined(); + expect(JSON.parse(String(follow!.init?.body)).parentTaskId).toBe('task-late-prepared'); + }); + + it('does not reopen admission when close timeout wins during an in-flight cancel', async () => { + const be = makeBackend({ resumeParentTaskId: 'task-timeout-parent', injectStatusLines: false }); + (be as any).destroyDeadlineMs = 10; + let resolveCancel!: (response: Response) => void; + fetchMock.mockImplementation(async (url: string | URL, init?: RequestInit) => { + const u = String(url); + calls.push({ url: u, init }); + if (u.includes('/api/task-cancel')) { + return new Promise(resolve => { resolveCancel = resolve; }); + } + if (u.includes('/api/task-follow-up')) return taskResponse('task-after-timeout'); + if (u.includes('/api2/task-stream')) return pendingSseResponse(); + if (u.includes('/api/task-detail')) return Response.json({ success: true, data: { task: {} } }); + return taskResponse('unexpected-create'); + }); + + let settled = false; + const closeP = be.destroySession().then(result => { settled = true; return result; }); + await new Promise(resolve => setTimeout(resolve, 25)); + expect(settled).toBe(false); + be.write('must remain fenced'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(0); + + resolveCancel(Response.json({ success: true, data: {} })); + const result = await closeP; + expect(result).toEqual({ ok: false, taskId: 'task-timeout-parent', error: 'close_timeout' }); + expect((be as any).closing).toBe(false); + + be.write('admitted after cancel settled'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up')).length).toBe(1); + }); + + it('cannot publish a prepared close after an abort races a late successful cancel', async () => { + const be = makeBackend({ resumeParentTaskId: 'task-abort-race', injectStatusLines: false }); + let resolveCancel!: (response: Response) => void; + fetchMock.mockImplementation(async (url: string | URL, init?: RequestInit) => { + const u = String(url); + calls.push({ url: u, init }); + if (u.includes('/api/task-cancel')) { + return new Promise(resolve => { resolveCancel = resolve; }); + } + if (u.includes('/api/task-follow-up')) return taskResponse('task-after-abort-race'); + if (u.includes('/api2/task-stream')) return pendingSseResponse(); + if (u.includes('/api/task-detail')) return Response.json({ success: true, data: { task: {} } }); + return taskResponse('unexpected-create'); + }); + + let closeSettled = false; + const close = be.destroySession().then(result => { + closeSettled = true; + return result; + }); + await flush(); + expect(resolveCancel).toBeTypeOf('function'); + + let abortSettled = false; + const abort = be.abortDestroySession().then(() => { abortSettled = true; }); + await flush(); + expect(closeSettled).toBe(false); + expect(abortSettled).toBe(false); + expect((be as any).closing).toBe(true); + + resolveCancel(Response.json({ success: true, data: {} })); + await expect(close).resolves.toEqual({ + ok: false, + taskId: 'task-abort-race', + error: 'close_aborted', + }); + await abort; + expect((be as any).closePrepared).toBe(false); + expect((be as any).closing).toBe(false); + + be.write('admitted after exact abort'); + await flush(); + expect(calls.filter(c => c.url.includes('/api/task-follow-up'))).toHaveLength(1); }); }); diff --git a/test/riff-explicit-close.test.ts b/test/riff-explicit-close.test.ts new file mode 100644 index 000000000..b65957b4d --- /dev/null +++ b/test/riff-explicit-close.test.ts @@ -0,0 +1,283 @@ +import { EventEmitter } from 'node:events'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { activeSessionKey, type DaemonSession } from '../src/core/types.js'; + +const { getBotMock, cancelRiffTaskMock } = vi.hoisted(() => ({ + getBotMock: vi.fn(), + cancelRiffTaskMock: vi.fn(async () => true), +})); + +vi.mock('../src/bot-registry.js', () => ({ + getBot: getBotMock, + getBotBrand: vi.fn(() => 'feishu'), + getAllBots: vi.fn(() => []), + loadBotConfigs: vi.fn(), + resolveBrandLabel: vi.fn(() => undefined), +})); + +vi.mock('../src/adapters/backend/riff-backend.js', () => ({ + hashUrlForLog: vi.fn(() => 'riffhash'), + cancelRiffTaskById: cancelRiffTaskMock, +})); + +vi.mock('../src/im/lark/client.js', () => ({ + updateMessage: vi.fn(), + deleteMessage: vi.fn(), + sendEphemeralCard: vi.fn(), + sendUserMessage: vi.fn(), + addReaction: vi.fn(), + removeReaction: vi.fn(), + getMessageChatId: vi.fn(), + MessageWithdrawnError: class extends Error {}, +})); + +vi.mock('../src/services/frozen-card-store.js', () => ({ + loadFrozenCards: vi.fn(() => new Map()), + saveFrozenCards: vi.fn(), + deleteFrozenCards: vi.fn(), +})); + +vi.mock('../src/utils/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn() }, +})); + +import { config } from '../src/config.js'; +import { + __testOnly_setupWorkerHandlers, + closeSession, + initWorkerPool, + killWorker, + setActiveSessionsRegistry, +} from '../src/core/worker-pool.js'; +import * as sessionStore from '../src/services/session-store.js'; + +let dataDir: string; +let previousDataDir: string; + +function createFixture(options: { + liveWorker?: boolean; + closeOk?: boolean; + resultTaskId?: string; +} = {}) { + sessionStore.init('app'); + const session = sessionStore.createSession('oc_riff', 'om_riff', 'riff close', 'group'); + session.larkAppId = 'app'; + session.scope = 'chat'; + session.backendType = 'riff'; + session.riffParentTaskId = 'task-riff-123'; + sessionStore.updateSession(session); + + const worker = options.liveWorker ? new EventEmitter() as any : null; + if (worker) { + worker.killed = false; + worker.exitCode = null; + worker.signalCode = null; + worker.kill = vi.fn(); + worker.send = vi.fn((message: any) => { + if (message.type === 'close_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'close_abort_result', + requestId: message.requestId, + ok: true, + })); + return; + } + if (message.type !== 'close' || !message.requestId) return; + queueMicrotask(() => worker.emit('message', { + type: 'close_result', + requestId: message.requestId, + ok: options.closeOk ?? true, + taskId: options.resultTaskId ?? 'task-riff-123', + ...((options.closeOk ?? true) ? {} : { error: 'task-cancel HTTP 500' }), + })); + }); + } + + const ds = { + larkAppId: 'app', + chatId: session.chatId, + chatType: 'group', + scope: 'chat', + worker, + session, + initConfig: { backendType: 'riff' }, + } as unknown as DaemonSession; + if (worker) __testOnly_setupWorkerHandlers(ds, worker); + const registry = new Map([[activeSessionKey(ds), ds]]); + setActiveSessionsRegistry(registry); + return { session, ds, worker, registry }; +} + +beforeEach(() => { + vi.clearAllMocks(); + dataDir = mkdtempSync(join(tmpdir(), 'botmux-riff-close-')); + previousDataDir = config.session.dataDir; + config.session.dataDir = dataDir; + getBotMock.mockReturnValue({ + resolvedAllowedUsers: [], + config: { riff: { baseUrl: 'https://riff.invalid', jwt: 'test' } }, + }); + cancelRiffTaskMock.mockResolvedValue(true); + initWorkerPool({ + sessionReply: vi.fn(async () => 'om_reply'), + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + setActiveSessionsRegistry(new Map()); + config.session.dataDir = previousDataDir; + sessionStore.init(); + rmSync(dataDir, { recursive: true, force: true }); +}); + +describe('Riff explicit close', () => { + it('awaits worker-less cancellation before atomically clearing lineage and closing', async () => { + const fixture = createFixture(); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: true, + alreadyClosed: false, + }); + expect(cancelRiffTaskMock).toHaveBeenCalledWith( + expect.objectContaining({ baseUrl: 'https://riff.invalid' }), + 'task-riff-123', + ); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ status: 'closed' }); + expect(sessionStore.getSession(fixture.session.sessionId)?.riffParentTaskId).toBeUndefined(); + expect(fixture.registry.size).toBe(0); + }); + + it('preserves the active row, route and retry lineage when cancellation fails', async () => { + cancelRiffTaskMock.mockResolvedValue(false); + const fixture = createFixture(); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: false, + alreadyClosed: false, + error: 'riff_cancel_failed', + retryable: true, + taskId: 'task-riff-123', + }); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + expect(fixture.registry.get(activeSessionKey(fixture.ds))).toBe(fixture.ds); + }); + + it('commits a live worker close only after its matching prepare result', async () => { + const fixture = createFixture({ liveWorker: true, resultTaskId: 'task-riff-child' }); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: true, + alreadyClosed: false, + }); + expect(fixture.worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'close', + requestId: expect.any(String), + })); + expect(fixture.worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'close_commit', + requestId: expect.any(String), + })); + expect(cancelRiffTaskMock).not.toHaveBeenCalled(); + expect(sessionStore.getSession(fixture.session.sessionId)?.riffParentTaskId).toBeUndefined(); + }); + + it('aborts a failed live prepare and keeps the session retryable', async () => { + const fixture = createFixture({ liveWorker: true, closeOk: false }); + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: false, + alreadyClosed: false, + error: 'riff_worker_close_failed', + retryable: true, + taskId: 'task-riff-123', + }); + expect(fixture.worker.send).toHaveBeenCalledWith(expect.objectContaining({ + type: 'close_abort', + requestId: expect.any(String), + })); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + expect(fixture.ds.riffCloseState).toBeUndefined(); + }); + + it('refuses an unprepared generic retirement of a live Riff worker', () => { + const fixture = createFixture({ liveWorker: true }); + + killWorker(fixture.ds); + + expect(fixture.ds.worker).toBe(fixture.worker); + expect(fixture.worker.send).not.toHaveBeenCalled(); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + }); + + it('refuses explicit close while the daemon shutdown fence owns the Riff worker', async () => { + const fixture = createFixture({ liveWorker: true }); + fixture.ds.riffShutdownState = { + phase: 'preparing', + requestId: 'shutdown-riff', + taskId: 'task-riff-123', + }; + + expect(await closeSession(fixture.session.sessionId)).toEqual({ + ok: false, + alreadyClosed: false, + error: 'riff_shutdown_fence_in_progress', + retryable: true, + taskId: 'task-riff-123', + }); + expect(fixture.worker.send).not.toHaveBeenCalled(); + expect(sessionStore.getSession(fixture.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-riff-123', + }); + }); + + it('keeps the newest runtime lineage when its durable save fails', async () => { + const fixture = createFixture({ liveWorker: true }); + vi.spyOn(sessionStore, 'updateSession').mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + fixture.worker.emit('message', { + type: 'riff_task_id', + taskId: 'task-riff-child', + }); + await new Promise(resolve => setImmediate(resolve)); + + expect(fixture.session.riffParentTaskId).toBe('task-riff-child'); + const durable = JSON.parse(readFileSync(join(dataDir, 'sessions-app.json'), 'utf8')); + expect(durable[fixture.session.sessionId].riffParentTaskId).toBe('task-riff-123'); + }); + + it('restores the prior runtime lineage when clearing its durable row fails', async () => { + const fixture = createFixture({ liveWorker: true }); + vi.spyOn(sessionStore, 'updateSession').mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + fixture.worker.emit('message', { + type: 'riff_task_id', + taskId: null, + }); + await new Promise(resolve => setImmediate(resolve)); + + expect(fixture.session.riffParentTaskId).toBe('task-riff-123'); + const durable = JSON.parse(readFileSync(join(dataDir, 'sessions-app.json'), 'utf8')); + expect(durable[fixture.session.sessionId].riffParentTaskId).toBe('task-riff-123'); + }); +}); diff --git a/test/riff-shutdown-detach.test.ts b/test/riff-shutdown-detach.test.ts new file mode 100644 index 000000000..d31c37627 --- /dev/null +++ b/test/riff-shutdown-detach.test.ts @@ -0,0 +1,1093 @@ +import { EventEmitter } from 'node:events'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { config } from '../src/config.js'; +import type { DaemonSession } from '../src/core/types.js'; +import { + abortRiffShutdownFleet, + abortPreparedRiffShutdown, + canAbortVerifiedExitedRiffPreparation, + collectUniqueDaemonShutdownSessions, + commitPreparedRiffShutdown, + detachRiffWorkerForShutdown, + persistPreparedRiffShutdown, + persistPreparedRiffShutdownFleet, + prepareRiffFleetForShutdown, + prepareRiffSessionForShutdown, + type FencedRiffShutdownParticipant, + type PreparedRiffShutdown, +} from '../src/core/riff-shutdown-detach.js'; +import { sendWorkerInput } from '../src/core/worker-pool.js'; +import * as sessionStore from '../src/services/session-store.js'; + +vi.mock('../src/utils/logger.js', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +type FakeWorker = EventEmitter & { + killed: boolean; + exitCode: number | null; + signalCode: NodeJS.Signals | null; + send: ReturnType; + kill: ReturnType; +}; + +describe('Riff graceful daemon-shutdown detach coordinator', () => { + let dataDir: string; + let previousDataDir: string; + + beforeEach(() => { + dataDir = mkdtempSync(join(tmpdir(), 'botmux-riff-shutdown-')); + previousDataDir = config.session.dataDir; + config.session.dataDir = dataDir; + sessionStore.init('app'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + config.session.dataDir = previousDataDir; + sessionStore.init(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + function fixture( + initialTaskId: string | undefined, + onSend: (worker: FakeWorker, message: any) => void, + ): { ds: DaemonSession; worker: FakeWorker; messages: any[] } { + const session = sessionStore.createSession('oc_riff', 'om_riff', 'riff shutdown', 'group'); + session.larkAppId = 'app'; + session.backendType = 'riff'; + session.riffParentTaskId = initialTaskId; + sessionStore.updateSession(session); + const messages: any[] = []; + const worker = new EventEmitter() as FakeWorker; + worker.killed = false; + worker.exitCode = null; + worker.signalCode = null; + worker.kill = vi.fn(); + worker.send = vi.fn((message: any) => { + messages.push(message); + onSend(worker, message); + }); + const ds = { + larkAppId: 'app', + chatId: session.chatId, + chatType: 'group', + scope: 'chat', + session, + worker, + workerPort: 4100, + workerToken: 'write', + workerViewToken: 'view', + managedTurnOrigin: { capability: 'cap' }, + initConfig: { backendType: 'riff' }, + } as unknown as DaemonSession; + return { ds, worker, messages }; + } + + function asPrepared( + result: Awaited>, + ): PreparedRiffShutdown { + if (!result.ok) throw new Error(`expected prepared Riff shutdown: ${result.error}`); + return result; + } + + function asFenced( + result: Awaited>, + ): FencedRiffShutdownParticipant { + if (result.fence === 'none') throw new Error(`expected fenced Riff shutdown: ${result.error}`); + return result; + } + + it('transactional lineage persistence rolls back its in-memory field when save throws', () => { + const session = sessionStore.createSession('oc_txn', 'om_txn', 'txn', 'group'); + session.backendType = 'riff'; + session.riffParentTaskId = 'task-before'; + sessionStore.updateSession(session); + const blockedDataDir = join(dataDir, 'not-a-directory'); + writeFileSync(blockedDataDir, 'block'); + config.session.dataDir = blockedDataDir; + + expect(() => sessionStore.persistActiveRiffLineageExact(session.sessionId, 'task-after')).toThrow(); + expect(session.riffParentTaskId).toBe('task-before'); + + config.session.dataDir = dataDir; + expect(sessionStore.getSessionFresh(session.sessionId)?.riffParentTaskId).toBe('task-before'); + }); + + it('deduplicates multiple registry aliases to the exact same daemon session object', () => { + const f = fixture('task-parent', () => { /* no IPC */ }); + + expect(collectUniqueDaemonShutdownSessions([f.ds, f.ds, f.ds])).toEqual({ + ok: true, + sessions: [f.ds], + }); + }); + + it('fails closed when distinct daemon session objects claim the same session id', () => { + const f = fixture('task-parent', () => { /* no IPC */ }); + const competing = { + ...f.ds, + session: { ...f.ds.session }, + } as DaemonSession; + + expect(collectUniqueDaemonShutdownSessions([f.ds, competing])).toMatchObject({ + ok: false, + sessionId: f.ds.session.sessionId, + error: expect.stringContaining('distinct daemon session generations'), + }); + }); + + it('retains only the ambiguous session fence and restores an unrelated prepared peer', async () => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } else if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'abort', ok: true, taskId: 'task-second-child', + })); + } + }); + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds]); + const competingFirst = { + ...first.ds, + session: { ...first.ds.session }, + } as DaemonSession; + const current = collectUniqueDaemonShutdownSessions([ + first.ds, + competingFirst, + second.ds, + ]); + if (current.ok) throw new Error('expected ambiguous daemon session id'); + + const restored = await abortRiffShutdownFleet(prepared + .filter(({ ds }) => ds.session.sessionId !== current.sessionId) + .map(({ ds, result }) => ({ ds, result: asFenced(result) }))); + + expect(restored).toHaveLength(1); + expect(restored[0].result.ok).toBe(true); + expect(first.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(first.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(second.ds.riffShutdownState).toBeUndefined(); + expect(second.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + }); + + it('refuses an initial durable-read failure before installing any worker fence', async () => { + vi.spyOn(sessionStore, 'getActiveRiffShutdownSnapshotsBatch').mockImplementation(() => { + throw new Error('lock unavailable'); + }); + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + + await expect(prepareRiffSessionForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-parent', + error: 'durable_session_read_failed:lock unavailable', + }); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.ds.worker).toBe(f.worker); + expect(f.messages).toEqual([]); + }); + + it('takes one all-owner snapshot before publishing any fleet prepare request', async () => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } + }); + first.ds.session.pid = 101; + second.ds.session.pid = 202; + sessionStore.updateSession(first.ds.session); + sessionStore.updateSession(second.ds.session); + const originalSnapshot = sessionStore.getActiveRiffShutdownSnapshotsBatch; + const snapshot = vi.spyOn(sessionStore, 'getActiveRiffShutdownSnapshotsBatch') + .mockImplementation((sessionIds, options) => { + expect(first.messages).toEqual([]); + expect(second.messages).toEqual([]); + return originalSnapshot(sessionIds, options); + }); + + const results = await prepareRiffFleetForShutdown([first.ds, second.ds]); + + expect(snapshot).toHaveBeenCalledTimes(1); + expect(snapshot).toHaveBeenCalledWith([ + first.ds.session.sessionId, + second.ds.session.sessionId, + ], expect.any(Object)); + expect(results.every(entry => entry.result.ok)).toBe(true); + expect(first.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(second.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it('does not inline-abort a timed-out prepare and restores all owners in one concurrent wave', async () => { + let firstAbortRequestId: string | undefined; + let secondAbortRequestId: string | undefined; + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } else if (message.type === 'riff_shutdown_abort') { + firstAbortRequestId = message.requestId; + } + }); + const second = fixture('task-second', (_worker, message) => { + if (message.type === 'riff_shutdown_abort') secondAbortRequestId = message.requestId; + // Deliberately never ACK prepare so its fence state is ambiguous. + }); + first.ds.session.pid = 101; + second.ds.session.pid = 202; + sessionStore.updateSession(first.ds.session); + sessionStore.updateSession(second.ds.session); + + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds], { + drainTimeoutMs: 5, + abortTimeoutMs: 200, + }); + expect(prepared[0].result).toMatchObject({ ok: true, fence: 'prepared' }); + expect(prepared[1].result).toMatchObject({ ok: false, fence: 'possible' }); + expect(first.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(second.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + + const aborting = abortRiffShutdownFleet(prepared.map(({ ds, result }) => ({ + ds, + result: asFenced(result), + })), { abortTimeoutMs: 200 }); + await vi.waitFor(() => { + expect(firstAbortRequestId).toEqual(expect.any(String)); + expect(secondAbortRequestId).toEqual(expect.any(String)); + }); + expect(first.messages.filter(message => message.type === 'riff_shutdown_abort')).toHaveLength(1); + expect(second.messages.filter(message => message.type === 'riff_shutdown_abort')).toHaveLength(1); + + first.worker.emit('message', { + type: 'riff_shutdown_result', requestId: firstAbortRequestId, + phase: 'abort', ok: true, taskId: 'task-first-child', + }); + second.worker.emit('message', { + type: 'riff_shutdown_result', requestId: secondAbortRequestId, + phase: 'abort', ok: true, taskId: 'task-second', + }); + await expect(aborting).resolves.toSatisfy( + (results: Array<{ result: { ok: boolean } }>) => results.every(entry => entry.result.ok), + ); + expect(first.ds.riffShutdownState).toBeUndefined(); + expect(second.ds.riffShutdownState).toBeUndefined(); + + // A prepare ACK arriving after the exact abort wave is inert. + second.worker.emit('message', { + type: 'riff_shutdown_result', requestId: secondAbortRequestId, + phase: 'prepare', ok: true, taskId: 'task-late-ack', + }); + expect(second.ds.riffShutdownState).toBeUndefined(); + expect(second.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + }); + + it.each(['explicit_close_in_progress', 'not_riff_backend'])( + 'classifies exact pre-fence worker refusal %s without sending abort', + async (error) => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: false, taskId: null, error, + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + fence: 'none', + error, + }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.ds.worker).toBe(f.worker); + }, + ); + + it('clears the synthetic daemon fence when prepare IPC throws before it can be queued', async () => { + const f = fixture('task-parent', () => { /* send is replaced below */ }); + f.worker.send.mockImplementation(() => { throw new Error('IPC channel closed'); }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + fence: 'none', + error: 'riff_shutdown_prepare_send_failed', + }); + expect(f.messages).toEqual([]); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('does not misclassify an existing older shutdown fence as a pre-fence refusal', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: false, taskId: null, + error: 'shutdown detach already prepared as older-request', + })); + } + }); + + await expect(prepareRiffSessionForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + fence: 'possible', + error: 'shutdown detach already prepared as older-request', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'preparing' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it('refuses before a worker fence unless phase-2 plus the abort reserve remain', async () => { + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + const now = 10_000; + + await expect(prepareRiffFleetForShutdown([f.ds], { + deadlineMs: now + 1_500, + abortTimeoutMs: 1_000, + now: () => now, + })).resolves.toMatchObject([{ + result: { + ok: false, + fence: 'none', + error: 'insufficient_abort_budget_before_fence', + }, + }]); + expect(f.messages).toEqual([]); + expect(f.ds.riffShutdownState).toBeUndefined(); + }); + + it('persists all prepared owners through one phase-2 batch call', async () => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } + }); + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds]); + const entries = prepared.map(({ ds, result }) => ({ ds, result: asPrepared(result) })); + const batch = vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch'); + + expect(persistPreparedRiffShutdownFleet(entries)).toEqual({ ok: true }); + expect(batch).toHaveBeenCalledTimes(1); + expect(sessionStore.getSessionFresh(first.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-first-child'); + expect(sessionStore.getSessionFresh(second.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-second-child'); + expect(entries.every(({ result }) => result.lineageVerified)).toBe(true); + }); + + it('retains every fence when verified phase-2 persistence returns after the absolute deadline', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + const originalBatch = sessionStore.persistActiveRiffLineagesExactBatch; + let now = 10_000; + const deadlineMs = 10_100; + vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch') + .mockImplementation((updates, options) => { + originalBatch(updates, options); + now = deadlineMs; + }); + + const result = persistPreparedRiffShutdownFleet( + [{ ds: f.ds, result: prepared }], + { deadlineMs, now: () => now }, + ); + + expect(result).toMatchObject({ + ok: false, + error: 'shutdown_deadline_elapsed_after_batch_persist', + rollbackDisposition: 'retain_fence', + sessionIds: [f.ds.session.sessionId], + retainFencedSessionIds: [f.ds.session.sessionId], + }); + expect(prepared.lineageVerified).toBe(true); + expect(f.ds.session.riffParentTaskId).toBe('task-child'); + expect(f.ds.riffShutdownState).toMatchObject({ requestId: prepared.requestId }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it.each([ + ['prewrite_ownership', 'affected'] as const, + ['prewrite_io', 'none'] as const, + ['postrename_ambiguity', 'all'] as const, + ])('maps %s batch failure to the exact retain-fence set', async (stage, expected) => { + const first = fixture('task-first', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-first-child', + })); + } + }); + const second = fixture('task-second', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-second-child', + })); + } + }); + const prepared = await prepareRiffFleetForShutdown([first.ds, second.ds]); + const entries = prepared.map(({ ds, result }) => ({ ds, result: asPrepared(result) })); + const affected = stage === 'postrename_ambiguity' + ? [first.ds.session.sessionId, second.ds.session.sessionId] + : [second.ds.session.sessionId]; + vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch').mockImplementation(() => { + throw new sessionStore.RiffLineageBatchError(stage, affected, `forced ${stage}`); + }); + + const result = persistPreparedRiffShutdownFleet(entries); + expect(result).toMatchObject({ ok: false }); + if (result.ok) throw new Error('expected batch failure'); + expect(result.retainFencedSessionIds).toEqual( + expected === 'all' + ? [first.ds.session.sessionId, second.ds.session.sessionId] + : expected === 'affected' + ? [second.ds.session.sessionId] + : [], + ); + }); + + it('retains the exact fence without writing when runtime owner changes after prepare', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + f.ds.session.pid = 999_999; + const batch = vi.spyOn(sessionStore, 'persistActiveRiffLineagesExactBatch'); + + const result = persistPreparedRiffShutdownFleet([{ ds: f.ds, result: prepared }]); + + expect(result).toMatchObject({ + ok: false, + retainFencedSessionIds: [f.ds.session.sessionId], + error: expect.stringContaining('runtime_owner_changed'), + }); + expect(batch).not.toHaveBeenCalled(); + expect(f.ds.riffShutdownState).toMatchObject({ requestId: prepared.requestId }); + }); + + it('persists and fresh-verifies the exact late child before commit', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: 'task-late-child', + })); + } + }); + + const result = await detachRiffWorkerForShutdown(f.ds); + expect(result).toMatchObject({ + ok: true, + taskId: 'task-late-child', + disposition: 'lineage_persisted', + }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_commit', + ]); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'task-late-child', + }); + expect(f.ds.worker).toBeNull(); + expect(f.worker.kill).not.toHaveBeenCalled(); + }); + + it('treats null as authoritative and clears a stale durable parent before commit', async () => { + const f = fixture('task-stale-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: null, + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: true, + taskId: null, + disposition: 'lineage_persisted', + }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId).toBeUndefined(); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_commit', + ]); + }); + + it('restores the prepared worker without cancellation when durable persistence fails', async () => { + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation(() => { + throw new Error('disk full'); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: 'task-exact-child', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'abort', + ok: true, + taskId: 'task-exact-child', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-exact-child', + error: expect.stringContaining('lineage_persist_failed:disk full'), + }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + // Persistence never advanced, so the accepted remote task and its exact + // worker generation remain live with admission restored. + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId).toBe('task-parent'); + expect(f.ds.worker).toBe(f.worker); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(f.messages.some(message => message.type === 'riff_shutdown_cancel')).toBe(false); + }); + + it('fails closed without commit or signal when persistence fails and abort is not ACKed', async () => { + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation(() => { + throw new Error('disk full'); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'prepare', + ok: true, + taskId: 'task-uncancellable', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'abort', + ok: false, + taskId: 'task-uncancellable', + error: 'admission restore refused', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-uncancellable', + error: expect.stringContaining('admission_restore_failed:admission restore refused'), + }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + expect(f.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + }); + + it('keeps the fence when an abort ACK reports a different task lineage', async () => { + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation(() => { + throw new Error('disk full'); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-prepared-child', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'abort', ok: true, taskId: 'task-unexpected-child', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('abort_task_lineage_mismatch'), + }); + expect(f.ds.worker).toBe(f.worker); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + }); + + it('prepares and persists workerless runtime lineage before allowing commit', async () => { + const f = fixture('task-durable-parent', () => { /* workerless: no IPC */ }); + f.ds.worker = null; + f.ds.workerPort = null; + f.ds.workerToken = null; + f.ds.workerViewToken = null; + // Simulate a prior ordinary riff_task_id save failure: runtime owns the + // child while the durable row still names its parent. + f.ds.session.riffParentTaskId = 'task-runtime-child'; + + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + expect(prepared.worker).toBeNull(); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-durable-parent'); + + expect(persistPreparedRiffShutdown(f.ds, prepared)).toEqual({ ok: true }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-runtime-child'); + expect(commitPreparedRiffShutdown(f.ds, prepared)).toBe(true); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(f.messages).toEqual([]); + }); + + it('aborts every prepared peer and commits none when one workerless lineage write fails', async () => { + const live = fixture('task-live-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-live-child', + })); + } + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'abort', ok: true, taskId: 'task-live-child', + })); + } + }); + const workerless = fixture('task-workerless-parent', () => { /* no IPC */ }); + workerless.ds.worker = null; + workerless.ds.session.riffParentTaskId = 'task-workerless-runtime-child'; + + const [livePrepared, workerlessPrepared] = await Promise.all([ + prepareRiffSessionForShutdown(live.ds), + prepareRiffSessionForShutdown(workerless.ds), + ]).then(results => results.map(asPrepared)); + const originalPersist = sessionStore.persistActiveRiffLineageExact; + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation( + (sessionId, taskId) => { + if (sessionId === workerless.ds.session.sessionId) throw new Error('target disk full'); + return originalPersist(sessionId, taskId); + }, + ); + + const persistence = [ + persistPreparedRiffShutdown(live.ds, livePrepared!), + persistPreparedRiffShutdown(workerless.ds, workerlessPrepared!), + ]; + expect(persistence[0]).toEqual({ ok: true }); + expect(persistence[1]).toMatchObject({ + ok: false, + error: expect.stringContaining('target disk full'), + }); + + await Promise.all([ + abortPreparedRiffShutdown(live.ds, livePrepared!), + abortPreparedRiffShutdown(workerless.ds, workerlessPrepared!), + ]); + expect(live.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + expect(live.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(live.ds.worker).toBe(live.worker); + expect(live.ds.riffShutdownState).toBeUndefined(); + expect(workerless.ds.riffShutdownState).toBeUndefined(); + expect(sessionStore.getSessionFresh(workerless.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-workerless-parent'); + }); + + it('retains a fail-closed fence when an unverified prepared worker exits before abort', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-unverified-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + f.worker.exitCode = 1; + // worker-pool clears the exact dead child handle but retains this request's + // shutdown fence for the coordinator. + f.ds.worker = null; + + expect(persistPreparedRiffShutdown(f.ds, prepared)).toMatchObject({ + ok: false, + error: 'stale_worker_generation', + }); + await expect(abortPreparedRiffShutdown(f.ds, prepared)).resolves.toMatchObject({ + ok: false, + error: 'worker_exited_before_admission_restore', + }); + expect(f.ds.worker).toBeNull(); + expect(f.ds.riffShutdownState).toMatchObject({ + phase: 'prepared', + requestId: prepared.requestId, + }); + expect(sendWorkerInput(f.ds, 'must stay fenced', 'turn-exited-unverified')).toBe(false); + }); + + it('classifies an unexpected durable lineage as ownership loss and never overwrites it', async () => { + const f = fixture('task-durable-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => { + // Simulate a different process advancing the durable owner after + // prepare, without changing this daemon's runtime generation. + const sessionsPath = join(dataDir, 'sessions-app.json'); + const projection = JSON.parse(readFileSync(sessionsPath, 'utf8')) as Record; + projection[f.ds.session.sessionId]!.riffParentTaskId = 'task-external-owner'; + writeFileSync(sessionsPath, JSON.stringify(projection, null, 2)); + worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-prepared-child', + }); + }); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('compare-and-set failed'), + rollbackDisposition: 'retain_fence', + }); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-external-owner'); + expect(f.ds.riffShutdownState).toMatchObject({ + phase: 'prepared', + }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + }); + + it('retains the fence when durable worker ownership changes with the same lineage', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => { + const sessionsPath = join(dataDir, 'sessions-app.json'); + const projection = JSON.parse(readFileSync(sessionsPath, 'utf8')) as Record; + projection[f.ds.session.sessionId]!.pid = 999_999; + writeFileSync(sessionsPath, JSON.stringify(projection, null, 2)); + worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-prepared-child', + }); + }); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('owner compare-and-set failed'), + rollbackDisposition: 'retain_fence', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)).toMatchObject({ + pid: 999_999, + riffParentTaskId: 'task-parent', + }); + }); + + it('retains the fence when ownership is replaced after CAS with the same task id', async () => { + const originalPersist = sessionStore.persistActiveRiffLineageExact; + vi.spyOn(sessionStore, 'persistActiveRiffLineageExact').mockImplementation( + (sessionId, taskId, options) => { + const result = originalPersist(sessionId, taskId, options); + // Exact replacement race: CAS wrote the expected task, then a new + // durable owner published the same task before the separate readback. + const sessionsPath = join(dataDir, 'sessions-app.json'); + const projection = JSON.parse(readFileSync(sessionsPath, 'utf8')) as Record; + projection[sessionId]!.pid = 888_888; + writeFileSync(sessionsPath, JSON.stringify(projection, null, 2)); + return result; + }, + ); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-same-after-replacement', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + taskId: 'task-same-after-replacement', + error: expect.stringContaining('fresh_lineage_verification_failed:'), + rollbackDisposition: 'retain_fence', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)).toMatchObject({ + pid: 888_888, + riffParentTaskId: 'task-same-after-replacement', + }); + }); + + it('retains the fence when the post-write fresh verification cannot be read', async () => { + const originalFresh = sessionStore.getSessionFresh; + let reads = 0; + vi.spyOn(sessionStore, 'getSessionFresh').mockImplementation(sessionId => { + reads++; + if (reads === 1) throw new Error('lock unavailable'); + return originalFresh(sessionId); + }); + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-written-not-verified', + })); + } + }); + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: 'fresh_lineage_verification_failed:lock unavailable', + rollbackDisposition: 'retain_fence', + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'prepared' }); + expect(f.messages.map(message => message.type)).toEqual(['riff_shutdown_prepare']); + expect(originalFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-written-not-verified'); + }); + + it('can release an exited prepared generation only after its lineage was durably verified', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-verified-child', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + expect(persistPreparedRiffShutdown(f.ds, prepared)).toEqual({ ok: true }); + f.worker.exitCode = 0; + f.ds.worker = null; + + expect(canAbortVerifiedExitedRiffPreparation(f.ds, prepared)).toBe(true); + + await expect(abortPreparedRiffShutdown(f.ds, prepared)).resolves.toEqual({ + ok: true, + taskId: 'task-verified-child', + }); + expect(f.ds.worker).toBeNull(); + expect(f.ds.riffShutdownState).toBeUndefined(); + expect(sessionStore.getSessionFresh(f.ds.session.sessionId)?.riffParentTaskId) + .toBe('task-verified-child'); + }); + + it('does not bless a replacement generation when the verified prepared worker exits', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_prepare') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', requestId: message.requestId, + phase: 'prepare', ok: true, taskId: 'task-verified-before-replacement', + })); + } + }); + const prepared = asPrepared(await prepareRiffSessionForShutdown(f.ds)); + expect(persistPreparedRiffShutdown(f.ds, prepared)).toEqual({ ok: true }); + f.worker.exitCode = 0; + const replacement = new EventEmitter() as FakeWorker; + replacement.killed = false; + replacement.exitCode = null; + replacement.signalCode = null; + replacement.send = vi.fn(); + replacement.kill = vi.fn(); + f.ds.worker = replacement; + + expect(canAbortVerifiedExitedRiffPreparation(f.ds, prepared)).toBe(false); + + await expect(abortPreparedRiffShutdown(f.ds, prepared)).resolves.toMatchObject({ + ok: false, + taskId: 'task-verified-before-replacement', + error: 'new_worker_generation', + }); + expect(f.ds.worker).toBe(replacement); + expect(f.ds.riffShutdownState).toMatchObject({ + phase: 'prepared', + requestId: prepared.requestId, + }); + expect(replacement.send).not.toHaveBeenCalled(); + }); + + it('times out boundedly and restores the worker instead of falling through to generic kill', async () => { + const f = fixture('task-parent', (worker, message) => { + if (message.type === 'riff_shutdown_abort') { + queueMicrotask(() => worker.emit('message', { + type: 'riff_shutdown_result', + requestId: message.requestId, + phase: 'abort', + ok: true, + taskId: 'task-parent', + })); + } + }); + const result = await detachRiffWorkerForShutdown(f.ds, { + drainTimeoutMs: 10, + abortTimeoutMs: 100, + }); + + expect(result).toMatchObject({ ok: false, error: 'riff_shutdown_prepare_timeout' }); + expect(f.messages.map(message => message.type)).toEqual([ + 'riff_shutdown_prepare', + 'riff_shutdown_abort', + ]); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('keeps daemon admission fenced until delayed abort restoration is ACKed', async () => { + let abortRequestId: string | undefined; + const f = fixture('task-parent', (_worker, message) => { + if (message.type === 'riff_shutdown_abort') abortRequestId = message.requestId; + }); + const detach = detachRiffWorkerForShutdown(f.ds, { + drainTimeoutMs: 5, + abortTimeoutMs: 200, + }); + await vi.waitFor(() => expect(abortRequestId).toEqual(expect.any(String))); + + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'preparing' }); + expect(sendWorkerInput(f.ds, 'must remain fenced', 'turn-during-abort')).toBe(false); + expect(f.messages.some(message => message.type === 'riff_shutdown_commit')).toBe(false); + expect(f.worker.kill).not.toHaveBeenCalled(); + + f.worker.emit('message', { + type: 'riff_shutdown_result', + requestId: abortRequestId, + phase: 'abort', + ok: true, + taskId: 'task-parent', + }); + await expect(detach).resolves.toMatchObject({ + ok: false, + error: 'riff_shutdown_prepare_timeout', + }); + expect(f.ds.riffShutdownState).toBeUndefined(); + }); + + it('refuses before worker prepare when daemon-owned raw/follow-up input is pending', async () => { + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + f.ds.pendingRawInput = '/goal keep this'; + f.ds.pendingFollowUpInput = { + userPrompt: 'follow-up', + cliInput: 'follow-up', + }; + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('daemon_inputs_not_drained:'), + }); + expect(f.messages).toEqual([]); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('refuses before worker prepare while a durable queued backlog is pending', async () => { + const f = fixture('task-parent', () => { /* no worker IPC is safe */ }); + f.ds.session.queued = true; + + await expect(detachRiffWorkerForShutdown(f.ds)).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('queued=1'), + }); + expect(f.messages).toEqual([]); + expect(f.worker.kill).not.toHaveBeenCalled(); + expect(f.ds.worker).toBe(f.worker); + }); + + it('retains the fail-closed fence when abort restoration cannot be confirmed', async () => { + const f = fixture('task-parent', () => { /* prepare and abort both held */ }); + const result = await detachRiffWorkerForShutdown(f.ds, { + drainTimeoutMs: 5, + abortTimeoutMs: 5, + }); + + expect(result).toMatchObject({ + ok: false, + error: expect.stringContaining('admission_restore_failed:riff_shutdown_abort_timeout'), + }); + expect(f.ds.riffShutdownState).toMatchObject({ phase: 'preparing' }); + expect(sendWorkerInput(f.ds, 'still blocked', 'turn-fail-closed')).toBe(false); + expect(f.worker.kill).not.toHaveBeenCalled(); + }); +}); diff --git a/test/riff-worker-shutdown-readiness.test.ts b/test/riff-worker-shutdown-readiness.test.ts new file mode 100644 index 000000000..df3c5fda7 --- /dev/null +++ b/test/riff-worker-shutdown-readiness.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { riffWorkerShutdownInputBlocker } from '../src/core/riff-worker-shutdown-readiness.js'; + +const idle = { + initPromptMaterialized: true, + isFlushing: false, + pendingMessages: 0, + pendingRawInputs: 0, + pendingSessionRename: false, + sessionRenameInFlight: false, + commandLineWritesPending: 0, +}; + +describe('Riff worker shutdown input ownership', () => { + it('allows the current backend task itself when no unsent worker input exists', () => { + expect(riffWorkerShutdownInputBlocker(idle)).toBeNull(); + }); + + it('refuses current-task detach when a follow-up is still in pendingMessages', () => { + expect(riffWorkerShutdownInputBlocker({ ...idle, pendingMessages: 1 })) + .toBe('messages=1'); + }); + + it('accounts for every worker-owned pre-backend input surface', () => { + expect(riffWorkerShutdownInputBlocker({ + initPromptMaterialized: false, + isFlushing: true, + pendingMessages: 2, + pendingRawInputs: 1, + pendingSessionRename: true, + sessionRenameInFlight: true, + commandLineWritesPending: 1, + })).toBe( + 'init=materializing,flushing=1,messages=2,raw=1,rename=1,rename_inflight=1,command_writes=1', + ); + }); +}); diff --git a/test/session-store.test.ts b/test/session-store.test.ts index 7315ec950..20aac00b4 100644 --- a/test/session-store.test.ts +++ b/test/session-store.test.ts @@ -361,6 +361,44 @@ describe('closeSession()', () => { expect(reloaded!.closedAt).toBeDefined(); }); + it('clears Riff lineage atomically with the durable closed row', () => { + const session = createSession('chat1', 'root1', 'Close Riff'); + session.backendType = 'riff'; + session.riffParentTaskId = 'riff-task-prepared'; + updateSession(session); + + closeSession(session.sessionId, { clearRiffParentTaskId: true }); + init(); + + expect(getSession(session.sessionId)).toMatchObject({ status: 'closed' }); + expect(getSession(session.sessionId)?.riffParentTaskId).toBeUndefined(); + }); + + it('restores Riff close state in memory when the atomic save fails', () => { + const session = createSession('chat1', 'root1', 'Close Riff Save Failure'); + session.backendType = 'riff'; + session.riffParentTaskId = 'riff-task-retry'; + updateSession(session); + fsControl.failSessionWrite = true; + + expect(() => closeSession( + session.sessionId, + { clearRiffParentTaskId: true }, + )).toThrow(/simulated session repair write failure/); + expect(getSession(session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'riff-task-retry', + }); + expect(mockDeleteFrozenCards).not.toHaveBeenCalled(); + + fsControl.failSessionWrite = false; + init(); + expect(getSession(session.sessionId)).toMatchObject({ + status: 'active', + riffParentTaskId: 'riff-task-retry', + }); + }); + it('should call deleteFrozenCards with the sessionId', () => { const session = createSession('chat1', 'root1', 'Frozen'); closeSession(session.sessionId); diff --git a/test/worker-riff-retirement-protocol.test.ts b/test/worker-riff-retirement-protocol.test.ts new file mode 100644 index 000000000..671c2fede --- /dev/null +++ b/test/worker-riff-retirement-protocol.test.ts @@ -0,0 +1,134 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; + +const workerSource = readFileSync(new URL('../src/worker.ts', import.meta.url), 'utf8'); +const workerPoolSource = readFileSync(new URL('../src/core/worker-pool.ts', import.meta.url), 'utf8'); + +describe('worker Riff retirement protocol', () => { + it('refuses Riff generation restart before the local restart helper can run', () => { + const start = workerSource.indexOf("case 'restart':"); + const end = workerSource.indexOf("case 'expire_durable_turn':", start); + const restart = workerSource.slice(start, end); + + const riffGuard = restart.indexOf("if (effectiveBackendType === 'riff')"); + const refusal = restart.indexOf('Refused Riff generation restart', riffGuard); + const guardBreak = restart.indexOf('break;', refusal); + const replacement = restart.indexOf('await restartCliProcess(', guardBreak); + + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(riffGuard).toBeGreaterThanOrEqual(0); + expect(refusal).toBeGreaterThan(riffGuard); + expect(guardBreak).toBeGreaterThan(refusal); + expect(replacement).toBeGreaterThan(guardBreak); + }); + + it('refuses request-less Riff close before destroy or process exit', () => { + const start = workerSource.indexOf("case 'close':"); + const end = workerSource.indexOf("case 'close_commit':", start); + const close = workerSource.slice(start, end); + + const riffBranch = close.indexOf("if (effectiveBackendType === 'riff')"); + const requestlessGuard = close.indexOf('if (!msg.requestId)', riffBranch); + const refusal = close.indexOf('Refused unsafe request-less Riff close', requestlessGuard); + const guardBreak = close.indexOf('break;', refusal); + const localDestroy = close.lastIndexOf('backend?.destroySession?.()'); + const localExit = close.lastIndexOf('process.exit(0)'); + + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(riffBranch).toBeGreaterThanOrEqual(0); + expect(requestlessGuard).toBeGreaterThan(riffBranch); + expect(refusal).toBeGreaterThan(requestlessGuard); + expect(guardBreak).toBeGreaterThan(refusal); + expect(localDestroy).toBeGreaterThan(guardBreak); + expect(localExit).toBeGreaterThan(localDestroy); + }); + + it('refuses request-less Riff suspend before teardown or process exit', () => { + const start = workerSource.indexOf("case 'suspend':"); + const end = workerSource.indexOf('\n }\n});', start); + const suspend = workerSource.slice(start, end); + + const riffGuard = suspend.indexOf("if (effectiveBackendType === 'riff')"); + const refusal = suspend.indexOf('Refused unsafe Riff suspend', riffGuard); + const guardBreak = suspend.indexOf('break;', refusal); + const localDestroy = suspend.indexOf('(backend?.destroySession ?? backend?.kill)', guardBreak); + const localExit = suspend.indexOf('process.exit(0)', localDestroy); + + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + expect(riffGuard).toBeGreaterThanOrEqual(0); + expect(refusal).toBeGreaterThan(riffGuard); + expect(guardBreak).toBeGreaterThan(refusal); + expect(localDestroy).toBeGreaterThan(guardBreak); + expect(localExit).toBeGreaterThan(localDestroy); + }); + + it('checks every unsent input buffer before fencing the backend or allowing commit', () => { + const prepareStart = workerSource.indexOf("case 'riff_shutdown_prepare':"); + const prepareEnd = workerSource.indexOf("case 'riff_shutdown_commit':", prepareStart); + const prepare = workerSource.slice(prepareStart, prepareEnd); + const readiness = prepare.indexOf('riffWorkerShutdownInputBlocker({'); + const queueCount = prepare.indexOf('pendingMessages: pendingMessages.length', readiness); + const rawCount = prepare.indexOf('pendingRawInputs: pendingRawInputs.length', readiness); + const initFence = prepare.indexOf('initPromptMaterialized', readiness); + const refusal = prepare.indexOf('worker_inputs_not_drained:', readiness); + const backendPrepare = prepare.indexOf('backend?.prepareShutdownDetach?.()', refusal); + + expect(readiness).toBeGreaterThanOrEqual(0); + expect(initFence).toBeGreaterThan(readiness); + expect(queueCount).toBeGreaterThan(readiness); + expect(rawCount).toBeGreaterThan(queueCount); + expect(refusal).toBeGreaterThan(rawCount); + expect(backendPrepare).toBeGreaterThan(refusal); + + const commitStart = workerSource.indexOf("case 'riff_shutdown_commit':", prepareEnd); + const commitEnd = workerSource.indexOf("case 'riff_shutdown_abort':", commitStart); + const commit = workerSource.slice(commitStart, commitEnd); + expect(commit).toContain("shutdownDetachPhase !== 'prepared'"); + expect(commit.indexOf("shutdownDetachPhase !== 'prepared'")) + .toBeLessThan(commit.indexOf('process.exit(0)')); + }); + + it('has no shutdown cancellation command that can discard accepted Riff work', () => { + expect(workerSource).not.toContain("case 'riff_shutdown_cancel':"); + expect(workerSource).not.toContain('cancelShutdownDetach'); + }); + + it('ACKs shutdown and explicit-close abort only after backend admission restoration', () => { + const shutdownStart = workerSource.indexOf("case 'riff_shutdown_abort':"); + const shutdownEnd = workerSource.indexOf("case 'close_commit':", shutdownStart); + const shutdown = workerSource.slice(shutdownStart, shutdownEnd); + const shutdownRestore = shutdown.indexOf('await backend?.abortShutdownDetach?.()'); + expect(shutdownRestore).toBeGreaterThanOrEqual(0); + expect(shutdown.indexOf("phase: 'abort'", shutdownRestore)) + .toBeGreaterThan(shutdownRestore); + + const closeStart = workerSource.indexOf("case 'close_abort':"); + const closeEnd = workerSource.indexOf("case 'suspend':", closeStart); + const close = workerSource.slice(closeStart, closeEnd); + const closeRestore = close.indexOf('await backend?.abortDestroySession?.()'); + expect(closeRestore).toBeGreaterThanOrEqual(0); + expect(close.indexOf("type: 'close_abort_result'", closeRestore)) + .toBeGreaterThan(closeRestore); + }); + + it('retains close and shutdown generations across worker error and preserves close fence on exit', () => { + const errorStart = workerPoolSource.indexOf("worker.on('error', (err) => {"); + const errorEnd = workerPoolSource.indexOf("worker.stdout?.on('data'", errorStart); + const errorHandler = workerPoolSource.slice(errorStart, errorEnd); + expect(errorStart).toBeGreaterThanOrEqual(0); + expect(errorHandler).toContain('ds.riffShutdownState !== undefined'); + expect(errorHandler).toContain('|| ds.riffCloseState !== undefined'); + expect(errorHandler.indexOf('if (!retainExactRetirementGeneration)')) + .toBeLessThan(errorHandler.indexOf('ds.riffCloseState = undefined')); + + const exitStart = workerPoolSource.indexOf("worker.on('exit', (code, signal) => {"); + const exitEnd = workerPoolSource.indexOf('\n return worker;', exitStart); + const exitHandler = workerPoolSource.slice(exitStart, exitEnd); + expect(exitStart).toBeGreaterThanOrEqual(0); + expect(exitHandler).toContain("phase: 'uncertain'"); + expect(exitHandler).not.toContain('ds.riffCloseState = undefined'); + }); +}); From b6bc26ff1536588e7a624d6068d4c80ea4279dab Mon Sep 17 00:00:00 2001 From: Xiaoxue Sun <54162759+xiaoxueSunn@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:23:16 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(riff):=20=E8=A1=A5=E9=BD=90=E5=85=B3?= =?UTF-8?q?=E9=97=AD=E6=8F=90=E7=A4=BA=E5=B9=B6=E6=8B=92=E7=BB=9D=E6=97=A0?= =?UTF-8?q?=E6=95=88=E9=87=8D=E5=90=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/command-handler.ts | 7 ++++- src/core/dashboard-ipc-server.ts | 11 +++++++ src/core/persistent-backend.ts | 11 +++++-- src/core/worker-pool.ts | 28 +++++++++++++++++- src/dashboard/web/sessions-page.tsx | 2 +- src/dashboard/web/sessions.ts | 2 +- src/i18n/en.ts | 2 ++ src/i18n/zh.ts | 2 ++ src/im/lark/card-builder.ts | 3 +- src/im/lark/card-handler.ts | 17 ++++++++++- test/card-builder.test.ts | 8 ++++++ test/card-integration.test.ts | 25 ++++++++++++++++ test/command-handler.test.ts | 22 ++++++++++++++ test/crash-loop-diagnostic.test.ts | 43 +++++++++++++++++++++++++++- test/dashboard-ipc.test.ts | 24 ++++++++++++++++ test/dashboard-sessions-ui.test.ts | 1 + test/persistent-backend-type.test.ts | 13 +++++++++ test/riff-explicit-close.test.ts | 32 ++++++++++++++++++++- 18 files changed, 242 insertions(+), 11 deletions(-) diff --git a/src/core/command-handler.ts b/src/core/command-handler.ts index 4d8545875..2c20e40c3 100644 --- a/src/core/command-handler.ts +++ b/src/core/command-handler.ts @@ -14,7 +14,7 @@ import * as scheduler from './scheduler.js'; import { scanProjects, scanMultipleProjects, describeProjectDir } from '../services/project-scanner.js'; import { createRepoWorktree, pushWorktreeBranch } from '../services/git-worktree.js'; import { worktreeSlugFromContextAI } from '../services/worktree-slug-ai.js'; -import { resolvePairedSpawnBackendType } from './persistent-backend.js'; +import { isRiffBackendSession, resolvePairedSpawnBackendType } from './persistent-backend.js'; import { buildRepoSelectCard, buildAdoptSelectCard, buildCodexAppThreadSelectCard, buildSlashListCard, getCliDisplayName, buildConfigCard } from '../im/lark/card-builder.js'; import { handleDashboardCommand } from './dashboard-command/index.js'; import { createCliAdapterSync } from '../adapters/cli/registry.js'; @@ -1325,6 +1325,11 @@ export async function handleCommand( case '/restart': { if (ds) { + if (isRiffBackendSession(ds)) { + logger.info(`[${logTag}] Rejected /restart for Riff backend session`); + await sessionReply(rootId, t('cmd.restart.riff_unsupported', undefined, loc)); + break; + } if (ds.worker && !ds.worker.killed) { ds.worker.send({ type: 'restart' } as DaemonToWorker); const cliName = getCliDisplayName(getBot(ds.larkAppId).config.cliId); diff --git a/src/core/dashboard-ipc-server.ts b/src/core/dashboard-ipc-server.ts index 80df951e3..9bc82a734 100644 --- a/src/core/dashboard-ipc-server.ts +++ b/src/core/dashboard-ipc-server.ts @@ -137,6 +137,7 @@ import { updateSessionTitle } from './session-title.js'; import { requestAgentSessionRename } from './session-rename.js'; import type { DaemonToWorker, ScheduledTask, ParsedSchedule, ScheduleExecutionPosition, Session } from '../types.js'; import { sessionAnchorId, type DaemonSession } from './types.js'; +import { isRiffBackendSession } from './persistent-backend.js'; import { attachSkillPolicy, detachSkillPolicy } from './skills/im-command.js'; import { readSkillRegistry } from '../services/skill-registry-store.js'; import { @@ -539,6 +540,16 @@ ipcRoute('POST', '/api/sessions/:sessionId/restart', (_req, res, params) => { if (ds.adoptedFrom || ds.initConfig?.adoptMode) { return jsonRes(res, 409, { ok: false, error: 'adopt_restart_unsupported' }); } + // Riff owns a remote task lineage. Its worker deliberately refuses restart + // because destroy + respawn would sever or replace that lineage. Reject at + // the daemon boundary so the dashboard never reports HTTP 200 for a no-op. + if (isRiffBackendSession(ds)) { + return jsonRes(res, 409, { + ok: false, + error: 'riff_restart_unsupported', + message: t('cmd.restart.riff_unsupported', undefined, localeForBot(ds.larkAppId)), + }); + } const cliId = ds.session.cliId ?? 'unknown'; if (ds.worker && !ds.worker.killed) { // Live worker → in-place CLI restart (kills the CLI, respawns with --resume). diff --git a/src/core/persistent-backend.ts b/src/core/persistent-backend.ts index 88c886489..302f2dd31 100644 --- a/src/core/persistent-backend.ts +++ b/src/core/persistent-backend.ts @@ -110,6 +110,14 @@ export function resolvePairedSpawnBackendType( ); } +/** Whether the live/persisted session is frozen onto the remote Riff backend. + * Restart guards must use the session stamp rather than the bot's mutable + * current config: changing a bot later must not make an existing Riff + * generation look locally restartable. */ +export function isRiffBackendSession(ds: DaemonSession): boolean { + return (ds.initConfig?.backendType ?? ds.session.backendType) === 'riff'; +} + /** * How a session's worker is torn down at daemon shutdown, branched on the * session's FROZEN backend (via getSessionPersistentBackendType), NOT live config: @@ -123,8 +131,7 @@ export function shutdownBackendDisposition(ds: DaemonSession): 'riff-drain-detac // Riff 不能落进普通 detach 的直接 SIGTERM：create/follow-up 最长 10s 才返回 // task id，而 worker SIGTERM 会立即 exit，丢掉唯一血缘。独立 disposition 迫使 // daemon 先走 drain → durable ACK → commit 协议；类型检查防止未来回归。 - const frozen = ds.initConfig?.backendType ?? ds.session.backendType; - if (frozen === 'riff') return 'riff-drain-detach'; + if (isRiffBackendSession(ds)) return 'riff-drain-detach'; return getSessionPersistentBackendType(ds) ? 'detach' : 'close'; } diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 2629f2f02..c7655ab85 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -36,7 +36,7 @@ import { listDocSubscriptionsForSession, removeDocSubscription } from '../servic import { TmuxBackend } from '../adapters/backend/tmux-backend.js'; import { HerdrBackend } from '../adapters/backend/herdr-backend.js'; import { sandboxEnabled } from '../adapters/backend/sandbox.js'; -import { isSuspendableBackendType, getSessionPersistentBackendType, persistentBackendTargetForSession, killPersistentBackendTarget, managedTargetsForCliChange, resolvePairedSpawnBackendType, resolvePersistentBackendTarget } from './persistent-backend.js'; +import { isRiffBackendSession, isSuspendableBackendType, getSessionPersistentBackendType, persistentBackendTargetForSession, killPersistentBackendTarget, managedTargetsForCliChange, resolvePairedSpawnBackendType, resolvePersistentBackendTarget } from './persistent-backend.js'; import { getBot, getAllBots, loadBotConfigs, resolveBrandLabel } from '../bot-registry.js'; /** A random id minted once per daemon process (this lifetime). Stamped onto @@ -3820,6 +3820,32 @@ function setupWorkerHandlers( break; } + // Riff is a remote lineage-owning backend, not a local CLI process. + // The worker intentionally refuses restart because tearing it down can + // destroy or orphan the remote sandbox. Stop before crash-loop + // accounting and tell the user how to recover, rather than logging an + // "auto-restart" whose IPC is guaranteed to be a no-op. + if (isRiffBackendSession(ds)) { + const retirementPhase = riffRetirementAdmissionPhase(ds); + logger.warn( + `[${t}] Riff backend exited; automatic restart is unsupported` + + (retirementPhase ? ` (${retirementPhase})` : ''), + ); + // Explicit /close and daemon shutdown already own the user-visible + // lifecycle. Only an unexpected backend exit needs recovery guidance. + if (!retirementPhase && !suppressExitUi) { + try { + await scopedReply(tr('cmd.restart.riff_unsupported', undefined, loc), 'text', undefined); + } catch (replyErr) { + if (replyErr instanceof MessageWithdrawnError) { + logger.warn(`[${t}] Root message withdrawn, closing stale session`); + cb.closeSession(ds); + } + } + } + break; + } + // Rate-limit auto-restart to prevent crash loops const key = ds.session.sessionId; const rc = restartCounts.get(key) ?? { count: 0, lastAt: 0 }; diff --git a/src/dashboard/web/sessions-page.tsx b/src/dashboard/web/sessions-page.tsx index c3e68dbde..bd44d60ec 100644 --- a/src/dashboard/web/sessions-page.tsx +++ b/src/dashboard/web/sessions-page.tsx @@ -2588,7 +2588,7 @@ function SessionsPage(): JSX.Element { const r = await fetch(`/api/sessions/${encodeURIComponent(row.sessionId)}/restart`, { method: 'POST' }); const body = await r.json().catch(() => ({})); if (!r.ok || body?.ok === false) { - if (r.status !== 401) alert(`${t('sessions.restartFailed')}: ${body?.error ?? r.status}`); + if (r.status !== 401) alert(`${t('sessions.restartFailed')}: ${body?.message ?? body?.error ?? r.status}`); return false; } restartCooldownIds.current.add(row.sessionId); diff --git a/src/dashboard/web/sessions.ts b/src/dashboard/web/sessions.ts index 2a2a37f47..32b6e8e09 100644 --- a/src/dashboard/web/sessions.ts +++ b/src/dashboard/web/sessions.ts @@ -339,7 +339,7 @@ export function restartConfirmMessage(s: any): string { } export function canRestartSession(s: any): boolean { - return s.status !== 'closed' && !s.adopt && !s.pendingRepo; + return s.status !== 'closed' && !s.adopt && !s.pendingRepo && s.cliId !== 'riff'; } export interface PickerBot { larkAppId: string; botName: string; } diff --git a/src/i18n/en.ts b/src/i18n/en.ts index a5db8e1b3..fc427611f 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -228,6 +228,7 @@ export const messages: Record = { 'cmd.substitute.usage': 'Usage: @me /substitute status | on | off', 'cmd.restart.in_progress': '🔄 Restarting {cliName}…', 'cmd.restart.terminated': '{cliName} has been terminated; it will auto-resume on your next message.', + 'cmd.restart.riff_unsupported': '⚠️ Riff sessions cannot be restarted. Use /close to close the current remote session, then send a new message to create one.', 'cmd.cd.usage': 'Usage: /cd \nExample: /cd ~/projects/my-app', 'cmd.cd.switched': 'Working directory switched to {path}. It will resume there on your next message.', 'cmd.cd.created_switched': '📁 Directory did not exist — created it and switched to {path}. It will resume there on your next message.', @@ -709,6 +710,7 @@ export const messages: Record = { // ─── Worker → daemon notices ───────────────────────────────────────────── 'worker.adopted_session_exited': '⏏ Adopted CLI session has exited.', + 'worker.riff_close_in_progress': '⏳ The remote Riff session is closing. Wait for the close result before sending another message.', 'worker.crash_loop_stopped': '⚠️ {cliName} crashed {count} times in 1 minute. Auto-restart disabled. Send a message to retry.', 'worker.crash_diagnostic_terminal': 'The web terminal, where available, preserves the last startup output. Fix the issue, then send a new message to retry.', 'worker.crash_recent_output': 'Recent terminal output:', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 190e891c4..540fd90c9 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -231,6 +231,7 @@ export const messages: Record = { 'cmd.substitute.usage': '用法：@我 /substitute status｜on｜off', 'cmd.restart.in_progress': '🔄 正在重启 {cliName}...', 'cmd.restart.terminated': '{cliName} 进程已终止，下次发消息时将自动恢复。', + 'cmd.restart.riff_unsupported': '⚠️ Riff 会话不支持重启。请先用 /close 关闭当前远程会话，再发送新消息创建会话。', 'cmd.cd.usage': '用法：/cd \n例如：/cd ~/projects/my-app', 'cmd.cd.switched': '工作目录已切换到 {path}，下次发消息时将在新目录下恢复。', 'cmd.cd.created_switched': '📁 目录不存在，已自动创建并切换到 {path}，下次发消息时将在新目录下恢复。', @@ -712,6 +713,7 @@ export const messages: Record = { // ─── Worker → daemon notices ───────────────────────────────────────────── 'worker.adopted_session_exited': '⏏ /adopt的 CLI 会话已断开', + 'worker.riff_close_in_progress': '⏳ Riff 远程会话正在关闭，请等待关闭结果后再发送消息。', 'worker.crash_loop_stopped': '⚠️ {cliName} 在 1 分钟内崩溃 {count} 次，已停止自动重启。发消息可触发重新启动。', 'worker.crash_diagnostic_terminal': 'Web 终端（若可用）保留了最后一次启动输出，可打开查看；修复问题后发新消息会重新启动。', 'worker.crash_recent_output': '最近终端输出：', diff --git a/src/im/lark/card-builder.ts b/src/im/lark/card-builder.ts index 40c37d487..6b4a34175 100644 --- a/src/im/lark/card-builder.ts +++ b/src/im/lark/card-builder.ts @@ -314,7 +314,7 @@ export function buildSessionCard( value: { action: 'get_write_link', ...actionBase }, }); } - if (showManageButtons && !adoptMode) { + if (showManageButtons && !adoptMode && effectiveCliId !== 'riff') { actions.push({ tag: 'button', text: { tag: 'plain_text', content: t('card.btn.restart_cli', { cliName }, locale) }, @@ -1989,4 +1989,3 @@ export function buildCodexAppThreadSelectCard(threads: CodexAppThreadSummary[], }; return JSON.stringify(card); } - diff --git a/src/im/lark/card-handler.ts b/src/im/lark/card-handler.ts index 373b7d9e8..60c47dda8 100644 --- a/src/im/lark/card-handler.ts +++ b/src/im/lark/card-handler.ts @@ -70,7 +70,7 @@ import { buildTerminalUrl } from '../../core/terminal-url.js'; import type { ProjectInfo } from '../../services/project-scanner.js'; import { createRepoWorktree, removeRepoWorktree, dirSuffixForBranch, pushWorktreeBranch } from '../../services/git-worktree.js'; import { withCodexAppContext } from '../../utils/codex-app-context.js'; -import { resolvePairedSpawnBackendType } from '../../core/persistent-backend.js'; +import { isRiffBackendSession, resolvePairedSpawnBackendType } from '../../core/persistent-backend.js'; import { worktreeSlugFromContextAI } from '../../services/worktree-slug-ai.js'; import { t, localeForBot, isLocale, type Locale } from '../../i18n/index.js'; import { @@ -1616,6 +1616,21 @@ export async function handleCardAction(data: CardActionData, deps: CardHandlerDe await sessionReply(rootId, t('card.action.adopt_no_restart', undefined, locDs)); return; } + // New Riff cards omit this button, but old/stale cards remain clickable. + // Surface the same explicit close-and-recreate guidance as /restart + // instead of forwarding an IPC the Riff worker must silently refuse. + if (isRiffBackendSession(ds)) { + logger.warn(`[${tag(ds)}] Rejected restart on Riff backend session`); + const unsupported = t('cmd.restart.riff_unsupported', undefined, locDs); + await deliverEphemeralOrReply( + ds, + operatorOpenId, + unsupported, + 'text', + () => sessionReply(rootId, unsupported), + ); + return; + } const botCfg = getBot(ds.larkAppId).config; const effectiveCliId = sessionCliId(ds); if (ds.worker) { diff --git a/test/card-builder.test.ts b/test/card-builder.test.ts index 40bf6f544..78552aba1 100644 --- a/test/card-builder.test.ts +++ b/test/card-builder.test.ts @@ -399,6 +399,14 @@ describe('buildSessionCard', () => { expect(restartBtn.value.session_id).toBe(SID); }); + it('should omit restart for Riff management cards', () => { + const card = parse(buildSessionCard(SID, ROOT, URL, TITLE, 'riff', true)); + const actions = findActions(card); + + expect(actions.find((a: any) => a.value?.action === 'restart')).toBeUndefined(); + expect(actions.map((a: any) => a.value?.action ?? 'url')).toEqual(['url', 'close']); + }); + it('should NOT include "get write link" button', () => { const card = parse(buildSessionCard(SID, ROOT, URL, TITLE, undefined, true)); const actions = findActions(card); diff --git a/test/card-integration.test.ts b/test/card-integration.test.ts index f5bf3943d..44185c6ae 100644 --- a/test/card-integration.test.ts +++ b/test/card-integration.test.ts @@ -480,6 +480,31 @@ describe('Card integration: full event flow', () => { expect(deps.sessionReply).not.toHaveBeenCalled(); }); + it('restart from a stale Riff card should explain close-and-recreate without IPC', async () => { + const clientMod = await import('../src/im/lark/client.js'); + const workerSend = vi.fn(); + const ds = makeDaemonSession({ + worker: { killed: false, send: workerSend } as any, + }); + ds.session.cliId = 'riff'; + ds.session.backendType = 'riff'; + const sessions = new Map(); + sessions.set(sessionKey(ROOT_ID, APP_ID), ds); + const deps = makeDeps(sessions); + + await handleCardAction(makeRestartEvent(ROOT_ID), deps, APP_ID); + + expect(workerSend).not.toHaveBeenCalled(); + expect(forkWorker).not.toHaveBeenCalled(); + expect(vi.mocked(clientMod.sendEphemeralCard)).toHaveBeenCalledWith( + APP_ID, + ds.chatId, + 'ou_user', + expect.stringMatching(/Riff.*不支持重启.*\/close/), + ); + expect(deps.sessionReply).not.toHaveBeenCalled(); + }); + it('restart without worker should re-fork', async () => { const ds = makeDaemonSession({ worker: null }); const sessions = new Map(); diff --git a/test/command-handler.test.ts b/test/command-handler.test.ts index 050e1c6e8..81f2bf920 100644 --- a/test/command-handler.test.ts +++ b/test/command-handler.test.ts @@ -1468,6 +1468,28 @@ describe('handleCommand', () => { // ─── /restart ─────────────────────────────────────────────────────────── describe('/restart', () => { + it('should reject Riff sessions with close-and-recreate guidance', async () => { + const workerSend = vi.fn(); + const ds = makeDaemonSession({ + worker: { killed: false, send: workerSend } as any, + }); + ds.session.cliId = 'riff'; + ds.session.backendType = 'riff'; + const deps = makeDeps(ds); + + await handleCommand('/restart', ROOT_ID, makeLarkMessage('/restart'), deps, LARK_APP_ID); + + expect(workerSend).not.toHaveBeenCalled(); + expect(killWorker).not.toHaveBeenCalled(); + expect(deps.sessionReply).toHaveBeenCalledWith( + ROOT_ID, + expect.stringMatching(/Riff.*不支持重启.*\/close/), + undefined, + LARK_APP_ID, + 'msg_001', + ); + }); + it('should send restart IPC when worker is alive', async () => { const workerSend = vi.fn(); const ds = makeDaemonSession({ diff --git a/test/crash-loop-diagnostic.test.ts b/test/crash-loop-diagnostic.test.ts index d79c97787..21776bc0d 100644 --- a/test/crash-loop-diagnostic.test.ts +++ b/test/crash-loop-diagnostic.test.ts @@ -53,6 +53,7 @@ vi.mock('../src/config.js', () => ({ })); const updateSessionMock = vi.fn(); +const sessionReplyMock = vi.fn(async () => 'om_reply'); vi.mock('../src/services/session-store.js', () => ({ closeSession: vi.fn(), updateSession: (...args: any[]) => updateSessionMock(...args), @@ -164,13 +165,53 @@ describe("crash-loop diagnostic terminal (daemon 'claude_exit' handler)", () => vi.clearAllMocks(); restartCounts.clear(); initWorkerPool({ - sessionReply: vi.fn(async () => 'om_reply'), + sessionReply: sessionReplyMock, getSessionWorkingDir: () => '/tmp', getActiveCount: () => 1, closeSession: vi.fn(), } as any); }); + it('does not auto-restart a crashed Riff worker and tells the user how to recover', async () => { + const worker = makeFakeWorker(); + const ds = makeDs('sid-riff-no-restart', worker); + ds.session.cliId = 'riff'; + ds.session.backendType = 'riff'; + __testOnly_setupWorkerHandlers(ds, worker); + + await crashTimes(worker, 1); + + expect(worker.send).not.toHaveBeenCalledWith({ type: 'restart' }); + expect(restartCounts.has('sid-riff-no-restart')).toBe(false); + expect(sessionReplyMock).toHaveBeenCalledWith( + 'om_root', + expect.stringMatching(/Riff.*不支持重启.*\/close/), + 'text', + 'app_test', + undefined, + undefined, + ); + }); + + it('does not duplicate recovery guidance while an explicit Riff close owns the exit', async () => { + const worker = makeFakeWorker(); + const ds = makeDs('sid-riff-closing', worker); + ds.session.cliId = 'riff'; + ds.session.backendType = 'riff'; + ds.riffCloseState = { + phase: 'preparing', + requestId: 'close-riff', + taskId: 'task-riff', + }; + __testOnly_setupWorkerHandlers(ds, worker); + + await crashTimes(worker, 1); + + expect(worker.send).not.toHaveBeenCalledWith({ type: 'restart' }); + expect(restartCounts.has('sid-riff-closing')).toBe(false); + expect(sessionReplyMock).not.toHaveBeenCalled(); + }); + it('after >3 tmux crashes: keeps the worker + marks lazy cold-resume (survives restart)', async () => { const worker = makeFakeWorker(); const ds = makeDs('sid-diag-keep', worker); diff --git a/test/dashboard-ipc.test.ts b/test/dashboard-ipc.test.ts index afece1f41..5c6c4d42b 100644 --- a/test/dashboard-ipc.test.ts +++ b/test/dashboard-ipc.test.ts @@ -704,6 +704,30 @@ describe('POST /api/sessions/:sessionId/restart', () => { forkSpy.mockRestore(); }); + it('rejects Riff sessions with close-and-recreate guidance', async () => { + const send = vi.fn(); + const forkSpy = vi.spyOn(workerPool, 'forkWorker').mockImplementation(() => {}); + const findSpy = vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue({ + session: { sessionId: 's-riff', cliId: 'riff', backendType: 'riff' }, + worker: { send, killed: false }, + adoptedFrom: undefined, + } as any); + + handle = await startIpcServer({ port: 0, host: '127.0.0.1' }); + const res = await fetch(`http://127.0.0.1:${handle.port}/api/sessions/s-riff/restart`, { method: 'POST' }); + + expect(res.status).toBe(409); + expect(await res.json()).toMatchObject({ + ok: false, + error: 'riff_restart_unsupported', + message: expect.stringMatching(/Riff.*不支持重启.*\/close/), + }); + expect(send).not.toHaveBeenCalled(); + expect(forkSpy).not.toHaveBeenCalled(); + findSpy.mockRestore(); + forkSpy.mockRestore(); + }); + it('revives a worker-less but active session by re-forking (matches the Feishu card path)', async () => { const forkSpy = vi.spyOn(workerPool, 'forkWorker').mockImplementation(() => {}); const findSpy = vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue({ diff --git a/test/dashboard-sessions-ui.test.ts b/test/dashboard-sessions-ui.test.ts index 1cb3974b3..51c608c35 100644 --- a/test/dashboard-sessions-ui.test.ts +++ b/test/dashboard-sessions-ui.test.ts @@ -294,6 +294,7 @@ describe('dashboard sessions filters', () => { expect(canRestartSession({ status: 'closed', adopt: false })).toBe(false); expect(canRestartSession({ status: 'idle', adopt: true })).toBe(false); expect(canRestartSession({ status: 'starting', pendingRepo: true })).toBe(false); + expect(canRestartSession({ status: 'idle', adopt: false, cliId: 'riff' })).toBe(false); }); it('formats session location labels for group chats and direct chats', () => { diff --git a/test/persistent-backend-type.test.ts b/test/persistent-backend-type.test.ts index 065b95b43..0b62db95e 100644 --- a/test/persistent-backend-type.test.ts +++ b/test/persistent-backend-type.test.ts @@ -20,6 +20,7 @@ vi.mock('../src/bot-registry.js', () => ({ import { getSessionPersistentBackendType, + isRiffBackendSession, killPersistentBackendTarget, managedTargetsForCliChange, probePersistentBackendTarget, @@ -167,3 +168,15 @@ describe('shutdownBackendDisposition (shutdown freeze-once)', () => { expect(shutdownBackendDisposition(ds({ sessionBackend: 'riff' }))).toBe('riff-drain-detach'); }); }); + +describe('isRiffBackendSession (restart guard freeze-once)', () => { + it('prefers the live worker stamp over a stale persisted backend', () => { + expect(isRiffBackendSession(ds({ initBackend: 'riff', sessionBackend: 'tmux' }))).toBe(true); + expect(isRiffBackendSession(ds({ initBackend: 'tmux', sessionBackend: 'riff' }))).toBe(false); + }); + + it('falls back to the persisted backend for a restored worker', () => { + expect(isRiffBackendSession(ds({ sessionBackend: 'riff' }))).toBe(true); + expect(isRiffBackendSession(ds({ sessionBackend: 'tmux' }))).toBe(false); + }); +}); diff --git a/test/riff-explicit-close.test.ts b/test/riff-explicit-close.test.ts index b65957b4d..4c9a2a27d 100644 --- a/test/riff-explicit-close.test.ts +++ b/test/riff-explicit-close.test.ts @@ -50,12 +50,14 @@ import { closeSession, initWorkerPool, killWorker, + sendWorkerInput, setActiveSessionsRegistry, } from '../src/core/worker-pool.js'; import * as sessionStore from '../src/services/session-store.js'; let dataDir: string; let previousDataDir: string; +const sessionReplyMock = vi.fn(async () => 'om_reply'); function createFixture(options: { liveWorker?: boolean; @@ -122,7 +124,7 @@ beforeEach(() => { }); cancelRiffTaskMock.mockResolvedValue(true); initWorkerPool({ - sessionReply: vi.fn(async () => 'om_reply'), + sessionReply: sessionReplyMock, getSessionWorkingDir: () => '/repo', getActiveCount: () => 1, closeSession: vi.fn(), @@ -247,6 +249,34 @@ describe('Riff explicit close', () => { }); }); + it('shows a localized close-in-progress notice instead of leaking the i18n key', async () => { + const fixture = createFixture({ liveWorker: true }); + fixture.ds.riffCloseState = { + phase: 'preparing', + requestId: 'close-riff', + taskId: 'task-riff-123', + }; + + expect(sendWorkerInput(fixture.ds, 'late turn', 'om_late_turn')).toBe(false); + await new Promise(resolve => setImmediate(resolve)); + + expect(fixture.worker.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'input' })); + expect(sessionReplyMock).toHaveBeenCalledWith( + 'oc_riff', + expect.stringMatching(/Riff.*正在关闭/), + 'text', + 'app', + 'om_late_turn', + ); + expect(sessionReplyMock).not.toHaveBeenCalledWith( + expect.anything(), + 'worker.riff_close_in_progress', + expect.anything(), + expect.anything(), + expect.anything(), + ); + }); + it('keeps the newest runtime lineage when its durable save fails', async () => { const fixture = createFixture({ liveWorker: true }); vi.spyOn(sessionStore, 'updateSession').mockImplementationOnce(() => {