From e335ba3ca5d8974bd79660e2af4ef6c072376ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=86=E9=80=8A?= <211125649+UncertaintyDeterminesYou4ndMe@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:32:27 +0800 Subject: [PATCH] test(cli): run ordinary run-command semantics in process Part of #2387. The maka run test suite paid a Node subprocess startup per assertion, including for scenarios that only exercise argument handling, session selection, and outcome-to-exit-code mapping through the injectable MakaRunDeps seam. - Extract the scenario fake from run-command-fixture.ts into run-command-fake.ts, parameterized by an options object instead of environment variables; the fixture becomes a thin subprocess wrapper over the same fake, so the two routes cannot drift apart. - Move 21 ordinary-semantics tests in process through runMakaTextCli with injected stdin/stdout/stderr; assertions are unchanged. - Retain real-subprocess coverage for every distinct process contract: piped non-TTY stdin (3), SIGINT delivery with exit 130 (2), and the fail-closed sandbox boundary path plus exit codes and stdout observed through a real process boundary (1). - The graph-runtime-error scenario's "graph wait must not run" marker now travels through the thrown error message so the negative assertion stays observable on the captured stderr channel. Timing (node --test dist/__tests__/run-command.test.js, local): before 15.06s, after 3.60s. 34/34 tests pass, 5 consecutive rounds. --- .../cli/src/__tests__/run-command-fake.ts | 400 +++++++++++++++++ .../cli/src/__tests__/run-command-fixture.ts | 422 +----------------- .../cli/src/__tests__/run-command.test.ts | 245 +++++----- 3 files changed, 528 insertions(+), 539 deletions(-) create mode 100644 packages/cli/src/__tests__/run-command-fake.ts diff --git a/packages/cli/src/__tests__/run-command-fake.ts b/packages/cli/src/__tests__/run-command-fake.ts new file mode 100644 index 0000000000..0556d711ec --- /dev/null +++ b/packages/cli/src/__tests__/run-command-fake.ts @@ -0,0 +1,400 @@ +import type { SessionEvent } from '@maka/core/events'; +import { createWorkspaceWritePermissionProfile } from '@maka/core/permission-profile'; +import type { SessionSummary } from '@maka/core/session'; +import type { InvocationResult } from '@maka/runtime'; +import type { MakaRunAdapter } from '../run-command-core.js'; +import type { MakaRunContext, MakaRunContextInput, MakaRunRuntime } from '../run-command.js'; +import { + invocationHasSandboxBoundaryFailure, + invocationRecoveredSandboxBoundaryFailure, +} from '../sandbox-boundary-failure.js'; +import type { ReadySessionTarget } from '../connection-target.js'; + +export interface RunCommandFakeOptions { + scenario?: string; + sessions?: SessionSummary[]; + expectNoCreate?: boolean; + expectPermissionMode?: string; + expectNoSend?: boolean; + expectSessionId?: string; + boundaryKind?: string; + expectBoundaryKind?: string; + expectGraph?: boolean; + graphBoundaryFailure?: boolean; + expectMaxSteps?: number; + expectContextCwd?: string; + expectContextConnection?: string; + expectContextModel?: string; + expectCwdOverride?: string; + onReady?: () => void; +} + +export function createRunCommandFake(options: RunCommandFakeOptions = {}): MakaRunAdapter { + const scenario = options.scenario ?? 'completed'; + let observer: MakaRunContextInput['runOutcomeObserver']; + let permissionDenied = false; + let releaseStop: (() => void) | undefined; + let releaseGraphWait: (() => void) | undefined; + let graphActivityReleased = false; + + const target = { + connection: { + slug: 'fixture', + name: 'Fixture', + providerType: 'ollama', + enabled: true, + defaultModel: 'fixture-model', + }, + apiKey: '', + model: 'fixture-model', + } as ReadySessionTarget; + + const summary = { + id: 'session-fixture', + cwd: process.cwd(), + name: 'fixture', + isFlagged: false, + isArchived: false, + labels: [], + hasUnread: false, + status: 'active', + backend: 'ai-sdk', + llmConnectionSlug: 'fixture', + connectionLocked: false, + model: 'fixture-model', + permissionMode: 'explore', + } satisfies SessionSummary; + + async function notify(result: InvocationResult): Promise { + await observer?.({ + outcomeId: result.invocationId, + status: result.status === 'completed' ? 'completed' : 'failed', + ...(result.finalOutput !== undefined ? { finalOutput: result.finalOutput } : {}), + ...(result.failure ? { failure: result.failure } : {}), + sandboxBoundary: invocationRecoveredSandboxBoundaryFailure(result) + ? 'recovered' + : invocationHasSandboxBoundaryFailure(result) + ? 'unresolved' + : 'none', + }); + } + + function completedResult(finalOutput: string): InvocationResult { + return { + invocationId: 'invocation-fixture', + runId: 'run-fixture', + sessionId: summary.id, + turnId: 'turn-fixture', + status: 'completed', + finalOutput, + events: [], + startedAt: 1, + finishedAt: 2, + }; + } + + function failedResult(failureClass: string, message: string): InvocationResult { + return { + invocationId: 'invocation-fixture', + runId: 'run-fixture', + sessionId: summary.id, + turnId: 'turn-fixture', + status: 'failed', + events: [], + failure: { class: failureClass, message }, + startedAt: 1, + finishedAt: 2, + }; + } + + function functionResponseEvent( + toolUseId: string, + isError: boolean, + result: unknown, + ): InvocationResult['events'][number] { + return { + id: `event-${toolUseId}`, + invocationId: 'invocation-fixture', + runId: 'run-fixture', + sessionId: summary.id, + turnId: 'turn-fixture', + ts: 1, + partial: false, + role: 'tool', + author: 'tool', + content: { + kind: 'function_response', + id: toolUseId, + name: 'Bash', + result, + isError, + }, + }; + } + + const runtime: MakaRunRuntime = { + async createSession(input) { + if (options.expectNoCreate) throw new Error('unexpected createSession call'); + if (options.expectPermissionMode && input.permissionMode !== options.expectPermissionMode) { + throw new Error(`unexpected permissionMode ${input.permissionMode}`); + } + return summary; + }, + async readExecutionBoundary() { + const kind = options.boundaryKind ?? 'managed'; + return kind === 'managed' + ? { + kind, + profile: createWorkspaceWritePermissionProfile(), + revision: 0, + } + : { kind: kind as 'bypass' | 'external', revision: 0 }; + }, + async setExecutionBoundaryKind(_sessionId, kind) { + if (options.expectBoundaryKind && kind !== options.expectBoundaryKind) { + throw new Error(`unexpected boundary kind ${kind}`); + } + }, + async *sendMessage(sessionId, input): AsyncIterable { + if (options.expectNoSend) throw new Error('unexpected sendMessage call'); + if (options.expectSessionId && sessionId !== options.expectSessionId) { + throw new Error(`unexpected sessionId ${sessionId}`); + } + if (scenario === 'runtime-error') throw new Error('provider failed after startup'); + if (scenario === 'graph-runtime-error') { + if (input.turnOrchestration?.mode !== 'graph') { + throw new Error('expected graph orchestration'); + } + await notify(failedResult('provider_unavailable', 'provider failed before graph creation')); + return; + } + if (scenario === 'graph-wait') { + if (input.turnOrchestration?.mode !== 'graph') { + throw new Error('expected graph orchestration'); + } + await notify(completedResult('initial graph supervisor output')); + return; + } + if (options.expectGraph) { + if ( + input.turnOrchestration?.mode !== 'graph' || + input.turnOrchestration.source !== 'host_api' + ) { + throw new Error( + `unexpected graph orchestration ${JSON.stringify(input.turnOrchestration)}`, + ); + } + await notify(completedResult('initial graph supervisor output')); + return; + } + if (scenario === 'sandbox-boundary') { + yield { + type: 'sandbox_boundary_request', + id: 'event-boundary', + turnId: input.turnId, + ts: 1, + requestId: 'boundary-1', + toolUseId: 'tool-boundary', + justification: 'Read an external file.', + expansion: { + filesystem: { + entries: [{ path: '/outside/file.txt', access: 'read', scope: 'exact' }], + }, + }, + }; + if (!permissionDenied) throw new Error('sandbox boundary request was not denied'); + return; + } + if (scenario === 'sandbox-boundary-tool-result') { + yield { + type: 'tool_result', + id: 'event-boundary-result', + turnId: input.turnId, + ts: 1, + toolUseId: 'tool-boundary', + isError: true, + content: { + kind: 'text', + text: 'Bash requires an approved session sandbox boundary expansion.', + sandboxFailure: { + reason: 'sandbox_boundary_required', + requiredExpansion: { network: { enabled: true } }, + }, + }, + } as unknown as SessionEvent; + await notify(completedResult('should not be emitted')); + return; + } + if (scenario === 'sandbox-boundary-recovered') { + yield { + type: 'tool_result', + id: 'event-boundary-result', + turnId: input.turnId, + ts: 1, + toolUseId: 'tool-boundary', + isError: true, + content: { + kind: 'text', + text: 'Bash requires an approved session sandbox boundary expansion.', + sandboxFailure: { + reason: 'sandbox_boundary_required', + requiredExpansion: { network: { enabled: true } }, + }, + }, + } as unknown as SessionEvent; + yield { + type: 'tool_result', + id: 'event-safe-result', + turnId: input.turnId, + ts: 2, + toolUseId: 'tool-safe', + isError: false, + content: { kind: 'text', text: 'completed within the current boundary' }, + }; + await notify({ + ...completedResult('recovered safely'), + events: [ + functionResponseEvent('tool-boundary', true, { + sandboxFailure: { reason: 'sandbox_boundary_required' }, + }), + functionResponseEvent('tool-safe', false, 'completed within the current boundary'), + ], + }); + return; + } + if (scenario === 'slow') { + options.onReady?.(); + const keepAlive = setInterval(() => {}, 1_000); + await new Promise((resolve) => { + releaseStop = resolve; + }); + clearInterval(keepAlive); + await notify(failedResult('aborted', 'fixture stopped')); + return; + } + if (scenario === 'missing-output') { + await notify( + failedResult('missing_final_output', 'completed invocation produced no final output'), + ); + return; + } + if (scenario === 'step-limit') { + await notify( + failedResult('step_limit', 'explicit tool-step limit reached; send continue to resume'), + ); + return; + } + const output = + options.expectMaxSteps === undefined + ? `prompt=${input.text}` + : `maxSteps=${options.expectMaxSteps};prompt=${input.text}`; + await notify(completedResult(output)); + }, + async respondToSandboxBoundary(_sessionId, response) { + permissionDenied = response.decision === 'deny' && response.requestId === 'boundary-1'; + }, + async stopSession() { + releaseStop?.(); + }, + }; + + async function createContext(input: MakaRunContextInput): Promise { + if (scenario === 'config-error') throw new Error('unknown connection fixture-missing'); + if (options.expectMaxSteps !== undefined && input.maxSteps !== options.expectMaxSteps) { + throw new Error(`unexpected maxSteps ${String(input.maxSteps)}`); + } + if (options.expectContextCwd && input.cwd !== options.expectContextCwd) { + throw new Error(`unexpected context cwd ${input.cwd}`); + } + if ( + options.expectContextConnection && + input.requestedConnectionSlug !== options.expectContextConnection + ) { + throw new Error(`unexpected context connection ${String(input.requestedConnectionSlug)}`); + } + if (options.expectContextModel && input.requestedModel !== options.expectContextModel) { + throw new Error(`unexpected context model ${String(input.requestedModel)}`); + } + if (options.expectCwdOverride) { + const actual = JSON.stringify(input.sessionCwdOverride); + if (actual !== options.expectCwdOverride) { + throw new Error(`unexpected sessionCwdOverride ${actual}`); + } + } + observer = input.runOutcomeObserver; + if (options.expectGraph || scenario === 'graph-runtime-error' || scenario === 'graph-wait') { + if (!input.enableAgentGraph) throw new Error('Graph host was not enabled'); + return { + runtime, + target, + agentGraph: { + reserveActivity: () => ({ + release: () => { + graphActivityReleased = true; + }, + }), + waitForCompletion: async () => { + if (!graphActivityReleased) throw new Error('Graph activity was not released'); + if (scenario === 'graph-runtime-error') { + // Reaching here is the bug this scenario guards against; the + // thrown message surfaces on the captured stderr channel. + throw new Error('graph-wait-called: unexpected graph wait after failed invocation'); + } + if (scenario === 'graph-wait') { + options.onReady?.(); + const keepAlive = setInterval(() => {}, 1_000); + await new Promise((resolve) => { + releaseGraphWait = resolve; + }); + clearInterval(keepAlive); + return; + } + if (options.graphBoundaryFailure) { + await notify({ + ...completedResult('child could not complete'), + invocationId: 'invocation-child', + runId: 'run-child', + sessionId: 'session-child', + events: [ + { + id: 'event-child-tool', + invocationId: 'invocation-child', + runId: 'run-child', + sessionId: 'session-child', + turnId: 'turn-child', + ts: 1, + partial: false, + role: 'tool', + author: 'tool', + content: { + kind: 'function_response', + id: 'tool-child', + name: 'Write', + isError: true, + result: { + kind: 'text', + text: 'boundary required', + sandboxFailure: { reason: 'sandbox_boundary_required' }, + }, + }, + }, + ], + }); + } + await notify(completedResult('graph completed')); + }, + }, + close: async () => { + releaseGraphWait?.(); + }, + }; + } + return { runtime, target, close: async () => {} }; + } + + async function listSessions(): Promise { + return options.sessions ?? []; + } + + return { createContext, listSessions }; +} diff --git a/packages/cli/src/__tests__/run-command-fixture.ts b/packages/cli/src/__tests__/run-command-fixture.ts index 2407469685..660072e3f2 100644 --- a/packages/cli/src/__tests__/run-command-fixture.ts +++ b/packages/cli/src/__tests__/run-command-fixture.ts @@ -1,409 +1,19 @@ -import type { SessionEvent } from '@maka/core/events'; -import { createWorkspaceWritePermissionProfile } from '@maka/core/permission-profile'; -import type { SessionSummary } from '@maka/core/session'; -import type { InvocationResult } from '@maka/runtime'; -import { - runMakaTextCli, - type MakaRunContext, - type MakaRunContextInput, - type MakaRunRuntime, -} from '../run-command.js'; -import { - invocationHasSandboxBoundaryFailure, - invocationRecoveredSandboxBoundaryFailure, -} from '../sandbox-boundary-failure.js'; -import type { ReadySessionTarget } from '../connection-target.js'; - -const scenario = process.env.MAKA_RUN_FIXTURE_SCENARIO ?? 'completed'; -let observer: MakaRunContextInput['runOutcomeObserver']; -let permissionDenied = false; -let releaseStop: (() => void) | undefined; -let releaseGraphWait: (() => void) | undefined; -let graphActivityReleased = false; - -const target = { - connection: { - slug: 'fixture', - name: 'Fixture', - providerType: 'ollama', - enabled: true, - defaultModel: 'fixture-model', - }, - apiKey: '', - model: 'fixture-model', -} as ReadySessionTarget; - -const summary = { - id: 'session-fixture', - cwd: process.cwd(), - name: 'fixture', - isFlagged: false, - isArchived: false, - labels: [], - hasUnread: false, - status: 'active', - backend: 'ai-sdk', - llmConnectionSlug: 'fixture', - connectionLocked: false, - model: 'fixture-model', - permissionMode: 'explore', -} satisfies SessionSummary; - -const runtime: MakaRunRuntime = { - async createSession(input) { - if (process.env.MAKA_RUN_EXPECT_NO_CREATE === '1') { - throw new Error('unexpected createSession call'); - } - if ( - process.env.MAKA_RUN_EXPECT_PERMISSION_MODE && - input.permissionMode !== process.env.MAKA_RUN_EXPECT_PERMISSION_MODE - ) { - throw new Error(`unexpected permissionMode ${input.permissionMode}`); - } - return summary; - }, - async readExecutionBoundary() { - const kind = process.env.MAKA_RUN_BOUNDARY_KIND ?? 'managed'; - return kind === 'managed' - ? { - kind, - profile: createWorkspaceWritePermissionProfile(), - revision: 0, - } - : { kind: kind as 'bypass' | 'external', revision: 0 }; - }, - async setExecutionBoundaryKind(_sessionId, kind) { - if ( - process.env.MAKA_RUN_EXPECT_BOUNDARY_KIND && - kind !== process.env.MAKA_RUN_EXPECT_BOUNDARY_KIND - ) { - throw new Error(`unexpected boundary kind ${kind}`); - } - }, - async *sendMessage(sessionId, input): AsyncIterable { - if (process.env.MAKA_RUN_EXPECT_NO_SEND === '1') { - throw new Error('unexpected sendMessage call'); - } - if ( - process.env.MAKA_RUN_EXPECT_SESSION_ID && - sessionId !== process.env.MAKA_RUN_EXPECT_SESSION_ID - ) { - throw new Error(`unexpected sessionId ${sessionId}`); - } - if (scenario === 'runtime-error') throw new Error('provider failed after startup'); - if (scenario === 'graph-runtime-error') { - if (input.turnOrchestration?.mode !== 'graph') { - throw new Error('expected graph orchestration'); - } - await notify(failedResult('provider_unavailable', 'provider failed before graph creation')); - return; - } - if (scenario === 'graph-wait') { - if (input.turnOrchestration?.mode !== 'graph') { - throw new Error('expected graph orchestration'); - } - await notify(completedResult('initial graph supervisor output')); - return; - } - if (process.env.MAKA_RUN_EXPECT_GRAPH === '1') { - if ( - input.turnOrchestration?.mode !== 'graph' || - input.turnOrchestration.source !== 'host_api' - ) { - throw new Error( - `unexpected graph orchestration ${JSON.stringify(input.turnOrchestration)}`, - ); - } - await notify(completedResult('initial graph supervisor output')); - return; - } - if (scenario === 'sandbox-boundary') { - yield { - type: 'sandbox_boundary_request', - id: 'event-boundary', - turnId: input.turnId, - ts: 1, - requestId: 'boundary-1', - toolUseId: 'tool-boundary', - justification: 'Read an external file.', - expansion: { - filesystem: { - entries: [{ path: '/outside/file.txt', access: 'read', scope: 'exact' }], - }, - }, - }; - if (!permissionDenied) throw new Error('sandbox boundary request was not denied'); - return; - } - if (scenario === 'sandbox-boundary-tool-result') { - yield { - type: 'tool_result', - id: 'event-boundary-result', - turnId: input.turnId, - ts: 1, - toolUseId: 'tool-boundary', - isError: true, - content: { - kind: 'text', - text: 'Bash requires an approved session sandbox boundary expansion.', - sandboxFailure: { - reason: 'sandbox_boundary_required', - requiredExpansion: { network: { enabled: true } }, - }, - }, - } as unknown as SessionEvent; - await notify(completedResult('should not be emitted')); - return; - } - if (scenario === 'sandbox-boundary-recovered') { - yield { - type: 'tool_result', - id: 'event-boundary-result', - turnId: input.turnId, - ts: 1, - toolUseId: 'tool-boundary', - isError: true, - content: { - kind: 'text', - text: 'Bash requires an approved session sandbox boundary expansion.', - sandboxFailure: { - reason: 'sandbox_boundary_required', - requiredExpansion: { network: { enabled: true } }, - }, - }, - } as unknown as SessionEvent; - yield { - type: 'tool_result', - id: 'event-safe-result', - turnId: input.turnId, - ts: 2, - toolUseId: 'tool-safe', - isError: false, - content: { kind: 'text', text: 'completed within the current boundary' }, - }; - await notify({ - ...completedResult('recovered safely'), - events: [ - functionResponseEvent('tool-boundary', true, { - sandboxFailure: { reason: 'sandbox_boundary_required' }, - }), - functionResponseEvent('tool-safe', false, 'completed within the current boundary'), - ], - }); - return; - } - if (scenario === 'slow') { - process.stderr.write('fixture-ready\n'); - const keepAlive = setInterval(() => {}, 1_000); - await new Promise((resolve) => { - releaseStop = resolve; - }); - clearInterval(keepAlive); - await notify(failedResult('aborted', 'fixture stopped')); - return; - } - if (scenario === 'missing-output') { - await notify( - failedResult('missing_final_output', 'completed invocation produced no final output'), - ); - return; - } - if (scenario === 'step-limit') { - await notify( - failedResult('step_limit', 'explicit tool-step limit reached; send continue to resume'), - ); - return; - } - const maxSteps = process.env.MAKA_RUN_EXPECT_MAX_STEPS; - const output = maxSteps ? `maxSteps=${maxSteps};prompt=${input.text}` : `prompt=${input.text}`; - await notify(completedResult(output)); - }, - async respondToSandboxBoundary(_sessionId, response) { - permissionDenied = response.decision === 'deny' && response.requestId === 'boundary-1'; - }, - async stopSession() { - releaseStop?.(); - }, -}; - -async function createContext(input: MakaRunContextInput): Promise { - if (scenario === 'config-error') throw new Error('unknown connection fixture-missing'); - if ( - process.env.MAKA_RUN_EXPECT_MAX_STEPS && - input.maxSteps !== Number(process.env.MAKA_RUN_EXPECT_MAX_STEPS) - ) { - throw new Error(`unexpected maxSteps ${String(input.maxSteps)}`); - } - if ( - process.env.MAKA_RUN_EXPECT_CONTEXT_CWD && - input.cwd !== process.env.MAKA_RUN_EXPECT_CONTEXT_CWD - ) { - throw new Error(`unexpected context cwd ${input.cwd}`); - } - if ( - process.env.MAKA_RUN_EXPECT_CONTEXT_CONNECTION && - input.requestedConnectionSlug !== process.env.MAKA_RUN_EXPECT_CONTEXT_CONNECTION - ) { - throw new Error(`unexpected context connection ${String(input.requestedConnectionSlug)}`); - } - if ( - process.env.MAKA_RUN_EXPECT_CONTEXT_MODEL && - input.requestedModel !== process.env.MAKA_RUN_EXPECT_CONTEXT_MODEL - ) { - throw new Error(`unexpected context model ${String(input.requestedModel)}`); - } - if (process.env.MAKA_RUN_EXPECT_CWD_OVERRIDE) { - const actual = JSON.stringify(input.sessionCwdOverride); - if (actual !== process.env.MAKA_RUN_EXPECT_CWD_OVERRIDE) { - throw new Error(`unexpected sessionCwdOverride ${actual}`); - } - } - observer = input.runOutcomeObserver; - if ( - process.env.MAKA_RUN_EXPECT_GRAPH === '1' || - scenario === 'graph-runtime-error' || - scenario === 'graph-wait' - ) { - if (!input.enableAgentGraph) throw new Error('Graph host was not enabled'); - return { - runtime, - target, - agentGraph: { - reserveActivity: () => ({ - release: () => { - graphActivityReleased = true; - }, - }), - waitForCompletion: async () => { - if (!graphActivityReleased) throw new Error('Graph activity was not released'); - if (scenario === 'graph-runtime-error') { - process.stderr.write('graph-wait-called\n'); - throw new Error('unexpected graph wait after failed invocation'); - } - if (scenario === 'graph-wait') { - process.stderr.write('fixture-ready\n'); - const keepAlive = setInterval(() => {}, 1_000); - await new Promise((resolve) => { - releaseGraphWait = resolve; - }); - clearInterval(keepAlive); - return; - } - if (process.env.MAKA_RUN_GRAPH_BOUNDARY_FAILURE === '1') { - await notify({ - ...completedResult('child could not complete'), - invocationId: 'invocation-child', - runId: 'run-child', - sessionId: 'session-child', - events: [ - { - id: 'event-child-tool', - invocationId: 'invocation-child', - runId: 'run-child', - sessionId: 'session-child', - turnId: 'turn-child', - ts: 1, - partial: false, - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: 'tool-child', - name: 'Write', - isError: true, - result: { - kind: 'text', - text: 'boundary required', - sandboxFailure: { reason: 'sandbox_boundary_required' }, - }, - }, - }, - ], - }); - } - await notify(completedResult('graph completed')); - }, - }, - close: async () => { - releaseGraphWait?.(); - }, - }; - } - return { runtime, target, close: async () => {} }; -} - -async function listSessions(): Promise { - return JSON.parse(process.env.MAKA_RUN_FIXTURE_SESSIONS ?? '[]') as SessionSummary[]; -} - -function completedResult(finalOutput: string): InvocationResult { - return { - invocationId: 'invocation-fixture', - runId: 'run-fixture', - sessionId: summary.id, - turnId: 'turn-fixture', - status: 'completed', - finalOutput, - events: [], - startedAt: 1, - finishedAt: 2, - }; -} - -function failedResult(failureClass: string, message: string): InvocationResult { - return { - invocationId: 'invocation-fixture', - runId: 'run-fixture', - sessionId: summary.id, - turnId: 'turn-fixture', - status: 'failed', - events: [], - failure: { class: failureClass, message }, - startedAt: 1, - finishedAt: 2, - }; -} - -function functionResponseEvent( - toolUseId: string, - isError: boolean, - result: unknown, -): InvocationResult['events'][number] { - return { - id: `event-${toolUseId}`, - invocationId: 'invocation-fixture', - runId: 'run-fixture', - sessionId: summary.id, - turnId: 'turn-fixture', - ts: 1, - partial: false, - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: toolUseId, - name: 'Bash', - result, - isError, - }, - }; -} - -async function notify(result: InvocationResult): Promise { - await observer?.({ - outcomeId: result.invocationId, - status: result.status === 'completed' ? 'completed' : 'failed', - ...(result.finalOutput !== undefined ? { finalOutput: result.finalOutput } : {}), - ...(result.failure ? { failure: result.failure } : {}), - sandboxBoundary: invocationRecoveredSandboxBoundaryFailure(result) - ? 'recovered' - : invocationHasSandboxBoundaryFailure(result) - ? 'unresolved' - : 'none', - }); -} - -runMakaTextCli(process.argv.slice(2), { createContext, listSessions }).then( +import { runMakaTextCli } from '../run-command.js'; +import { createRunCommandFake } from './run-command-fake.js'; + +// Subprocess entry for the retained process-contract tests: real stdin +// piping, a fail-closed sandbox boundary, and SIGINT delivery. Ordinary +// command semantics run in process through the same fake via +// createRunCommandFake — keep this wrapper limited to what a real child +// process is genuinely needed for. +const fake = createRunCommandFake({ + ...(process.env.MAKA_RUN_FIXTURE_SCENARIO + ? { scenario: process.env.MAKA_RUN_FIXTURE_SCENARIO } + : {}), + onReady: () => process.stderr.write('fixture-ready\n'), +}); + +runMakaTextCli(process.argv.slice(2), fake).then( (code) => { process.exitCode = code; }, diff --git a/packages/cli/src/__tests__/run-command.test.ts b/packages/cli/src/__tests__/run-command.test.ts index 46738e6ae4..36d0c4cf0d 100644 --- a/packages/cli/src/__tests__/run-command.test.ts +++ b/packages/cli/src/__tests__/run-command.test.ts @@ -5,23 +5,11 @@ import { realpath } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; import { describe, test } from 'node:test'; import type { SessionSummary } from '@maka/core/session'; -import { parseMakaRunArgs } from '../run-command.js'; +import { parseMakaRunArgs, runMakaTextCli } from '../run-command.js'; +import { createRunCommandFake, type RunCommandFakeOptions } from './run-command-fake.js'; const fixturePath = fileURLToPath(new URL('./run-command-fixture.js', import.meta.url)); -/** Node may print ExperimentalWarning for node:sqlite on stderr; ignore it in process contracts. */ -function processContractStderr(stderr: string): string { - return stderr - .split('\n') - .filter( - (line) => - !line.includes('ExperimentalWarning: SQLite is an experimental feature') && - !line.includes('Use `node --trace-warnings') && - line.trim().length > 0, - ) - .join('\n'); -} - describe('maka run argument parsing', () => { test('parses prompt, target, thinking, timeout, and max steps', () => { assert.deepEqual( @@ -119,28 +107,31 @@ describe('maka run argument parsing', () => { }); }); -describe('maka run process contract', () => { +// Ordinary command semantics run in process against the injectable +// MakaRunDeps seam: same fake runtime as the subprocess fixture, but without +// paying a Node startup per assertion. Real-process coverage for stdin +// piping, SIGINT, and fail-closed boundary handling stays below in +// 'maka run process contract'. +describe('maka run command semantics (in process)', () => { test('writes only the final answer to stdout', async () => { - const result = await runFixture(['hello'], { input: '' }); + const result = await runInProcess(['hello']); assert.equal(result.code, 0, result.stderr); assert.equal(result.stdout, 'prompt=hello\n'); - assert.equal(processContractStderr(result.stderr), ''); + // The injected stderr channel sees only the CLI's own writes, so no Node + // runtime warning can leak in here — the exact-empty assertion is safe. + assert.equal(result.stderr, ''); }); test('waits for the complete Graph before printing the final supervisor output', async () => { - const result = await runFixture(['implement it', '--graph'], { - input: '', - env: { MAKA_RUN_EXPECT_GRAPH: '1' }, - }); + const result = await runInProcess(['implement it', '--graph'], { expectGraph: true }); assert.equal(result.code, 0, result.stderr); assert.equal(result.stdout, 'graph completed\n'); - assert.equal(processContractStderr(result.stderr), ''); + assert.equal(result.stderr, ''); }); test('does not wait for Graph completion after the root invocation fails', async () => { - const result = await runFixture(['implement it', '--graph'], { + const result = await runInProcess(['implement it', '--graph'], { scenario: 'graph-runtime-error', - input: '', }); assert.equal(result.code, 1); @@ -149,109 +140,69 @@ describe('maka run process contract', () => { assert.doesNotMatch(result.stderr, /graph-wait-called/); }); - test('uses stdin as the complete prompt for run -', async () => { - const result = await runFixture(['-'], { input: 'from stdin\nsecond line' }); - assert.equal(result.code, 0, result.stderr); - assert.equal(result.stdout, 'prompt=from stdin\nsecond line\n'); - }); - - test('uses non-TTY stdin as the prompt when no positional prompt is provided', async () => { - const result = await runFixture([], { input: 'implicit stdin prompt' }); - assert.equal(result.code, 0, result.stderr); - assert.equal(result.stdout, 'prompt=implicit stdin prompt\n'); - }); - - test('combines a positional instruction with piped stdin context', async () => { - const result = await runFixture(['summarize'], { input: 'document body' }); - assert.equal(result.code, 0, result.stderr); - assert.equal(result.stdout, 'prompt=summarize\n\ndocument body\n'); - }); - test('returns exit 2 for missing input and pre-invocation configuration errors', async () => { - const missing = await runFixture([], { input: '' }); + const missing = await runInProcess([]); assert.equal(missing.code, 2); assert.match(missing.stderr, /missing prompt input/); - const config = await runFixture(['hello'], { scenario: 'config-error', input: '' }); + const config = await runInProcess(['hello'], { scenario: 'config-error' }); assert.equal(config.code, 2); assert.match(config.stderr, /unknown connection/); }); test('returns exit 1 for runtime failure and missing final output', async () => { - const runtime = await runFixture(['hello'], { scenario: 'runtime-error', input: '' }); + const runtime = await runInProcess(['hello'], { scenario: 'runtime-error' }); assert.equal(runtime.code, 1); assert.match(runtime.stderr, /provider failed after startup/); - const missing = await runFixture(['hello'], { scenario: 'missing-output', input: '' }); + const missing = await runInProcess(['hello'], { scenario: 'missing-output' }); assert.equal(missing.code, 1); assert.match(missing.stderr, /no final output/); }); test('returns exit 1 without successful output when the explicit step limit is reached', async () => { - const result = await runFixture(['hello'], { scenario: 'step-limit', input: '' }); + const result = await runInProcess(['hello'], { scenario: 'step-limit' }); assert.equal(result.code, 1); assert.equal(result.stdout, ''); assert.match(result.stderr, /tool-step limit reached/); }); - test('fails closed when a sandbox boundary request reaches non-interactive run', async () => { - const result = await runFixture(['hello'], { scenario: 'sandbox-boundary', input: '' }); - assert.equal(result.code, 1); - assert.match(result.stderr, /sandbox boundary expansion is unavailable/); - assert.equal(result.stdout, ''); - }); - test('fails closed when a tool reports an unresolved sandbox boundary requirement', async () => { - const result = await runFixture(['hello'], { - scenario: 'sandbox-boundary-tool-result', - input: '', - }); + const result = await runInProcess(['hello'], { scenario: 'sandbox-boundary-tool-result' }); assert.equal(result.code, 1); assert.match(result.stderr, /sandbox boundary expansion is unavailable/); assert.equal(result.stdout, ''); }); test('accepts a completed boundary-safe alternative', async () => { - const result = await runFixture(['hello'], { - scenario: 'sandbox-boundary-recovered', - input: '', - }); + const result = await runInProcess(['hello'], { scenario: 'sandbox-boundary-recovered' }); assert.equal(result.code, 0, result.stderr); assert.equal(result.stdout, 'recovered safely\n'); }); test('creates an Auto boundary by default', async () => { - const result = await runFixture(['hello'], { - input: '', - env: { MAKA_RUN_EXPECT_PERMISSION_MODE: 'ask' }, - }); + const result = await runInProcess(['hello'], { expectPermissionMode: 'ask' }); assert.equal(result.code, 0, result.stderr); assert.equal(result.stdout, 'prompt=hello\n'); }); test('passes max steps as an invocation-local context limit', async () => { - const result = await runFixture(['hello', '--max-steps', '3'], { - input: '', - env: { MAKA_RUN_EXPECT_MAX_STEPS: '3' }, - }); + const result = await runInProcess(['hello', '--max-steps', '3'], { expectMaxSteps: 3 }); assert.equal(result.code, 0, result.stderr); assert.match(result.stdout, /^maxSteps=3;/); }); test('creates a bypass boundary only when --yolo is explicit', async () => { - const result = await runFixture(['hello', '--yolo'], { - input: '', - env: { MAKA_RUN_EXPECT_PERMISSION_MODE: 'bypass' }, - }); + const result = await runInProcess(['hello', '--yolo'], { expectPermissionMode: 'bypass' }); assert.equal(result.code, 0, result.stderr); assert.equal(result.stdout, 'prompt=hello\n'); }); test('returns exit 2 for removed permission flags before runtime startup', async () => { - const result = await runFixture(['hello', '--permission-mode', 'ask'], { input: '' }); + const result = await runInProcess(['hello', '--permission-mode', 'ask']); assert.equal(result.code, 2); assert.match(result.stderr, /unknown option: --permission-mode/); assert.equal(result.stdout, ''); @@ -266,7 +217,7 @@ describe('maka run process contract', () => { model: 'fixture-model', permissionMode: 'execute', }); - const result = await runFixture( + const result = await runInProcess( [ 'continue this', '--resume', @@ -277,16 +228,13 @@ describe('maka run process contract', () => { resumed.model, ], { - input: '', - env: { - MAKA_RUN_FIXTURE_SESSIONS: JSON.stringify([resumed]), - MAKA_RUN_EXPECT_NO_CREATE: '1', - MAKA_RUN_EXPECT_SESSION_ID: resumed.id, - MAKA_RUN_EXPECT_CONTEXT_CWD: cwd, - MAKA_RUN_EXPECT_CONTEXT_CONNECTION: resumed.llmConnectionSlug, - MAKA_RUN_EXPECT_CONTEXT_MODEL: resumed.model, - MAKA_RUN_EXPECT_CWD_OVERRIDE: JSON.stringify({ sessionId: resumed.id, cwd }), - }, + sessions: [resumed], + expectNoCreate: true, + expectSessionId: resumed.id, + expectContextCwd: cwd, + expectContextConnection: resumed.llmConnectionSlug, + expectContextModel: resumed.model, + expectCwdOverride: JSON.stringify({ sessionId: resumed.id, cwd }), }, ); @@ -295,7 +243,7 @@ describe('maka run process contract', () => { }); test('names the mode the same way the desktop and TUI do (#1616)', async () => { - const help = await runFixture(['--help'], { input: '' }); + const help = await runInProcess(['--help']); assert.equal(help.code, 0, help.stderr); assert.match(help.stdout, /--yolo\s+Give this session full access to your files and network/); @@ -310,13 +258,10 @@ describe('maka run process contract', () => { cwd, permissionMode: 'bypass', }); - const result = await runFixture(['continue this', '--resume', resumed.id], { - input: '', - env: { - MAKA_RUN_FIXTURE_SESSIONS: JSON.stringify([resumed]), - MAKA_RUN_BOUNDARY_KIND: 'bypass', - MAKA_RUN_EXPECT_NO_SEND: '1', - }, + const result = await runInProcess(['continue this', '--resume', resumed.id], { + sessions: [resumed], + boundaryKind: 'bypass', + expectNoSend: true, }); assert.equal(result.code, 2); @@ -334,13 +279,10 @@ describe('maka run process contract', () => { cwd, permissionMode: 'bypass', }); - const result = await runFixture(['continue this', '--resume', resumed.id, '--yolo'], { - input: '', - env: { - MAKA_RUN_FIXTURE_SESSIONS: JSON.stringify([resumed]), - MAKA_RUN_BOUNDARY_KIND: 'bypass', - MAKA_RUN_EXPECT_BOUNDARY_KIND: 'bypass', - }, + const result = await runInProcess(['continue this', '--resume', resumed.id, '--yolo'], { + sessions: [resumed], + boundaryKind: 'bypass', + expectBoundaryKind: 'bypass', }); assert.equal(result.code, 0, result.stderr); @@ -349,12 +291,9 @@ describe('maka run process contract', () => { test('returns exit 2 when explicit configuration conflicts with a resumed session', async () => { const resumed = fixtureSession({ id: 'resume-me', cwd: process.cwd() }); - const result = await runFixture( + const result = await runInProcess( ['continue this', '--resume', resumed.id, '--model', 'different-model'], - { - input: '', - env: { MAKA_RUN_FIXTURE_SESSIONS: JSON.stringify([resumed]) }, - }, + { sessions: [resumed] }, ); assert.equal(result.code, 2); @@ -369,17 +308,14 @@ describe('maka run process contract', () => { fixtureSession({ id: 'a', cwd, lastMessageAt: 200, status: 'aborted' }), fixtureSession({ id: 'newer-other-cwd', cwd: '/missing-other', lastMessageAt: 300 }), ]; - const result = await runFixture(['continue this', '--continue'], { - input: '', - env: { - MAKA_RUN_FIXTURE_SESSIONS: JSON.stringify(sessions), - MAKA_RUN_EXPECT_NO_CREATE: '1', - MAKA_RUN_EXPECT_SESSION_ID: 'a', - MAKA_RUN_EXPECT_CONTEXT_CWD: cwd, - MAKA_RUN_EXPECT_CONTEXT_CONNECTION: 'fixture', - MAKA_RUN_EXPECT_CONTEXT_MODEL: 'fixture-model', - MAKA_RUN_EXPECT_CWD_OVERRIDE: JSON.stringify({ sessionId: 'a', cwd }), - }, + const result = await runInProcess(['continue this', '--continue'], { + sessions, + expectNoCreate: true, + expectSessionId: 'a', + expectContextCwd: cwd, + expectContextConnection: 'fixture', + expectContextModel: 'fixture-model', + expectCwdOverride: JSON.stringify({ sessionId: 'a', cwd }), }); assert.equal(result.code, 0, result.stderr); @@ -387,10 +323,7 @@ describe('maka run process contract', () => { }); test('returns exit 2 when continue finds no compatible session', async () => { - const result = await runFixture(['continue this', '--continue'], { - input: '', - env: { MAKA_RUN_FIXTURE_SESSIONS: '[]' }, - }); + const result = await runInProcess(['continue this', '--continue'], { sessions: [] }); assert.equal(result.code, 2); assert.match(result.stderr, /no compatible session found for cwd/); @@ -398,26 +331,50 @@ describe('maka run process contract', () => { }); test('returns exit 1 when the invocation timeout stops the run', async () => { - const result = await runFixture(['hello', '--timeout', '0.05'], { - scenario: 'slow', - input: '', - }); + const result = await runInProcess(['hello', '--timeout', '0.05'], { scenario: 'slow' }); assert.equal(result.code, 1); assert.match(result.stderr, /timed out after 50ms/); }); test('returns exit 1 when a graph descendant leaves a boundary failure unresolved', async () => { - const result = await runFixture(['graph task', '--graph'], { - input: '', - env: { - MAKA_RUN_EXPECT_GRAPH: '1', - MAKA_RUN_GRAPH_BOUNDARY_FAILURE: '1', - }, + const result = await runInProcess(['graph task', '--graph'], { + expectGraph: true, + graphBoundaryFailure: true, }); assert.equal(result.code, 1, result.stderr); assert.equal(result.stdout, ''); }); +}); + +// Retained real-subprocess coverage. Each test here exercises a contract the +// in-process seam cannot: piped non-TTY stdin, OS signal delivery, and the +// fail-closed path observed through a real process boundary. +describe('maka run process contract', () => { + test('uses stdin as the complete prompt for run -', async () => { + const result = await runFixture(['-'], { input: 'from stdin\nsecond line' }); + assert.equal(result.code, 0, result.stderr); + assert.equal(result.stdout, 'prompt=from stdin\nsecond line\n'); + }); + + test('uses non-TTY stdin as the prompt when no positional prompt is provided', async () => { + const result = await runFixture([], { input: 'implicit stdin prompt' }); + assert.equal(result.code, 0, result.stderr); + assert.equal(result.stdout, 'prompt=implicit stdin prompt\n'); + }); + + test('combines a positional instruction with piped stdin context', async () => { + const result = await runFixture(['summarize'], { input: 'document body' }); + assert.equal(result.code, 0, result.stderr); + assert.equal(result.stdout, 'prompt=summarize\n\ndocument body\n'); + }); + + test('fails closed when a sandbox boundary request reaches non-interactive run', async () => { + const result = await runFixture(['hello'], { scenario: 'sandbox-boundary', input: '' }); + assert.equal(result.code, 1); + assert.match(result.stderr, /sandbox boundary expansion is unavailable/); + assert.equal(result.stdout, ''); + }); test('returns exit 130 on SIGINT', async () => { const child = spawn(process.execPath, [fixturePath, 'hello'], { @@ -485,12 +442,35 @@ describe('maka run process contract', () => { }); }); +async function runInProcess( + args: string[], + options: RunCommandFakeOptions = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const fake = createRunCommandFake(options); + let stdout = ''; + let stderr = ''; + const code = await runMakaTextCli(args, { + createContext: fake.createContext, + listSessions: fake.listSessions, + // Mirror the subprocess fixture's environment: stdin is a piped (non-TTY) + // stream that is already at EOF. + stdinIsTTY: () => false, + readStdin: async () => '', + writeStdout: (text) => { + stdout += text; + }, + writeStderr: (text) => { + stderr += text; + }, + }); + return { code, stdout, stderr }; +} + function runFixture( args: string[], options: { scenario?: string; input?: string; - env?: NodeJS.ProcessEnv; } = {}, ): Promise<{ code: number | null; stdout: string; stderr: string }> { return new Promise((resolve) => { @@ -498,7 +478,6 @@ function runFixture( env: { ...process.env, ...(options.scenario ? { MAKA_RUN_FIXTURE_SCENARIO: options.scenario } : {}), - ...options.env, }, stdio: ['pipe', 'pipe', 'pipe'], });