From e424d569b51ea018056102d47e2f2d5faf79a4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=86=E9=80=8A?= <211125649+UncertaintyDeterminesYou4ndMe@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:42:22 +0800 Subject: [PATCH] test(headless): run ordinary CLI command semantics in process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of #2387. The Headless CLI and contamination-scan suites started a Node subprocess per assertion for scenarios that only exercise argument validation, task-run business logic, and report verdicts — semantics the exported entry points already expose. - cli.test.ts: 10 of 11 tests now run through mapLegacyMakaHeadlessArgs + runMakaEvalCli with stdout/stderr captured at the process-stream seam and env overrides applied and restored around the call. The non-Headless-root test keeps the real bin route as the representative wiring contract (real exit code, stack-free stderr). - contamination-scan-cli.test.ts: 13 of 14 tests call the script's exported main(argv), mirroring the executable footer exactly (thrown error -> stderr + exit 2). The no-argument rejection keeps the real subprocess as representative coverage of that footer itself, including its realpath main-module guard. - New shared helper withCapturedProcessIo swaps and restores the process-wide stream writers; safe for the sequential node:test runs these files use. runtime-policy-ab-cli.test.ts is intentionally untouched: its single test is the representative subprocess for run-runtime-policy-ab.mjs, whose main() is not exported, and adding an export to shave 0.6s is not warranted. (harness-ab-cli.test.ts was part of this change until #2462 deleted that suite on main.) Timing (node --test, local, warm build): cli.test.js 21.8s -> 13.1s (spawns 20 -> 2) contamination-scan-cli.test.js 1.05s -> 0.63s (spawns 14 -> 1) Tests pass across 3 consecutive rounds. --- packages/headless/src/__tests__/cli.test.ts | 83 ++++++++++++++----- .../__tests__/contamination-scan-cli.test.ts | 77 +++++++++-------- .../__tests__/helpers/capture-process-io.ts | 30 +++++++ 3 files changed, 132 insertions(+), 58 deletions(-) create mode 100644 packages/headless/src/__tests__/helpers/capture-process-io.ts diff --git a/packages/headless/src/__tests__/cli.test.ts b/packages/headless/src/__tests__/cli.test.ts index 1bd5202184..69d7f65357 100644 --- a/packages/headless/src/__tests__/cli.test.ts +++ b/packages/headless/src/__tests__/cli.test.ts @@ -7,7 +7,9 @@ import { fileURLToPath } from 'node:url'; import { describe, test } from 'node:test'; import { createSessionStore } from '@maka/storage'; import { validateHarborCellOutput } from '../cell-output.js'; +import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js'; import { openHeadlessStorageForWrite } from '../headless-storage.js'; +import { withCapturedProcessIo } from './helpers/capture-process-io.js'; import { readResults } from '../results.js'; import type { TaskEvent } from '../task-contracts.js'; import { taskRunLocator } from '../task-run-identity.js'; @@ -15,13 +17,10 @@ import { taskRunLocator } from '../task-run-identity.js'; const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url)); const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)); -function runCli( - args: string[], - options: { env?: NodeJS.ProcessEnv } = {}, -): Promise<{ code: number | null; stdout: string; stderr: string }> { +function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> { return new Promise((resolve) => { const child = spawn(process.execPath, [cliPath, ...args], { - env: { ...process.env, ...options.env }, + env: { ...process.env }, }); let stdout = ''; let stderr = ''; @@ -35,6 +34,46 @@ function runCli( }); } +// In-process route for ordinary command semantics: the same legacy argv +// mapping and canonical router the bin runs, with stdout/stderr captured at +// the process-stream seam and env overrides applied around the call. The real +// subprocess route above stays for the representative bin-wiring contract. +async function runCliInProcess( + args: string[], + options: { env?: NodeJS.ProcessEnv } = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const mapped = mapLegacyMakaHeadlessArgs(args); + assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command'); + const envOverrides = Object.entries(options.env ?? {}); + const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const); + for (const [key, value] of envOverrides) process.env[key] = value; + const savedExitCode = process.exitCode; + try { + const { + result: code, + stdout, + stderr, + } = await withCapturedProcessIo(async () => { + try { + return await runMakaEvalCli(mapped); + } catch (error) { + // Mirror the bin's fatal handler: report the error and fail with 1. + process.stderr.write( + `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`, + ); + return 1; + } + }); + return { code, stdout, stderr }; + } finally { + process.exitCode = savedExitCode; + for (const [key, value] of savedEnv) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + describe('maka-headless CLI', () => { test('task-run readers report an actionable error for a non-Headless root', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-')); @@ -72,7 +111,7 @@ describe('maka-headless CLI', () => { }; const specPath = join(dir, 'spec.json'); await writeFile(specPath, JSON.stringify(spec), 'utf8'); - const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]); + const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]); assert.equal(result.code, 1); assert.match(result.stderr, /protectedPaths/); } finally { @@ -84,7 +123,7 @@ describe('maka-headless CLI', () => { const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-')); try { await mkdir(join(dir, 'fixture'), { recursive: true }); - const result = await runCli([ + const result = await runCliInProcess([ 'harbor', 'run', '--backend', @@ -105,7 +144,7 @@ describe('maka-headless CLI', () => { const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-')); try { await mkdir(join(dir, 'fixture'), { recursive: true }); - const missingUrl = await runCli([ + const missingUrl = await runCliInProcess([ 'harbor', 'run', '--backend', @@ -120,7 +159,7 @@ describe('maka-headless CLI', () => { assert.equal(missingUrl.code, 1); assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/); - const missingToken = await runCli( + const missingToken = await runCliInProcess( [ 'harbor', 'run', @@ -151,7 +190,7 @@ describe('maka-headless CLI', () => { await mkdir(fixture, { recursive: true }); await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8'); - const result = await runCli( + const result = await runCliInProcess( [ 'harbor', 'run', @@ -239,7 +278,7 @@ describe('maka-headless CLI', () => { await mkdir(fixture, { recursive: true }); await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8'); - const result = await runCli( + const result = await runCliInProcess( [ 'harbor', 'run', @@ -309,7 +348,7 @@ describe('maka-headless CLI', () => { }; const specPath = join(dir, 'spec.json'); await writeFile(specPath, JSON.stringify(spec), 'utf8'); - const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]); + const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]); assert.equal(result.code, 0, result.stderr); const records = await readResults(join(dir, 'out', 'results.jsonl')); assert.equal(records[0]?.passed, false); @@ -347,7 +386,7 @@ describe('maka-headless CLI', () => { const outDir = join(dir, 'out'); await writeFile(specPath, JSON.stringify(spec), 'utf8'); - const run = await runCli([ + const run = await runCliInProcess([ 'task', 'run', specPath, @@ -364,7 +403,7 @@ describe('maka-headless CLI', () => { assert.equal(run.code, 1); assert.match(run.stdout, /taskRunId: task-run-1/); - const inspect = await runCli([ + const inspect = await runCliInProcess([ 'task', 'inspect', 'task-run-1', @@ -379,7 +418,7 @@ describe('maka-headless CLI', () => { assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed'); assert.ok(Array.isArray(inspectDocument.attempts)); - const humanInspect = await runCli([ + const humanInspect = await runCliInProcess([ 'task', 'inspect', 'task-run-1', @@ -391,7 +430,7 @@ describe('maka-headless CLI', () => { assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/); const exportDir = join(dir, 'manual-export'); - const exported = await runCli([ + const exported = await runCliInProcess([ 'task', 'export', 'task-run-1', @@ -422,7 +461,7 @@ describe('maka-headless CLI', () => { } const aheExportDir = join(dir, 'ahe-export'); - const aheExported = await runCli([ + const aheExported = await runCliInProcess([ 'ahe', 'export', 'task-run-1', @@ -513,7 +552,7 @@ describe('maka-headless CLI', () => { const taskRunId = `long-task-run-${'x'.repeat(320)}`; await writeFile(specPath, JSON.stringify(spec), 'utf8'); - const run = await runCli([ + const run = await runCliInProcess([ 'task', 'run', specPath, @@ -529,7 +568,7 @@ describe('maka-headless CLI', () => { ]); assert.equal(run.code, 0, run.stderr); - const inspect = await runCli([ + const inspect = await runCliInProcess([ 'task', 'inspect', taskRunId, @@ -638,7 +677,7 @@ describe('maka-headless CLI', () => { const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs')); for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event); - const resumed = await runCli([ + const resumed = await runCliInProcess([ 'task', 'resume', taskRunId, @@ -651,7 +690,7 @@ describe('maka-headless CLI', () => { assert.match(resumed.stdout, /resumed: parked-run/); assert.match(resumed.stdout, /status: completed/); - const inspect = await runCli([ + const inspect = await runCliInProcess([ 'task', 'inspect', taskRunId, @@ -738,7 +777,7 @@ describe('maka-headless CLI', () => { ); const outDir = join(dir, 'out'); - const result = await runCli([ + const result = await runCliInProcess([ 'task', 'retry-failed', priorPath, diff --git a/packages/headless/src/__tests__/contamination-scan-cli.test.ts b/packages/headless/src/__tests__/contamination-scan-cli.test.ts index 3f0a555c03..ca00bd5a25 100644 --- a/packages/headless/src/__tests__/contamination-scan-cli.test.ts +++ b/packages/headless/src/__tests__/contamination-scan-cli.test.ts @@ -13,6 +13,7 @@ import { scheduledCellLogPath, trialCellLogPath, } from '../trial-cell-log.js'; +import { withCapturedProcessIo } from './helpers/capture-process-io.js'; const execFileAsync = promisify(execFile); const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'); @@ -123,14 +124,38 @@ async function withRunRoot( } } +/** + * The script's exported entry, invoked in process with the executable + * footer's contract mirrored exactly: a thrown error lands on stderr and + * exits 2. The real-subprocess route stays in 'refuses an invocation it + * cannot act on' as the representative coverage of that footer itself. + */ +async function runScanScript( + argv: string[], +): Promise<{ code: number; stdout: string; stderr: string }> { + const { main } = (await import( + new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href + )) as { main: (argv?: string[]) => Promise }; + const { + result: code, + stdout, + stderr, + } = await withCapturedProcessIo(async () => { + try { + return await main(argv); + } catch (error) { + process.stderr.write( + `${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`, + ); + return 2; + } + }); + return { code, stdout, stderr }; +} + async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> { const jsonPath = join(runRoot, 'report.json'); - let code = 0; - try { - await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]); - } catch (error) { - code = (error as { code?: number }).code ?? -1; - } + const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]); return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport }; } @@ -184,13 +209,8 @@ describe('run-contamination-scan', () => { [{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }], async (runRoot, armIds) => { const markdownPath = join(runRoot, 'report.md'); - await execFileAsync(process.execPath, [ - SCRIPT, - '--run-root', - runRoot, - '--markdown', - markdownPath, - ]); + const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]); + assert.equal(code, 0); assert.match( await readFile(markdownPath, 'utf8'), new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`), @@ -200,13 +220,8 @@ describe('run-contamination-scan', () => { }); test('refuses a flag it does not know', async () => { - await assert.rejects( - execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']), - (error: { code?: number }) => { - assert.equal(error.code, 2); - return true; - }, - ); + const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']); + assert.equal(code, 2); }); // A cell whose trajectory never landed was not searched, and a zero exit here @@ -282,14 +297,9 @@ describe('run-contamination-scan', () => { test('says what is missing when a run root is not one', async () => { const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-')); try { - await assert.rejects( - execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]), - (error: { code?: number; stderr?: string }) => { - assert.equal(error.code, 2); - assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/); - return true; - }, - ); + const { code, stderr } = await runScanScript(['--run-root', empty]); + assert.equal(code, 2); + assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/); } finally { await rm(empty, { recursive: true, force: true }); } @@ -323,14 +333,9 @@ describe('run-contamination-scan', () => { const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-')); try { await writeFile(scheduledCellLogPath(runRoot), '', 'utf8'); - await assert.rejects( - execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]), - (error: { code?: number; stderr?: string }) => { - assert.equal(error.code, 2); - assert.match(error.stderr ?? '', /names no cells/); - return true; - }, - ); + const { code, stderr } = await runScanScript(['--run-root', runRoot]); + assert.equal(code, 2); + assert.match(stderr, /names no cells/); } finally { await rm(runRoot, { recursive: true, force: true }); } diff --git a/packages/headless/src/__tests__/helpers/capture-process-io.ts b/packages/headless/src/__tests__/helpers/capture-process-io.ts new file mode 100644 index 0000000000..c08d04ae5f --- /dev/null +++ b/packages/headless/src/__tests__/helpers/capture-process-io.ts @@ -0,0 +1,30 @@ +/** + * Capture process.stdout/stderr writes around an in-process CLI invocation. + * + * Tests that used to spawn a Node subprocess to observe a command's output can + * run the command in process and read the same two channels here. Only safe + * for sequential tests: the capture swaps the process-wide stream writers. + */ +export async function withCapturedProcessIo( + fn: () => Promise, +): Promise<{ result: T; stdout: string; stderr: string }> { + const savedStdoutWrite = process.stdout.write; + const savedStderrWrite = process.stderr.write; + let stdout = ''; + let stderr = ''; + process.stdout.write = ((chunk: string | Uint8Array) => { + stdout += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'); + return true; + }) as typeof process.stdout.write; + process.stderr.write = ((chunk: string | Uint8Array) => { + stderr += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'); + return true; + }) as typeof process.stderr.write; + try { + const result = await fn(); + return { result, stdout, stderr }; + } finally { + process.stdout.write = savedStdoutWrite; + process.stderr.write = savedStderrWrite; + } +}