Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 61 additions & 22 deletions packages/headless/src/__tests__/cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,20 @@ import { fileURLToPath } from 'node:url';
import { describe, test } from 'node:test';
import { createSessionStore } from '@maka/storage';
import { validateHarborCellOutput } from '../cell-output.js';
import { mapLegacyMakaHeadlessArgs, runMakaEvalCli } from '../cli.js';
import { openHeadlessStorageForWrite } from '../headless-storage.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';
import { readResults } from '../results.js';
import type { TaskEvent } from '../task-contracts.js';
import { taskRunLocator } from '../task-run-identity.js';

const cliPath = fileURLToPath(new URL('../cli.js', import.meta.url));
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url));

function runCli(
args: string[],
options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number | null; stdout: string; stderr: string }> {
function runCli(args: string[]): Promise<{ code: number | null; stdout: string; stderr: string }> {
return new Promise((resolve) => {
const child = spawn(process.execPath, [cliPath, ...args], {
env: { ...process.env, ...options.env },
env: { ...process.env },
});
let stdout = '';
let stderr = '';
Expand All @@ -35,6 +34,46 @@ function runCli(
});
}

// In-process route for ordinary command semantics: the same legacy argv
// mapping and canonical router the bin runs, with stdout/stderr captured at
// the process-stream seam and env overrides applied around the call. The real
// subprocess route above stays for the representative bin-wiring contract.
async function runCliInProcess(
args: string[],
options: { env?: NodeJS.ProcessEnv } = {},
): Promise<{ code: number; stdout: string; stderr: string }> {
const mapped = mapLegacyMakaHeadlessArgs(args);
assert.ok(mapped && mapped.length > 0, 'in-process runner expects a canonical command');
const envOverrides = Object.entries(options.env ?? {});
const savedEnv = envOverrides.map(([key]) => [key, process.env[key]] as const);
for (const [key, value] of envOverrides) process.env[key] = value;
const savedExitCode = process.exitCode;
try {
const {
result: code,
stdout,
stderr,
} = await withCapturedProcessIo(async () => {
try {
return await runMakaEvalCli(mapped);
} catch (error) {
// Mirror the bin's fatal handler: report the error and fail with 1.
process.stderr.write(
`${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
);
return 1;
}
});
return { code, stdout, stderr };
} finally {
process.exitCode = savedExitCode;
for (const [key, value] of savedEnv) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
}

describe('maka-headless CLI', () => {
test('task-run readers report an actionable error for a non-Headless root', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-headless-unmarked-'));
Expand Down Expand Up @@ -72,7 +111,7 @@ describe('maka-headless CLI', () => {
};
const specPath = join(dir, 'spec.json');
await writeFile(specPath, JSON.stringify(spec), 'utf8');
const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
assert.equal(result.code, 1);
assert.match(result.stderr, /protectedPaths/);
} finally {
Expand All @@ -84,7 +123,7 @@ describe('maka-headless CLI', () => {
const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
try {
await mkdir(join(dir, 'fixture'), { recursive: true });
const result = await runCli([
const result = await runCliInProcess([
'harbor',
'run',
'--backend',
Expand All @@ -105,7 +144,7 @@ describe('maka-headless CLI', () => {
const dir = await mkdtemp(join(tmpdir(), 'maka-headless-harbor-cli-'));
try {
await mkdir(join(dir, 'fixture'), { recursive: true });
const missingUrl = await runCli([
const missingUrl = await runCliInProcess([
'harbor',
'run',
'--backend',
Expand All @@ -120,7 +159,7 @@ describe('maka-headless CLI', () => {
assert.equal(missingUrl.code, 1);
assert.match(missingUrl.stderr, /MAKA_HARBOR_TOOL_EXECUTOR_URL is required/);

const missingToken = await runCli(
const missingToken = await runCliInProcess(
[
'harbor',
'run',
Expand Down Expand Up @@ -151,7 +190,7 @@ describe('maka-headless CLI', () => {
await mkdir(fixture, { recursive: true });
await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

const result = await runCli(
const result = await runCliInProcess(
[
'harbor',
'run',
Expand Down Expand Up @@ -239,7 +278,7 @@ describe('maka-headless CLI', () => {
await mkdir(fixture, { recursive: true });
await writeFile(join(fixture, 'README.txt'), 'Harbor owns the task workspace.\n', 'utf8');

const result = await runCli(
const result = await runCliInProcess(
[
'harbor',
'run',
Expand Down Expand Up @@ -309,7 +348,7 @@ describe('maka-headless CLI', () => {
};
const specPath = join(dir, 'spec.json');
await writeFile(specPath, JSON.stringify(spec), 'utf8');
const result = await runCli(['eval', specPath, '--out', join(dir, 'out')]);
const result = await runCliInProcess(['eval', specPath, '--out', join(dir, 'out')]);
assert.equal(result.code, 0, result.stderr);
const records = await readResults(join(dir, 'out', 'results.jsonl'));
assert.equal(records[0]?.passed, false);
Expand Down Expand Up @@ -347,7 +386,7 @@ describe('maka-headless CLI', () => {
const outDir = join(dir, 'out');
await writeFile(specPath, JSON.stringify(spec), 'utf8');

const run = await runCli([
const run = await runCliInProcess([
'task',
'run',
specPath,
Expand All @@ -364,7 +403,7 @@ describe('maka-headless CLI', () => {
assert.equal(run.code, 1);
assert.match(run.stdout, /taskRunId: task-run-1/);

const inspect = await runCli([
const inspect = await runCliInProcess([
'task',
'inspect',
'task-run-1',
Expand All @@ -379,7 +418,7 @@ describe('maka-headless CLI', () => {
assert.equal(inspectDocument.taskRun.result.taxonomy, 'verification_failed');
assert.ok(Array.isArray(inspectDocument.attempts));

const humanInspect = await runCli([
const humanInspect = await runCliInProcess([
'task',
'inspect',
'task-run-1',
Expand All @@ -391,7 +430,7 @@ describe('maka-headless CLI', () => {
assert.match(humanInspect.stdout, /Task Events task_event:task-run-1/);

const exportDir = join(dir, 'manual-export');
const exported = await runCli([
const exported = await runCliInProcess([
'task',
'export',
'task-run-1',
Expand Down Expand Up @@ -422,7 +461,7 @@ describe('maka-headless CLI', () => {
}

const aheExportDir = join(dir, 'ahe-export');
const aheExported = await runCli([
const aheExported = await runCliInProcess([
'ahe',
'export',
'task-run-1',
Expand Down Expand Up @@ -513,7 +552,7 @@ describe('maka-headless CLI', () => {
const taskRunId = `long-task-run-${'x'.repeat(320)}`;
await writeFile(specPath, JSON.stringify(spec), 'utf8');

const run = await runCli([
const run = await runCliInProcess([
'task',
'run',
specPath,
Expand All @@ -529,7 +568,7 @@ describe('maka-headless CLI', () => {
]);
assert.equal(run.code, 0, run.stderr);

const inspect = await runCli([
const inspect = await runCliInProcess([
'task',
'inspect',
taskRunId,
Expand Down Expand Up @@ -638,7 +677,7 @@ describe('maka-headless CLI', () => {
const { taskRunStore } = await openHeadlessStorageForWrite(join(outDir, 'runs'));
for (const event of initialEvents) await taskRunStore.appendEvent(taskRunId, event);

const resumed = await runCli([
const resumed = await runCliInProcess([
'task',
'resume',
taskRunId,
Expand All @@ -651,7 +690,7 @@ describe('maka-headless CLI', () => {
assert.match(resumed.stdout, /resumed: parked-run/);
assert.match(resumed.stdout, /status: completed/);

const inspect = await runCli([
const inspect = await runCliInProcess([
'task',
'inspect',
taskRunId,
Expand Down Expand Up @@ -738,7 +777,7 @@ describe('maka-headless CLI', () => {
);

const outDir = join(dir, 'out');
const result = await runCli([
const result = await runCliInProcess([
'task',
'retry-failed',
priorPath,
Expand Down
77 changes: 41 additions & 36 deletions packages/headless/src/__tests__/contamination-scan-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
scheduledCellLogPath,
trialCellLogPath,
} from '../trial-cell-log.js';
import { withCapturedProcessIo } from './helpers/capture-process-io.js';

const execFileAsync = promisify(execFile);
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..');
Expand Down Expand Up @@ -123,14 +124,38 @@ async function withRunRoot<T>(
}
}

/**
* The script's exported entry, invoked in process with the executable
* footer's contract mirrored exactly: a thrown error lands on stderr and
* exits 2. The real-subprocess route stays in 'refuses an invocation it
* cannot act on' as the representative coverage of that footer itself.
*/
async function runScanScript(
argv: string[],
): Promise<{ code: number; stdout: string; stderr: string }> {
const { main } = (await import(
new URL('../../harbor/run-contamination-scan.mjs', import.meta.url).href
)) as { main: (argv?: string[]) => Promise<number> };
const {
result: code,
stdout,
stderr,
} = await withCapturedProcessIo(async () => {
try {
return await main(argv);
} catch (error) {
process.stderr.write(
`${error instanceof Error ? (error.stack ?? String(error)) : String(error)}\n`,
);
return 2;
}
});
return { code, stdout, stderr };
}

async function scan(runRoot: string): Promise<{ code: number; report: ContaminationScanReport }> {
const jsonPath = join(runRoot, 'report.json');
let code = 0;
try {
await execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot, '--json', jsonPath]);
} catch (error) {
code = (error as { code?: number }).code ?? -1;
}
const { code } = await runScanScript(['--run-root', runRoot, '--json', jsonPath]);
return { code, report: JSON.parse(await readFile(jsonPath, 'utf8')) as ContaminationScanReport };
}

Expand Down Expand Up @@ -184,13 +209,8 @@ describe('run-contamination-scan', () => {
[{ agent: 'maka', taskId: 'cobol-modernization', messages: ['ran the tests'] }],
async (runRoot, armIds) => {
const markdownPath = join(runRoot, 'report.md');
await execFileAsync(process.execPath, [
SCRIPT,
'--run-root',
runRoot,
'--markdown',
markdownPath,
]);
const { code } = await runScanScript(['--run-root', runRoot, '--markdown', markdownPath]);
assert.equal(code, 0);
assert.match(
await readFile(markdownPath, 'utf8'),
new RegExp(`Searched ${armIds.length} of ${armIds.length} recorded cells\\.`),
Expand All @@ -200,13 +220,8 @@ describe('run-contamination-scan', () => {
});

test('refuses a flag it does not know', async () => {
await assert.rejects(
execFileAsync(process.execPath, [SCRIPT, '--run-root', '/tmp', '--depth', '2']),
(error: { code?: number }) => {
assert.equal(error.code, 2);
return true;
},
);
const { code } = await runScanScript(['--run-root', '/tmp', '--depth', '2']);
assert.equal(code, 2);
});

// A cell whose trajectory never landed was not searched, and a zero exit here
Expand Down Expand Up @@ -282,14 +297,9 @@ describe('run-contamination-scan', () => {
test('says what is missing when a run root is not one', async () => {
const empty = await mkdtemp(join(tmpdir(), 'maka-contamination-empty-'));
try {
await assert.rejects(
execFileAsync(process.execPath, [SCRIPT, '--run-root', empty]),
(error: { code?: number; stderr?: string }) => {
assert.equal(error.code, 2);
assert.match(error.stderr ?? '', /no schedule recorded at .*scheduled-cells\.jsonl/);
return true;
},
);
const { code, stderr } = await runScanScript(['--run-root', empty]);
assert.equal(code, 2);
assert.match(stderr, /no schedule recorded at .*scheduled-cells\.jsonl/);
} finally {
await rm(empty, { recursive: true, force: true });
}
Expand Down Expand Up @@ -323,14 +333,9 @@ describe('run-contamination-scan', () => {
const runRoot = await mkdtemp(join(tmpdir(), 'maka-contamination-torn-'));
try {
await writeFile(scheduledCellLogPath(runRoot), '', 'utf8');
await assert.rejects(
execFileAsync(process.execPath, [SCRIPT, '--run-root', runRoot]),
(error: { code?: number; stderr?: string }) => {
assert.equal(error.code, 2);
assert.match(error.stderr ?? '', /names no cells/);
return true;
},
);
const { code, stderr } = await runScanScript(['--run-root', runRoot]);
assert.equal(code, 2);
assert.match(stderr, /names no cells/);
} finally {
await rm(runRoot, { recursive: true, force: true });
}
Expand Down
30 changes: 30 additions & 0 deletions packages/headless/src/__tests__/helpers/capture-process-io.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/**
* Capture process.stdout/stderr writes around an in-process CLI invocation.
*
* Tests that used to spawn a Node subprocess to observe a command's output can
* run the command in process and read the same two channels here. Only safe
* for sequential tests: the capture swaps the process-wide stream writers.
*/
export async function withCapturedProcessIo<T>(
fn: () => Promise<T>,
): Promise<{ result: T; stdout: string; stderr: string }> {
const savedStdoutWrite = process.stdout.write;
const savedStderrWrite = process.stderr.write;
let stdout = '';
let stderr = '';
process.stdout.write = ((chunk: string | Uint8Array) => {
stdout += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8');
return true;
}) as typeof process.stdout.write;
process.stderr.write = ((chunk: string | Uint8Array) => {
stderr += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8');
return true;
}) as typeof process.stderr.write;
try {
const result = await fn();
return { result, stdout, stderr };
} finally {
process.stdout.write = savedStdoutWrite;
process.stderr.write = savedStderrWrite;
}
}
Loading