From e3d090c84d532592ebe782ba6c53d661ba4810d8 Mon Sep 17 00:00:00 2001 From: Thomas Hart Date: Fri, 7 Aug 2026 22:07:15 +0000 Subject: [PATCH 1/2] feat: Add airlock run CLI with structured JSON results Closes #10 --- README.md | 27 +++++++- package.json | 5 ++ src/cli.ts | 157 ++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 15 ++++ src/run-file.ts | 173 +++++++++++++++++++++++++++++++++++++++++++++++ test/cli.test.ts | 114 +++++++++++++++++++++++++++++++ tsup.config.ts | 27 +++++--- 7 files changed, 509 insertions(+), 9 deletions(-) create mode 100644 src/cli.ts create mode 100644 src/run-file.ts create mode 100644 test/cli.test.ts diff --git a/README.md b/README.md index 2e89e41..641e2d3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Ephemeral, zero-credential, self-verifying execution for untrusted or agent-writ An airlock is the safe way to run code you do not trust: an LLM-generated snippet, a plugin, a user-submitted function. This repo builds that primitive from the ground up in TypeScript. The guarantee is that a caller never reads an output unless the run stayed inside its resource ceilings and its output satisfies a post-condition the caller supplied. Untrusted code is guilty until proven correct, and the type system makes you prove it before you can touch the value. -The first slice was the contract and the in-process runner that enforces it. The second slice added `run(code, opts)`, which executes untrusted source in a fresh `node:vm` context with no ambient authority. The third slice added `runInWorker(code, opts)`: the same contract on a `worker_threads` isolate with frozen globals and an empty env. The fourth slice hardens the **resource limit** layer: wall-clock deadline with hard terminate on abort, V8 heap cap, and output size caps. This slice adds a **deny-by-default module loader**: untrusted code has no `require` unless the caller opts in with `allowedModules`, and only the listed builtin or package ids resolve. Relative and absolute paths are always refused. Later slices add a Docker-backed tier and a growing suite of documented escape-attempt tests. +The first slice was the contract and the in-process runner that enforces it. The second slice added `run(code, opts)`, which executes untrusted source in a fresh `node:vm` context with no ambient authority. The third slice added `runInWorker(code, opts)`: the same contract on a `worker_threads` isolate with frozen globals and an empty env. The fourth slice hardens the **resource limit** layer: wall-clock deadline with hard terminate on abort, V8 heap cap, and output size caps. Module loading is deny-by-default via `allowedModules`. This slice adds the **`airlock run ` CLI** and a programmatic `runFile` / `runSource` API that return a structured JSON result, so agents and shell pipelines can consume every outcome without special-casing thrown `Error` values. ## Concepts demonstrated @@ -26,6 +26,8 @@ The first slice was the contract and the in-process runner that enforces it. The - **Deny-by-default module loading.** `require` is unbound unless the caller sets `allowedModules`. An empty list injects a gate that refuses every specifier; a non-empty list is an exact-match allowlist (with bare/`node:` equivalence), never a prefix grant. - **Capability allowlists.** Module loading is treated as ambient authority: the host's real `require` is reachable only after the gate admits the id, so unlisted builtins like `fs` stay closed even when a sibling id is granted. - **Path-specifier refusal.** Relative and absolute paths are dropped from the allowlist and rejected at load time so filesystem resolution cannot re-open host I/O through a crafty entry. +- **Structured result DTOs + exit codes.** `RunResult` maps to a JSON envelope (`JsonRunResult`) with Error DTOs; exit `0` only for `ok`, `1` for sandbox refusals, `2` for CLI/IO failures. +- **CLI as a process-boundary adapter.** Argv and file I/O stay outside the sandbox; the untrusted payload is the file body, while `--assert` is a host-side expression over the returned value. - **Strict TypeScript.** `strict`, `noUncheckedIndexedAccess`, and `exactOptionalPropertyTypes`, no `any`. ## The primitive contract @@ -149,6 +151,28 @@ await run("require('path')", { // -> { status: "error", error: ModuleNotAllowedError } ``` +### CLI and structured JSON + +`airlock run ` prints one JSON result to stdout. Exit `0` only for `status: "ok"`, `1` for sandbox refusals, `2` for usage/IO errors. + +```bash +pnpm run build +echo '21 * 2' > /tmp/snippet.js +node dist/cli.js run /tmp/snippet.js --timeout 200 --assert 'value === 42' +# {"status":"ok","value":42,"durationMs":...} +``` + +```ts +import { runFile } from "airlock"; +const result = await runFile("./snippet.js", { + timeoutMs: 200, + assertExpr: "value === 42", +}); +if (result.status === "ok") console.log(result.value); +``` + +Flags: `--tier sandbox|worker`, `--grant`, `--allow-module`, `--max-output-bytes`, `--max-old-gen-mb`. + ## Develop ```bash @@ -165,3 +189,4 @@ pnpm run build - `src/worker.ts`: `runInWorker(code, opts)` runs untrusted source in a `worker_threads` isolate started with an empty `process.env` and frozen globals, caps the heap with `maxOldGenerationSizeMb` (reported as `out-of-memory`), and hard-kills the thread on the deadline so a sync spin and a never-settling async task are both preempted. An escape-attempt test confirms the constructor walk that reaches the host realm in-process reaches only the credential-free worker realm here. - `src/limits.ts`: shared resource ceilings for every tier. Wall-clock timeout aborts the task signal and, on the worker tier, calls `worker.terminate()` on both deadline and caller abort. Heap cap via V8 `resourceLimits`. Output size caps (`maxOutputBytes`) measure UTF-8 payload with a budgeted walk (cycle-safe, early-exit) and refuse with `output-too-large` before the post-condition runs. - `src/modules.ts`: deny-by-default module loader with an explicit `allowedModules` allowlist. Omitted means no `require`; `[]` or a list injects `createGatedRequire` over the host/worker require. Exact match only (bare and `node:` equivalent), path specifiers always refused, and the gate wins over a grant-supplied `require`. Wired into both `run` and `runInWorker`. +- `src/cli.ts` + `src/run-file.ts`: `airlock run ` CLI and programmatic `runFile` / `runSource` API with structured JSON results, exit-code mapping, host-side `--assert` expressions, and tier/grant/module flags. diff --git a/package.json b/package.json index 131bbdb..56c7d2f 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,9 @@ "import": "./dist/index.js" } }, + "bin": { + "airlock": "./dist/cli.js" + }, "files": [ "dist" ], @@ -29,8 +32,10 @@ "untrusted-code", "agent-safety", "capability-security", + "cli", "typescript" ], + "license": "MIT", "packageManager": "pnpm@9.15.0", "devDependencies": { diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..a1dee97 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,157 @@ +import { pathToFileURL } from "node:url"; +import { + exitCodeFor, + runFile, + stringifyJsonResult, + type ExecutorTier, + type JsonRunResult, +} from "./run-file.js"; + +export interface CliArgs { + file: string; + timeoutMs: number; + assertExpr?: string; + grantJson?: string; + allowedModules: string[]; + maxOutputBytes?: number; + maxOldGenerationSizeMb?: number; + tier: ExecutorTier; +} + +export type ParseResult = + | { ok: true; args: CliArgs } + | { ok: false; message: string; showHelp?: boolean }; + +const USAGE = `Usage: airlock run [options] + --timeout --assert --tier sandbox|worker + --grant --allow-module --max-output-bytes + --max-old-gen-mb -h, --help +Exit: 0 ok, 1 refusal, 2 usage/io. +`; + +function fail(message: string, showHelp?: boolean): ParseResult { + return showHelp ? { ok: false, message, showHelp } : { ok: false, message }; +} + +export function parseArgv(argv: string[]): ParseResult { + if (argv.length === 0 || argv[0] === "-h" || argv[0] === "--help") { + return fail(USAGE, true); + } + if (argv[0] !== "run") return fail(`unknown command: ${argv[0]}\n\n${USAGE}`); + + const args: CliArgs = { + file: "", + timeoutMs: 5_000, + allowedModules: [], + tier: "sandbox", + }; + let file: string | undefined; + + for (let i = 1; i < argv.length; i++) { + const arg = argv[i]!; + if (arg === "-h" || arg === "--help") return fail(USAGE, true); + if (!arg.startsWith("-")) { + if (file !== undefined) return fail(`unexpected argument: ${arg}\n\n${USAGE}`); + file = arg; + continue; + } + const value = argv[i + 1]; + if (value === undefined || value.startsWith("-")) { + return fail(`${arg} requires a value`); + } + i++; + if (arg === "--timeout" || arg === "--max-output-bytes" || arg === "--max-old-gen-mb") { + if (!/^\d+$/.test(value)) return fail(`${arg} must be a non-negative integer`); + const n = Number(value); + if (!Number.isSafeInteger(n)) return fail(`${arg} is out of range`); + if (arg === "--timeout") args.timeoutMs = n; + else if (arg === "--max-output-bytes") args.maxOutputBytes = n; + else args.maxOldGenerationSizeMb = n; + continue; + } + if (arg === "--assert") { args.assertExpr = value; continue; } + if (arg === "--grant") { args.grantJson = value; continue; } + if (arg === "--allow-module") { args.allowedModules.push(value); continue; } + if (arg === "--tier") { + if (value !== "sandbox" && value !== "worker") { + return fail(`--tier must be "sandbox" or "worker"`); + } + args.tier = value; + continue; + } + return fail(`unknown option: ${arg}\n\n${USAGE}`); + } + + if (file === undefined) return fail(`missing \n\n${USAGE}`); + args.file = file; + return { ok: true, args }; +} + +function parseGrant( + raw: string, +): { ok: true; grant: Record } | { ok: false; message: string } { + try { + const parsed: unknown = JSON.parse(raw); + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + return { ok: false, message: "--grant must be a JSON object" }; + } + return { ok: true, grant: parsed as Record }; + } catch (error) { + return { + ok: false, + message: `--grant: ${error instanceof Error ? error.message : "invalid JSON"}`, + }; + } +} + +export interface CliIo { + stdout: { write(chunk: string): void }; + stderr: { write(chunk: string): void }; +} + +export async function main(argv: string[], io: CliIo = process): Promise { + const parsed = parseArgv(argv); + if (!parsed.ok) { + io.stderr.write(parsed.message.endsWith("\n") ? parsed.message : `${parsed.message}\n`); + return parsed.showHelp ? 0 : 2; + } + + const { args } = parsed; + let grant: Record | undefined; + if (args.grantJson !== undefined) { + const g = parseGrant(args.grantJson); + if (!g.ok) { + io.stderr.write(`${g.message}\n`); + return 2; + } + grant = g.grant; + } + + let result: JsonRunResult; + try { + result = await runFile(args.file, { + timeoutMs: args.timeoutMs, + tier: args.tier, + ...(args.assertExpr !== undefined ? { assertExpr: args.assertExpr } : {}), + ...(grant ? { grant } : {}), + ...(args.allowedModules.length > 0 ? { allowedModules: args.allowedModules } : {}), + ...(args.maxOutputBytes !== undefined ? { maxOutputBytes: args.maxOutputBytes } : {}), + ...(args.maxOldGenerationSizeMb !== undefined + ? { maxOldGenerationSizeMb: args.maxOldGenerationSizeMb } + : {}), + }); + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 2; + } + + io.stdout.write(`${stringifyJsonResult(result)}\n`); + return exitCodeFor(result); +} + +const entry = process.argv[1]; +if (entry !== undefined && import.meta.url === pathToFileURL(entry).href) { + void main(process.argv.slice(2)).then((code) => { + process.exitCode = code; + }); +} diff --git a/src/index.ts b/src/index.ts index 0b399c5..29f82dc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,21 @@ export { checkOutputSize, } from "./limits.js"; export type { ResourceLimits, OutputSizeCheck } from "./limits.js"; +export { + serializeError, + toJsonResult, + exitCodeFor, + stringifyJsonResult, + compileAssertExpr, + runSource, + runFile, +} from "./run-file.js"; +export type { + JsonError, + JsonRunResult, + ExecutorTier, + RunFileOptions, +} from "./run-file.js"; export type { Assertion, RunResult, diff --git a/src/run-file.ts b/src/run-file.ts new file mode 100644 index 0000000..6e251dc --- /dev/null +++ b/src/run-file.ts @@ -0,0 +1,173 @@ +import { readFile } from "node:fs/promises"; +import type { Assertion, RunResult } from "./contract.js"; +import { run } from "./sandbox.js"; +import { runInWorker } from "./worker.js"; + +export interface JsonError { + name: string; + message: string; + code?: string; +} + +/** JSON-serializable run outcome. Errors are DTOs, never live Error instances. */ +export type JsonRunResult = + | { status: "ok"; value: unknown; durationMs: number } + | { status: "timeout"; timeoutMs: number } + | { status: "assertion-failed"; value: unknown } + | { status: "error"; error: JsonError } + | { status: "out-of-memory"; maxOldGenerationSizeMb: number } + | { status: "output-too-large"; maxOutputBytes: number; actualBytes: number } + | { status: "io-error"; message: string }; + +export type ExecutorTier = "sandbox" | "worker"; + +export interface RunFileOptions { + timeoutMs?: number; + assert?: Assertion; + /** Host-side expression with `value` in scope (trusted operator input). */ + assertExpr?: string; + grant?: Readonly>; + allowedModules?: readonly string[]; + signal?: AbortSignal; + maxOutputBytes?: number; + maxOldGenerationSizeMb?: number; + tier?: ExecutorTier; + filename?: string; +} + +const DEFAULT_TIMEOUT_MS = 5_000; + +export function serializeError(error: unknown): JsonError { + // Other-realm Errors fail `instanceof`; duck-type name/message/code. + if (typeof error === "object" && error !== null) { + const rec = error as { name?: unknown; message?: unknown; code?: unknown }; + if (typeof rec.message === "string") { + const base: JsonError = { + name: typeof rec.name === "string" && rec.name ? rec.name : "Error", + message: rec.message, + }; + if (typeof rec.code === "string" || typeof rec.code === "number") { + return { ...base, code: String(rec.code) }; + } + return base; + } + } + if (typeof error === "string") return { name: "Error", message: error }; + return { name: "Error", message: String(error) }; +} + +export function toJsonResult(result: RunResult): JsonRunResult { + switch (result.status) { + case "ok": + return { status: "ok", value: result.value, durationMs: result.durationMs }; + case "timeout": + return { status: "timeout", timeoutMs: result.timeoutMs }; + case "assertion-failed": + return { status: "assertion-failed", value: result.value }; + case "error": + return { status: "error", error: serializeError(result.error) }; + case "out-of-memory": + return { status: "out-of-memory", maxOldGenerationSizeMb: result.maxOldGenerationSizeMb }; + case "output-too-large": + return { + status: "output-too-large", + maxOutputBytes: result.maxOutputBytes, + actualBytes: result.actualBytes, + }; + } +} + +/** 0 = ok, 1 = sandbox refusal, 2 = CLI/IO failure. */ +export function exitCodeFor(result: JsonRunResult): number { + if (result.status === "ok") return 0; + if (result.status === "io-error") return 2; + return 1; +} + +function jsonReplacer(_key: string, value: unknown): unknown { + if (typeof value === "bigint") return value.toString(); + if (typeof value === "undefined") return null; + if (typeof value === "function") { + return { __type: "Function", name: value.name || "anonymous" }; + } + return value; +} + +export function stringifyJsonResult(result: JsonRunResult): string { + try { + return JSON.stringify(result, jsonReplacer); + } catch (error) { + return JSON.stringify({ + status: "error", + error: serializeError(error), + } satisfies JsonRunResult); + } +} + +export function compileAssertExpr(expr: string): Assertion { + const trimmed = expr.trim(); + if (!trimmed) throw new Error("assert expression must not be empty"); + const fn = new Function("value", `return (${trimmed});`) as ( + value: unknown, + ) => unknown; + return (value) => Boolean(fn(value)); +} + +export async function runSource( + code: string, + opts: RunFileOptions = {}, +): Promise { + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const assert = + opts.assert ?? + (opts.assertExpr !== undefined + ? compileAssertExpr(opts.assertExpr) + : () => true); + const common = { + timeoutMs, + assert, + filename: opts.filename ?? "airlock-run.js", + ...(opts.grant ? { grant: opts.grant } : {}), + ...(opts.allowedModules !== undefined + ? { allowedModules: opts.allowedModules } + : {}), + ...(opts.signal ? { signal: opts.signal } : {}), + ...(opts.maxOutputBytes !== undefined + ? { maxOutputBytes: opts.maxOutputBytes } + : {}), + }; + + try { + if ((opts.tier ?? "sandbox") === "worker") { + return toJsonResult( + await runInWorker(code, { + ...common, + ...(opts.maxOldGenerationSizeMb !== undefined + ? { maxOldGenerationSizeMb: opts.maxOldGenerationSizeMb } + : {}), + }), + ); + } + return toJsonResult(await run(code, common)); + } catch (error) { + return { status: "error", error: serializeError(error) }; + } +} + +export async function runFile( + path: string, + opts: RunFileOptions = {}, +): Promise { + if (!path.trim()) { + return { status: "io-error", message: "path must not be empty" }; + } + try { + const code = await readFile(path, "utf8"); + return runSource(code, { ...opts, filename: opts.filename ?? path }); + } catch (error) { + return { + status: "io-error", + message: error instanceof Error ? error.message : `failed to read ${path}`, + }; + } +} diff --git a/test/cli.test.ts b/test/cli.test.ts new file mode 100644 index 0000000..70c9422 --- /dev/null +++ b/test/cli.test.ts @@ -0,0 +1,114 @@ +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { main, parseArgv } from "../src/cli.js"; +import { + compileAssertExpr, + exitCodeFor, + runFile, + runSource, + serializeError, + stringifyJsonResult, + toJsonResult, + type JsonRunResult, +} from "../src/run-file.js"; + +function capture() { + let out = ""; + return { + io: { + stdout: { write(c: string) { out += c; } }, + stderr: { write(_c: string) {} }, + }, + get out() { return out; }, + }; +} + +async function tempFile(body: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "airlock-cli-")); + const path = join(dir, "s.js"); + await writeFile(path, body, "utf8"); + return path; +} + +describe("runFile JSON API + cli", () => { + it("covers serialization, execution edges, and CLI exit codes", async () => { + expect(serializeError({ name: "Error", message: "boom" })).toEqual({ + name: "Error", + message: "boom", + }); + expect(toJsonResult({ status: "ok", value: 7, durationMs: 1 })).toEqual({ + status: "ok", + value: 7, + durationMs: 1, + }); + expect(exitCodeFor({ status: "ok", value: 1, durationMs: 0 })).toBe(0); + expect(exitCodeFor({ status: "timeout", timeoutMs: 1 })).toBe(1); + expect(exitCodeFor({ status: "io-error", message: "x" })).toBe(2); + expect( + JSON.parse( + stringifyJsonResult({ status: "ok", value: { n: 1n }, durationMs: 0 }), + ).value, + ).toEqual({ n: "1" }); + const cycle: { self?: unknown } = {}; + cycle.self = cycle; + expect( + (JSON.parse( + stringifyJsonResult({ status: "ok", value: cycle, durationMs: 0 }), + ) as JsonRunResult).status, + ).toBe("error"); + + expect(compileAssertExpr("value > 0")(2)).toBe(true); + expect(() => compileAssertExpr(" ")).toThrow(/empty/); + expect( + await runSource("1 + 2", { timeoutMs: 100, assert: (v) => v === 3 }), + ).toMatchObject({ status: "ok", value: 3 }); + expect( + await runSource("41", { timeoutMs: 100, assertExpr: "value === 42" }), + ).toEqual({ status: "assertion-failed", value: 41 }); + expect(await runSource("while (true) {}", { timeoutMs: 25 })).toEqual({ + status: "timeout", + timeoutMs: 25, + }); + expect( + await runSource('throw new Error("boom")', { timeoutMs: 100 }), + ).toMatchObject({ status: "error", error: { message: "boom" } }); + expect( + await runSource("rows.reduce((s, n) => s + n, 0)", { + timeoutMs: 500, + tier: "worker", + grant: { rows: [1, 2, 3] }, + assertExpr: "value === 6", + }), + ).toMatchObject({ status: "ok", value: 6 }); + + const path = await tempFile("2 * 21"); + expect(await runFile(path, { timeoutMs: 100, assertExpr: "value === 42" })) + .toMatchObject({ status: "ok", value: 42 }); + expect(await runFile("/no/such/airlock-file.js")).toMatchObject({ status: "io-error" }); + expect(await runFile(" ")).toMatchObject({ status: "io-error" }); + + expect(parseArgv(["run", "a.js"])).toMatchObject({ + ok: true, + args: { file: "a.js", tier: "sandbox" }, + }); + expect(parseArgv(["run"]).ok).toBe(false); + expect(parseArgv(["run", "a.js", "--tier", "docker"]).ok).toBe(false); + + const ok = capture(); + const okPath = await tempFile("21 * 2"); + expect(await main(["run", okPath, "--timeout", "200", "--assert", "value === 42"], ok.io)).toBe(0); + expect(JSON.parse(ok.out.trim())).toMatchObject({ status: "ok", value: 42 }); + expect(await main(["run", okPath, "--assert", "value === 0"], capture().io)).toBe(1); + expect(await main(["run", "/no/such/airlock-cli.js"], capture().io)).toBe(2); + expect(await main(["run"], capture().io)).toBe(2); + expect( + await main( + ["run", await tempFile("n + 1"), "--grant", '{"n":41}', "--assert", "value === 42"], + capture().io, + ), + ).toBe(0); + expect(await main(["run", okPath, "--grant", "nope"], capture().io)).toBe(2); + }); +}); diff --git a/tsup.config.ts b/tsup.config.ts index 75bf903..1bc066e 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,10 +1,21 @@ import { defineConfig } from "tsup"; -export default defineConfig({ - entry: ["src/index.ts"], - format: ["esm"], - target: "node20", - dts: true, - clean: true, - sourcemap: true, -}); +export default defineConfig([ + { + entry: ["src/index.ts"], + format: ["esm"], + target: "node20", + dts: true, + clean: true, + sourcemap: true, + }, + { + entry: ["src/cli.ts"], + format: ["esm"], + target: "node20", + dts: false, + clean: false, + sourcemap: true, + banner: { js: "#!/usr/bin/env node" }, + }, +]); From 77d5e248fafe5b056722b90a571f963daa9de581 Mon Sep 17 00:00:00 2001 From: Thomas Hart Date: Fri, 7 Aug 2026 22:13:36 +0000 Subject: [PATCH 2/2] fix: Make assertExpr await settle and always return JSON envelopes Boolean(fn()) treated Promise-returning asserts as always true. Await the expression result before coercing, compile assertExpr inside runSource's try so bad exprs become status error not rejections, and await runSource from runFile. CLI prints one JSON result for assert compile failures (exit 1) instead of bare stderr + exit 2. --- README.md | 6 +- package.json | 1 - src/cli.ts | 44 ++++---- src/run-file.ts | 45 +++++---- test/cli.test.ts | 255 +++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 286 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 641e2d3..039b07b 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ await run("require('path')", { ### CLI and structured JSON -`airlock run ` prints one JSON result to stdout. Exit `0` only for `status: "ok"`, `1` for sandbox refusals, `2` for usage/IO errors. +`airlock run ` prints **one JSON result** to stdout for every run invocation (including failed post-conditions and assert compile failures). Exit `0` only for `status: "ok"`, `1` for sandbox refusals and assert/runtime errors, `2` only for usage parse failures and file IO (`io-error`). ```bash pnpm run build @@ -173,6 +173,10 @@ if (result.status === "ok") console.log(result.value); Flags: `--tier sandbox|worker`, `--grant`, `--allow-module`, `--max-output-bytes`, `--max-old-gen-mb`. +**Host-privileged assert.** `--assert` / `assertExpr` is compiled with `new Function` in the **host** realm, not inside the sandbox. The expression can reach `process` and other host globals. Treat it as trusted operator input, same trust level as the caller process. Untrusted code is only the file body. + +Invalid or empty assert expressions do not reject: they return `{ status: "error", error: ... }` (CLI exit `1`) so agents always parse one JSON envelope. + ## Develop ```bash diff --git a/package.json b/package.json index 56c7d2f..4f65627 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "cli", "typescript" ], - "license": "MIT", "packageManager": "pnpm@9.15.0", "devDependencies": { diff --git a/src/cli.ts b/src/cli.ts index a1dee97..78fddb6 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,7 +26,8 @@ const USAGE = `Usage: airlock run [options] --timeout --assert --tier sandbox|worker --grant --allow-module --max-output-bytes --max-old-gen-mb -h, --help -Exit: 0 ok, 1 refusal, 2 usage/io. +Exit: 0 ok, 1 refusal (incl. assert compile/runtime), 2 usage/io. +--assert is host-privileged (new Function in the host realm). `; function fail(message: string, showHelp?: boolean): ParseResult { @@ -127,23 +128,17 @@ export async function main(argv: string[], io: CliIo = process): Promise grant = g.grant; } - let result: JsonRunResult; - try { - result = await runFile(args.file, { - timeoutMs: args.timeoutMs, - tier: args.tier, - ...(args.assertExpr !== undefined ? { assertExpr: args.assertExpr } : {}), - ...(grant ? { grant } : {}), - ...(args.allowedModules.length > 0 ? { allowedModules: args.allowedModules } : {}), - ...(args.maxOutputBytes !== undefined ? { maxOutputBytes: args.maxOutputBytes } : {}), - ...(args.maxOldGenerationSizeMb !== undefined - ? { maxOldGenerationSizeMb: args.maxOldGenerationSizeMb } - : {}), - }); - } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - return 2; - } + const result = await runFile(args.file, { + timeoutMs: args.timeoutMs, + tier: args.tier, + ...(args.assertExpr !== undefined ? { assertExpr: args.assertExpr } : {}), + ...(grant ? { grant } : {}), + ...(args.allowedModules.length > 0 ? { allowedModules: args.allowedModules } : {}), + ...(args.maxOutputBytes !== undefined ? { maxOutputBytes: args.maxOutputBytes } : {}), + ...(args.maxOldGenerationSizeMb !== undefined + ? { maxOldGenerationSizeMb: args.maxOldGenerationSizeMb } + : {}), + }); io.stdout.write(`${stringifyJsonResult(result)}\n`); return exitCodeFor(result); @@ -151,7 +146,14 @@ export async function main(argv: string[], io: CliIo = process): Promise const entry = process.argv[1]; if (entry !== undefined && import.meta.url === pathToFileURL(entry).href) { - void main(process.argv.slice(2)).then((code) => { - process.exitCode = code; - }); + void main(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 2; + }); } diff --git a/src/run-file.ts b/src/run-file.ts index 6e251dc..779f8f4 100644 --- a/src/run-file.ts +++ b/src/run-file.ts @@ -110,34 +110,35 @@ export function compileAssertExpr(expr: string): Assertion { const fn = new Function("value", `return (${trimmed});`) as ( value: unknown, ) => unknown; - return (value) => Boolean(fn(value)); + // Await then coerce: Promise-returning exprs must settle before Boolean(). + return async (value) => Boolean(await fn(value)); } export async function runSource( code: string, opts: RunFileOptions = {}, ): Promise { - const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; - const assert = - opts.assert ?? - (opts.assertExpr !== undefined - ? compileAssertExpr(opts.assertExpr) - : () => true); - const common = { - timeoutMs, - assert, - filename: opts.filename ?? "airlock-run.js", - ...(opts.grant ? { grant: opts.grant } : {}), - ...(opts.allowedModules !== undefined - ? { allowedModules: opts.allowedModules } - : {}), - ...(opts.signal ? { signal: opts.signal } : {}), - ...(opts.maxOutputBytes !== undefined - ? { maxOutputBytes: opts.maxOutputBytes } - : {}), - }; - try { + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const assert = + opts.assert ?? + (opts.assertExpr !== undefined + ? compileAssertExpr(opts.assertExpr) + : () => true); + const common = { + timeoutMs, + assert, + filename: opts.filename ?? "airlock-run.js", + ...(opts.grant ? { grant: opts.grant } : {}), + ...(opts.allowedModules !== undefined + ? { allowedModules: opts.allowedModules } + : {}), + ...(opts.signal ? { signal: opts.signal } : {}), + ...(opts.maxOutputBytes !== undefined + ? { maxOutputBytes: opts.maxOutputBytes } + : {}), + }; + if ((opts.tier ?? "sandbox") === "worker") { return toJsonResult( await runInWorker(code, { @@ -163,7 +164,7 @@ export async function runFile( } try { const code = await readFile(path, "utf8"); - return runSource(code, { ...opts, filename: opts.filename ?? path }); + return await runSource(code, { ...opts, filename: opts.filename ?? path }); } catch (error) { return { status: "io-error", diff --git a/test/cli.test.ts b/test/cli.test.ts index 70c9422..d924913 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -16,12 +16,26 @@ import { function capture() { let out = ""; + let err = ""; return { io: { - stdout: { write(c: string) { out += c; } }, - stderr: { write(_c: string) {} }, + stdout: { + write(c: string) { + out += c; + }, + }, + stderr: { + write(c: string) { + err += c; + }, + }, + }, + get out() { + return out; + }, + get err() { + return err; }, - get out() { return out; }, }; } @@ -32,12 +46,15 @@ async function tempFile(body: string): Promise { return path; } -describe("runFile JSON API + cli", () => { - it("covers serialization, execution edges, and CLI exit codes", async () => { +describe("serializeError / toJsonResult / exitCodeFor", () => { + it("serializes duck-typed errors", () => { expect(serializeError({ name: "Error", message: "boom" })).toEqual({ name: "Error", message: "boom", }); + }); + + it("maps ok and refusal statuses", () => { expect(toJsonResult({ status: "ok", value: 7, durationMs: 1 })).toEqual({ status: "ok", value: 7, @@ -45,7 +62,11 @@ describe("runFile JSON API + cli", () => { }); expect(exitCodeFor({ status: "ok", value: 1, durationMs: 0 })).toBe(0); expect(exitCodeFor({ status: "timeout", timeoutMs: 1 })).toBe(1); + expect(exitCodeFor({ status: "error", error: { name: "Error", message: "x" } })).toBe(1); expect(exitCodeFor({ status: "io-error", message: "x" })).toBe(2); + }); + + it("stringifies bigints and recovers from cycles", () => { expect( JSON.parse( stringifyJsonResult({ status: "ok", value: { n: 1n }, durationMs: 0 }), @@ -54,19 +75,120 @@ describe("runFile JSON API + cli", () => { const cycle: { self?: unknown } = {}; cycle.self = cycle; expect( - (JSON.parse( - stringifyJsonResult({ status: "ok", value: cycle, durationMs: 0 }), - ) as JsonRunResult).status, + ( + JSON.parse( + stringifyJsonResult({ status: "ok", value: cycle, durationMs: 0 }), + ) as JsonRunResult + ).status, ).toBe("error"); + }); + + it("maps out-of-memory and output-too-large DTOs", () => { + expect( + toJsonResult({ status: "out-of-memory", maxOldGenerationSizeMb: 8 }), + ).toEqual({ status: "out-of-memory", maxOldGenerationSizeMb: 8 }); + expect( + toJsonResult({ + status: "output-too-large", + maxOutputBytes: 10, + actualBytes: 99, + }), + ).toEqual({ + status: "output-too-large", + maxOutputBytes: 10, + actualBytes: 99, + }); + expect( + exitCodeFor({ status: "out-of-memory", maxOldGenerationSizeMb: 8 }), + ).toBe(1); + expect( + exitCodeFor({ + status: "output-too-large", + maxOutputBytes: 10, + actualBytes: 99, + }), + ).toBe(1); + }); +}); + +describe("compileAssertExpr", () => { + it("evaluates sync expressions", async () => { + expect(await compileAssertExpr("value > 0")(2)).toBe(true); + expect(await compileAssertExpr("value > 0")(0)).toBe(false); + }); - expect(compileAssertExpr("value > 0")(2)).toBe(true); + it("rejects empty expressions at compile time", () => { expect(() => compileAssertExpr(" ")).toThrow(/empty/); + expect(() => compileAssertExpr("")).toThrow(/empty/); + }); + + it("awaits Promise-returning expressions before Boolean coerce", async () => { + expect(await compileAssertExpr("Promise.resolve(false)")(1)).toBe(false); + expect(await compileAssertExpr("Promise.resolve(true)")(1)).toBe(true); + }); +}); + +describe("runSource / runFile assertExpr edges", () => { + it("returns ok when assert passes", async () => { expect( await runSource("1 + 2", { timeoutMs: 100, assert: (v) => v === 3 }), ).toMatchObject({ status: "ok", value: 3 }); + }); + + it("returns assertion-failed for false sync assertExpr", async () => { expect( await runSource("41", { timeoutMs: 100, assertExpr: "value === 42" }), ).toEqual({ status: "assertion-failed", value: 41 }); + }); + + it("returns assertion-failed for Promise.resolve(false) assertExpr", async () => { + const result = await runSource("99", { + timeoutMs: 100, + assertExpr: "Promise.resolve(false)", + }); + expect(result).toEqual({ status: "assertion-failed", value: 99 }); + }); + + it("does not reject on empty assertExpr; returns status error", async () => { + const result = await runSource("1", { timeoutMs: 100, assertExpr: " " }); + expect(result.status).toBe("error"); + if (result.status === "error") { + expect(result.error.message).toMatch(/empty/i); + } + }); + + it("does not reject on syntax-invalid assertExpr; returns status error", async () => { + const result = await runSource("1", { + timeoutMs: 100, + assertExpr: "value ===", + }); + expect(result.status).toBe("error"); + if (result.status === "error") { + expect(result.error.name).toMatch(/SyntaxError|Error/); + expect(result.error.message.length).toBeGreaterThan(0); + } + }); + + it("runFile does not reject on bad assertExpr", async () => { + const path = await tempFile("1 + 1"); + const result = await runFile(path, { + timeoutMs: 100, + assertExpr: "value ===", + }); + expect(result.status).toBe("error"); + }); + + it("runFile Promise.resolve(false) assertExpr is assertion-failed", async () => { + const path = await tempFile("7"); + expect( + await runFile(path, { + timeoutMs: 100, + assertExpr: "Promise.resolve(false)", + }), + ).toEqual({ status: "assertion-failed", value: 7 }); + }); + + it("timeouts and thrown errors become structured statuses", async () => { expect(await runSource("while (true) {}", { timeoutMs: 25 })).toEqual({ status: "timeout", timeoutMs: 25, @@ -74,6 +196,9 @@ describe("runFile JSON API + cli", () => { expect( await runSource('throw new Error("boom")', { timeoutMs: 100 }), ).toMatchObject({ status: "error", error: { message: "boom" } }); + }); + + it("worker tier + grant + assertExpr", async () => { expect( await runSource("rows.reduce((s, n) => s + n, 0)", { timeoutMs: 500, @@ -82,33 +207,123 @@ describe("runFile JSON API + cli", () => { assertExpr: "value === 6", }), ).toMatchObject({ status: "ok", value: 6 }); + }); + it("runFile happy path and missing file", async () => { const path = await tempFile("2 * 21"); - expect(await runFile(path, { timeoutMs: 100, assertExpr: "value === 42" })) - .toMatchObject({ status: "ok", value: 42 }); - expect(await runFile("/no/such/airlock-file.js")).toMatchObject({ status: "io-error" }); + expect( + await runFile(path, { timeoutMs: 100, assertExpr: "value === 42" }), + ).toMatchObject({ status: "ok", value: 42 }); + expect(await runFile("/no/such/airlock-file.js")).toMatchObject({ + status: "io-error", + }); expect(await runFile(" ")).toMatchObject({ status: "io-error" }); + }); +}); +describe("parseArgv", () => { + it("parses run with defaults", () => { expect(parseArgv(["run", "a.js"])).toMatchObject({ ok: true, args: { file: "a.js", tier: "sandbox" }, }); + }); + + it("rejects missing file and bad tier", () => { expect(parseArgv(["run"]).ok).toBe(false); expect(parseArgv(["run", "a.js", "--tier", "docker"]).ok).toBe(false); + }); +}); - const ok = capture(); - const okPath = await tempFile("21 * 2"); - expect(await main(["run", okPath, "--timeout", "200", "--assert", "value === 42"], ok.io)).toBe(0); - expect(JSON.parse(ok.out.trim())).toMatchObject({ status: "ok", value: 42 }); - expect(await main(["run", okPath, "--assert", "value === 0"], capture().io)).toBe(1); - expect(await main(["run", "/no/such/airlock-cli.js"], capture().io)).toBe(2); +describe("CLI main", () => { + it("prints JSON and exits 0 on success", async () => { + const cap = capture(); + const path = await tempFile("21 * 2"); + expect( + await main( + ["run", path, "--timeout", "200", "--assert", "value === 42"], + cap.io, + ), + ).toBe(0); + expect(JSON.parse(cap.out.trim())).toMatchObject({ + status: "ok", + value: 42, + }); + }); + + it("exits 1 on assertion-failed with JSON envelope", async () => { + const path = await tempFile("21 * 2"); + const cap = capture(); + expect(await main(["run", path, "--assert", "value === 0"], cap.io)).toBe( + 1, + ); + expect(JSON.parse(cap.out.trim())).toMatchObject({ + status: "assertion-failed", + value: 42, + }); + }); + + it("exits 1 with JSON when --assert is syntax-invalid", async () => { + const path = await tempFile("1"); + const cap = capture(); + const code = await main(["run", path, "--assert", "value ==="], cap.io); + expect(code).toBe(1); + const parsed = JSON.parse(cap.out.trim()) as JsonRunResult; + expect(parsed.status).toBe("error"); + if (parsed.status === "error") { + expect(parsed.error.message.length).toBeGreaterThan(0); + } + }); + + it("exits 1 with JSON when --assert is empty whitespace", async () => { + const path = await tempFile("1"); + const cap = capture(); + const code = await main(["run", path, "--assert", " "], cap.io); + expect(code).toBe(1); + const parsed = JSON.parse(cap.out.trim()) as JsonRunResult; + expect(parsed.status).toBe("error"); + if (parsed.status === "error") { + expect(parsed.error.message).toMatch(/empty/i); + } + }); + + it("exits 1 with JSON when --assert is Promise.resolve(false)", async () => { + const path = await tempFile("5"); + const cap = capture(); + const code = await main( + ["run", path, "--assert", "Promise.resolve(false)"], + cap.io, + ); + expect(code).toBe(1); + expect(JSON.parse(cap.out.trim())).toEqual({ + status: "assertion-failed", + value: 5, + }); + }); + + it("exits 2 on missing file and bad usage without requiring stdout JSON", async () => { + expect(await main(["run", "/no/such/airlock-cli.js"], capture().io)).toBe( + 2, + ); expect(await main(["run"], capture().io)).toBe(2); + expect(await main(["run", await tempFile("1"), "--grant", "nope"], capture().io)).toBe( + 2, + ); + }); + + it("accepts --grant and --assert together", async () => { expect( await main( - ["run", await tempFile("n + 1"), "--grant", '{"n":41}', "--assert", "value === 42"], + [ + "run", + await tempFile("n + 1"), + "--grant", + '{"n":41}', + "--assert", + "value === 42", + ], capture().io, ), ).toBe(0); - expect(await main(["run", okPath, "--grant", "nope"], capture().io)).toBe(2); }); });