From d278736f98da97aab90c083d999c8f4d43f134e6 Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 6 Sep 2026 20:36:55 -0400 Subject: [PATCH 1/2] fix: isolate benchmark runners from coordinator data --- docs/agent-benchmark.md | 16 +++ scripts/agent-benchmark/README.md | 15 ++ scripts/agent-benchmark/driver.mjs | 88 ++++++++---- scripts/agent-benchmark/runner-isolation.mjs | 132 +++++++++++++++++ .../agent-benchmark/runner-isolation.test.mjs | 134 ++++++++++++++++++ 5 files changed, 360 insertions(+), 25 deletions(-) create mode 100644 scripts/agent-benchmark/runner-isolation.mjs create mode 100644 scripts/agent-benchmark/runner-isolation.test.mjs diff --git a/docs/agent-benchmark.md b/docs/agent-benchmark.md index 185cfdc8..c2a263d6 100644 --- a/docs/agent-benchmark.md +++ b/docs/agent-benchmark.md @@ -100,6 +100,22 @@ independently pinned `agent-device` skill. The control profile exposes only `agent-device`; the Stim binary and skill are unavailable. Both profiles have the same model settings, filesystem authority, and app task. +The macOS coordinator launches both runners and their child processes under +the same filesystem policy. It denies coordinator configuration, golden-state +and sibling-run file contents and writes, while permitting parent-directory +listings. Each run retains its own worktree, proof, temporary files, runner +home, selected tools, and the configured shared native-cache/device state. +Coordinator metadata and transcripts remain outside those writable grants. +Before timing starts, real read/write and child-process probes must verify the +policy; an unavailable sandbox or overbroad grant refuses dispatch. The policy +and its digest are retained with the private run metadata. + +This boundary prevents accidental benchmark-data access, not adversarial host +access: native system services and pre-existing processes are not sandboxed by +the runner policy. Strong isolation from a malicious agent requires a separate +host or VM. The coordinator still independently verifies app/device evidence. +Historical attempts keep the policy and verdict under which they were run. + The Stim agent creates its own worktree with Git after dispatch: ```text diff --git a/scripts/agent-benchmark/README.md b/scripts/agent-benchmark/README.md index 37f027e8..5671488f 100644 --- a/scripts/agent-benchmark/README.md +++ b/scripts/agent-benchmark/README.md @@ -90,6 +90,21 @@ or CLI digest mismatch and refuses a control shell that can resolve Stim. Golden cache validation hashes the fixture with the pinned CLI's fingerprint dependency, not the fixture's potentially different version. +Both runners are launched through macOS `sandbox-exec` with a verified, +run-scoped policy. Configuration, golden files, coordinator evidence and +sibling worktrees/results cannot be read or written by the runner process tree. +Parent-directory listing is permitted. The current worktree, proof, temporary +files, runner home, selected tool runtime and configured shared Gradle/AVD/device +state remain accessible. Keep those shared paths narrowly scoped; dispatch +refuses a grant that exposes the protected coordinator probes. Do not use a +protected coordinator directory as a shared cache root. + +`smoke stim` and `smoke control` verify real filesystem denials and the Codex +skill profile without running a model task. Timed dispatch repeats the same +checks before starting the clock and records the policy digest. This is a +macOS benchmark-data boundary, not a security sandbox for hostile code or +already-running native services. Unsupported hosts fail closed. + ## Run a cell Prepare the platform golden, then dispatch, collect, and clean one cell: diff --git a/scripts/agent-benchmark/driver.mjs b/scripts/agent-benchmark/driver.mjs index 78a80356..19fddeb9 100644 --- a/scripts/agent-benchmark/driver.mjs +++ b/scripts/agent-benchmark/driver.mjs @@ -34,6 +34,7 @@ import { matchesExpectedIosSimulator, } from './watch-app-selection.mjs'; import { completedCleanupRecord, durableRunRecord } from './run-record.mjs'; +import { isolatedRunnerInvocation, prepareRunnerIsolation, runnerIsolationPolicy } from './runner-isolation.mjs'; import { agentDeviceIsolationInvalidReasons, benchmarkSetupInvalidReasons, @@ -892,7 +893,7 @@ function promptFor(arm, variant, runId, runDir, crash = null, requestedPlatform const targetFlag = platform === 'ios' ? '--udid' : '--serial'; const deviceProof = ` After the app launches, you MUST use the agent-device skill and CLI. Codex does not forward the coordinator's agent-device environment into shell tools, so every agent-device command below includes the required prefix. Never run a bare \`agent-device\` command. Read the exact run ${targetDescription} from the launch output. Handle any Expo onboarding shown and navigate to the Settings tab using semantic refs or labels between steps 2 and 3 below. Do not stop or restart the agent-device daemon; report a failure if the isolated session refuses to open. The explicit state and session assignments and device identifier prevent cross-run ownership.`; const proofProtocol = `\n\nFINAL PROOF PROTOCOL: The proof directory already exists. For each numbered shell command below, send the displayed line alone as the entire Bash \`command\` string. Do not prepend \`mkdir\`, append \`ls\`, combine it with another command, use redirection, or wrap it in a script or interactive shell. Replace only the angle-bracketed value in step 1.\n\n1. \`${agentDevicePrefix} open com.appandflow.trailhead --foreground --platform ${platform} ${targetFlag} \`\n2. \`${agentDevicePrefix} record start ${recordingScratch} --scope device --quality high --hide-touches\`\n3. \`${agentDevicePrefix} wait text ${JSON.stringify(expected)}\`\n4. \`${agentDevicePrefix} screenshot ${screenshotScratch}\`\n5. \`cp ${screenshotScratch} ${screenshot}\`\n6. \`${agentDevicePrefix} record stop\`\n7. \`cp ${recordingScratch} ${recording}\`\n8. \`${agentDevicePrefix} close\`\n\nDo not claim completion before all eight commands succeed in order, the wait finds the expected text, recording stop reports the saved video, and the copied screenshot and recording exist.`; - const suffix = ` Stay in this turn until the Settings screenshot is saved; do not stop to await a background notification. Do not use subagents. Do not read or write outside the fixture checkout, the run worktree, ${runDir}, ${screenshotScratch}, and ${recordingScratch}. Report the run worktree and screenshot paths, then stop; the coordinator will verify and clean up.${proofProtocol}`; + const suffix = ` Stay in this turn until the Settings screenshot is saved; do not stop to await a background notification. Do not use subagents. Work only in the fixture checkout, run worktree, and the current run's proof, temporary, runtime, and tool-state paths. Coordinator configuration, golden caches, other worktrees, and other runs' file contents are protected by the runner filesystem policy. Parent directory listings are permitted; do not try to bypass a denied read or use another process or service to access protected files. Report the run worktree and screenshot paths, then stop; the coordinator will verify and clean up.${proofProtocol}`; if (variant === launchCrashVariant) { const launch = arm === 'stim' @@ -1004,9 +1005,39 @@ function makeRunnerHome(runDir, arm) { return { codexHome }; } -function verifyRunnerProfile(codexHome, env, arm, runDir) { +function prepareRunIsolation(runId, runDir, env, arm, crash = null, claudeGuidance = null) { + const runTmp = join(runDir, 'tmp'); + return prepareRunnerIsolation({ + policy: runnerIsolationPolicy({ + protectedRoots: [root, worktreeParent], + readPaths: [ + crash?.fixtureCheckout ?? main, + allowedBin, + ...(arm === 'stim' ? [stimBin, join(root, 'runtime'), stimPackage] : []), + ...(claudeGuidance ? [claudeGuidance.path] : []), + ], + writePaths: [ + join(worktreeParent, arm === 'stim' ? `bench-${runId}` : runId), + join(runDir, 'runner-home'), + join(runDir, 'shell-home'), + join(runDir, 'proof'), + runTmp, + agentDeviceState, + ...[env.STIM_HOME, env.GRADLE_USER_HOME, env.ANDROID_AVD_HOME].filter(Boolean), + ], + }), + profilePath: join(runDir, 'runner-isolation.sb'), + probeRoots: [root, golden, results], + probeParent: runTmp, + execute: (file, args) => run(file, args, { cwd: main, env }), + }); +} + +function verifyRunnerProfile(codexHome, env, arm, runDir, isolation) { const path = join(runDir, 'prompt-input.json'); - const output = run(executablePath(codexBin), ['debug', 'prompt-input', 'profile smoke'], { + const invocation = { command: executablePath(codexBin), args: ['debug', 'prompt-input', 'profile smoke'] }; + const checked = isolatedRunnerInvocation(isolation, invocation.command, invocation.args); + const output = run(checked.command, checked.args, { cwd: main, env, timeout: 30_000, @@ -1045,7 +1076,8 @@ function smoke(arm) { }; if (arm === 'stim') env.STIM_POOL_IOS_PARKED_MAX = '1'; env.BENCH_STIM_HOME = env.STIM_HOME; - const profile = verifyRunnerProfile(codexHome, env, arm, runDir); + const isolation = prepareRunIsolation(runId, runDir, env, arm); + const profile = verifyRunnerProfile(codexHome, env, arm, runDir, isolation); const resolvedStim = run('/bin/sh', ['-c', 'command -v stim || true'], { cwd: main, env, @@ -1057,7 +1089,7 @@ function smoke(arm) { throw new Error(`Stim profile resolved unexpected binary: ${resolvedStim}`); } const stimStatus = arm === 'stim' ? JSON.parse(run('stim', ['status', '--json'], { cwd: main, env })) : null; - const smokeRecord = { checkedAt: new Date().toISOString(), arm, profile, resolvedStim, stimStatus }; + const smokeRecord = { checkedAt: new Date().toISOString(), arm, profile, isolation, resolvedStim, stimStatus }; writeFileSync(join(runDir, 'smoke.json'), `${JSON.stringify(smokeRecord, null, 2)}\n`); process.stdout.write(`${JSON.stringify(smokeRecord, null, 2)}\n`); } @@ -1071,25 +1103,29 @@ async function runnerSmoke(arm) { const env = { ...cleanRubyEnvironment(process.env), CODEX_HOME: codexHome, + STIM_HOME: join(runDir, 'stim-home'), + TMPDIR: join(runDir, 'tmp'), PATH: `${arm === 'stim' ? `${stimBin}:` : ''}${allowedBin}:/usr/bin:/bin:/usr/sbin:/sbin`, }; + const isolation = prepareRunIsolation(`runner-smoke-${arm}`, runDir, env, arm); + const invocation = isolatedRunnerInvocation(isolation, executablePath(codexBin), [ + '--ask-for-approval', + 'never', + 'exec', + '--strict-config', + '--ignore-rules', + '--json', + '--model', + 'gpt-5.6-luna', + '--sandbox', + 'read-only', + '--cd', + main, + '-', + ]); const result = await spawnStamped( - executablePath(codexBin), - [ - '--ask-for-approval', - 'never', - 'exec', - '--strict-config', - '--ignore-rules', - '--json', - '--model', - 'gpt-5.6-luna', - '--sandbox', - 'read-only', - '--cd', - main, - '-', - ], + invocation.command, + invocation.args, join(runDir, 'events.jsonl'), { cwd: main, env, stdio: ['pipe', 'pipe', 'pipe'] }, 'Reply OK only.', @@ -1280,8 +1316,9 @@ async function dispatch(model, arm, variant, stage = 'pilot', requestedPlatform const prompt = promptFor(arm, variant, runId, runDir, crash, platform); writeFileSync(join(runDir, 'prompt.txt'), `${prompt}\n`); const shellProvenance = verifyRunnerShell(arm, env); - const profile = verifyRunnerProfile(codexHome, env, arm, runDir); const claudeGuidance = runnerKind === 'claude' ? writeClaudeGuidance(codexHome, arm, runDir) : null; + const isolation = prepareRunIsolation(runId, runDir, env, arm, crash, claudeGuidance); + const profile = verifyRunnerProfile(codexHome, env, arm, runDir, isolation); const agentDevice = prepareAgentDeviceRun(runId, platform, expectedParkedSimulator?.udid ?? null); const dispatchAt = new Date().toISOString(); const meta = { @@ -1301,7 +1338,7 @@ async function dispatch(model, arm, variant, stage = 'pilot', requestedPlatform preflight: preflightReport, expectedStimShellProvenance: arm === 'stim' ? expectedStimShellProvenance() : null, stimShellProvenance: shellProvenance, - profile: { ...profile, claudeGuidance }, + profile: { ...profile, claudeGuidance, isolation }, expectedBuildCache, expectedParkedSimulator, expectedStimDevice, @@ -1375,9 +1412,10 @@ async function dispatch(model, arm, variant, stage = 'pilot', requestedPlatform runnerCwd, '-', ]; + const isolatedRunner = isolatedRunnerInvocation(isolation, runnerCommand, runnerArgs); const runner = spawnStamped( - runnerCommand, - runnerArgs, + isolatedRunner.command, + isolatedRunner.args, join(runDir, 'events.jsonl'), { cwd: runnerCwd, env, stdio: ['pipe', 'pipe', 'pipe'] }, prompt, diff --git a/scripts/agent-benchmark/runner-isolation.mjs b/scripts/agent-benchmark/runner-isolation.mjs new file mode 100644 index 00000000..140868f7 --- /dev/null +++ b/scripts/agent-benchmark/runner-isolation.mjs @@ -0,0 +1,132 @@ +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'; + +const launcher = '/usr/bin/sandbox-exec'; + +function canonicalPath(path) { + if (!isAbsolute(path)) throw new Error('benchmark isolation paths must be absolute'); + try { + return realpathSync(path); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + return join(canonicalPath(dirname(path)), basename(path)); + } +} + +function contains(parent, path) { + const child = relative(parent, path); + return child === '' || (!child.startsWith('../') && child !== '..' && !isAbsolute(child)); +} + +function pathFilter(paths) { + return `(require-any ${paths.map((path) => `(subpath ${JSON.stringify(path)})`).join(' ')})`; +} + +export function runnerIsolationPolicy({ protectedRoots, readPaths, writePaths }) { + const roots = [...new Set(protectedRoots.map(canonicalPath))]; + if (roots.length === 0 || roots.includes('/')) throw new Error('benchmark isolation needs bounded protected roots'); + const writes = [...new Set(writePaths.map(canonicalPath))]; + const reads = [...new Set([...readPaths.map(canonicalPath), ...writes])]; + if (reads.some((path) => roots.some((root) => contains(path, root)))) { + throw new Error('benchmark isolation grant covers a protected root'); + } + return [ + '(version 1)', + '(allow default)', + `(deny file-read-data (require-all ${pathFilter(roots)} (require-not (vnode-type DIRECTORY))${reads.length ? ` (require-not ${pathFilter(reads)})` : ''}))`, + `(deny file-write* (require-all ${pathFilter(roots)}${writes.length ? ` (require-not ${pathFilter(writes)})` : ''}))`, + '', + ].join('\n'); +} + +const probeScript = ` +const fs = require('node:fs'); +const {execFileSync} = require('node:child_process'); +const {protectedFiles, allowedFile, alias} = JSON.parse(process.argv[1]); +function denied(fn) { + try { fn(); } catch (error) { + if (['EPERM', 'EACCES'].includes(error.code)) return; + throw error; + } + throw new Error('protected benchmark access was allowed'); +} +for (const file of [...protectedFiles, alias]) { + denied(() => fs.readFileSync(file)); + denied(() => fs.writeFileSync(file, 'forbidden-write')); +} +for (const file of protectedFiles) fs.readdirSync(require('node:path').dirname(file)); +fs.writeFileSync(allowedFile, 'allowed'); +if (fs.readFileSync(allowedFile, 'utf8') !== 'allowed') throw new Error('run access failed'); +for (const file of protectedFiles) { + try { + execFileSync('/bin/cat', [file], {stdio: 'pipe'}); + throw new Error('child read protected benchmark data'); + } catch (error) { + if (error.status !== 1 || error.stdout.length) throw error; + } +} +process.stdout.write('benchmark-isolation-verified'); +`; + +export function prepareRunnerIsolation({ policy, profilePath, probeRoots, probeParent, execute }) { + if (process.platform !== 'darwin' || !existsSync(launcher)) { + throw new Error('benchmark isolation requires the verified macOS sandbox launcher'); + } + mkdirSync(probeParent, { recursive: true }); + const temporary = []; + try { + const protectedFiles = probeRoots.map((root) => { + const directory = mkdtempSync(join(root, '.isolation-probe-')); + temporary.push(directory); + const path = join(directory, 'private.txt'); + writeFileSync(path, 'private-benchmark-probe'); + return path; + }); + const directory = mkdtempSync(join(probeParent, '.isolation-probe-')); + temporary.push(directory); + const allowedFile = join(directory, 'allowed.txt'); + const alias = join(directory, 'alias.txt'); + symlinkSync(protectedFiles[0], alias); + const output = execute(launcher, [ + '-p', + policy, + process.execPath, + '-e', + probeScript, + JSON.stringify({ protectedFiles, allowedFile, alias }), + ]); + if (output.trim() !== 'benchmark-isolation-verified') throw new Error('benchmark isolation verification failed'); + writeFileSync(profilePath, policy); + return { + backend: 'macos-sandbox-exec', + verified: true, + profilePath: resolve(profilePath), + profileSha256: createHash('sha256').update(policy).digest('hex'), + directoryListings: 'allowed', + }; + } finally { + for (const path of temporary) rmSync(path, { recursive: true, force: true }); + } +} + +export function isolatedRunnerInvocation(isolation, command, args) { + const policy = readFileSync(isolation.profilePath, 'utf8'); + if ( + isolation.verified !== true || + isolation.backend !== 'macos-sandbox-exec' || + createHash('sha256').update(policy).digest('hex') !== isolation.profileSha256 + ) { + throw new Error('benchmark isolation policy changed after verification'); + } + return { command: launcher, args: ['-p', policy, command, ...args] }; +} diff --git a/scripts/agent-benchmark/runner-isolation.test.mjs b/scripts/agent-benchmark/runner-isolation.test.mjs new file mode 100644 index 00000000..e61717ab --- /dev/null +++ b/scripts/agent-benchmark/runner-isolation.test.mjs @@ -0,0 +1,134 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { isolatedRunnerInvocation, prepareRunnerIsolation, runnerIsolationPolicy } from './runner-isolation.mjs'; + +const roots = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'stim-runner-isolation-')); + roots.push(root); + const golden = join(root, 'golden'); + const results = join(root, 'results'); + const run = join(results, 'current'); + const tools = join(root, 'tools with "quotes"'); + for (const path of [golden, run, tools]) mkdirSync(path, { recursive: true }); + const privateFile = join(root, 'pins.env'); + writeFileSync(privateFile, 'private'); + writeFileSync(join(tools, 'version.txt'), 'fixture-tool'); + const policy = runnerIsolationPolicy({ protectedRoots: [root], readPaths: [tools], writePaths: [run] }); + return { root, golden, results, run, tools, privateFile, policy }; +} + +function prepare(paths, policy = paths.policy) { + return prepareRunnerIsolation({ + policy, + profilePath: join(paths.root, 'runner-isolation.sb'), + probeRoots: [paths.root, paths.golden, paths.results], + probeParent: paths.run, + execute: (file, args) => execFileSync(file, args, { encoding: 'utf8', timeout: 10_000, stdio: 'pipe' }), + }); +} + +describe('benchmark runner filesystem isolation', () => { + it.skipIf(process.platform !== 'darwin')( + 'allows Git to create the exact future worktree without exposing siblings', + () => { + const paths = fixture(); + const repository = mkdtempSync(join(tmpdir(), 'stim-isolation-git-')); + roots.push(repository); + const execute = (args) => execFileSync('git', args, { cwd: repository, encoding: 'utf8', stdio: 'pipe' }); + execute(['init']); + execute([ + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.invalid', + 'commit', + '--allow-empty', + '-m', + 'fixture', + ]); + const parent = join(paths.root, 'worktrees'); + mkdirSync(parent); + const target = join(parent, 'current'); + const policy = runnerIsolationPolicy({ + protectedRoots: [paths.root, parent], + readPaths: [paths.tools], + writePaths: [paths.run, target], + }); + const isolation = prepare(paths, policy); + const invocation = isolatedRunnerInvocation(isolation, '/usr/bin/git', [ + 'worktree', + 'add', + '--detach', + target, + 'HEAD', + ]); + execFileSync(invocation.command, invocation.args, { cwd: repository, stdio: 'pipe' }); + expect(execute(['-C', target, 'rev-parse', 'HEAD'])).toBe(execute(['rev-parse', 'HEAD'])); + execute(['worktree', 'remove', target]); + }, + ); + it('rejects relative paths and grants that would expose a protected root, including symlink aliases', () => { + const paths = fixture(); + const alias = join(paths.root, 'alias'); + symlinkSync(paths.root, alias); + for (const grant of [paths.root, tmpdir(), alias]) { + expect(() => runnerIsolationPolicy({ protectedRoots: [paths.root], readPaths: [grant], writePaths: [] })).toThrow( + 'grant covers a protected root', + ); + } + expect(() => runnerIsolationPolicy({ protectedRoots: ['relative'], readPaths: [], writePaths: [] })).toThrow( + 'must be absolute', + ); + expect(() => runnerIsolationPolicy({ protectedRoots: [], readPaths: [], writePaths: [] })).toThrow('bounded'); + }); + + it.skipIf(process.platform !== 'darwin')( + 'denies coordinator and sibling contents through native and child reads while allowing run/tool access', + () => { + const paths = fixture(); + const isolation = prepare(paths); + expect(isolation.verified).toBe(true); + const invocation = isolatedRunnerInvocation(isolation, process.execPath, [ + '-e', + `const fs=require('node:fs'); + const [tool, secret, run] = process.argv.slice(1); + if(fs.readFileSync(tool,'utf8')!=='fixture-tool')throw Error('tool unreadable'); + try{fs.writeFileSync(tool,'changed');throw Error('tool writable');}catch(e){if(e.code!=='EPERM')throw e;} + try{fs.readFileSync(secret);throw Error('secret readable');}catch(e){if(e.code!=='EPERM')throw e;} + fs.writeFileSync(run,'output'); + process.stdout.write('ok');`, + join(paths.tools, 'version.txt'), + paths.privateFile, + join(paths.run, 'proof.txt'), + ]); + expect(execFileSync(invocation.command, invocation.args, { encoding: 'utf8' })).toBe('ok'); + expect(readFileSync(paths.privateFile, 'utf8')).toBe('private'); + expect(readFileSync(join(paths.run, 'proof.txt'), 'utf8')).toBe('output'); + writeFileSync(isolation.profilePath, '(version 1) (allow default)'); + expect(() => isolatedRunnerInvocation(isolation, '/bin/true', [])).toThrow('changed after verification'); + }, + ); + + it.skipIf(process.platform !== 'darwin')( + 'refuses dispatch when a grant exposes golden or sibling result data', + () => { + const paths = fixture(); + for (const grant of [paths.golden, paths.results]) { + const policy = runnerIsolationPolicy({ + protectedRoots: [paths.root], + readPaths: [paths.tools, grant], + writePaths: [paths.run], + }); + expect(() => prepare(paths, policy)).toThrow('protected benchmark access was allowed'); + } + }, + ); +}); From ecfdadd581cc4564fbce4f8beebe72c47bcb722a Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 6 Sep 2026 20:44:36 -0400 Subject: [PATCH 2/2] fix: reject private benchmark cache grants --- scripts/agent-benchmark/README.md | 7 ++-- scripts/agent-benchmark/driver.mjs | 26 ++++++++------- scripts/agent-benchmark/runner-isolation.mjs | 27 ++++++++++++++-- .../agent-benchmark/runner-isolation.test.mjs | 32 +++++++++++++++++++ 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/scripts/agent-benchmark/README.md b/scripts/agent-benchmark/README.md index 5671488f..5ffa8914 100644 --- a/scripts/agent-benchmark/README.md +++ b/scripts/agent-benchmark/README.md @@ -95,9 +95,10 @@ run-scoped policy. Configuration, golden files, coordinator evidence and sibling worktrees/results cannot be read or written by the runner process tree. Parent-directory listing is permitted. The current worktree, proof, temporary files, runner home, selected tool runtime and configured shared Gradle/AVD/device -state remain accessible. Keep those shared paths narrowly scoped; dispatch -refuses a grant that exposes the protected coordinator probes. Do not use a -protected coordinator directory as a shared cache root. +state remain accessible. Configured tool/cache paths cannot overlap golden, +results, coordinator state or worktree directories, including through symlinks. +Only the coordinator's selected run paths receive scoped exceptions. Dispatch +also refuses a policy that exposes the protected coordinator probes. `smoke stim` and `smoke control` verify real filesystem denials and the Codex skill profile without running a model task. Timed dispatch repeats the same diff --git a/scripts/agent-benchmark/driver.mjs b/scripts/agent-benchmark/driver.mjs index 19fddeb9..d02a38e9 100644 --- a/scripts/agent-benchmark/driver.mjs +++ b/scripts/agent-benchmark/driver.mjs @@ -1010,21 +1010,25 @@ function prepareRunIsolation(runId, runDir, env, arm, crash = null, claudeGuidan return prepareRunnerIsolation({ policy: runnerIsolationPolicy({ protectedRoots: [root, worktreeParent], + reservedRoots: [golden, results, state, worktreeParent], readPaths: [ - crash?.fixtureCheckout ?? main, + ...(!crash ? [main] : []), allowedBin, ...(arm === 'stim' ? [stimBin, join(root, 'runtime'), stimPackage] : []), - ...(claudeGuidance ? [claudeGuidance.path] : []), - ], - writePaths: [ - join(worktreeParent, arm === 'stim' ? `bench-${runId}` : runId), - join(runDir, 'runner-home'), - join(runDir, 'shell-home'), - join(runDir, 'proof'), - runTmp, - agentDeviceState, - ...[env.STIM_HOME, env.GRADLE_USER_HOME, env.ANDROID_AVD_HOME].filter(Boolean), ], + writePaths: [env.GRADLE_USER_HOME, env.ANDROID_AVD_HOME].filter(Boolean), + scopedAccess: { + readPaths: [crash?.fixtureCheckout, claudeGuidance?.path].filter(Boolean), + writePaths: [ + join(worktreeParent, arm === 'stim' ? `bench-${runId}` : runId), + join(runDir, 'runner-home'), + join(runDir, 'shell-home'), + join(runDir, 'proof'), + runTmp, + agentDeviceState, + env.STIM_HOME, + ], + }, }), profilePath: join(runDir, 'runner-isolation.sb'), probeRoots: [root, golden, results], diff --git a/scripts/agent-benchmark/runner-isolation.mjs b/scripts/agent-benchmark/runner-isolation.mjs index 140868f7..d3d8aef2 100644 --- a/scripts/agent-benchmark/runner-isolation.mjs +++ b/scripts/agent-benchmark/runner-isolation.mjs @@ -32,11 +32,32 @@ function pathFilter(paths) { return `(require-any ${paths.map((path) => `(subpath ${JSON.stringify(path)})`).join(' ')})`; } -export function runnerIsolationPolicy({ protectedRoots, readPaths, writePaths }) { +export function runnerIsolationPolicy({ + protectedRoots, + reservedRoots = [], + readPaths, + writePaths, + scopedAccess = {}, +}) { const roots = [...new Set(protectedRoots.map(canonicalPath))]; if (roots.length === 0 || roots.includes('/')) throw new Error('benchmark isolation needs bounded protected roots'); - const writes = [...new Set(writePaths.map(canonicalPath))]; - const reads = [...new Set([...readPaths.map(canonicalPath), ...writes])]; + const reserved = reservedRoots.map(canonicalPath); + const configuredReads = readPaths.map(canonicalPath); + const configuredWrites = writePaths.map(canonicalPath); + if ( + [...configuredReads, ...configuredWrites].some((path) => + reserved.some((root) => contains(path, root) || contains(root, path)), + ) + ) { + throw new Error('benchmark isolation configured grant overlaps a reserved directory'); + } + const scopedReads = (scopedAccess.readPaths ?? []).map(canonicalPath); + const scopedWrites = (scopedAccess.writePaths ?? []).map(canonicalPath); + if ([...scopedReads, ...scopedWrites].some((path) => reserved.some((root) => contains(path, root)))) { + throw new Error('benchmark isolation scoped grant covers a reserved directory'); + } + const writes = [...new Set([...configuredWrites, ...scopedWrites])]; + const reads = [...new Set([...configuredReads, ...scopedReads, ...writes])]; if (reads.some((path) => roots.some((root) => contains(path, root)))) { throw new Error('benchmark isolation grant covers a protected root'); } diff --git a/scripts/agent-benchmark/runner-isolation.test.mjs b/scripts/agent-benchmark/runner-isolation.test.mjs index e61717ab..5d2fdf7a 100644 --- a/scripts/agent-benchmark/runner-isolation.test.mjs +++ b/scripts/agent-benchmark/runner-isolation.test.mjs @@ -36,6 +36,38 @@ function prepare(paths, policy = paths.policy) { } describe('benchmark runner filesystem isolation', () => { + it('rejects configured grants inside reserved data, including future paths and aliases, but allows scoped run paths', () => { + const paths = fixture(); + const worktrees = join(paths.root, 'worktrees'); + mkdirSync(worktrees); + const alias = join(paths.root, 'golden-alias'); + symlinkSync(paths.golden, alias); + const options = { + protectedRoots: [paths.root, worktrees], + reservedRoots: [paths.golden, paths.results, worktrees], + readPaths: [paths.tools], + writePaths: [], + scopedAccess: { writePaths: [paths.run, join(worktrees, 'current')] }, + }; + expect(() => runnerIsolationPolicy(options)).not.toThrow(); + for (const grant of [ + paths.root, + paths.golden, + join(paths.golden, 'android', 'stim-home'), + join(alias, 'android'), + join(paths.results, 'pilot', 'previous-run'), + join(worktrees, 'previous-run'), + ]) { + for (const key of ['readPaths', 'writePaths']) { + expect(() => runnerIsolationPolicy({ ...options, [key]: [grant] })).toThrow( + 'configured grant overlaps a reserved directory', + ); + } + } + expect(() => runnerIsolationPolicy({ ...options, scopedAccess: { writePaths: [paths.results] } })).toThrow( + 'scoped grant covers a reserved directory', + ); + }); it.skipIf(process.platform !== 'darwin')( 'allows Git to create the exact future worktree without exposing siblings', () => {