From ccbb0db2bd3a00429cc032c93dfa2058ef0360f1 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:41:08 +0000 Subject: [PATCH] test(runtime): cover command and capture boundaries --- CHANGELOG.md | 3 + apps/headless/Tests/agent-runtime.test.mjs | 123 +++++++++++++++++++++ docs/roadmap/improvements-backlog.md | 6 +- 3 files changed, 130 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3de6726..522e4bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ Cutting that release is tracked in diagnostics, flow, network emulation, reporting, and local command paths. - Core regression coverage for bounded artifact reads, non-regular artifacts, non-replayable fill values in flows, and the ffmpeg visual-difference path. +- Agent-runtime regression coverage for interaction commands, page state, + tours, screenshot-plan bounds and deduplication, budget fallback, unsafe + links, and stale element references. - Progressive context pruning: `inspect --context summary|outline|text|actions|full` with `--task` ranking, `--within @rN` scoping, and `--limit` / `--budget` / `--depth` bounds. Every focused response reports `contextStats` and `omitted`. diff --git a/apps/headless/Tests/agent-runtime.test.mjs b/apps/headless/Tests/agent-runtime.test.mjs index efc67e1..bdc57fd 100644 --- a/apps/headless/Tests/agent-runtime.test.mjs +++ b/apps/headless/Tests/agent-runtime.test.mjs @@ -129,6 +129,128 @@ const fresh = agent.snapshot(false, false, {context: 'actions', limit: 5}); assert(fresh.elements.length > 0, 'actions context should return executable controls'); assert.equal(agent.click({target: fresh.elements[0].ref}).clicked, fresh.elements[0].ref); +// Exercise the command surface against controls with unique semantic names. +const controls = window.document.createElement('section'); +controls.innerHTML = ` + + + Unsafe +
Read only
+`; +window.document.body.prepend(controls); +const button = controls.querySelector('button'); +const input = controls.querySelector('input'); +let clicks = 0; +let inputs = 0; +let changes = 0; +const pressed = []; +button.addEventListener('click', () => { clicks += 1; }); +input.addEventListener('input', () => { inputs += 1; }); +input.addEventListener('change', () => { changes += 1; }); +input.addEventListener('keydown', event => pressed.push(`down:${event.key}`)); +input.addEventListener('keyup', event => pressed.push(`up:${event.key}`)); + +const clicked = agent.click({role: 'button', name: 'Runtime action'}); +assert.match(clicked.clicked, /^@e\d+$/); +assert.equal(clicks, 1, 'click should dispatch exactly once'); +const filled = agent.fill({role: 'textbox', name: 'Runtime input', value: 'private value'}); +assert.equal(filled.valueLength, 13); +assert.equal(filled.value, undefined, 'fill responses must not echo values'); +assert.equal(input.value, 'private value'); +assert.equal(inputs, 1); +assert.equal(changes, 1); +assert.equal(agent.press('A').pressed, 'A'); +assert.deepEqual(pressed, ['down:A', 'up:A']); +assert.throws( + () => agent.fill({role: 'button', name: 'Read only runtime control', value: 'no'}), + /NOT_EDITABLE/, +); +assert.throws( + () => agent.click({role: 'link', name: 'Unsafe runtime link'}), + /UNSAFE_NAVIGATION:javascript:/, +); + +window.scrollY = 0; +const downward = agent.scroll({direction: 'down', amount: 300}); +assert.equal(downward.direction, 'down'); +assert.equal(downward.amount, 300); +assert.equal(window.scrollY, 300); +agent.scroll({direction: 'up', amount: 125}); +assert.equal(window.scrollY, 175); +agent.scroll({direction: 'top'}); +assert.equal(window.scrollY, 0); +agent.scroll({direction: 'bottom'}); +assert.equal(window.scrollY, 32000); + +const pageState = agent.state(); +assert.equal(pageState.url, 'http://127.0.0.1:41739/large-document'); +assert.equal(pageState.contentHeight, 32000); +assert.equal(pageState.runningAnimations, 0); +assert(pageState.text.includes('Run')); +assert(pageState.text.length <= 30000, 'state text must stay bounded'); + +const stationaryTour = await agent.tour({fullPage: false}); +assert.equal(stationaryTour.start, stationaryTour.end); +assert.equal(stationaryTour.durationMs, 0); +Object.defineProperty(window.document.documentElement, 'scrollHeight', { + value: 1000, + configurable: true, +}); +const fullTour = await agent.tour({fullPage: true, pace: 5000}); +assert.equal(fullTour.start, 0); +assert.equal(fullTour.end, 240); +assert.equal(fullTour.durationMs, 500); +assert.equal(window.scrollY, 240); + +// A very tall page must produce an explicit, end-anchored 80-point cap. +Object.defineProperty(window.document.documentElement, 'scrollHeight', { + value: 100000, + configurable: true, +}); +window.scrollY = 321; +const viewportPlan = agent.screenshotPlan({mode: 'viewport'}); +assert.equal(viewportPlan.initialY, 321); +assert.equal(viewportPlan.points.length, 80); +assert.equal(viewportPlan.truncated, true); +assert(viewportPlan.totalPoints > viewportPlan.points.length); +assert.equal(viewportPlan.points.at(-1).y, 100000 - window.innerHeight); + +// Section plans deduplicate points within 96 px and apply the same hard cap. +const sectionRoot = window.document.createElement('main'); +for (let index = 0; index < 100; index += 1) { + const heading = window.document.createElement('h2'); + heading.textContent = `Runtime section ${index + 1}`; + heading.getBoundingClientRect = () => ({ + x: 24, + y: index * 200, + top: index * 200, + left: 24, + right: 824, + bottom: index * 200 + 48, + width: 800, + height: 48, + }); + sectionRoot.append(heading); +} +window.document.body.append(sectionRoot); +window.scrollY = 0; +const sectionPlan = agent.screenshotPlan({mode: 'section'}); +assert.equal(sectionPlan.points.length, 80); +assert.equal(sectionPlan.truncated, true); +for (let index = 1; index < sectionPlan.points.length; index += 1) { + assert( + sectionPlan.points[index].y - sectionPlan.points[index - 1].y > 96, + 'section capture points within 96 px should be deduplicated', + ); +} + +// Once result arrays are exhausted, budget pruning must fall back to chopping +// page text instead of returning an oversized response. +const budgetedText = agent.snapshot(false, true, {context: 'full', limit: 1, budget: 256}); +assert(budgetedText.contextStats.encodedBytes <= 1024); +assert(budgetedText.text.length < 30000, 'text fallback should be shortened to fit the budget'); +assert.equal(budgetedText.contextStats.budgetApplied, true); + console.log(JSON.stringify({ selectedRegion: targetRegion.ref, full: full.contextStats, @@ -136,4 +258,5 @@ console.log(JSON.stringify({ outline: outline.contextStats, scopedText: scopedText.contextStats, scopedActions: scopedActions.contextStats, + budgetedText: budgetedText.contextStats, })); diff --git a/docs/roadmap/improvements-backlog.md b/docs/roadmap/improvements-backlog.md index de91cc7..41615a6 100644 --- a/docs/roadmap/improvements-backlog.md +++ b/docs/roadmap/improvements-backlog.md @@ -281,8 +281,10 @@ which currently passes for the wrong reason. command matrix is covered, and the unexpected-field test first proves its current-version control request decodes before adding the forbidden field. Artifact read boundaries, non-regular-file rejection, flow replay safety, and -visual-comparison invocation are now covered as well. Recording, runtime, -transport, JavaScript-runtime, and host-specific gaps remain. +visual-comparison invocation are now covered as well. Runtime coverage now +locks click/fill/press/scroll/state/tour behavior, unsafe-link rejection, +screenshot-plan caps and 96 px deduplication, budget text fallback, and stale +element-reference errors. Recording, transport, and host-specific gaps remain. **D3. Web CI:** ~~`next build` + eslint on PR (site can break invisibly today).~~ **Done** — the `web` job in `ci.yml` runs `pnpm --filter @headless/web lint`