From 84fbf3f01749c101a757d2c25b7824d8ebf9a556 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:49:01 +0000 Subject: [PATCH] perf(runtime): install agent script once per document --- CHANGELOG.md | 3 + apps/headless/Dockerfile.linux | 2 + apps/headless/Host/AgentBridge.swift | 76 +- apps/headless/LinuxHost/BrowserProcess.swift | 74 +- apps/headless/Package.swift | 5 +- .../HeadlessProtocol/AgentRuntime.swift | 777 +----------------- .../Resources/AgentRuntime.js | 763 +++++++++++++++++ apps/headless/Tests/agent-runtime.test.mjs | 6 +- apps/headless/Tests/linux-e2e.sh | 13 +- apps/headless/build-linux.sh | 3 +- apps/headless/build.sh | 8 + apps/headless/docs/P2.md | 5 + apps/headless/install-linux.sh | 7 +- apps/headless/main.swift | 6 + docs/roadmap/architecture-decisions.md | 7 + docs/roadmap/improvements-backlog.md | 7 +- 16 files changed, 947 insertions(+), 815 deletions(-) create mode 100644 apps/headless/Sources/HeadlessProtocol/Resources/AgentRuntime.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 10882fd..11b26e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,9 @@ Cutting that release is tracked in ### Changed +- Both engines now install the compiled agent-runtime resource once per + document; Chromium caches and safely invalidates its isolated context instead + of resending the runtime for every command. - Context-budget pruning now measures each candidate once, removes oversized entries regardless of array position, and byte-budgets text fallback. - Removed stale screenshot and JSON conversion paths, honored per-operation diff --git a/apps/headless/Dockerfile.linux b/apps/headless/Dockerfile.linux index 1c10486..a0735eb 100644 --- a/apps/headless/Dockerfile.linux +++ b/apps/headless/Dockerfile.linux @@ -26,6 +26,7 @@ RUN apt-get update \ COPY --from=builder /src/.build/release/headless /usr/local/bin/headless COPY --from=builder /src/.build/release/headless-linux-host /usr/local/bin/headless-host COPY --from=builder /src/.build/release/headless-mcp /usr/local/bin/headless-mcp +COPY --from=builder /src/.build/release/Headless_HeadlessProtocol.resources /usr/local/bin/Headless_HeadlessProtocol.resources RUN chmod 0755 /usr/local/bin/headless /usr/local/bin/headless-host /usr/local/bin/headless-mcp USER headless ENV HEADLESS_HOST_EXECUTABLE=/usr/local/bin/headless-host @@ -44,6 +45,7 @@ COPY install-linux.sh /opt/headless/package/install-linux.sh COPY --from=builder /src/.build/release/headless /opt/headless/package/headless COPY --from=builder /src/.build/release/headless-linux-host /opt/headless/package/headless-host COPY --from=builder /src/.build/release/headless-mcp /opt/headless/package/headless-mcp +COPY --from=builder /src/.build/release/Headless_HeadlessProtocol.resources /opt/headless/package/Headless_HeadlessProtocol.resources RUN chmod 0755 /opt/headless/linux-e2e.sh \ /opt/headless/package/install-linux.sh \ /opt/headless/package/headless \ diff --git a/apps/headless/Host/AgentBridge.swift b/apps/headless/Host/AgentBridge.swift index 98e3431..4bd9497 100644 --- a/apps/headless/Host/AgentBridge.swift +++ b/apps/headless/Host/AgentBridge.swift @@ -4,7 +4,7 @@ import CoreFoundation import Foundation import WebKit -private let agentWorld = WKContentWorld.world(name: "HeadlessAgent") +let agentWorld = WKContentWorld.world(name: "HeadlessAgent") struct ScreenshotArtifactData { let data: Data @@ -378,37 +378,57 @@ extension BrowserWindowController { arguments: [String: Any] = [:], timeout: TimeInterval = 10 ) throws -> JSONValue { - let semaphore = DispatchSemaphore(value: 0) - let lock = NSLock() - var capturedResult: Result? - DispatchQueue.main.async { - self.webView.callAsyncJavaScript( - agentRuntimeJavaScript + "\n" + agentEvaluationBody(body), - arguments: arguments, - in: nil, - in: agentWorld - ) { result in - lock.lock() - capturedResult = result.map { $0 as Any } - lock.unlock() - semaphore.signal() + func evaluate(_ source: String) throws -> JSONValue { + let semaphore = DispatchSemaphore(value: 0) + let lock = NSLock() + var capturedResult: Result? + DispatchQueue.main.async { + self.webView.callAsyncJavaScript( + source, + arguments: arguments, + in: nil, + in: agentWorld + ) { result in + lock.lock() + capturedResult = result.map { $0 as Any } + lock.unlock() + semaphore.signal() + } + } + guard semaphore.wait(timeout: .now() + timeout) == .success else { + throw HostError(code: .timedOut, message: "Timed out while waiting for browser operation") + } + lock.lock() + let result = capturedResult + lock.unlock() + guard let result else { + throw HostError(code: .operationFailed, message: "Browser returned an invalid agent result") + } + do { + return try unwrapAgentEvaluationResult(JSONValue.foundationValue(result.get())) + } catch let error as HostError { + throw error + } catch { + throw HostError(code: .operationFailed, message: String(describing: error)) } } - guard semaphore.wait(timeout: .now() + timeout) == .success else { - throw HostError(code: .timedOut, message: "Timed out while waiting for browser operation") - } - lock.lock() - let result = capturedResult - lock.unlock() - guard let result else { - throw HostError(code: .operationFailed, message: "Browser returned an invalid agent result") + + let runtimeUnavailable = "Headless agent runtime is not installed in this document" + let guardedBody = """ + if (!globalThis.__headlessAgent) { + const error = new Error('\(runtimeUnavailable)'); + error.headlessCode = 'OPERATION_FAILED'; + throw error; } + \(body) + """ do { - return try unwrapAgentEvaluationResult(JSONValue.foundationValue(result.get())) - } catch let error as HostError { - throw error - } catch { - throw HostError(code: .operationFailed, message: String(describing: error)) + return try evaluate(agentEvaluationBody(guardedBody)) + } catch let error as HostError where error.code == .operationFailed && error.message == runtimeUnavailable { + // WKWebView can expose its initial about:blank document before + // document-start scripts run. Install once in that document, then + // subsequent calls use the cached isolated-world runtime. + return try evaluate(agentRuntimeJavaScript + "\n" + agentEvaluationBody(body)) } } } diff --git a/apps/headless/LinuxHost/BrowserProcess.swift b/apps/headless/LinuxHost/BrowserProcess.swift index 624e372..790be25 100644 --- a/apps/headless/LinuxHost/BrowserProcess.swift +++ b/apps/headless/LinuxHost/BrowserProcess.swift @@ -268,6 +268,7 @@ final class LinuxBrowserSession: @unchecked Sendable { private var mainFrameID: String? private var lastSafeURL: String? private var navigationRecoveryPending = false + private var isolatedContextID: Int? private let mockLock = NSLock() private var networkMocks: [NetworkMock] = [] @@ -278,6 +279,11 @@ final class LinuxBrowserSession: @unchecked Sendable { _ = try command("Page.enable") _ = try command("Runtime.enable") _ = try command("Log.enable") + _ = try command("Page.addScriptToEvaluateOnNewDocument", parameters: [ + "source": agentRuntimeJavaScript, + "worldName": "HeadlessAgent", + "runImmediately": true, + ]) _ = try command("Network.enable", parameters: [ "maxTotalBufferSize": 10_000_000, "maxResourceBufferSize": 1_000_000, @@ -690,6 +696,11 @@ final class LinuxBrowserSession: @unchecked Sendable { message: exception?["description"] as? String ?? details["text"] as? String ?? "JavaScript exception", url: details["url"] as? String ) + case "Runtime.executionContextsCleared": + clearIsolatedContext() + case "Runtime.executionContextDestroyed": + let identifier = (parameters["executionContextId"] as? NSNumber)?.intValue + clearIsolatedContext(matching: identifier) case "Log.entryAdded": let entry = parameters["entry"] as? [String: Any] ?? [:] diagnostics.append(kind: "console", level: entry["level"] as? String, @@ -736,6 +747,7 @@ final class LinuxBrowserSession: @unchecked Sendable { navigationLock.lock() if mainFrameID == nil { mainFrameID = frameID } guard mainFrameID == frameID else { navigationLock.unlock(); return } + if committed { isolatedContextID = nil } if let parsed = URL(string: url), agentMayNavigate(to: parsed) { if committed { lastSafeURL = url } navigationLock.unlock() @@ -788,21 +800,33 @@ final class LinuxBrowserSession: @unchecked Sendable { const args = __input.args; const key = __input.key; const options = __input.options || {}; - \(agentRuntimeJavaScript) \(agentEvaluationBody(body)) })() """ - let response = try command( - "Runtime.evaluate", - parameters: [ + func evaluateParameters() throws -> [String: Any] { + [ "expression": expression, "awaitPromise": true, "returnByValue": true, "userGesture": true, "contextId": try isolatedExecutionContextID(), - ], - timeoutMilliseconds: timeoutMilliseconds - ) + ] + } + let response: [String: Any] + do { + response = try command( + "Runtime.evaluate", + parameters: evaluateParameters(), + timeoutMilliseconds: timeoutMilliseconds + ) + } catch let error as CDPError where isTransientNavigationContext(error) { + clearIsolatedContext() + response = try command( + "Runtime.evaluate", + parameters: evaluateParameters(), + timeoutMilliseconds: timeoutMilliseconds + ) + } if let exception = response["exceptionDetails"] as? [String: Any] { throw CDPError.commandFailed(exception["text"] as? String ?? String(describing: exception)) } @@ -817,11 +841,14 @@ final class LinuxBrowserSession: @unchecked Sendable { ) } - /// Evaluate agent helpers in a fresh isolated world. Page scripts cannot - /// discover or replace `__headlessAgent`, while the world still has DOM - /// access. Recreating the world also avoids stale context IDs after a - /// cross-document navigation. + /// Reuse one isolated world per document. Page scripts cannot discover or + /// replace `__headlessAgent`, while the world still has DOM access. private func isolatedExecutionContextID() throws -> Int { + navigationLock.lock() + let cached = isolatedContextID + navigationLock.unlock() + if let cached { return cached } + let frameTree = try command("Page.getFrameTree") guard let tree = frameTree["frameTree"] as? [String: Any], let frame = tree["frame"] as? [String: Any], @@ -836,7 +863,30 @@ final class LinuxBrowserSession: @unchecked Sendable { guard let contextID = result["executionContextId"] as? NSNumber else { throw CDPError.invalidResponse("Page.createIsolatedWorld did not return an execution context") } - return contextID.intValue + let identifier = contextID.intValue + let installed = try command("Runtime.evaluate", parameters: [ + "expression": "typeof globalThis.__headlessAgent === 'object'", + "returnByValue": true, + "contextId": identifier, + ]) + let installedValue = (installed["result"] as? [String: Any])?["value"] as? Bool ?? false + if !installedValue { + _ = try command("Runtime.evaluate", parameters: [ + "expression": agentRuntimeJavaScript, + "returnByValue": true, + "contextId": identifier, + ]) + } + navigationLock.lock() + isolatedContextID = identifier + navigationLock.unlock() + return identifier + } + + private func clearIsolatedContext(matching identifier: Int? = nil) { + navigationLock.lock() + if identifier == nil || isolatedContextID == identifier { isolatedContextID = nil } + navigationLock.unlock() } private func targetArguments(_ parameters: [String: JSONValue]) throws -> [String: Any] { diff --git a/apps/headless/Package.swift b/apps/headless/Package.swift index 9185208..66278fd 100644 --- a/apps/headless/Package.swift +++ b/apps/headless/Package.swift @@ -15,7 +15,10 @@ let package = Package( .library(name: "HeadlessProtocol", targets: ["HeadlessProtocol"]), ], targets: [ - .target(name: "HeadlessProtocol"), + .target( + name: "HeadlessProtocol", + resources: [.process("Resources")] + ), .executableTarget( name: "HeadlessCLI", dependencies: ["HeadlessProtocol"] diff --git a/apps/headless/Sources/HeadlessProtocol/AgentRuntime.swift b/apps/headless/Sources/HeadlessProtocol/AgentRuntime.swift index f3c5560..4ea9316 100644 --- a/apps/headless/Sources/HeadlessProtocol/AgentRuntime.swift +++ b/apps/headless/Sources/HeadlessProtocol/AgentRuntime.swift @@ -1,765 +1,16 @@ -public let agentRuntimeJavaScript = #""" -if (!globalThis.__headlessAgent) { - globalThis.__headlessAgent = (() => { - let nextRef = 1; - let nextRegionRef = 1; - const refs = new WeakMap(); - const regionRefs = new WeakMap(); - let current = new Map(); - let currentRegions = new Map(); - // Every reference ever handed out, so a failed lookup can say whether the - // reference expired or was never issued at all. - const issuedRefs = new Set(); - const issuedRegionRefs = new Set(); - const maximumTrackedRegions = 256; - let lastMutation = performance.now(); - new MutationObserver(() => { lastMutation = performance.now(); }) - .observe(document, {subtree: true, childList: true, attributes: true, characterData: true}); +import Foundation - const normalize = value => String(value || '').replace(/\s+/g, ' ').trim(); - const clipped = (value, maximum) => normalize(value).slice(0, maximum); - const clippedURL = value => String(value || '').slice(0, 512); - const fail = (code, message) => { - const error = new Error(message); - error.headlessCode = code; - throw error; - }; - const blockedResourceExtensions = new Set([ - 'apk','app','bat','cmd','com','deb','dll','dmg','dylib','exe','img','iso','jar', - 'msi','msp','pif','pkg','ps1','psm1','scr','sh','so','vbe','vbs','wsf','zsh' - ]); - const cautionResourceExtensions = new Set(['7z','bz2','gz','rar','rpm','tar','tgz','xz','zip']); - const defaultStyleProperties = [ - 'display','visibility','opacity','position','z-index','overflow','box-sizing', - 'width','height','min-width','min-height','max-width','max-height', - 'margin-top','margin-right','margin-bottom','margin-left', - 'padding-top','padding-right','padding-bottom','padding-left', - 'color','background-color','border-top-width','border-top-style','border-top-color', - 'font-family','font-size','font-weight','line-height','text-align', - 'flex-direction','justify-content','align-items','gap','transform' - ]; - const resourceSafety = value => { - try { - const parsed = new URL(value, document.baseURI); - const extension = parsed.pathname.split('.').pop().toLowerCase(); - if (blockedResourceExtensions.has(extension)) return {level: 'blocked', extension}; - if (cautionResourceExtensions.has(extension)) return {level: 'caution', extension}; - } catch (_) { return {level: 'unknown'}; } - return {level: 'allowed'}; - }; - const visible = element => { - if (!(element instanceof Element) || !element.isConnected) return false; - const style = getComputedStyle(element); - if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false; - const rect = element.getBoundingClientRect(); - return rect.width > 0 && rect.height > 0; - }; - const role = element => { - const explicit = element.getAttribute('role'); - if (explicit) return explicit.split(/\s+/)[0].toLowerCase().slice(0, 64); - const tag = element.tagName.toLowerCase(); - if (tag === 'a' && element.hasAttribute('href')) return 'link'; - if (tag === 'button') return 'button'; - if (tag === 'textarea') return 'textbox'; - if (tag === 'select') return 'combobox'; - if (tag === 'img') return 'img'; - if (/^h[1-6]$/.test(tag)) return 'heading'; - if (tag === 'input') { - const type = (element.getAttribute('type') || 'text').toLowerCase(); - if (['button', 'submit', 'reset', 'image'].includes(type)) return 'button'; - if (type === 'checkbox') return 'checkbox'; - if (type === 'radio') return 'radio'; - if (type === 'range') return 'slider'; - return 'textbox'; - } - return tag; - }; - const actionHints = element => { - const hints = []; - const tag = element.tagName.toLowerCase(); - const elementRole = role(element); - if (tag === 'a' && element.hasAttribute('href')) hints.push('click'); - if (tag === 'button' || elementRole === 'button') hints.push('click'); - if (tag === 'summary' || elementRole === 'tab' || elementRole === 'menuitem') hints.push('click'); - // Only advertise verbs implemented by the public Headless protocol. - // Unsupported controls can still appear for context, but must not route - // an agent toward nonexistent select/upload/slide commands. - if (element instanceof HTMLTextAreaElement || element.isContentEditable) hints.push('fill'); - if (element instanceof HTMLInputElement) { - const type = (element.getAttribute('type') || 'text').toLowerCase(); - if (type === 'file') return hints; - else if (['checkbox', 'radio'].includes(type)) hints.push('click'); - else if (type === 'range') hints.push('fill'); - else if (['button', 'submit', 'reset', 'image'].includes(type)) hints.push('click'); - else hints.push('fill'); - } - if (element.tabIndex >= 0 && hints.length === 0) hints.push('click'); - return Array.from(new Set(hints)); - }; - const isInteractive = element => actionHints(element).length > 0 - || ['link','button','textbox','checkbox','radio','combobox','slider'].includes(role(element)); - const name = element => { - const labelledBy = element.getAttribute('aria-labelledby'); - if (labelledBy) { - const labelled = labelledBy.split(/\s+/).map(id => document.getElementById(id)?.innerText || '').join(' '); - if (normalize(labelled)) return clipped(labelled, 256); - } - if (normalize(element.getAttribute('aria-label'))) return clipped(element.getAttribute('aria-label'), 256); - if (element.labels?.length) return clipped(Array.from(element.labels).map(label => label.innerText).join(' '), 256); - if (normalize(element.getAttribute('alt'))) return clipped(element.getAttribute('alt'), 256); - if (normalize(element.getAttribute('placeholder'))) return clipped(element.getAttribute('placeholder'), 256); - if (normalize(element.getAttribute('title'))) return clipped(element.getAttribute('title'), 256); - if (normalize(element.value) && ['button', 'submit', 'reset'].includes((element.type || '').toLowerCase())) { - return clipped(element.value, 256); - } - return clipped(element.innerText || element.textContent, 256); - }; - const refFor = element => { - let ref = refs.get(element); - if (!ref) { ref = `@e${nextRef++}`; refs.set(element, ref); } - issuedRefs.add(ref); - current.set(ref, element); - return ref; - }; - const elementSelector = 'a[href],button,input,textarea,select,summary,[role],[contenteditable="true"],[tabindex],img,video,audio,canvas,svg'; - const regionSelector = 'main,nav,aside,header,footer,section,article,form,dialog,[role="main"],[role="navigation"],[role="region"],[role="dialog"],[role="form"],[role="search"],[role="complementary"],h1,h2,h3,h4,h5,h6'; - const headingLevel = element => { - const match = element?.tagName?.match(/^H([1-6])$/i); - return match ? Number(match[1]) : null; - }; - const scopeRoots = scope => { - if (!scope || scope === document || scope === document.documentElement) return [document.documentElement]; - const level = headingLevel(scope); - if (!level) return [scope]; - const roots = [scope]; - for (let sibling = scope.nextElementSibling; sibling; sibling = sibling.nextElementSibling) { - const siblingLevel = headingLevel(sibling); - if (siblingLevel && siblingLevel <= level) break; - roots.push(sibling); - } - return roots; - }; - const scopedQuery = (scope, selector) => { - const result = []; - const seen = new Set(); - for (const root of scopeRoots(scope)) { - if (!root) continue; - const add = element => { - if (!seen.has(element)) { seen.add(element); result.push(element); } - }; - if (root.matches?.(selector)) add(root); - for (const element of root.querySelectorAll?.(selector) || []) add(element); - } - return result; - }; - const candidates = (scope = document) => scopedQuery(scope, elementSelector).filter(visible); - const regionCandidates = (scope = document) => scopedQuery(scope, regionSelector).filter(visible); - const taskTerms = task => normalize(task).toLowerCase().split(/[^a-z0-9]+/).filter(term => term.length > 1).slice(0, 24); - const rankingText = element => [ - role(element), name(element), element.getAttribute('placeholder'), element.getAttribute('aria-label'), - element.getAttribute('title'), element.getAttribute('type'), element.getAttribute('name'), - element.getAttribute('id'), element.href - ].map(normalize).join(' ').toLowerCase(); - const rank = (element, terms, phrase) => { - const rect = element.getBoundingClientRect(); - const text = rankingText(element); - const hints = actionHints(element); - let score = 0; - const reasons = []; - if (hints.length > 0) { score += 50; reasons.push('interactive'); } - if (rect.top >= 0 && rect.top <= innerHeight && rect.left < innerWidth && rect.right > 0) { - score += 12; reasons.push('in viewport'); - } - if (phrase && text.includes(phrase)) { score += 35; reasons.push('task phrase match'); } - for (const term of terms) { - if (text.includes(term)) score += 14; - } - const termSet = new Set(terms); - const elementRole = role(element); - const inputType = element instanceof HTMLInputElement ? (element.getAttribute('type') || 'text').toLowerCase() : ''; - if (termSet.has('search')) { - if (inputType === 'search' || text.includes('search')) { score += 60; reasons.push('search control'); } - else if (hints.includes('fill')) score += 20; - } - if (termSet.has('login') || termSet.has('signin') || termSet.has('sign') || termSet.has('account')) { - if (inputType === 'password' || text.includes('password')) { score += 50; reasons.push('credential field'); } - if (hints.includes('fill')) score += 20; - if (elementRole === 'button' || elementRole === 'link') score += 12; - } - if (termSet.has('click') && (hints.includes('click') || elementRole === 'button' || elementRole === 'link')) score += 20; - score += Math.max(0, 8 - Math.floor(Math.max(0, rect.top) / 300)); - return {element, score, reasons: Array.from(new Set(reasons)).slice(0, 4)}; - }; - const rankedElements = (elements, task) => { - const terms = taskTerms(task); - if (terms.length === 0) return elements.map(element => ({element, score: 0, reasons: []})); - const phrase = normalize(task).toLowerCase().slice(0, 128); - return elements.map(element => rank(element, terms, phrase)) - .sort((left, right) => right.score - left.score); - }; - const regionRole = element => { - const explicit = element.getAttribute('role'); - if (explicit) return explicit.split(/\s+/)[0].toLowerCase().slice(0, 64); - const tag = element.tagName.toLowerCase(); - return ({main: 'main', nav: 'navigation', aside: 'complementary', header: 'banner', - footer: 'contentinfo', section: 'region', article: 'article', form: 'form', dialog: 'dialog'}[tag] - || (/^h[1-6]$/.test(tag) ? 'heading' : tag)); - }; - const regionName = element => { - const labelledBy = element.getAttribute('aria-labelledby'); - if (labelledBy) { - const labelled = labelledBy.split(/\s+/).map(id => document.getElementById(id)?.innerText || '').join(' '); - if (normalize(labelled)) return clipped(labelled, 160); - } - if (normalize(element.getAttribute('aria-label'))) return clipped(element.getAttribute('aria-label'), 160); - if (headingLevel(element)) return clipped(element.innerText || element.textContent, 160); - const heading = element.querySelector('h1,h2,h3,h4,h5,h6'); - if (heading && normalize(heading.innerText || heading.textContent)) { - return clipped(heading.innerText || heading.textContent, 160); - } - return clipped(element.getAttribute('title') || element.getAttribute('id') || regionRole(element), 160); - }; - const regionRefFor = element => { - let ref = regionRefs.get(element); - if (!ref) { ref = `@r${nextRegionRef++}`; regionRefs.set(element, ref); } - issuedRegionRefs.add(ref); - currentRegions.set(ref, element); - // Region references stay resolvable across inspections so a scoped - // `--within @rN` workflow keeps working, but this map holds elements - // strongly. Drop the oldest so a long-lived page cannot grow it without - // bound, and never drop the reference just issued. - while (currentRegions.size > maximumTrackedRegions) { - const oldest = currentRegions.keys().next().value; - if (oldest === undefined || oldest === ref) break; - currentRegions.delete(oldest); - } - return ref; - }; - const resolveRegion = reference => { - if (!reference) return document; - const element = currentRegions.get(reference); - if (!element) { - fail('REGION_NOT_FOUND', issuedRegionRefs.has(reference) - ? `REGION_NOT_FOUND:${reference} (expired: inspect again to refresh region references)` - : `REGION_NOT_FOUND:${reference} (unknown: no inspection has issued this reference)`); - } - if (!element.isConnected || !visible(element)) { - currentRegions.delete(reference); - fail('REGION_NOT_FOUND', `REGION_NOT_FOUND:${reference} (detached: the region is no longer visible on this page)`); - } - return element; - }; - const regionDepth = (element, scope) => { - if (headingLevel(element)) return Math.max(0, headingLevel(element) - 1); - let depth = 0; - for (let parent = element.parentElement; parent && parent !== scope && parent !== document.body; parent = parent.parentElement) { - if (parent.matches?.(regionSelector) && !headingLevel(parent)) depth += 1; - } - return depth; - }; - const rankRegion = (element, task) => { - const terms = taskTerms(task); - const phrase = normalize(task).toLowerCase().slice(0, 128); - const sample = normalize(`${regionRole(element)} ${regionName(element)} ${element.innerText || ''}`).toLowerCase().slice(0, 2000); - let score = 0; - const reasons = []; - if (phrase && sample.includes(phrase)) { score += 40; reasons.push('task phrase match'); } - for (const term of terms) if (sample.includes(term)) score += 14; - const rect = element.getBoundingClientRect(); - if (rect.top >= 0 && rect.top <= innerHeight) { score += 10; reasons.push('in viewport'); } - if (regionRole(element) === 'main') score += 4; - return {element, score, reasons: Array.from(new Set(reasons)).slice(0, 4)}; - }; - const describeRegion = (entry, scope, includeRelevance) => { - const element = entry.element || entry; - const rect = element.getBoundingClientRect(); - const actions = candidates(element).filter(isInteractive).length; - const text = normalize(scopeRoots(element).map(root => root.innerText || '').join(' ')); - const item = { - ref: regionRefFor(element), role: regionRole(element), name: regionName(element), - depth: regionDepth(element, scope), actions, textChars: text.length, - bounds: {x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height)} - }; - const level = headingLevel(element); - if (level) item.headingLevel = level; - if (includeRelevance) item.relevance = {score: Math.round(entry.score), reasons: entry.reasons}; - return item; - }; - const textCandidates = scope => { - const selector = 'h1,h2,h3,h4,h5,h6,p,li,dt,dd,blockquote,pre,code,caption,th,td'; - const seen = new Set(); - return scopedQuery(scope, selector).filter(visible).flatMap(element => { - const text = clipped(element.innerText || element.textContent, 600); - const key = text.toLowerCase(); - if (!text || seen.has(key)) return []; - seen.add(key); - return [{element, text}]; - }); - }; - const rankText = (entry, task) => { - const terms = taskTerms(task); - const phrase = normalize(task).toLowerCase().slice(0, 128); - const value = entry.text.toLowerCase(); - let score = 0; - const reasons = []; - if (phrase && value.includes(phrase)) { score += 40; reasons.push('task phrase match'); } - for (const term of terms) if (value.includes(term)) score += 16; - if (headingLevel(entry.element)) { score += 8; reasons.push('heading'); } - const rect = entry.element.getBoundingClientRect(); - if (rect.top >= 0 && rect.top <= innerHeight) { score += 6; reasons.push('in viewport'); } - return {...entry, score, reasons: Array.from(new Set(reasons)).slice(0, 4)}; - }; - const describeText = (entry, includeRelevance) => { - const item = {ref: refFor(entry.element), role: role(entry.element), text: entry.text}; - if (includeRelevance) item.relevance = {score: Math.round(entry.score), reasons: entry.reasons}; - return item; - }; - const describe = (entry, includeRelevance) => { - const element = entry.element || entry; - const rect = element.getBoundingClientRect(); - const item = { - ref: refFor(element), role: role(element), name: name(element), - actions: actionHints(element), - disabled: Boolean(element.disabled || element.getAttribute('aria-disabled') === 'true'), - bounds: {x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height)} - }; - if (element instanceof HTMLInputElement) item.inputType = (element.getAttribute('type') || 'text').toLowerCase().slice(0, 32); - if (includeRelevance) item.relevance = {score: Math.round(entry.score), reasons: entry.reasons}; - if ('value' in element && typeof element.value === 'string' && element.type !== 'password') item.value = element.value.slice(0, 500); - if (element.tagName === 'A') { - item.href = clippedURL(element.href); - item.resourceSafety = resourceSafety(element.href); - } - if (element.tagName === 'IMG') { - item.src = clippedURL(element.currentSrc || element.src); - item.media = { - kind: 'image', loaded: Boolean(element.complete && element.naturalWidth > 0), - width: element.naturalWidth || null, height: element.naturalHeight || null - }; - item.resourceSafety = resourceSafety(element.currentSrc || element.src); - } - if (element instanceof HTMLMediaElement) { - item.src = clippedURL(element.currentSrc || element.src || element.querySelector('source')?.src); - item.media = { - kind: element instanceof HTMLVideoElement ? 'video' : 'audio', - loaded: element.readyState >= HTMLMediaElement.HAVE_METADATA, - readyState: element.readyState, - paused: element.paused, - muted: element.muted, - currentTime: Number.isFinite(element.currentTime) ? element.currentTime : null, - duration: Number.isFinite(element.duration) ? element.duration : null, - width: element instanceof HTMLVideoElement ? element.videoWidth || null : null, - height: element instanceof HTMLVideoElement ? element.videoHeight || null : null - }; - item.resourceSafety = resourceSafety(element.currentSrc || element.src || element.querySelector('source')?.src); - if (element instanceof HTMLVideoElement && element.poster) item.poster = clippedURL(element.poster); - } - return item; - }; - const encodedBytes = value => new TextEncoder().encode(JSON.stringify(value)).length; - const pruneToBudget = (result, available, requestedBudget) => { - const budget = requestedBudget || null; - const arrays = ['regions', 'elements', 'snippets'].filter(key => Array.isArray(result[key])); - const refresh = () => { - const omitted = {}; - for (const key of arrays) omitted[key] = Math.max(0, (available[key] || 0) - result[key].length); - result.omitted = omitted; - result.truncated = Object.values(omitted).some(value => value > 0); - result.contextStats = { - encodedBytes: 0, - estimatedTokens: 0, - budget, - budgetApplied: Boolean(budget) - }; - const baseBytes = encodedBytes(result); - let bytes = baseBytes; - let estimatedTokens = Math.ceil(bytes / 4); - while (true) { - const measured = baseBytes + String(bytes).length - 1 + String(estimatedTokens).length - 1; - const measuredTokens = Math.ceil(measured / 4); - if (measured === bytes && measuredTokens === estimatedTokens) break; - bytes = measured; - estimatedTokens = measuredTokens; - } - result.contextStats.encodedBytes = bytes; - result.contextStats.estimatedTokens = estimatedTokens; - return bytes; - }; - let currentBytes = refresh(); - if (!budget) return result; +public let agentRuntimeJavaScript: String = { + if let resourceURL = Bundle.main.resourceURL? + .appendingPathComponent("Headless_HeadlessProtocol.bundle", isDirectory: true) + .appendingPathComponent("AgentRuntime.js", isDirectory: false), + let source = try? String(contentsOf: resourceURL, encoding: .utf8) { + return source + } - const maximumBytes = budget * 4; - if (currentBytes > maximumBytes) { - const candidates = arrays.flatMap(key => result[key].map((value, index) => ({ - key, index, bytes: encodedBytes(value) + (result[key].length > 1 ? 1 : 0) - }))).sort((left, right) => right.bytes - left.bytes); - const removed = new Map(arrays.map(key => [key, new Set()])); - // Reserve a small allowance for omitted-count digit growth. Each item - // is encoded once; removal then preserves the order of retained items. - const targetBytes = Math.max(0, maximumBytes - 32); - for (const candidate of candidates) { - if (currentBytes <= targetBytes) break; - removed.get(candidate.key).add(candidate.index); - currentBytes -= candidate.bytes; - } - for (const key of arrays) { - const indexes = removed.get(key); - if (indexes.size > 0) result[key] = result[key].filter((_, index) => !indexes.has(index)); - } - currentBytes = refresh(); - } - - if (currentBytes > maximumBytes && typeof result.text === 'string' && result.text.length > 0) { - const textBytes = Math.max(0, encodedBytes(result.text) - 2); - const targetTextBytes = Math.max(0, textBytes - (currentBytes - maximumBytes) - 32); - let lower = 0; - let upper = result.text.length; - while (lower < upper) { - const middle = Math.ceil((lower + upper) / 2); - const prefixBytes = Math.max(0, encodedBytes(result.text.slice(0, middle)) - 2); - if (prefixBytes <= targetTextBytes) lower = middle; - else upper = middle - 1; - } - result.text = result.text.slice(0, lower); - currentBytes = refresh(); - } - - // Metadata is deliberately bounded, but keep the budget fail-closed if - // its final digit widths ever outgrow the allowance above. - if (currentBytes > maximumBytes) { - for (const key of arrays) result[key] = []; - if (typeof result.text === 'string') result.text = ''; - refresh(); - } - return result; - }; - const snapshot = (interactiveOnly, includeText, options = {}) => { - current = new Map(); - const allowedContexts = new Set(['summary', 'outline', 'text', 'actions', 'full']); - const requestedContext = allowedContexts.has(options.context) ? options.context : 'full'; - const contextMode = interactiveOnly && requestedContext === 'full' ? 'actions' : requestedContext; - const task = clipped(options.task, 512); - const scope = resolveRegion(clipped(options.within, 16)); - const defaultLimits = {summary: 12, outline: 40, text: 12, actions: 20, full: 250}; - const defaultBudgets = {summary: 1200, outline: 1600, text: 1500, actions: 1200, full: null}; - const limit = Math.min(250, Math.max(1, Number(options.limit || defaultLimits[contextMode]))); - const budget = options.budget ? Math.min(16000, Math.max(256, Number(options.budget))) : defaultBudgets[contextMode]; - const depth = Math.min(8, Math.max(0, Number(options.depth ?? 3))); - const root = document.documentElement; - const result = { - url: String(location.href).slice(0, 8192), title: String(document.title).slice(0, 512), - contextMode, - viewport: {width: innerWidth, height: innerHeight, scrollX, scrollY, - contentWidth: root ? root.scrollWidth : 0, contentHeight: root ? root.scrollHeight : 0}, - untrustedContent: true - }; - if (task) result.task = task; - if (options.within) result.within = options.within; - const available = {}; - if (contextMode === 'actions' || contextMode === 'full' || contextMode === 'summary') { - let elements = candidates(scope); - if (contextMode !== 'full') elements = elements.filter(isInteractive); - const ranked = rankedElements(elements, task); - const maximum = contextMode === 'summary' ? Math.min(limit, 8) : limit; - result.elements = ranked.slice(0, maximum).map(entry => describe(entry, Boolean(task))); - available.elements = ranked.length; - } - if (contextMode === 'outline' || contextMode === 'summary') { - const ranked = regionCandidates(scope).map(element => rankRegion(element, task)) - .filter(entry => regionDepth(entry.element, scope) <= depth) - .sort((left, right) => task ? right.score - left.score : left.element.getBoundingClientRect().top - right.element.getBoundingClientRect().top); - result.regions = ranked.slice(0, limit).map(entry => describeRegion(entry, scope, Boolean(task))); - available.regions = ranked.length; - } - if (contextMode === 'text') { - const ranked = textCandidates(scope).map(entry => rankText(entry, task)) - .sort((left, right) => task ? right.score - left.score : left.element.getBoundingClientRect().top - right.element.getBoundingClientRect().top); - result.snippets = ranked.slice(0, limit).map(entry => describeText(entry, Boolean(task))); - available.snippets = ranked.length; - } - if (includeText) { - result.text = normalize(scopeRoots(scope).map(root => root.innerText || root.textContent || '').join(' ')).slice(0, 30000); - } - return pruneToBudget(result, available, budget); - }; - const resolve = target => { - const element = current.get(target); - // Element references describe the most recent inspection only. Saying so - // is the difference between an agent re-inspecting and an agent retrying - // the same dead reference. - if (!element) { - fail('ELEMENT_NOT_FOUND', issuedRefs.has(target) - ? `ELEMENT_NOT_FOUND:${target} (expired: element references come from the most recent inspection — inspect again to refresh)` - : `ELEMENT_NOT_FOUND:${target} (unknown: no inspection has issued this reference)`); - } - if (!element.isConnected) { - current.delete(target); - fail('ELEMENT_NOT_FOUND', `ELEMENT_NOT_FOUND:${target} (detached: the element is no longer in the page)`); - } - return element; - }; - const find = (wantedRole, wantedName) => { - const normalizedRole = normalize(wantedRole).toLowerCase(); - const normalizedName = normalize(wantedName).toLowerCase(); - const matches = candidates().filter(element => - (!normalizedRole || role(element) === normalizedRole) && - (!normalizedName || name(element).toLowerCase() === normalizedName) - ); - if (matches.length === 0) fail('ELEMENT_NOT_FOUND', `ELEMENT_NOT_FOUND:${wantedRole || ''}/${wantedName || ''}`); - if (matches.length > 1) throw new Error(`ELEMENT_AMBIGUOUS:${matches.length}`); - refFor(matches[0]); - return matches[0]; - }; - const target = args => args.target ? resolve(args.target) : find(args.role, args.name); - const rectangle = args => { - const element = target(args); - element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'}); - const rect = element.getBoundingClientRect(); - if (rect.width <= 0 || rect.height <= 0) throw new Error('ELEMENT_NOT_VISIBLE'); - return { - viewport: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}, - document: {x: rect.x + scrollX, y: rect.y + scrollY, width: rect.width, height: rect.height} - }; - }; - const styles = args => { - const element = target(args); - const computed = getComputedStyle(element); - const requested = Array.isArray(args.properties) && args.properties.length > 0 - ? args.properties : defaultStyleProperties; - const values = {}; - for (const property of requested.slice(0, 64)) { - const propertyName = String(property || '').slice(0, 128); - if (propertyName) values[propertyName] = String(computed.getPropertyValue(propertyName) || '').slice(0, 2048); - } - const rect = element.getBoundingClientRect(); - return { - ref: refFor(element), role: role(element), name: name(element), - box: {x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height)}, - styles: values - }; - }; - const storage = args => { - const includeValues = Boolean(args.includeValues); - const describe = (store, scope) => { - const entries = []; - for (let index = 0; index < store.length && index < 200; index++) { - const key = String(store.key(index) || '').slice(0, 512); - const value = String(store.getItem(key) || ''); - entries.push(includeValues - ? {key, value: value.slice(0, 16384), bytes: value.length} - : {key, bytes: value.length}); - } - return {scope, count: store.length, entries, truncated: store.length > entries.length}; - }; - const scope = args.scope || 'all'; - const stores = []; - try { - if (scope === 'local' || scope === 'all') stores.push(describe(localStorage, 'local')); - if (scope === 'session' || scope === 'all') stores.push(describe(sessionStorage, 'session')); - } catch (error) { - return {origin: String(location.origin), stores, error: String(error).slice(0, 512)}; - } - return {origin: String(location.origin).slice(0, 2048), stores}; - }; - const click = args => { - const element = target(args); - if (element instanceof HTMLAnchorElement && element.href) { - const destination = new URL(element.href, document.baseURI); - const scheme = destination.protocol.toLowerCase(); - if (!['http:', 'https:'].includes(scheme) || destination.username || destination.password) { - fail('UNSAFE_NAVIGATION', `UNSAFE_NAVIGATION:${scheme}`); - } - const safety = resourceSafety(destination.href); - if (safety.level === 'blocked') fail('UNSAFE_RESOURCE_TYPE', `UNSAFE_RESOURCE_TYPE:${safety.extension}`); - } - element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'}); - element.focus({preventScroll: true}); - element.click(); - return {clicked: refFor(element), role: role(element), name: name(element)}; - }; - const fill = args => { - const element = target(args); - if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || element.isContentEditable)) { - throw new Error('NOT_EDITABLE'); - } - element.focus({preventScroll: false}); - if (element.isContentEditable) { - element.textContent = args.value; - } else { - const prototype = element instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; - Object.getOwnPropertyDescriptor(prototype, 'value').set.call(element, args.value); - } - element.dispatchEvent(new InputEvent('input', {bubbles: true, inputType: 'insertText', data: args.value})); - element.dispatchEvent(new Event('change', {bubbles: true})); - return {filled: refFor(element), valueLength: String(args.value).length}; - }; - const press = key => { - const element = document.activeElement || document.body; - const options = {key, code: key, bubbles: true, cancelable: true}; - element.dispatchEvent(new KeyboardEvent('keydown', options)); - if (key === 'Enter') { - const form = element.form || element.closest?.('form'); - if (form?.requestSubmit) form.requestSubmit(); - else if (element.tagName === 'BUTTON') element.click(); - } else if (key === ' ' && element.tagName === 'BUTTON') { - element.click(); - } - element.dispatchEvent(new KeyboardEvent('keyup', options)); - return {pressed: key}; - }; - const scroll = args => { - const amount = Number(args.amount || Math.max(240, innerHeight * 0.8)); - if (args.direction === 'top') scrollTo({top: 0, behavior: 'smooth'}); - else if (args.direction === 'bottom') scrollTo({top: document.documentElement.scrollHeight, behavior: 'smooth'}); - else scrollBy({top: args.direction === 'up' ? -amount : amount, behavior: 'smooth'}); - return {direction: args.direction, amount}; - }; - const blockingAnimationCount = () => document.getAnimations().filter(animation => { - if (animation.playState !== 'running') return false; - const timing = animation.effect?.getComputedTiming?.(); - // Perpetual ambient animation must not prevent a loaded page from ever - // settling. Finite transitions still block until they finish. - return Number.isFinite(timing?.endTime); - }).length; - const state = () => { - const root = document.documentElement; - return { - url: String(location.href).slice(0, 8192), title: String(document.title).slice(0, 512), - // A page can be between documents immediately after navigation. Report - // it as loading so the host retries instead of dereferencing a missing - // root element and turning a normal navigation race into a failure. - readyState: root ? document.readyState : 'loading', - text: normalize(document.body?.innerText).slice(0, 30000), - runningAnimations: blockingAnimationCount(), - mutationQuietMs: Math.round(performance.now() - lastMutation), - scrollY, contentHeight: root ? root.scrollHeight : 0 - }; - }; - const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); - const tour = async args => { - const pace = Math.min(5000, Math.max(100, Number(args.pace || 800))); - if (args.fullPage === false) { - return {start: scrollY, end: scrollY, durationMs: 0}; - } - const maximum = Math.max(0, document.documentElement.scrollHeight - innerHeight); - scrollTo({top: 0, behavior: 'instant'}); - await sleep(250); - const desiredDuration = maximum / pace; - const duration = Math.min(60, Math.max(0.5, desiredDuration)); - const steps = Math.max(1, Math.ceil(duration * 20)); - for (let index = 1; index <= steps; index++) { - scrollTo({top: maximum * index / steps, behavior: 'instant'}); - await sleep(duration * 1000 / steps); - } - return {start: 0, end: maximum, durationMs: Math.round(duration * 1000)}; - }; - const capturePoint = (y, label, kind) => ({ - y: Math.round(Math.max(0, y)), - label: clipped(label || kind || 'viewport', 80), - kind: clipped(kind || 'viewport', 32) - }); - const dedupePoints = points => { - const maximum = Math.max(0, document.documentElement.scrollHeight - innerHeight); - const result = []; - let truncated = false; - for (const point of points) { - const normalizedY = Math.round(Math.min(maximum, Math.max(0, point.y || 0))); - if (result.some(existing => Math.abs(existing.y - normalizedY) <= 96)) continue; - if (result.length >= 80) { truncated = true; break; } - result.push({...point, y: normalizedY}); - } - if (result.length === 0) result.push(capturePoint(0, 'viewport', 'viewport')); - return {points: result, truncated, totalPoints: truncated ? points.length : result.length}; - }; - const screenshotPlan = args => { - const mode = args.mode === 'section' ? 'section' : 'viewport'; - const root = document.documentElement; - const maximum = Math.max(0, root.scrollHeight - innerHeight); - const initialY = Math.round(scrollY); - if (mode === 'viewport') { - const step = Math.max(1, innerHeight); - const totalPoints = maximum === 0 ? 1 : Math.ceil(maximum / step) + 1; - const truncated = totalPoints > 80; - const points = []; - const leadingCount = truncated ? 79 : Math.max(0, totalPoints - 1); - for (let index = 0; index < leadingCount; index += 1) { - points.push(capturePoint(index * step, `viewport ${index + 1}`, 'viewport')); - } - if (points.length === 0 || points[points.length - 1].y !== maximum) { - points.push(capturePoint(maximum, `viewport ${totalPoints}`, 'viewport')); - } - return {mode, initialY, viewportHeight: innerHeight, contentHeight: root.scrollHeight, - points, truncated, totalPoints}; - } - const candidates = Array.from(document.querySelectorAll('main h1,main h2,main h3,h1,h2,h3,section,[role="region"]')); - const points = candidates.filter(visible).map(element => { - const rect = element.getBoundingClientRect(); - const label = name(element) || element.getAttribute('id') || element.tagName.toLowerCase(); - const kind = /^h[1-6]$/i.test(element.tagName) ? element.tagName.toLowerCase() : (role(element) || element.tagName.toLowerCase()); - return capturePoint(rect.top + scrollY - 16, label, kind); - }).sort((left, right) => left.y - right.y); - const deduped = dedupePoints(points); - return {mode, initialY, viewportHeight: innerHeight, contentHeight: root.scrollHeight, - points: deduped.points, truncated: deduped.truncated, totalPoints: deduped.totalPoints}; - }; - const scrollToCapturePoint = async args => { - const maximum = Math.max(0, document.documentElement.scrollHeight - innerHeight); - const requestedY = Math.min(maximum, Math.max(0, Number(args.y || 0))); - scrollTo({top: requestedY, behavior: 'instant'}); - await sleep(180); - return {scrollY: Math.round(scrollY), requestedY: Math.round(requestedY)}; - }; - const performanceSummary = () => { - const navigation = performance.getEntriesByType('navigation')[0]; - const paint = performance.getEntriesByType('paint'); - const resources = performance.getEntriesByType('resource'); - const byName = name => paint.find(entry => entry.name === name)?.startTime ?? null; - let lcp = null; - let cls = 0; - try { - for (const entry of performance.getEntriesByType('largest-contentful-paint')) lcp = Math.max(lcp || 0, entry.startTime); - for (const entry of performance.getEntriesByType('layout-shift')) if (!entry.hadRecentInput) cls += entry.value || 0; - } catch (_) {} - const failedResources = resources.filter(item => item.transferSize === 0 && item.duration > 0).length; - return { - url: String(location.href).slice(0, 8192), - timing: navigation ? { - dnsMs: Math.max(0, navigation.domainLookupEnd - navigation.domainLookupStart), - connectMs: Math.max(0, navigation.connectEnd - navigation.connectStart), - ttfbMs: Math.max(0, navigation.responseStart - navigation.requestStart), - domContentLoadedMs: navigation.domContentLoadedEventEnd, - loadMs: navigation.loadEventEnd || null, - transferBytes: navigation.transferSize, encodedBytes: navigation.encodedBodySize, decodedBytes: navigation.decodedBodySize - } : null, - webVitals: {fcpMs: byName('first-contentful-paint'), lcpMs: lcp, cls: Number(cls.toFixed(4))}, - resources: {count: resources.length, transferBytes: resources.reduce((sum, item) => sum + (item.transferSize || 0), 0), zeroTransferTimedEntries: failedResources}, - caveat: 'Values are browser timing data for the current document; cross-origin resources may be subject to timing privacy limits.' - }; - }; - const animations = () => { - const finiteNumber = value => typeof value === 'number' && Number.isFinite(value) ? value : null; - const all = document.getAnimations().slice(0, 100).map(animation => { - const effect = animation.effect; - const timing = effect?.getComputedTiming?.() || {}; - const target = effect?.target; - return { - target: target instanceof Element ? {ref: refFor(target), role: role(target), name: name(target)} : null, - playState: animation.playState, - currentTime: finiteNumber(animation.currentTime) === null ? null : Math.round(animation.currentTime), - playbackRate: finiteNumber(animation.playbackRate), - durationMs: finiteNumber(timing.duration), - delayMs: finiteNumber(timing.delay), - progress: finiteNumber(timing.progress) === null ? null : Number(timing.progress.toFixed(4)), - iterations: finiteNumber(timing.iterations) - }; - }); - return {count: document.getAnimations().length, animations: all, truncated: document.getAnimations().length > all.length}; - }; - return { - snapshot, click, fill, press, scroll, state, tour, screenshotPlan, - scrollToCapturePoint, rectangle, styles, storage, - performance: performanceSummary, animations - }; - })(); -} -"""# + guard let url = Bundle.module.url(forResource: "AgentRuntime", withExtension: "js"), + let source = try? String(contentsOf: url, encoding: .utf8) else { + fatalError("HeadlessProtocol is missing its compiled AgentRuntime.js resource") + } + return source +}() diff --git a/apps/headless/Sources/HeadlessProtocol/Resources/AgentRuntime.js b/apps/headless/Sources/HeadlessProtocol/Resources/AgentRuntime.js new file mode 100644 index 0000000..90d0283 --- /dev/null +++ b/apps/headless/Sources/HeadlessProtocol/Resources/AgentRuntime.js @@ -0,0 +1,763 @@ +if (!globalThis.__headlessAgent) { + globalThis.__headlessAgent = (() => { + let nextRef = 1; + let nextRegionRef = 1; + const refs = new WeakMap(); + const regionRefs = new WeakMap(); + let current = new Map(); + let currentRegions = new Map(); + // Every reference ever handed out, so a failed lookup can say whether the + // reference expired or was never issued at all. + const issuedRefs = new Set(); + const issuedRegionRefs = new Set(); + const maximumTrackedRegions = 256; + let lastMutation = performance.now(); + new MutationObserver(() => { lastMutation = performance.now(); }) + .observe(document, {subtree: true, childList: true, attributes: true, characterData: true}); + + const normalize = value => String(value || '').replace(/\s+/g, ' ').trim(); + const clipped = (value, maximum) => normalize(value).slice(0, maximum); + const clippedURL = value => String(value || '').slice(0, 512); + const fail = (code, message) => { + const error = new Error(message); + error.headlessCode = code; + throw error; + }; + const blockedResourceExtensions = new Set([ + 'apk','app','bat','cmd','com','deb','dll','dmg','dylib','exe','img','iso','jar', + 'msi','msp','pif','pkg','ps1','psm1','scr','sh','so','vbe','vbs','wsf','zsh' + ]); + const cautionResourceExtensions = new Set(['7z','bz2','gz','rar','rpm','tar','tgz','xz','zip']); + const defaultStyleProperties = [ + 'display','visibility','opacity','position','z-index','overflow','box-sizing', + 'width','height','min-width','min-height','max-width','max-height', + 'margin-top','margin-right','margin-bottom','margin-left', + 'padding-top','padding-right','padding-bottom','padding-left', + 'color','background-color','border-top-width','border-top-style','border-top-color', + 'font-family','font-size','font-weight','line-height','text-align', + 'flex-direction','justify-content','align-items','gap','transform' + ]; + const resourceSafety = value => { + try { + const parsed = new URL(value, document.baseURI); + const extension = parsed.pathname.split('.').pop().toLowerCase(); + if (blockedResourceExtensions.has(extension)) return {level: 'blocked', extension}; + if (cautionResourceExtensions.has(extension)) return {level: 'caution', extension}; + } catch (_) { return {level: 'unknown'}; } + return {level: 'allowed'}; + }; + const visible = element => { + if (!(element instanceof Element) || !element.isConnected) return false; + const style = getComputedStyle(element); + if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false; + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }; + const role = element => { + const explicit = element.getAttribute('role'); + if (explicit) return explicit.split(/\s+/)[0].toLowerCase().slice(0, 64); + const tag = element.tagName.toLowerCase(); + if (tag === 'a' && element.hasAttribute('href')) return 'link'; + if (tag === 'button') return 'button'; + if (tag === 'textarea') return 'textbox'; + if (tag === 'select') return 'combobox'; + if (tag === 'img') return 'img'; + if (/^h[1-6]$/.test(tag)) return 'heading'; + if (tag === 'input') { + const type = (element.getAttribute('type') || 'text').toLowerCase(); + if (['button', 'submit', 'reset', 'image'].includes(type)) return 'button'; + if (type === 'checkbox') return 'checkbox'; + if (type === 'radio') return 'radio'; + if (type === 'range') return 'slider'; + return 'textbox'; + } + return tag; + }; + const actionHints = element => { + const hints = []; + const tag = element.tagName.toLowerCase(); + const elementRole = role(element); + if (tag === 'a' && element.hasAttribute('href')) hints.push('click'); + if (tag === 'button' || elementRole === 'button') hints.push('click'); + if (tag === 'summary' || elementRole === 'tab' || elementRole === 'menuitem') hints.push('click'); + // Only advertise verbs implemented by the public Headless protocol. + // Unsupported controls can still appear for context, but must not route + // an agent toward nonexistent select/upload/slide commands. + if (element instanceof HTMLTextAreaElement || element.isContentEditable) hints.push('fill'); + if (element instanceof HTMLInputElement) { + const type = (element.getAttribute('type') || 'text').toLowerCase(); + if (type === 'file') return hints; + else if (['checkbox', 'radio'].includes(type)) hints.push('click'); + else if (type === 'range') hints.push('fill'); + else if (['button', 'submit', 'reset', 'image'].includes(type)) hints.push('click'); + else hints.push('fill'); + } + if (element.tabIndex >= 0 && hints.length === 0) hints.push('click'); + return Array.from(new Set(hints)); + }; + const isInteractive = element => actionHints(element).length > 0 + || ['link','button','textbox','checkbox','radio','combobox','slider'].includes(role(element)); + const name = element => { + const labelledBy = element.getAttribute('aria-labelledby'); + if (labelledBy) { + const labelled = labelledBy.split(/\s+/).map(id => document.getElementById(id)?.innerText || '').join(' '); + if (normalize(labelled)) return clipped(labelled, 256); + } + if (normalize(element.getAttribute('aria-label'))) return clipped(element.getAttribute('aria-label'), 256); + if (element.labels?.length) return clipped(Array.from(element.labels).map(label => label.innerText).join(' '), 256); + if (normalize(element.getAttribute('alt'))) return clipped(element.getAttribute('alt'), 256); + if (normalize(element.getAttribute('placeholder'))) return clipped(element.getAttribute('placeholder'), 256); + if (normalize(element.getAttribute('title'))) return clipped(element.getAttribute('title'), 256); + if (normalize(element.value) && ['button', 'submit', 'reset'].includes((element.type || '').toLowerCase())) { + return clipped(element.value, 256); + } + return clipped(element.innerText || element.textContent, 256); + }; + const refFor = element => { + let ref = refs.get(element); + if (!ref) { ref = `@e${nextRef++}`; refs.set(element, ref); } + issuedRefs.add(ref); + current.set(ref, element); + return ref; + }; + const elementSelector = 'a[href],button,input,textarea,select,summary,[role],[contenteditable="true"],[tabindex],img,video,audio,canvas,svg'; + const regionSelector = 'main,nav,aside,header,footer,section,article,form,dialog,[role="main"],[role="navigation"],[role="region"],[role="dialog"],[role="form"],[role="search"],[role="complementary"],h1,h2,h3,h4,h5,h6'; + const headingLevel = element => { + const match = element?.tagName?.match(/^H([1-6])$/i); + return match ? Number(match[1]) : null; + }; + const scopeRoots = scope => { + if (!scope || scope === document || scope === document.documentElement) return [document.documentElement]; + const level = headingLevel(scope); + if (!level) return [scope]; + const roots = [scope]; + for (let sibling = scope.nextElementSibling; sibling; sibling = sibling.nextElementSibling) { + const siblingLevel = headingLevel(sibling); + if (siblingLevel && siblingLevel <= level) break; + roots.push(sibling); + } + return roots; + }; + const scopedQuery = (scope, selector) => { + const result = []; + const seen = new Set(); + for (const root of scopeRoots(scope)) { + if (!root) continue; + const add = element => { + if (!seen.has(element)) { seen.add(element); result.push(element); } + }; + if (root.matches?.(selector)) add(root); + for (const element of root.querySelectorAll?.(selector) || []) add(element); + } + return result; + }; + const candidates = (scope = document) => scopedQuery(scope, elementSelector).filter(visible); + const regionCandidates = (scope = document) => scopedQuery(scope, regionSelector).filter(visible); + const taskTerms = task => normalize(task).toLowerCase().split(/[^a-z0-9]+/).filter(term => term.length > 1).slice(0, 24); + const rankingText = element => [ + role(element), name(element), element.getAttribute('placeholder'), element.getAttribute('aria-label'), + element.getAttribute('title'), element.getAttribute('type'), element.getAttribute('name'), + element.getAttribute('id'), element.href + ].map(normalize).join(' ').toLowerCase(); + const rank = (element, terms, phrase) => { + const rect = element.getBoundingClientRect(); + const text = rankingText(element); + const hints = actionHints(element); + let score = 0; + const reasons = []; + if (hints.length > 0) { score += 50; reasons.push('interactive'); } + if (rect.top >= 0 && rect.top <= innerHeight && rect.left < innerWidth && rect.right > 0) { + score += 12; reasons.push('in viewport'); + } + if (phrase && text.includes(phrase)) { score += 35; reasons.push('task phrase match'); } + for (const term of terms) { + if (text.includes(term)) score += 14; + } + const termSet = new Set(terms); + const elementRole = role(element); + const inputType = element instanceof HTMLInputElement ? (element.getAttribute('type') || 'text').toLowerCase() : ''; + if (termSet.has('search')) { + if (inputType === 'search' || text.includes('search')) { score += 60; reasons.push('search control'); } + else if (hints.includes('fill')) score += 20; + } + if (termSet.has('login') || termSet.has('signin') || termSet.has('sign') || termSet.has('account')) { + if (inputType === 'password' || text.includes('password')) { score += 50; reasons.push('credential field'); } + if (hints.includes('fill')) score += 20; + if (elementRole === 'button' || elementRole === 'link') score += 12; + } + if (termSet.has('click') && (hints.includes('click') || elementRole === 'button' || elementRole === 'link')) score += 20; + score += Math.max(0, 8 - Math.floor(Math.max(0, rect.top) / 300)); + return {element, score, reasons: Array.from(new Set(reasons)).slice(0, 4)}; + }; + const rankedElements = (elements, task) => { + const terms = taskTerms(task); + if (terms.length === 0) return elements.map(element => ({element, score: 0, reasons: []})); + const phrase = normalize(task).toLowerCase().slice(0, 128); + return elements.map(element => rank(element, terms, phrase)) + .sort((left, right) => right.score - left.score); + }; + const regionRole = element => { + const explicit = element.getAttribute('role'); + if (explicit) return explicit.split(/\s+/)[0].toLowerCase().slice(0, 64); + const tag = element.tagName.toLowerCase(); + return ({main: 'main', nav: 'navigation', aside: 'complementary', header: 'banner', + footer: 'contentinfo', section: 'region', article: 'article', form: 'form', dialog: 'dialog'}[tag] + || (/^h[1-6]$/.test(tag) ? 'heading' : tag)); + }; + const regionName = element => { + const labelledBy = element.getAttribute('aria-labelledby'); + if (labelledBy) { + const labelled = labelledBy.split(/\s+/).map(id => document.getElementById(id)?.innerText || '').join(' '); + if (normalize(labelled)) return clipped(labelled, 160); + } + if (normalize(element.getAttribute('aria-label'))) return clipped(element.getAttribute('aria-label'), 160); + if (headingLevel(element)) return clipped(element.innerText || element.textContent, 160); + const heading = element.querySelector('h1,h2,h3,h4,h5,h6'); + if (heading && normalize(heading.innerText || heading.textContent)) { + return clipped(heading.innerText || heading.textContent, 160); + } + return clipped(element.getAttribute('title') || element.getAttribute('id') || regionRole(element), 160); + }; + const regionRefFor = element => { + let ref = regionRefs.get(element); + if (!ref) { ref = `@r${nextRegionRef++}`; regionRefs.set(element, ref); } + issuedRegionRefs.add(ref); + currentRegions.set(ref, element); + // Region references stay resolvable across inspections so a scoped + // `--within @rN` workflow keeps working, but this map holds elements + // strongly. Drop the oldest so a long-lived page cannot grow it without + // bound, and never drop the reference just issued. + while (currentRegions.size > maximumTrackedRegions) { + const oldest = currentRegions.keys().next().value; + if (oldest === undefined || oldest === ref) break; + currentRegions.delete(oldest); + } + return ref; + }; + const resolveRegion = reference => { + if (!reference) return document; + const element = currentRegions.get(reference); + if (!element) { + fail('REGION_NOT_FOUND', issuedRegionRefs.has(reference) + ? `REGION_NOT_FOUND:${reference} (expired: inspect again to refresh region references)` + : `REGION_NOT_FOUND:${reference} (unknown: no inspection has issued this reference)`); + } + if (!element.isConnected || !visible(element)) { + currentRegions.delete(reference); + fail('REGION_NOT_FOUND', `REGION_NOT_FOUND:${reference} (detached: the region is no longer visible on this page)`); + } + return element; + }; + const regionDepth = (element, scope) => { + if (headingLevel(element)) return Math.max(0, headingLevel(element) - 1); + let depth = 0; + for (let parent = element.parentElement; parent && parent !== scope && parent !== document.body; parent = parent.parentElement) { + if (parent.matches?.(regionSelector) && !headingLevel(parent)) depth += 1; + } + return depth; + }; + const rankRegion = (element, task) => { + const terms = taskTerms(task); + const phrase = normalize(task).toLowerCase().slice(0, 128); + const sample = normalize(`${regionRole(element)} ${regionName(element)} ${element.innerText || ''}`).toLowerCase().slice(0, 2000); + let score = 0; + const reasons = []; + if (phrase && sample.includes(phrase)) { score += 40; reasons.push('task phrase match'); } + for (const term of terms) if (sample.includes(term)) score += 14; + const rect = element.getBoundingClientRect(); + if (rect.top >= 0 && rect.top <= innerHeight) { score += 10; reasons.push('in viewport'); } + if (regionRole(element) === 'main') score += 4; + return {element, score, reasons: Array.from(new Set(reasons)).slice(0, 4)}; + }; + const describeRegion = (entry, scope, includeRelevance) => { + const element = entry.element || entry; + const rect = element.getBoundingClientRect(); + const actions = candidates(element).filter(isInteractive).length; + const text = normalize(scopeRoots(element).map(root => root.innerText || '').join(' ')); + const item = { + ref: regionRefFor(element), role: regionRole(element), name: regionName(element), + depth: regionDepth(element, scope), actions, textChars: text.length, + bounds: {x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height)} + }; + const level = headingLevel(element); + if (level) item.headingLevel = level; + if (includeRelevance) item.relevance = {score: Math.round(entry.score), reasons: entry.reasons}; + return item; + }; + const textCandidates = scope => { + const selector = 'h1,h2,h3,h4,h5,h6,p,li,dt,dd,blockquote,pre,code,caption,th,td'; + const seen = new Set(); + return scopedQuery(scope, selector).filter(visible).flatMap(element => { + const text = clipped(element.innerText || element.textContent, 600); + const key = text.toLowerCase(); + if (!text || seen.has(key)) return []; + seen.add(key); + return [{element, text}]; + }); + }; + const rankText = (entry, task) => { + const terms = taskTerms(task); + const phrase = normalize(task).toLowerCase().slice(0, 128); + const value = entry.text.toLowerCase(); + let score = 0; + const reasons = []; + if (phrase && value.includes(phrase)) { score += 40; reasons.push('task phrase match'); } + for (const term of terms) if (value.includes(term)) score += 16; + if (headingLevel(entry.element)) { score += 8; reasons.push('heading'); } + const rect = entry.element.getBoundingClientRect(); + if (rect.top >= 0 && rect.top <= innerHeight) { score += 6; reasons.push('in viewport'); } + return {...entry, score, reasons: Array.from(new Set(reasons)).slice(0, 4)}; + }; + const describeText = (entry, includeRelevance) => { + const item = {ref: refFor(entry.element), role: role(entry.element), text: entry.text}; + if (includeRelevance) item.relevance = {score: Math.round(entry.score), reasons: entry.reasons}; + return item; + }; + const describe = (entry, includeRelevance) => { + const element = entry.element || entry; + const rect = element.getBoundingClientRect(); + const item = { + ref: refFor(element), role: role(element), name: name(element), + actions: actionHints(element), + disabled: Boolean(element.disabled || element.getAttribute('aria-disabled') === 'true'), + bounds: {x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height)} + }; + if (element instanceof HTMLInputElement) item.inputType = (element.getAttribute('type') || 'text').toLowerCase().slice(0, 32); + if (includeRelevance) item.relevance = {score: Math.round(entry.score), reasons: entry.reasons}; + if ('value' in element && typeof element.value === 'string' && element.type !== 'password') item.value = element.value.slice(0, 500); + if (element.tagName === 'A') { + item.href = clippedURL(element.href); + item.resourceSafety = resourceSafety(element.href); + } + if (element.tagName === 'IMG') { + item.src = clippedURL(element.currentSrc || element.src); + item.media = { + kind: 'image', loaded: Boolean(element.complete && element.naturalWidth > 0), + width: element.naturalWidth || null, height: element.naturalHeight || null + }; + item.resourceSafety = resourceSafety(element.currentSrc || element.src); + } + if (element instanceof HTMLMediaElement) { + item.src = clippedURL(element.currentSrc || element.src || element.querySelector('source')?.src); + item.media = { + kind: element instanceof HTMLVideoElement ? 'video' : 'audio', + loaded: element.readyState >= HTMLMediaElement.HAVE_METADATA, + readyState: element.readyState, + paused: element.paused, + muted: element.muted, + currentTime: Number.isFinite(element.currentTime) ? element.currentTime : null, + duration: Number.isFinite(element.duration) ? element.duration : null, + width: element instanceof HTMLVideoElement ? element.videoWidth || null : null, + height: element instanceof HTMLVideoElement ? element.videoHeight || null : null + }; + item.resourceSafety = resourceSafety(element.currentSrc || element.src || element.querySelector('source')?.src); + if (element instanceof HTMLVideoElement && element.poster) item.poster = clippedURL(element.poster); + } + return item; + }; + const encodedBytes = value => new TextEncoder().encode(JSON.stringify(value)).length; + const pruneToBudget = (result, available, requestedBudget) => { + const budget = requestedBudget || null; + const arrays = ['regions', 'elements', 'snippets'].filter(key => Array.isArray(result[key])); + const refresh = () => { + const omitted = {}; + for (const key of arrays) omitted[key] = Math.max(0, (available[key] || 0) - result[key].length); + result.omitted = omitted; + result.truncated = Object.values(omitted).some(value => value > 0); + result.contextStats = { + encodedBytes: 0, + estimatedTokens: 0, + budget, + budgetApplied: Boolean(budget) + }; + const baseBytes = encodedBytes(result); + let bytes = baseBytes; + let estimatedTokens = Math.ceil(bytes / 4); + while (true) { + const measured = baseBytes + String(bytes).length - 1 + String(estimatedTokens).length - 1; + const measuredTokens = Math.ceil(measured / 4); + if (measured === bytes && measuredTokens === estimatedTokens) break; + bytes = measured; + estimatedTokens = measuredTokens; + } + result.contextStats.encodedBytes = bytes; + result.contextStats.estimatedTokens = estimatedTokens; + return bytes; + }; + let currentBytes = refresh(); + if (!budget) return result; + + const maximumBytes = budget * 4; + if (currentBytes > maximumBytes) { + const candidates = arrays.flatMap(key => result[key].map((value, index) => ({ + key, index, bytes: encodedBytes(value) + (result[key].length > 1 ? 1 : 0) + }))).sort((left, right) => right.bytes - left.bytes); + const removed = new Map(arrays.map(key => [key, new Set()])); + // Reserve a small allowance for omitted-count digit growth. Each item + // is encoded once; removal then preserves the order of retained items. + const targetBytes = Math.max(0, maximumBytes - 32); + for (const candidate of candidates) { + if (currentBytes <= targetBytes) break; + removed.get(candidate.key).add(candidate.index); + currentBytes -= candidate.bytes; + } + for (const key of arrays) { + const indexes = removed.get(key); + if (indexes.size > 0) result[key] = result[key].filter((_, index) => !indexes.has(index)); + } + currentBytes = refresh(); + } + + if (currentBytes > maximumBytes && typeof result.text === 'string' && result.text.length > 0) { + const textBytes = Math.max(0, encodedBytes(result.text) - 2); + const targetTextBytes = Math.max(0, textBytes - (currentBytes - maximumBytes) - 32); + let lower = 0; + let upper = result.text.length; + while (lower < upper) { + const middle = Math.ceil((lower + upper) / 2); + const prefixBytes = Math.max(0, encodedBytes(result.text.slice(0, middle)) - 2); + if (prefixBytes <= targetTextBytes) lower = middle; + else upper = middle - 1; + } + result.text = result.text.slice(0, lower); + currentBytes = refresh(); + } + + // Metadata is deliberately bounded, but keep the budget fail-closed if + // its final digit widths ever outgrow the allowance above. + if (currentBytes > maximumBytes) { + for (const key of arrays) result[key] = []; + if (typeof result.text === 'string') result.text = ''; + refresh(); + } + return result; + }; + const snapshot = (interactiveOnly, includeText, options = {}) => { + current = new Map(); + const allowedContexts = new Set(['summary', 'outline', 'text', 'actions', 'full']); + const requestedContext = allowedContexts.has(options.context) ? options.context : 'full'; + const contextMode = interactiveOnly && requestedContext === 'full' ? 'actions' : requestedContext; + const task = clipped(options.task, 512); + const scope = resolveRegion(clipped(options.within, 16)); + const defaultLimits = {summary: 12, outline: 40, text: 12, actions: 20, full: 250}; + const defaultBudgets = {summary: 1200, outline: 1600, text: 1500, actions: 1200, full: null}; + const limit = Math.min(250, Math.max(1, Number(options.limit || defaultLimits[contextMode]))); + const budget = options.budget ? Math.min(16000, Math.max(256, Number(options.budget))) : defaultBudgets[contextMode]; + const depth = Math.min(8, Math.max(0, Number(options.depth ?? 3))); + const root = document.documentElement; + const result = { + url: String(location.href).slice(0, 8192), title: String(document.title).slice(0, 512), + contextMode, + viewport: {width: innerWidth, height: innerHeight, scrollX, scrollY, + contentWidth: root ? root.scrollWidth : 0, contentHeight: root ? root.scrollHeight : 0}, + untrustedContent: true + }; + if (task) result.task = task; + if (options.within) result.within = options.within; + const available = {}; + if (contextMode === 'actions' || contextMode === 'full' || contextMode === 'summary') { + let elements = candidates(scope); + if (contextMode !== 'full') elements = elements.filter(isInteractive); + const ranked = rankedElements(elements, task); + const maximum = contextMode === 'summary' ? Math.min(limit, 8) : limit; + result.elements = ranked.slice(0, maximum).map(entry => describe(entry, Boolean(task))); + available.elements = ranked.length; + } + if (contextMode === 'outline' || contextMode === 'summary') { + const ranked = regionCandidates(scope).map(element => rankRegion(element, task)) + .filter(entry => regionDepth(entry.element, scope) <= depth) + .sort((left, right) => task ? right.score - left.score : left.element.getBoundingClientRect().top - right.element.getBoundingClientRect().top); + result.regions = ranked.slice(0, limit).map(entry => describeRegion(entry, scope, Boolean(task))); + available.regions = ranked.length; + } + if (contextMode === 'text') { + const ranked = textCandidates(scope).map(entry => rankText(entry, task)) + .sort((left, right) => task ? right.score - left.score : left.element.getBoundingClientRect().top - right.element.getBoundingClientRect().top); + result.snippets = ranked.slice(0, limit).map(entry => describeText(entry, Boolean(task))); + available.snippets = ranked.length; + } + if (includeText) { + result.text = normalize(scopeRoots(scope).map(root => root.innerText || root.textContent || '').join(' ')).slice(0, 30000); + } + return pruneToBudget(result, available, budget); + }; + const resolve = target => { + const element = current.get(target); + // Element references describe the most recent inspection only. Saying so + // is the difference between an agent re-inspecting and an agent retrying + // the same dead reference. + if (!element) { + fail('ELEMENT_NOT_FOUND', issuedRefs.has(target) + ? `ELEMENT_NOT_FOUND:${target} (expired: element references come from the most recent inspection — inspect again to refresh)` + : `ELEMENT_NOT_FOUND:${target} (unknown: no inspection has issued this reference)`); + } + if (!element.isConnected) { + current.delete(target); + fail('ELEMENT_NOT_FOUND', `ELEMENT_NOT_FOUND:${target} (detached: the element is no longer in the page)`); + } + return element; + }; + const find = (wantedRole, wantedName) => { + const normalizedRole = normalize(wantedRole).toLowerCase(); + const normalizedName = normalize(wantedName).toLowerCase(); + const matches = candidates().filter(element => + (!normalizedRole || role(element) === normalizedRole) && + (!normalizedName || name(element).toLowerCase() === normalizedName) + ); + if (matches.length === 0) fail('ELEMENT_NOT_FOUND', `ELEMENT_NOT_FOUND:${wantedRole || ''}/${wantedName || ''}`); + if (matches.length > 1) throw new Error(`ELEMENT_AMBIGUOUS:${matches.length}`); + refFor(matches[0]); + return matches[0]; + }; + const target = args => args.target ? resolve(args.target) : find(args.role, args.name); + const rectangle = args => { + const element = target(args); + element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'}); + const rect = element.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) throw new Error('ELEMENT_NOT_VISIBLE'); + return { + viewport: {x: rect.x, y: rect.y, width: rect.width, height: rect.height}, + document: {x: rect.x + scrollX, y: rect.y + scrollY, width: rect.width, height: rect.height} + }; + }; + const styles = args => { + const element = target(args); + const computed = getComputedStyle(element); + const requested = Array.isArray(args.properties) && args.properties.length > 0 + ? args.properties : defaultStyleProperties; + const values = {}; + for (const property of requested.slice(0, 64)) { + const propertyName = String(property || '').slice(0, 128); + if (propertyName) values[propertyName] = String(computed.getPropertyValue(propertyName) || '').slice(0, 2048); + } + const rect = element.getBoundingClientRect(); + return { + ref: refFor(element), role: role(element), name: name(element), + box: {x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height)}, + styles: values + }; + }; + const storage = args => { + const includeValues = Boolean(args.includeValues); + const describe = (store, scope) => { + const entries = []; + for (let index = 0; index < store.length && index < 200; index++) { + const key = String(store.key(index) || '').slice(0, 512); + const value = String(store.getItem(key) || ''); + entries.push(includeValues + ? {key, value: value.slice(0, 16384), bytes: value.length} + : {key, bytes: value.length}); + } + return {scope, count: store.length, entries, truncated: store.length > entries.length}; + }; + const scope = args.scope || 'all'; + const stores = []; + try { + if (scope === 'local' || scope === 'all') stores.push(describe(localStorage, 'local')); + if (scope === 'session' || scope === 'all') stores.push(describe(sessionStorage, 'session')); + } catch (error) { + return {origin: String(location.origin), stores, error: String(error).slice(0, 512)}; + } + return {origin: String(location.origin).slice(0, 2048), stores}; + }; + const click = args => { + const element = target(args); + if (element instanceof HTMLAnchorElement && element.href) { + const destination = new URL(element.href, document.baseURI); + const scheme = destination.protocol.toLowerCase(); + if (!['http:', 'https:'].includes(scheme) || destination.username || destination.password) { + fail('UNSAFE_NAVIGATION', `UNSAFE_NAVIGATION:${scheme}`); + } + const safety = resourceSafety(destination.href); + if (safety.level === 'blocked') fail('UNSAFE_RESOURCE_TYPE', `UNSAFE_RESOURCE_TYPE:${safety.extension}`); + } + element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'}); + element.focus({preventScroll: true}); + element.click(); + return {clicked: refFor(element), role: role(element), name: name(element)}; + }; + const fill = args => { + const element = target(args); + if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || element.isContentEditable)) { + throw new Error('NOT_EDITABLE'); + } + element.focus({preventScroll: false}); + if (element.isContentEditable) { + element.textContent = args.value; + } else { + const prototype = element instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + Object.getOwnPropertyDescriptor(prototype, 'value').set.call(element, args.value); + } + element.dispatchEvent(new InputEvent('input', {bubbles: true, inputType: 'insertText', data: args.value})); + element.dispatchEvent(new Event('change', {bubbles: true})); + return {filled: refFor(element), valueLength: String(args.value).length}; + }; + const press = key => { + const element = document.activeElement || document.body; + const options = {key, code: key, bubbles: true, cancelable: true}; + element.dispatchEvent(new KeyboardEvent('keydown', options)); + if (key === 'Enter') { + const form = element.form || element.closest?.('form'); + if (form?.requestSubmit) form.requestSubmit(); + else if (element.tagName === 'BUTTON') element.click(); + } else if (key === ' ' && element.tagName === 'BUTTON') { + element.click(); + } + element.dispatchEvent(new KeyboardEvent('keyup', options)); + return {pressed: key}; + }; + const scroll = args => { + const amount = Number(args.amount || Math.max(240, innerHeight * 0.8)); + if (args.direction === 'top') scrollTo({top: 0, behavior: 'smooth'}); + else if (args.direction === 'bottom') scrollTo({top: document.documentElement.scrollHeight, behavior: 'smooth'}); + else scrollBy({top: args.direction === 'up' ? -amount : amount, behavior: 'smooth'}); + return {direction: args.direction, amount}; + }; + const blockingAnimationCount = () => document.getAnimations().filter(animation => { + if (animation.playState !== 'running') return false; + const timing = animation.effect?.getComputedTiming?.(); + // Perpetual ambient animation must not prevent a loaded page from ever + // settling. Finite transitions still block until they finish. + return Number.isFinite(timing?.endTime); + }).length; + const state = () => { + const root = document.documentElement; + return { + url: String(location.href).slice(0, 8192), title: String(document.title).slice(0, 512), + // A page can be between documents immediately after navigation. Report + // it as loading so the host retries instead of dereferencing a missing + // root element and turning a normal navigation race into a failure. + readyState: root ? document.readyState : 'loading', + text: normalize(document.body?.innerText).slice(0, 30000), + runningAnimations: blockingAnimationCount(), + mutationQuietMs: Math.round(performance.now() - lastMutation), + scrollY, contentHeight: root ? root.scrollHeight : 0 + }; + }; + const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + const tour = async args => { + const pace = Math.min(5000, Math.max(100, Number(args.pace || 800))); + if (args.fullPage === false) { + return {start: scrollY, end: scrollY, durationMs: 0}; + } + const maximum = Math.max(0, document.documentElement.scrollHeight - innerHeight); + scrollTo({top: 0, behavior: 'instant'}); + await sleep(250); + const desiredDuration = maximum / pace; + const duration = Math.min(60, Math.max(0.5, desiredDuration)); + const steps = Math.max(1, Math.ceil(duration * 20)); + for (let index = 1; index <= steps; index++) { + scrollTo({top: maximum * index / steps, behavior: 'instant'}); + await sleep(duration * 1000 / steps); + } + return {start: 0, end: maximum, durationMs: Math.round(duration * 1000)}; + }; + const capturePoint = (y, label, kind) => ({ + y: Math.round(Math.max(0, y)), + label: clipped(label || kind || 'viewport', 80), + kind: clipped(kind || 'viewport', 32) + }); + const dedupePoints = points => { + const maximum = Math.max(0, document.documentElement.scrollHeight - innerHeight); + const result = []; + let truncated = false; + for (const point of points) { + const normalizedY = Math.round(Math.min(maximum, Math.max(0, point.y || 0))); + if (result.some(existing => Math.abs(existing.y - normalizedY) <= 96)) continue; + if (result.length >= 80) { truncated = true; break; } + result.push({...point, y: normalizedY}); + } + if (result.length === 0) result.push(capturePoint(0, 'viewport', 'viewport')); + return {points: result, truncated, totalPoints: truncated ? points.length : result.length}; + }; + const screenshotPlan = args => { + const mode = args.mode === 'section' ? 'section' : 'viewport'; + const root = document.documentElement; + const maximum = Math.max(0, root.scrollHeight - innerHeight); + const initialY = Math.round(scrollY); + if (mode === 'viewport') { + const step = Math.max(1, innerHeight); + const totalPoints = maximum === 0 ? 1 : Math.ceil(maximum / step) + 1; + const truncated = totalPoints > 80; + const points = []; + const leadingCount = truncated ? 79 : Math.max(0, totalPoints - 1); + for (let index = 0; index < leadingCount; index += 1) { + points.push(capturePoint(index * step, `viewport ${index + 1}`, 'viewport')); + } + if (points.length === 0 || points[points.length - 1].y !== maximum) { + points.push(capturePoint(maximum, `viewport ${totalPoints}`, 'viewport')); + } + return {mode, initialY, viewportHeight: innerHeight, contentHeight: root.scrollHeight, + points, truncated, totalPoints}; + } + const candidates = Array.from(document.querySelectorAll('main h1,main h2,main h3,h1,h2,h3,section,[role="region"]')); + const points = candidates.filter(visible).map(element => { + const rect = element.getBoundingClientRect(); + const label = name(element) || element.getAttribute('id') || element.tagName.toLowerCase(); + const kind = /^h[1-6]$/i.test(element.tagName) ? element.tagName.toLowerCase() : (role(element) || element.tagName.toLowerCase()); + return capturePoint(rect.top + scrollY - 16, label, kind); + }).sort((left, right) => left.y - right.y); + const deduped = dedupePoints(points); + return {mode, initialY, viewportHeight: innerHeight, contentHeight: root.scrollHeight, + points: deduped.points, truncated: deduped.truncated, totalPoints: deduped.totalPoints}; + }; + const scrollToCapturePoint = async args => { + const maximum = Math.max(0, document.documentElement.scrollHeight - innerHeight); + const requestedY = Math.min(maximum, Math.max(0, Number(args.y || 0))); + scrollTo({top: requestedY, behavior: 'instant'}); + await sleep(180); + return {scrollY: Math.round(scrollY), requestedY: Math.round(requestedY)}; + }; + const performanceSummary = () => { + const navigation = performance.getEntriesByType('navigation')[0]; + const paint = performance.getEntriesByType('paint'); + const resources = performance.getEntriesByType('resource'); + const byName = name => paint.find(entry => entry.name === name)?.startTime ?? null; + let lcp = null; + let cls = 0; + try { + for (const entry of performance.getEntriesByType('largest-contentful-paint')) lcp = Math.max(lcp || 0, entry.startTime); + for (const entry of performance.getEntriesByType('layout-shift')) if (!entry.hadRecentInput) cls += entry.value || 0; + } catch (_) {} + const failedResources = resources.filter(item => item.transferSize === 0 && item.duration > 0).length; + return { + url: String(location.href).slice(0, 8192), + timing: navigation ? { + dnsMs: Math.max(0, navigation.domainLookupEnd - navigation.domainLookupStart), + connectMs: Math.max(0, navigation.connectEnd - navigation.connectStart), + ttfbMs: Math.max(0, navigation.responseStart - navigation.requestStart), + domContentLoadedMs: navigation.domContentLoadedEventEnd, + loadMs: navigation.loadEventEnd || null, + transferBytes: navigation.transferSize, encodedBytes: navigation.encodedBodySize, decodedBytes: navigation.decodedBodySize + } : null, + webVitals: {fcpMs: byName('first-contentful-paint'), lcpMs: lcp, cls: Number(cls.toFixed(4))}, + resources: {count: resources.length, transferBytes: resources.reduce((sum, item) => sum + (item.transferSize || 0), 0), zeroTransferTimedEntries: failedResources}, + caveat: 'Values are browser timing data for the current document; cross-origin resources may be subject to timing privacy limits.' + }; + }; + const animations = () => { + const finiteNumber = value => typeof value === 'number' && Number.isFinite(value) ? value : null; + const all = document.getAnimations().slice(0, 100).map(animation => { + const effect = animation.effect; + const timing = effect?.getComputedTiming?.() || {}; + const target = effect?.target; + return { + target: target instanceof Element ? {ref: refFor(target), role: role(target), name: name(target)} : null, + playState: animation.playState, + currentTime: finiteNumber(animation.currentTime) === null ? null : Math.round(animation.currentTime), + playbackRate: finiteNumber(animation.playbackRate), + durationMs: finiteNumber(timing.duration), + delayMs: finiteNumber(timing.delay), + progress: finiteNumber(timing.progress) === null ? null : Number(timing.progress.toFixed(4)), + iterations: finiteNumber(timing.iterations) + }; + }); + return {count: document.getAnimations().length, animations: all, truncated: document.getAnimations().length > all.length}; + }; + return { + snapshot, click, fill, press, scroll, state, tour, screenshotPlan, + scrollToCapturePoint, rectangle, styles, storage, + performance: performanceSummary, animations + }; + })(); +} diff --git a/apps/headless/Tests/agent-runtime.test.mjs b/apps/headless/Tests/agent-runtime.test.mjs index af92ec8..af7718f 100644 --- a/apps/headless/Tests/agent-runtime.test.mjs +++ b/apps/headless/Tests/agent-runtime.test.mjs @@ -3,11 +3,9 @@ import {readFile} from 'node:fs/promises'; import {JSDOM} from 'jsdom'; const runtimeSource = await readFile( - new URL('../Sources/HeadlessProtocol/AgentRuntime.swift', import.meta.url), + new URL('../Sources/HeadlessProtocol/Resources/AgentRuntime.js', import.meta.url), 'utf8', ); -const runtimeMatch = runtimeSource.match(/#"""\n([\s\S]*?)\n"""#/); -assert(runtimeMatch, 'embedded agent runtime should be extractable'); const fixture = await readFile(new URL('Fixtures/large-document.html', import.meta.url), 'utf8'); const dom = new JSDOM(fixture, { @@ -49,7 +47,7 @@ Object.defineProperties(window.document.documentElement, { scrollHeight: {value: 32000, configurable: true}, }); -window.eval(runtimeMatch[1]); +window.eval(runtimeSource); const agent = window.__headlessAgent; assert(agent, 'agent runtime should initialize'); diff --git a/apps/headless/Tests/linux-e2e.sh b/apps/headless/Tests/linux-e2e.sh index deb8edb..62ef480 100755 --- a/apps/headless/Tests/linux-e2e.sh +++ b/apps/headless/Tests/linux-e2e.sh @@ -26,6 +26,7 @@ trap cleanup EXIT INT TERM /opt/headless/package/install-linux.sh --prefix "$INSTALL_ROOT" | grep -q 'Headless installed' test -x "$INSTALL_ROOT/bin/headless" test -x "$INSTALL_ROOT/bin/headless-host" +test -r "$INSTALL_ROOT/bin/Headless_HeadlessProtocol.resources/AgentRuntime.js" if /opt/headless/package/install-linux.sh --prefix relative/path >/dev/null 2>&1; then echo "relative install prefix was not rejected" >&2 exit 1 @@ -170,8 +171,16 @@ headless --session qa flow start | grep -q '"recording":true' headless --session qa visit http://127.0.0.1:41739/designers/dashboard/ | grep -q 'Designers Dashboard' headless --session qa click --role button --name Continue | grep -q '"clicked"' headless --session qa flow stop --output dashboard-flow.json | grep -q '"name":"dashboard-flow.json"' -headless --session qa flow run dashboard-flow.json | grep -q '"completed":2' -headless --session qa report create --output pr-report.json | grep -q '"name":"pr-report.json"' +if ! FLOW_RUN="$(headless --session qa flow run dashboard-flow.json)"; then + echo "$FLOW_RUN" >&2 + exit 1 +fi +echo "$FLOW_RUN" | grep -q '"completed":2' +if ! REPORT_CREATE="$(headless --session qa report create --output pr-report.json)"; then + echo "$REPORT_CREATE" >&2 + exit 1 +fi +echo "$REPORT_CREATE" | grep -q '"name":"pr-report.json"' grep -q 'headless-qa-report-v1' "$HEADLESS_ARTIFACT_DIR/pr-report.json" headless --session qa visit http://127.0.0.1:41739/designers/dashboard/ | grep -q 'Designers Dashboard' test -s "$HEADLESS_ARTIFACT_DIR/viewport.png" diff --git a/apps/headless/build-linux.sh b/apps/headless/build-linux.sh index c707a09..351c12b 100755 --- a/apps/headless/build-linux.sh +++ b/apps/headless/build-linux.sh @@ -21,6 +21,7 @@ trap cleanup EXIT INT TERM docker cp "$CONTAINER:/usr/local/bin/headless" build/linux/headless docker cp "$CONTAINER:/usr/local/bin/headless-host" build/linux/headless-host docker cp "$CONTAINER:/usr/local/bin/headless-mcp" build/linux/headless-mcp +docker cp "$CONTAINER:/usr/local/bin/Headless_HeadlessProtocol.resources" build/linux/Headless_HeadlessProtocol.resources chmod 0755 build/linux/headless build/linux/headless-host build/linux/headless-mcp cp install-linux.sh build/linux/install-linux.sh cp docs/P1.md docs/P2.md build/linux/ @@ -29,7 +30,7 @@ PLATFORM_LABEL="${HEADLESS_LINUX_PLATFORM:-}" PLATFORM_LABEL="${PLATFORM_LABEL##*/}" if [ -z "$PLATFORM_LABEL" ]; then PLATFORM_LABEL="$(uname -m)"; fi ARCHIVE="build/headless-linux-$PLATFORM_LABEL.tar.gz" -tar -czf "$ARCHIVE" -C build/linux headless headless-host headless-mcp install-linux.sh P1.md P2.md +tar -czf "$ARCHIVE" -C build/linux headless headless-host headless-mcp Headless_HeadlessProtocol.resources install-linux.sh P1.md P2.md echo "Linux binaries: $PWD/build/linux" echo "Linux package: $PWD/$ARCHIVE" echo "The Docker image includes Debian Chromium at /usr/lib/chromium/chromium." diff --git a/apps/headless/build.sh b/apps/headless/build.sh index db501c7..96b5238 100755 --- a/apps/headless/build.sh +++ b/apps/headless/build.sh @@ -57,12 +57,20 @@ BIN_PATH="$(swift build "${SDK_ARGS[@]}" -c release --scratch-path "$SWIFT_SCRAT swift build "${SDK_ARGS[@]}" -c release --product headless-host --scratch-path "$SWIFT_SCRATCH" swift build "${SDK_ARGS[@]}" -c release --product headless --scratch-path "$SWIFT_SCRATCH" swift build "${SDK_ARGS[@]}" -c release --product headless-mcp --scratch-path "$SWIFT_SCRATCH" +RESOURCE_BUNDLE="$BIN_PATH/Headless_HeadlessProtocol.bundle" +if [[ ! -d "$RESOURCE_BUNDLE" ]]; then + echo "headless build: compiled HeadlessProtocol resource bundle was not found" >&2 + exit 70 +fi cp "$BIN_PATH/headless-host" "$APP/Contents/MacOS/Headless" mkdir -p "$APP/Contents/Resources/bin" cp "$BIN_PATH/headless" "$APP/Contents/Resources/bin/headless" cp "$BIN_PATH/headless" build/bin/headless cp "$BIN_PATH/headless-mcp" "$APP/Contents/Resources/bin/headless-mcp" cp "$BIN_PATH/headless-mcp" build/bin/headless-mcp +rm -rf "$APP/Headless_HeadlessProtocol.bundle" +rm -rf "$APP/Contents/Resources/Headless_HeadlessProtocol.bundle" +cp -R "$RESOURCE_BUNDLE" "$APP/Contents/Resources/Headless_HeadlessProtocol.bundle" cp "$ICON" "$APP/Contents/Resources/Headless.icns" cat > "$APP/Contents/Info.plist" <&2 @@ -125,6 +125,9 @@ install -d -m 0755 "$BIN_DIR" install -m 0755 "$SOURCE_DIR/headless" "$BIN_DIR/headless" install -m 0755 "$SOURCE_DIR/headless-host" "$BIN_DIR/headless-host" install -m 0755 "$SOURCE_DIR/headless-mcp" "$BIN_DIR/headless-mcp" +install -d -m 0755 "$BIN_DIR/Headless_HeadlessProtocol.resources" +install -m 0644 "$SOURCE_DIR/Headless_HeadlessProtocol.resources/AgentRuntime.js" \ + "$BIN_DIR/Headless_HeadlessProtocol.resources/AgentRuntime.js" echo "Headless installed in $BIN_DIR" echo "Browser runtime: $CHROMIUM (supported inherited DevTools pipe)" diff --git a/apps/headless/main.swift b/apps/headless/main.swift index e2d385f..8a264ed 100644 --- a/apps/headless/main.swift +++ b/apps/headless/main.swift @@ -234,6 +234,12 @@ final class BrowserWindowController: NSWindowController, NSWindowDelegate, conf.userContentController.addUserScript(WKUserScript( source: webKitQAScript, injectionTime: .atDocumentStart, forMainFrameOnly: false )) + conf.userContentController.addUserScript(WKUserScript( + source: agentRuntimeJavaScript, + injectionTime: .atDocumentStart, + forMainFrameOnly: false, + in: agentWorld + )) if !hasPasskeyEntitlement { let hideWebAuthn = WKUserScript( source: """ diff --git a/docs/roadmap/architecture-decisions.md b/docs/roadmap/architecture-decisions.md index 8710290..3043207 100644 --- a/docs/roadmap/architecture-decisions.md +++ b/docs/roadmap/architecture-decisions.md @@ -208,6 +208,13 @@ one implementation of page-side semantics for every engine (it is what makes resources) so tooling/tests stop regex-extracting it from a Swift string literal (`Tests/agent-runtime.test.mjs`'s `/#"""…"""#/` coupling). +**Status:** implemented 2026-08-10. The source is now the compiled +`Resources/AgentRuntime.js`; WebKit installs it in `HeadlessAgent` at document +start, while Chromium installs it with +`Page.addScriptToEvaluateOnNewDocument`, caches the isolated context for the +document, and invalidates or retries it across navigation races. Distribution +scripts install the generated resource bundle alongside each host executable. + ## 11. Session model: document shared-profile reality; isolation is a future opt-in **Decision:** sessions are windows (macOS) / tabs (Linux) sharing one diff --git a/docs/roadmap/improvements-backlog.md b/docs/roadmap/improvements-backlog.md index cd602bc..17939ca 100644 --- a/docs/roadmap/improvements-backlog.md +++ b/docs/roadmap/improvements-backlog.md @@ -217,11 +217,14 @@ through it; `press` length enforced in bridge only on macOS (`BrowserProcess.swift:521`). Generate `capabilities` from code (`CLI.swift:648-682` is a hand-written literal today) and assert it. -**B7. Runtime injection cost** ([#27](https://github.com/LockInTime/headless/issues/27)) — cache the isolated world / install runtime +**B7. Runtime injection cost** ([#27](https://github.com/LockInTime/headless/issues/27)) — ~~cache the isolated world / install runtime per-navigation instead of per-call on both engines; extract the JS to a compiled resource. Spec in [architecture-decisions §10](architecture-decisions.md). (`BrowserProcess.swift:777-834`, `AgentBridge.swift:382-420`, -`Tests/agent-runtime.test.mjs` regex extraction.) +`Tests/agent-runtime.test.mjs` regex extraction.)~~ **Done:** both engines +install the compiled JavaScript resource at document start, Linux caches and +invalidates its isolated context with one stale-context retry, and release / +installer paths carry the SwiftPM resource bundle. **B8. QA diagnostics bridge isolation (macOS).** ([#28](https://github.com/LockInTime/headless/issues/28)) Page-world injection is detectable/forgeable/spammable by a hostile page