From 1222a9429f143c0aceb41e40a92b48a0fbe96319 Mon Sep 17 00:00:00 2001 From: SarthakWade Date: Wed, 12 Aug 2026 19:32:44 +0530 Subject: [PATCH 1/2] test: add benchmark medians and JSON results --- CHANGELOG.md | 3 + apps/headless/Tests/Benchmarks/headless.sh | 1 + .../Tests/Benchmarks/headless_warm.sh | 1 + apps/headless/Tests/Benchmarks/summarize.mjs | 174 ++++++++++++ .../headless/Tests/benchmark-summary.test.mjs | 129 +++++++++ apps/headless/benchmark.sh | 33 ++- apps/headless/docs/BENCHMARK.md | 51 ++-- apps/headless/package.json | 2 +- docs/roadmap/improvements-backlog.md | 9 +- packages/benchmark-results/README.md | 12 + packages/benchmark-results/results.json | 250 ++++++++++++++++++ 11 files changed, 638 insertions(+), 27 deletions(-) create mode 100644 apps/headless/Tests/Benchmarks/summarize.mjs create mode 100644 apps/headless/Tests/benchmark-summary.test.mjs create mode 100644 packages/benchmark-results/README.md create mode 100644 packages/benchmark-results/results.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a6f257..101ab42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ Cutting that release is tracked in ### Added +- The comparison benchmark now emits a validated JSON results artifact with + raw samples and medians, and its refreshed Headless workflows include + task-aware action inspection. - A single cross-engine conformance scenario now runs against real WKWebView and Chromium hosts, locking shared response shapes and declared capability errors without mirrored platform assertions. diff --git a/apps/headless/Tests/Benchmarks/headless.sh b/apps/headless/Tests/Benchmarks/headless.sh index d5a0b4a..6632f9e 100755 --- a/apps/headless/Tests/Benchmarks/headless.sh +++ b/apps/headless/Tests/Benchmarks/headless.sh @@ -6,6 +6,7 @@ headless start >/dev/null headless session create bench >/dev/null headless --session bench visit "$BENCH_URL" >/dev/null headless --session bench record start --fps 5 >/dev/null +headless --session bench inspect --context actions --task 'continue to designer details' --limit 8 --budget 700 >/dev/null headless --session bench tour --full-page --pace 5000 >/dev/null headless --session bench click --role button --name Continue >/dev/null headless --session bench wait --url /next --text 'Designer details' --settled --timeout 10000 >/dev/null diff --git a/apps/headless/Tests/Benchmarks/headless_warm.sh b/apps/headless/Tests/Benchmarks/headless_warm.sh index 3377ba0..7df7d0a 100755 --- a/apps/headless/Tests/Benchmarks/headless_warm.sh +++ b/apps/headless/Tests/Benchmarks/headless_warm.sh @@ -3,6 +3,7 @@ set -eu headless --session bench visit "$BENCH_URL" >/dev/null headless --session bench record start --fps 5 >/dev/null +headless --session bench inspect --context actions --task 'continue to designer details' --limit 8 --budget 700 >/dev/null headless --session bench tour --full-page --pace 5000 >/dev/null headless --session bench click --role button --name Continue >/dev/null headless --session bench wait --url /next --text 'Designer details' --settled --timeout 10000 >/dev/null diff --git a/apps/headless/Tests/Benchmarks/summarize.mjs b/apps/headless/Tests/Benchmarks/summarize.mjs new file mode 100644 index 0000000..0b049ed --- /dev/null +++ b/apps/headless/Tests/Benchmarks/summarize.mjs @@ -0,0 +1,174 @@ +import { + chmodSync, + closeSync, + constants, + mkdirSync, + openSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { basename, dirname, join } from "node:path"; + +const CASES = [ + ["headless", "Headless, cold"], + ["headless-warm", "Headless, warm"], + ["selenium", "Selenium with Python"], + ["puppeteer", "Puppeteer"], +]; +const METRICS = [ + "wallMs", + "cpuMs", + "memoryPeakBytes", + "artifactBytes", + "workflowBytes", + "estimatedTokens", +]; +const EXPECTED_KEYS = ["case", ...METRICS].sort(); +const MAX_INPUT_BYTES = 8 * 1024 * 1024; + +function fail(message) { + console.error(`benchmark summary: ${message}`); + process.exit(65); +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function parsePositiveInteger(value, name, maximum) { + if (!/^[1-9][0-9]*$/.test(value)) fail(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + fail(`${name} must not exceed ${maximum}`); + } + return parsed; +} + +function parseRecords(inputPath, repeats) { + const inputStat = statSync(inputPath); + if (!inputStat.isFile() || inputStat.size === 0 || inputStat.size > MAX_INPUT_BYTES) { + fail(`input must be a non-empty regular file no larger than ${MAX_INPUT_BYTES} bytes`); + } + + const lines = readFileSync(inputPath, "utf8").split("\n").filter((line) => line.length > 0); + if (lines.length !== CASES.length * repeats) { + fail(`expected ${CASES.length * repeats} samples, received ${lines.length}`); + } + + return lines.map((line, index) => { + let record; + try { + record = JSON.parse(line); + } catch { + fail(`sample ${index + 1} is not valid JSON`); + } + if (record === null || Array.isArray(record) || typeof record !== "object") { + fail(`sample ${index + 1} must be an object`); + } + if (JSON.stringify(Object.keys(record).sort()) !== JSON.stringify(EXPECTED_KEYS)) { + fail(`sample ${index + 1} has an unexpected schema`); + } + if (!CASES.some(([caseName]) => caseName === record.case)) { + fail(`sample ${index + 1} has an unknown case`); + } + for (const metric of METRICS) { + if (!Number.isSafeInteger(record[metric]) || record[metric] < 0) { + fail(`sample ${index + 1} has an invalid ${metric}`); + } + } + return record; + }); +} + +function aggregate(records, repeats) { + return CASES.map(([caseName, label]) => { + const caseRecords = records.filter((record) => record.case === caseName); + if (caseRecords.length !== repeats) { + fail(`${caseName} must have exactly ${repeats} samples`); + } + for (const metric of ["workflowBytes", "estimatedTokens"]) { + if (new Set(caseRecords.map((record) => record[metric])).size !== 1) { + fail(`${caseName} has inconsistent ${metric} values`); + } + } + + return { + case: caseName, + label, + samples: caseRecords.map((record, index) => ({ + iteration: index + 1, + ...Object.fromEntries(METRICS.map((metric) => [metric, record[metric]])), + })), + median: Object.fromEntries( + METRICS.map((metric) => [metric, median(caseRecords.map((record) => record[metric]))]), + ), + }; + }); +} + +function writeAtomically(outputPath, document) { + const outputDirectory = dirname(outputPath); + mkdirSync(outputDirectory, { recursive: true }); + const temporaryPath = join( + outputDirectory, + `.${basename(outputPath)}.${process.pid}.${Date.now()}.tmp`, + ); + let descriptor; + try { + descriptor = openSync( + temporaryPath, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, + 0o600, + ); + writeFileSync(descriptor, `${JSON.stringify(document, null, 2)}\n`, "utf8"); + closeSync(descriptor); + descriptor = undefined; + chmodSync(temporaryPath, 0o644); + renameSync(temporaryPath, outputPath); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + rmSync(temporaryPath, { force: true }); + } +} + +if (process.argv.length !== 7) { + console.error("usage: summarize.mjs INPUT OUTPUT REPEATS GENERATED_AT PLATFORM"); + process.exit(64); +} + +const [, , inputPath, outputPath, repeatsValue, generatedAt, platform] = process.argv; +const repeats = parsePositiveInteger(repeatsValue, "repeats", 100); +const parsedTimestamp = new Date(generatedAt); +if ( + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/.test(generatedAt) + || Number.isNaN(parsedTimestamp.valueOf()) + || parsedTimestamp.toISOString() !== generatedAt.replace("Z", ".000Z") +) { + fail("generated timestamp must be UTC ISO 8601 without fractional seconds"); +} +if (!/^linux\/[a-z0-9][a-z0-9_-]{0,31}$/.test(platform)) { + fail("platform must identify a Linux container architecture"); +} + +const records = parseRecords(inputPath, repeats); +const document = { + schemaVersion: 1, + generatedAt, + provenance: { + generator: "apps/headless/benchmark.sh", + method: "apps/headless/docs/BENCHMARK.md", + platform, + repeats, + aggregation: "median", + taskAwareInspection: true, + }, + cases: aggregate(records, repeats), +}; +writeAtomically(outputPath, document); diff --git a/apps/headless/Tests/benchmark-summary.test.mjs b/apps/headless/Tests/benchmark-summary.test.mjs new file mode 100644 index 0000000..c0e1cc6 --- /dev/null +++ b/apps/headless/Tests/benchmark-summary.test.mjs @@ -0,0 +1,129 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const testRoot = mkdtempSync(join(tmpdir(), "headless-benchmark-summary.")); +const appRoot = fileURLToPath(new URL("..", import.meta.url)); +const summarizer = join(appRoot, "Tests/Benchmarks/summarize.mjs"); +const coldWorkflow = readFileSync(join(appRoot, "Tests/Benchmarks/headless.sh"), "utf8"); +const warmWorkflow = readFileSync(join(appRoot, "Tests/Benchmarks/headless_warm.sh"), "utf8"); + +function sample(caseName, value, workflowBytes) { + return { + case: caseName, + wallMs: value, + cpuMs: value + 10, + memoryPeakBytes: value + 20, + artifactBytes: value + 30, + workflowBytes, + estimatedTokens: Math.ceil(workflowBytes / 4), + }; +} + +function run(records, repeats, name = "results.json") { + const input = join(testRoot, `${name}.ndjson`); + const output = join(testRoot, name); + writeFileSync(input, `${records.map((record) => JSON.stringify(record)).join("\n")}\n`); + const result = spawnSync( + process.execPath, + [summarizer, input, output, String(repeats), "2026-08-12T10:00:00Z", "linux/arm64"], + { encoding: "utf8" }, + ); + return { ...result, output }; +} + +try { + for (const workflow of [coldWorkflow, warmWorkflow]) { + assert.match(workflow, /inspect --context actions --task 'continue to designer details'/); + assert.match(workflow, /--limit 8 --budget 700/); + } + + const oddRecords = []; + for (const [caseName, workflowBytes] of [ + ["headless", 800], + ["headless-warm", 600], + ["selenium", 1600], + ["puppeteer", 2000], + ]) { + oddRecords.push(sample(caseName, 30, workflowBytes)); + oddRecords.push(sample(caseName, 10, workflowBytes)); + oddRecords.push(sample(caseName, 20, workflowBytes)); + } + const odd = run(oddRecords, 3); + assert.equal(odd.status, 0, odd.stderr); + const document = JSON.parse(readFileSync(odd.output, "utf8")); + assert.equal(document.schemaVersion, 1); + assert.equal(document.provenance.repeats, 3); + assert.equal(document.provenance.aggregation, "median"); + assert.equal(document.provenance.taskAwareInspection, true); + assert.deepEqual(document.cases.map((entry) => entry.case), [ + "headless", + "headless-warm", + "selenium", + "puppeteer", + ]); + assert.equal(document.cases[0].median.wallMs, 20); + assert.deepEqual(document.cases[0].samples.map((entry) => entry.iteration), [1, 2, 3]); + const firstOutput = readFileSync(odd.output, "utf8"); + const oddAgain = run(oddRecords, 3); + assert.equal(oddAgain.status, 0, oddAgain.stderr); + assert.equal(readFileSync(oddAgain.output, "utf8"), firstOutput); + + const evenRecords = []; + for (const [caseName, workflowBytes] of [ + ["headless", 800], + ["headless-warm", 600], + ["selenium", 1600], + ["puppeteer", 2000], + ]) { + evenRecords.push(sample(caseName, 10, workflowBytes)); + evenRecords.push(sample(caseName, 11, workflowBytes)); + } + const even = run(evenRecords, 2, "even.json"); + assert.equal(even.status, 0, even.stderr); + assert.equal(JSON.parse(readFileSync(even.output, "utf8")).cases[0].median.wallMs, 10.5); + + const missing = run(oddRecords.slice(1), 3, "missing.json"); + assert.notEqual(missing.status, 0); + assert.match(missing.stderr, /expected 12 samples/); + + const unknown = run( + [{ ...oddRecords[0], case: "unknown" }, ...oddRecords.slice(1)], + 3, + "unknown.json", + ); + assert.notEqual(unknown.status, 0); + assert.match(unknown.stderr, /unknown case/); + + const inconsistent = run( + [{ ...oddRecords[0], workflowBytes: 801 }, ...oddRecords.slice(1)], + 3, + "inconsistent.json", + ); + assert.notEqual(inconsistent.status, 0); + assert.match(inconsistent.stderr, /inconsistent workflowBytes/); + + const malformed = run( + [{ ...oddRecords[0], wallMs: -1 }, ...oddRecords.slice(1)], + 3, + "malformed.json", + ); + assert.notEqual(malformed.status, 0); + assert.match(malformed.stderr, /invalid wallMs/); + + writeFileSync(malformed.output, "preserve existing result\n"); + const malformedAgain = run( + [{ ...oddRecords[0], wallMs: -1 }, ...oddRecords.slice(1)], + 3, + "malformed.json", + ); + assert.notEqual(malformedAgain.status, 0); + assert.equal(readFileSync(malformed.output, "utf8"), "preserve existing result\n"); +} finally { + rmSync(testRoot, { recursive: true, force: true }); +} + +console.log("Benchmark summary tests passed"); diff --git a/apps/headless/benchmark.sh b/apps/headless/benchmark.sh index b71cd00..c6e41fa 100755 --- a/apps/headless/benchmark.sh +++ b/apps/headless/benchmark.sh @@ -3,14 +3,41 @@ set -eu cd "$(dirname "$0")" REPEATS="${1:-3}" -case "$REPEATS" in *[!0-9]*|'') echo "usage: ./benchmark.sh [positive-repeat-count]" >&2; exit 64;; esac +OUTPUT="${2:-../../packages/benchmark-results/results.json}" +if [ "$#" -gt 2 ]; then + echo "usage: ./benchmark.sh [positive-repeat-count] [output-path]" >&2 + exit 64 +fi +case "$REPEATS" in *[!0-9]*|'') echo "usage: ./benchmark.sh [positive-repeat-count] [output-path]" >&2; exit 64;; esac test "$REPEATS" -gt 0 || { echo "repeat count must be positive" >&2; exit 64; } +test "$REPEATS" -le 100 || { echo "repeat count must not exceed 100" >&2; exit 64; } -docker build --quiet --target benchmark -f Dockerfile.linux -t headless-p1-benchmark . >/dev/null +for tool in docker node; do + command -v "$tool" >/dev/null 2>&1 || { echo "$tool is required" >&2; exit 69; } +done + +OUTPUT_DIRECTORY="$(dirname "$OUTPUT")" +mkdir -p "$OUTPUT_DIRECTORY" +OUTPUT="$(CDPATH='' cd -- "$OUTPUT_DIRECTORY" && pwd -P)/$(basename "$OUTPUT")" +RAW_RESULTS="$(mktemp "${TMPDIR:-/tmp}/headless-benchmark.XXXXXX")" +IMAGE="headless-p1-benchmark:run-$$" +cleanup() { + rm -f "$RAW_RESULTS" + docker image rm "$IMAGE" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +docker build --quiet --target benchmark -f Dockerfile.linux -t "$IMAGE" . >/dev/null for case_name in headless headless-warm selenium puppeteer; do iteration=1 while [ "$iteration" -le "$REPEATS" ]; do - docker run --rm --shm-size=1g --cap-add=SYS_ADMIN headless-p1-benchmark "$case_name" + echo "Benchmarking $case_name ($iteration/$REPEATS)" >&2 + docker run --rm --shm-size=1g --cap-add=SYS_ADMIN "$IMAGE" "$case_name" >> "$RAW_RESULTS" iteration=$((iteration + 1)) done done + +PLATFORM="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$IMAGE")" +GENERATED_AT="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" +node Tests/Benchmarks/summarize.mjs "$RAW_RESULTS" "$OUTPUT" "$REPEATS" "$GENERATED_AT" "$PLATFORM" +echo "Benchmark results: $OUTPUT" diff --git a/apps/headless/docs/BENCHMARK.md b/apps/headless/docs/BENCHMARK.md index 5551a2e..630e219 100644 --- a/apps/headless/docs/BENCHMARK.md +++ b/apps/headless/docs/BENCHMARK.md @@ -5,37 +5,42 @@ record through the `Continue` transition, and save a final screenshot. ## Current snapshot -One fresh container was run for each case on 17 July 2026, on Apple Silicon -with Docker Linux ARM64. These are point-in-time measurements, not medians; -repeat the benchmark before using them to compare a change. +Five fresh containers were run for each case on 12 August 2026, on Apple +Silicon with Docker Linux ARM64. The table reports the median of each metric. +These remain point-in-time measurements; repeat the benchmark before using +them to compare a change. The generated +[`results.json`](../../../packages/benchmark-results/results.json) preserves +all 20 raw samples and the aggregation provenance. | Workflow | Estimated tokens | Wall time | CPU time | Peak memory | | --- | ---: | ---: | ---: | ---: | -| Headless, cold | 194 | 5,002 ms | 1,478 ms | 279 MiB | -| Headless, warm | 147 | 4,753 ms | 842 ms | 276 MiB | -| Selenium with Python | 410 | 3,134 ms | 1,900 ms | 280 MiB | -| Puppeteer | 499 | 2,850 ms | 2,441 ms | 319 MiB | +| Headless, cold | 218 | 3,484 ms | 2,028 ms | 368 MiB | +| Headless, warm | 174 | 3,248 ms | 1,379 ms | 366 MiB | +| Selenium with Python | 410 | 2,880 ms | 2,010 ms | 363 MiB | +| Puppeteer | 499 | 2,402 ms | 1,860 ms | 358 MiB | Estimated tokens are `ceil(workflow source bytes / 4)`. They compare the agent workflow surface, not billed LLM tokens, tool schemas, prompts, or responses. -Headless has the smallest agent surface: the warm workflow uses about 64% -fewer estimated tokens than Selenium and 71% fewer than Puppeteer. In this -single sample it also had the lowest CPU time and memory peak. Puppeteer was -fastest. The reusable P2 flow command reduces orchestration work for real -agent-driven repeats, but this benchmark retains the comparable explicit CLI -workflow rather than claiming an unmeasured flow speedup. +Headless has the smallest measured agent surface: the warm workflow uses about +58% fewer estimated tokens than Selenium and 65% fewer than Puppeteer. Its +median CPU time is about 31% lower than Selenium and 26% lower than Puppeteer. +Puppeteer is fastest and has the lowest median peak memory; Headless does not +lead those dimensions. The reusable P2 flow command reduces orchestration work +for real agent-driven repeats, but this benchmark retains the comparable +explicit CLI workflow rather than claiming an unmeasured flow speedup. -Task-aware inspection was added after this benchmark. -`inspect --context actions --task "..."` prunes the page to visible controls -and ranks them by the agent's current goal. Re-run the benchmark before quoting -any new token number for that workflow. +Both Headless cases now run +`inspect --context actions --task "continue to designer details"` before the +semantic click. The estimated-token count includes that task-aware inspection +command. Selenium and Puppeteer retain their explicit selector-based action +lookup. ## Method -Each workflow uses Chromium 150 and FFmpeg 5.1 to produce the same two +Each workflow uses Chromium 151 and FFmpeg 5.1 to produce the same two artifacts: an MP4 that tours both pages and a final viewport PNG. Selenium 4.8.3 -uses ChromeDriver 150; Puppeteer Core is 22.15.0. All waits use page load or an +uses ChromeDriver 151; Puppeteer Core is 22.15.0. All waits use page load or an explicit URL condition. Every measured run gets a fresh container. The warm Headless case starts its host and session before timing; the cold case includes them. @@ -50,6 +55,14 @@ Run the benchmark with: ./apps/headless/benchmark.sh 5 ``` +The default output is `packages/benchmark-results/results.json`. Pass a second +argument to write a separate result document without replacing the canonical +snapshot: + +```sh +./apps/headless/benchmark.sh 5 /tmp/headless-benchmark.json +``` + ## VM compatibility result On 17 July 2026, the Linux ARM64 package was exercised on the Hermes VM diff --git a/apps/headless/package.json b/apps/headless/package.json index 7e6dd27..70e404f 100644 --- a/apps/headless/package.json +++ b/apps/headless/package.json @@ -9,7 +9,7 @@ "test": "./test.sh", "test:e2e:mac": "./Tests/macos-e2e.sh", "test:e2e:linux": "./Tests/linux-docker.sh", - "test:runtime": "node Tests/agent-runtime.test.mjs" + "test:runtime": "node Tests/agent-runtime.test.mjs && node Tests/benchmark-summary.test.mjs" }, "devDependencies": { "jsdom": "30.0.1" diff --git a/docs/roadmap/improvements-backlog.md b/docs/roadmap/improvements-backlog.md index 766d6f2..e29f502 100644 --- a/docs/roadmap/improvements-backlog.md +++ b/docs/roadmap/improvements-backlog.md @@ -346,10 +346,11 @@ suites. It locks common JSON fields across lifecycle, navigation, inspection, diagnostics, capture, flows, reports, and errors, branching only through the declared capability matrix for intentional differences. -**D5. Benchmark refresh discipline [exists: benchmark.sh].** ([#38](https://github.com/LockInTime/headless/issues/38)) Emit JSON -results artifact; re-run with the task-aware flow (BENCHMARK.md:29-31 says -current numbers predate `--task`); repeat-count medians instead of single -samples. +**D5. Benchmark refresh discipline [exists: benchmark.sh].** ([#38](https://github.com/LockInTime/headless/issues/38)) ~~Emit JSON +results artifact; re-run with the task-aware flow; report repeat-count medians +instead of single samples.~~ **Done:** the benchmark validates and preserves +every sample in a provenance-bearing JSON document, reports per-metric medians, +and the refreshed five-repeat snapshot includes task-aware action inspection. ## §E — Distribution (Phase 3) diff --git a/packages/benchmark-results/README.md b/packages/benchmark-results/README.md new file mode 100644 index 0000000..196b78a --- /dev/null +++ b/packages/benchmark-results/README.md @@ -0,0 +1,12 @@ +# Benchmark results + +`results.json` is generated by `apps/headless/benchmark.sh`. It contains every +raw sample and the median of each metric, plus the repeat count, container +platform, generation time, generator, and benchmark method. Do not edit the +result document by hand. + +Regenerate it from the repository root with: + +```sh +./apps/headless/benchmark.sh 5 +``` diff --git a/packages/benchmark-results/results.json b/packages/benchmark-results/results.json new file mode 100644 index 0000000..62d2896 --- /dev/null +++ b/packages/benchmark-results/results.json @@ -0,0 +1,250 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-12T13:50:38Z", + "provenance": { + "generator": "apps/headless/benchmark.sh", + "method": "apps/headless/docs/BENCHMARK.md", + "platform": "linux/arm64", + "repeats": 5, + "aggregation": "median", + "taskAwareInspection": true + }, + "cases": [ + { + "case": "headless", + "label": "Headless, cold", + "samples": [ + { + "iteration": 1, + "wallMs": 3930, + "cpuMs": 2187, + "memoryPeakBytes": 752189440, + "artifactBytes": 102472, + "workflowBytes": 870, + "estimatedTokens": 218 + }, + { + "iteration": 2, + "wallMs": 3484, + "cpuMs": 2028, + "memoryPeakBytes": 380952576, + "artifactBytes": 99135, + "workflowBytes": 870, + "estimatedTokens": 218 + }, + { + "iteration": 3, + "wallMs": 3473, + "cpuMs": 2036, + "memoryPeakBytes": 386068480, + "artifactBytes": 137356, + "workflowBytes": 870, + "estimatedTokens": 218 + }, + { + "iteration": 4, + "wallMs": 3492, + "cpuMs": 1908, + "memoryPeakBytes": 388087808, + "artifactBytes": 132276, + "workflowBytes": 870, + "estimatedTokens": 218 + }, + { + "iteration": 5, + "wallMs": 3479, + "cpuMs": 1972, + "memoryPeakBytes": 381677568, + "artifactBytes": 101808, + "workflowBytes": 870, + "estimatedTokens": 218 + } + ], + "median": { + "wallMs": 3484, + "cpuMs": 2028, + "memoryPeakBytes": 386068480, + "artifactBytes": 102472, + "workflowBytes": 870, + "estimatedTokens": 218 + } + }, + { + "case": "headless-warm", + "label": "Headless, warm", + "samples": [ + { + "iteration": 1, + "wallMs": 3248, + "cpuMs": 1379, + "memoryPeakBytes": 390807552, + "artifactBytes": 141710, + "workflowBytes": 693, + "estimatedTokens": 174 + }, + { + "iteration": 2, + "wallMs": 3237, + "cpuMs": 1349, + "memoryPeakBytes": 383737856, + "artifactBytes": 142986, + "workflowBytes": 693, + "estimatedTokens": 174 + }, + { + "iteration": 3, + "wallMs": 3233, + "cpuMs": 1414, + "memoryPeakBytes": 383971328, + "artifactBytes": 101808, + "workflowBytes": 693, + "estimatedTokens": 174 + }, + { + "iteration": 4, + "wallMs": 3256, + "cpuMs": 1376, + "memoryPeakBytes": 383102976, + "artifactBytes": 103840, + "workflowBytes": 693, + "estimatedTokens": 174 + }, + { + "iteration": 5, + "wallMs": 3277, + "cpuMs": 1455, + "memoryPeakBytes": 389758976, + "artifactBytes": 136998, + "workflowBytes": 693, + "estimatedTokens": 174 + } + ], + "median": { + "wallMs": 3248, + "cpuMs": 1379, + "memoryPeakBytes": 383971328, + "artifactBytes": 136998, + "workflowBytes": 693, + "estimatedTokens": 174 + } + }, + { + "case": "selenium", + "label": "Selenium with Python", + "samples": [ + { + "iteration": 1, + "wallMs": 2865, + "cpuMs": 1959, + "memoryPeakBytes": 380690432, + "artifactBytes": 173975, + "workflowBytes": 1638, + "estimatedTokens": 410 + }, + { + "iteration": 2, + "wallMs": 2813, + "cpuMs": 1893, + "memoryPeakBytes": 383242240, + "artifactBytes": 173975, + "workflowBytes": 1638, + "estimatedTokens": 410 + }, + { + "iteration": 3, + "wallMs": 2924, + "cpuMs": 2117, + "memoryPeakBytes": 380301312, + "artifactBytes": 173975, + "workflowBytes": 1638, + "estimatedTokens": 410 + }, + { + "iteration": 4, + "wallMs": 2897, + "cpuMs": 2078, + "memoryPeakBytes": 385925120, + "artifactBytes": 173975, + "workflowBytes": 1638, + "estimatedTokens": 410 + }, + { + "iteration": 5, + "wallMs": 2880, + "cpuMs": 2010, + "memoryPeakBytes": 378208256, + "artifactBytes": 173975, + "workflowBytes": 1638, + "estimatedTokens": 410 + } + ], + "median": { + "wallMs": 2880, + "cpuMs": 2010, + "memoryPeakBytes": 380690432, + "artifactBytes": 173975, + "workflowBytes": 1638, + "estimatedTokens": 410 + } + }, + { + "case": "puppeteer", + "label": "Puppeteer", + "samples": [ + { + "iteration": 1, + "wallMs": 2486, + "cpuMs": 1942, + "memoryPeakBytes": 377262080, + "artifactBytes": 178320, + "workflowBytes": 1996, + "estimatedTokens": 499 + }, + { + "iteration": 2, + "wallMs": 2437, + "cpuMs": 1942, + "memoryPeakBytes": 375758848, + "artifactBytes": 178320, + "workflowBytes": 1996, + "estimatedTokens": 499 + }, + { + "iteration": 3, + "wallMs": 2362, + "cpuMs": 1827, + "memoryPeakBytes": 375615488, + "artifactBytes": 178320, + "workflowBytes": 1996, + "estimatedTokens": 499 + }, + { + "iteration": 4, + "wallMs": 2402, + "cpuMs": 1860, + "memoryPeakBytes": 376266752, + "artifactBytes": 178320, + "workflowBytes": 1996, + "estimatedTokens": 499 + }, + { + "iteration": 5, + "wallMs": 2398, + "cpuMs": 1849, + "memoryPeakBytes": 374726656, + "artifactBytes": 178320, + "workflowBytes": 1996, + "estimatedTokens": 499 + } + ], + "median": { + "wallMs": 2402, + "cpuMs": 1860, + "memoryPeakBytes": 375758848, + "artifactBytes": 178320, + "workflowBytes": 1996, + "estimatedTokens": 499 + } + } + ] +} From a0fd3d9770715962c746acf913c9ee8b61354236 Mon Sep 17 00:00:00 2001 From: SarthakWade Date: Wed, 12 Aug 2026 19:57:42 +0530 Subject: [PATCH 2/2] test: keep benchmark aggregation Docker-only --- CHANGELOG.md | 6 +++--- apps/headless/benchmark.sh | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 101ab42..cac7182 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,15 +23,15 @@ Cutting that release is tracked in ### Added -- The comparison benchmark now emits a validated JSON results artifact with - raw samples and medians, and its refreshed Headless workflows include - task-aware action inspection. - A single cross-engine conformance scenario now runs against real WKWebView and Chromium hosts, locking shared response shapes and declared capability errors without mirrored platform assertions. - A generated WebKit/Chromium capability matrix now declares exhaustive command support and intentional engine differences; host ping responses include the active engine profile. +- The comparison benchmark now emits a validated JSON results artifact with + raw samples and medians, and its refreshed Headless workflows include + task-aware action inspection. - The single MCP tool now accurately declares its mutating, destructive, non-idempotent, open-world behavior; integration tests lock the metadata and deliberate `stop` / `session close` exposure. diff --git a/apps/headless/benchmark.sh b/apps/headless/benchmark.sh index c6e41fa..7f4069c 100755 --- a/apps/headless/benchmark.sh +++ b/apps/headless/benchmark.sh @@ -12,9 +12,7 @@ case "$REPEATS" in *[!0-9]*|'') echo "usage: ./benchmark.sh [positive-repeat-cou test "$REPEATS" -gt 0 || { echo "repeat count must be positive" >&2; exit 64; } test "$REPEATS" -le 100 || { echo "repeat count must not exceed 100" >&2; exit 64; } -for tool in docker node; do - command -v "$tool" >/dev/null 2>&1 || { echo "$tool is required" >&2; exit 69; } -done +command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 69; } OUTPUT_DIRECTORY="$(dirname "$OUTPUT")" mkdir -p "$OUTPUT_DIRECTORY" @@ -25,7 +23,9 @@ cleanup() { rm -f "$RAW_RESULTS" docker image rm "$IMAGE" >/dev/null 2>&1 || true } -trap cleanup EXIT INT TERM +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM docker build --quiet --target benchmark -f Dockerfile.linux -t "$IMAGE" . >/dev/null for case_name in headless headless-warm selenium puppeteer; do @@ -39,5 +39,16 @@ done PLATFORM="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$IMAGE")" GENERATED_AT="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" -node Tests/Benchmarks/summarize.mjs "$RAW_RESULTS" "$OUTPUT" "$REPEATS" "$GENERATED_AT" "$PLATFORM" +docker run --rm \ + --user "$(id -u):$(id -g)" \ + --mount "type=bind,src=$RAW_RESULTS,dst=/tmp/headless-benchmark.ndjson,readonly" \ + --mount "type=bind,src=$OUTPUT_DIRECTORY,dst=/output" \ + --entrypoint node \ + "$IMAGE" \ + /opt/headless/benchmarks/summarize.mjs \ + /tmp/headless-benchmark.ndjson \ + "/output/$(basename "$OUTPUT")" \ + "$REPEATS" \ + "$GENERATED_AT" \ + "$PLATFORM" echo "Benchmark results: $OUTPUT"