test: publish task-aware benchmark medians

CHANGELOG.md

@@ -29,6 +29,9 @@ Cutting that release is tracked in
 - A generated WebKit/Chromium capability matrix now declares exhaustive
   command support and intentional engine differences; host ping responses
   include the active engine profile.
+- The comparison benchmark now emits a validated JSON results artifact with
+  raw samples and medians, and its refreshed Headless workflows include
+  task-aware action inspection.
 - The single MCP tool now accurately declares its mutating, destructive,
   non-idempotent, open-world behavior; integration tests lock the metadata and
   deliberate `stop` / `session close` exposure.

apps/headless/Tests/Benchmarks/headless.sh

@@ -6,6 +6,7 @@ headless start >/dev/null
 headless session create bench >/dev/null
 headless --session bench visit "$BENCH_URL" >/dev/null
 headless --session bench record start --fps 5 >/dev/null
+headless --session bench inspect --context actions --task 'continue to designer details' --limit 8 --budget 700 >/dev/null
 headless --session bench tour --full-page --pace 5000 >/dev/null
 headless --session bench click --role button --name Continue >/dev/null
 headless --session bench wait --url /next --text 'Designer details' --settled --timeout 10000 >/dev/null

apps/headless/Tests/Benchmarks/headless_warm.sh

@@ -3,6 +3,7 @@ set -eu
 
 headless --session bench visit "$BENCH_URL" >/dev/null
 headless --session bench record start --fps 5 >/dev/null
+headless --session bench inspect --context actions --task 'continue to designer details' --limit 8 --budget 700 >/dev/null
 headless --session bench tour --full-page --pace 5000 >/dev/null
 headless --session bench click --role button --name Continue >/dev/null
 headless --session bench wait --url /next --text 'Designer details' --settled --timeout 10000 >/dev/null

apps/headless/Tests/Benchmarks/summarize.mjs

@@ -0,0 +1,174 @@
+import {
+  chmodSync,
+  closeSync,
+  constants,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { basename, dirname, join } from "node:path";
+
+const CASES = [
+  ["headless", "Headless, cold"],
+  ["headless-warm", "Headless, warm"],
+  ["selenium", "Selenium with Python"],
+  ["puppeteer", "Puppeteer"],
+];
+const METRICS = [
+  "wallMs",
+  "cpuMs",
+  "memoryPeakBytes",
+  "artifactBytes",
+  "workflowBytes",
+  "estimatedTokens",
+];
+const EXPECTED_KEYS = ["case", ...METRICS].sort();
+const MAX_INPUT_BYTES = 8 * 1024 * 1024;
+
+function fail(message) {
+  console.error(`benchmark summary: ${message}`);
+  process.exit(65);
+}
+
+function median(values) {
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1
+    ? sorted[middle]
+    : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+function parsePositiveInteger(value, name, maximum) {
+  if (!/^[1-9][0-9]*$/.test(value)) fail(`${name} must be a positive integer`);
+  const parsed = Number(value);
+  if (!Number.isSafeInteger(parsed) || parsed > maximum) {
+    fail(`${name} must not exceed ${maximum}`);
+  }
+  return parsed;
+}
+
+function parseRecords(inputPath, repeats) {
+  const inputStat = statSync(inputPath);
+  if (!inputStat.isFile() || inputStat.size === 0 || inputStat.size > MAX_INPUT_BYTES) {
+    fail(`input must be a non-empty regular file no larger than ${MAX_INPUT_BYTES} bytes`);
+  }
+
+  const lines = readFileSync(inputPath, "utf8").split("\n").filter((line) => line.length > 0);
+  if (lines.length !== CASES.length * repeats) {
+    fail(`expected ${CASES.length * repeats} samples, received ${lines.length}`);
+  }
+
+  return lines.map((line, index) => {
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch {
+      fail(`sample ${index + 1} is not valid JSON`);
+    }
+    if (record === null || Array.isArray(record) || typeof record !== "object") {
+      fail(`sample ${index + 1} must be an object`);
+    }
+    if (JSON.stringify(Object.keys(record).sort()) !== JSON.stringify(EXPECTED_KEYS)) {
+      fail(`sample ${index + 1} has an unexpected schema`);
+    }
+    if (!CASES.some(([caseName]) => caseName === record.case)) {
+      fail(`sample ${index + 1} has an unknown case`);
+    }
+    for (const metric of METRICS) {
+      if (!Number.isSafeInteger(record[metric]) || record[metric] < 0) {
+        fail(`sample ${index + 1} has an invalid ${metric}`);
+      }
+    }
+    return record;
+  });
+}
+
+function aggregate(records, repeats) {
+  return CASES.map(([caseName, label]) => {
+    const caseRecords = records.filter((record) => record.case === caseName);
+    if (caseRecords.length !== repeats) {
+      fail(`${caseName} must have exactly ${repeats} samples`);
+    }
+    for (const metric of ["workflowBytes", "estimatedTokens"]) {
+      if (new Set(caseRecords.map((record) => record[metric])).size !== 1) {
+        fail(`${caseName} has inconsistent ${metric} values`);
+      }
+    }
+
+    return {
+      case: caseName,
+      label,
+      samples: caseRecords.map((record, index) => ({
+        iteration: index + 1,
+        ...Object.fromEntries(METRICS.map((metric) => [metric, record[metric]])),
+      })),
+      median: Object.fromEntries(
+        METRICS.map((metric) => [metric, median(caseRecords.map((record) => record[metric]))]),
+      ),
+    };
+  });
+}
+
+function writeAtomically(outputPath, document) {
+  const outputDirectory = dirname(outputPath);
+  mkdirSync(outputDirectory, { recursive: true });
+  const temporaryPath = join(
+    outputDirectory,
+    `.${basename(outputPath)}.${process.pid}.${Date.now()}.tmp`,
+  );
+  let descriptor;
+  try {
+    descriptor = openSync(
+      temporaryPath,
+      constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
+      0o600,
+    );
+    writeFileSync(descriptor, `${JSON.stringify(document, null, 2)}\n`, "utf8");
+    closeSync(descriptor);
+    descriptor = undefined;
+    chmodSync(temporaryPath, 0o644);
+    renameSync(temporaryPath, outputPath);
+  } finally {
+    if (descriptor !== undefined) closeSync(descriptor);
+    rmSync(temporaryPath, { force: true });
+  }
+}
+
+if (process.argv.length !== 7) {
+  console.error("usage: summarize.mjs INPUT OUTPUT REPEATS GENERATED_AT PLATFORM");
+  process.exit(64);
+}
+
+const [, , inputPath, outputPath, repeatsValue, generatedAt, platform] = process.argv;
+const repeats = parsePositiveInteger(repeatsValue, "repeats", 100);
+const parsedTimestamp = new Date(generatedAt);
+if (
+  !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/.test(generatedAt)
+  || Number.isNaN(parsedTimestamp.valueOf())
+  || parsedTimestamp.toISOString() !== generatedAt.replace("Z", ".000Z")
+) {
+  fail("generated timestamp must be UTC ISO 8601 without fractional seconds");
+}
+if (!/^linux\/[a-z0-9][a-z0-9_-]{0,31}$/.test(platform)) {
+  fail("platform must identify a Linux container architecture");
+}
+
+const records = parseRecords(inputPath, repeats);
+const document = {
+  schemaVersion: 1,
+  generatedAt,
+  provenance: {
+    generator: "apps/headless/benchmark.sh",
+    method: "apps/headless/docs/BENCHMARK.md",
+    platform,
+    repeats,
+    aggregation: "median",
+    taskAwareInspection: true,
+  },
+  cases: aggregate(records, repeats),
+};
+writeAtomically(outputPath, document);

apps/headless/Tests/benchmark-summary.test.mjs

@@ -0,0 +1,129 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const testRoot = mkdtempSync(join(tmpdir(), "headless-benchmark-summary."));
+const appRoot = fileURLToPath(new URL("..", import.meta.url));
+const summarizer = join(appRoot, "Tests/Benchmarks/summarize.mjs");
+const coldWorkflow = readFileSync(join(appRoot, "Tests/Benchmarks/headless.sh"), "utf8");
+const warmWorkflow = readFileSync(join(appRoot, "Tests/Benchmarks/headless_warm.sh"), "utf8");
+
+function sample(caseName, value, workflowBytes) {
+  return {
+    case: caseName,
+    wallMs: value,
+    cpuMs: value + 10,
+    memoryPeakBytes: value + 20,
+    artifactBytes: value + 30,
+    workflowBytes,
+    estimatedTokens: Math.ceil(workflowBytes / 4),
+  };
+}
+
+function run(records, repeats, name = "results.json") {
+  const input = join(testRoot, `${name}.ndjson`);
+  const output = join(testRoot, name);
+  writeFileSync(input, `${records.map((record) => JSON.stringify(record)).join("\n")}\n`);
+  const result = spawnSync(
+    process.execPath,
+    [summarizer, input, output, String(repeats), "2026-08-12T10:00:00Z", "linux/arm64"],
+    { encoding: "utf8" },
+  );
+  return { ...result, output };
+}
+
+try {
+  for (const workflow of [coldWorkflow, warmWorkflow]) {
+    assert.match(workflow, /inspect --context actions --task 'continue to designer details'/);
+    assert.match(workflow, /--limit 8 --budget 700/);
+  }
+
+  const oddRecords = [];
+  for (const [caseName, workflowBytes] of [
+    ["headless", 800],
+    ["headless-warm", 600],
+    ["selenium", 1600],
+    ["puppeteer", 2000],
+  ]) {
+    oddRecords.push(sample(caseName, 30, workflowBytes));
+    oddRecords.push(sample(caseName, 10, workflowBytes));
+    oddRecords.push(sample(caseName, 20, workflowBytes));
+  }
+  const odd = run(oddRecords, 3);
+  assert.equal(odd.status, 0, odd.stderr);
+  const document = JSON.parse(readFileSync(odd.output, "utf8"));
+  assert.equal(document.schemaVersion, 1);
+  assert.equal(document.provenance.repeats, 3);
+  assert.equal(document.provenance.aggregation, "median");
+  assert.equal(document.provenance.taskAwareInspection, true);
+  assert.deepEqual(document.cases.map((entry) => entry.case), [
+    "headless",
+    "headless-warm",
+    "selenium",
+    "puppeteer",
+  ]);
+  assert.equal(document.cases[0].median.wallMs, 20);
+  assert.deepEqual(document.cases[0].samples.map((entry) => entry.iteration), [1, 2, 3]);
+  const firstOutput = readFileSync(odd.output, "utf8");
+  const oddAgain = run(oddRecords, 3);
+  assert.equal(oddAgain.status, 0, oddAgain.stderr);
+  assert.equal(readFileSync(oddAgain.output, "utf8"), firstOutput);
+
+  const evenRecords = [];
+  for (const [caseName, workflowBytes] of [
+    ["headless", 800],
+    ["headless-warm", 600],
+    ["selenium", 1600],
+    ["puppeteer", 2000],
+  ]) {
+    evenRecords.push(sample(caseName, 10, workflowBytes));
+    evenRecords.push(sample(caseName, 11, workflowBytes));
+  }
+  const even = run(evenRecords, 2, "even.json");
+  assert.equal(even.status, 0, even.stderr);
+  assert.equal(JSON.parse(readFileSync(even.output, "utf8")).cases[0].median.wallMs, 10.5);
+
+  const missing = run(oddRecords.slice(1), 3, "missing.json");
+  assert.notEqual(missing.status, 0);
+  assert.match(missing.stderr, /expected 12 samples/);
+
+  const unknown = run(
+    [{ ...oddRecords[0], case: "unknown" }, ...oddRecords.slice(1)],
+    3,
+    "unknown.json",
+  );
+  assert.notEqual(unknown.status, 0);
+  assert.match(unknown.stderr, /unknown case/);
+
+  const inconsistent = run(
+    [{ ...oddRecords[0], workflowBytes: 801 }, ...oddRecords.slice(1)],
+    3,
+    "inconsistent.json",
+  );
+  assert.notEqual(inconsistent.status, 0);
+  assert.match(inconsistent.stderr, /inconsistent workflowBytes/);
+
+  const malformed = run(
+    [{ ...oddRecords[0], wallMs: -1 }, ...oddRecords.slice(1)],
+    3,
+    "malformed.json",
+  );
+  assert.notEqual(malformed.status, 0);
+  assert.match(malformed.stderr, /invalid wallMs/);
+
+  writeFileSync(malformed.output, "preserve existing result\n");
+  const malformedAgain = run(
+    [{ ...oddRecords[0], wallMs: -1 }, ...oddRecords.slice(1)],
+    3,
+    "malformed.json",
+  );
+  assert.notEqual(malformedAgain.status, 0);
+  assert.equal(readFileSync(malformed.output, "utf8"), "preserve existing result\n");
+} finally {
+  rmSync(testRoot, { recursive: true, force: true });
+}
+
+console.log("Benchmark summary tests passed");

apps/headless/benchmark.sh

@@ -3,14 +3,52 @@ set -eu
 cd "$(dirname "$0")"
 
 REPEATS="${1:-3}"
-case "$REPEATS" in *[!0-9]*|'') echo "usage: ./benchmark.sh [positive-repeat-count]" >&2; exit 64;; esac
+OUTPUT="${2:-../../packages/benchmark-results/results.json}"
+if [ "$#" -gt 2 ]; then
+  echo "usage: ./benchmark.sh [positive-repeat-count] [output-path]" >&2
+  exit 64
+fi
+case "$REPEATS" in *[!0-9]*|'') echo "usage: ./benchmark.sh [positive-repeat-count] [output-path]" >&2; exit 64;; esac
 test "$REPEATS" -gt 0 || { echo "repeat count must be positive" >&2; exit 64; }
+test "$REPEATS" -le 100 || { echo "repeat count must not exceed 100" >&2; exit 64; }
 
-docker build --quiet --target benchmark -f Dockerfile.linux -t headless-p1-benchmark . >/dev/null
+command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 69; }
+
+OUTPUT_DIRECTORY="$(dirname "$OUTPUT")"
+mkdir -p "$OUTPUT_DIRECTORY"
+OUTPUT="$(CDPATH='' cd -- "$OUTPUT_DIRECTORY" && pwd -P)/$(basename "$OUTPUT")"
+RAW_RESULTS="$(mktemp "${TMPDIR:-/tmp}/headless-benchmark.XXXXXX")"
+IMAGE="headless-p1-benchmark:run-$$"
+cleanup() {
+  rm -f "$RAW_RESULTS"
+  docker image rm "$IMAGE" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+docker build --quiet --target benchmark -f Dockerfile.linux -t "$IMAGE" . >/dev/null
 for case_name in headless headless-warm selenium puppeteer; do
   iteration=1
   while [ "$iteration" -le "$REPEATS" ]; do
-    docker run --rm --shm-size=1g --cap-add=SYS_ADMIN headless-p1-benchmark "$case_name"
+    echo "Benchmarking $case_name ($iteration/$REPEATS)" >&2
+    docker run --rm --shm-size=1g --cap-add=SYS_ADMIN "$IMAGE" "$case_name" >> "$RAW_RESULTS"
     iteration=$((iteration + 1))
   done
 done
+
+PLATFORM="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$IMAGE")"
+GENERATED_AT="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+docker run --rm \
+  --user "$(id -u):$(id -g)" \
+  --mount "type=bind,src=$RAW_RESULTS,dst=/tmp/headless-benchmark.ndjson,readonly" \
+  --mount "type=bind,src=$OUTPUT_DIRECTORY,dst=/output" \
+  --entrypoint node \
+  "$IMAGE" \
+  /opt/headless/benchmarks/summarize.mjs \
+  /tmp/headless-benchmark.ndjson \
+  "/output/$(basename "$OUTPUT")" \
+  "$REPEATS" \
+  "$GENERATED_AT" \
+  "$PLATFORM"
+echo "Benchmark results: $OUTPUT"