LockInTime · yashranaway · Aug 10, 2026 · Aug 10, 2026 · Aug 10, 2026
@@ -23,6 +23,9 @@ Cutting that release is tracked in
 
 ### Added
 
+- A single cross-engine conformance scenario now runs against real WKWebView
+  and Chromium hosts, locking shared response shapes and declared capability
+  errors without mirrored platform assertions.
 - A generated WebKit/Chromium capability matrix now declares exhaustive
   command support and intentional engine differences; host ping responses
   include the active engine profile.

@@ -41,12 +41,13 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 COPY Tests/Fixtures /opt/headless/fixtures
 COPY Tests/linux-e2e.sh /opt/headless/linux-e2e.sh
+COPY Tests/conformance.sh /opt/headless/conformance.sh
 COPY install-linux.sh /opt/headless/package/install-linux.sh
 COPY --from=builder /src/.build/release/headless /opt/headless/package/headless
 COPY --from=builder /src/.build/release/headless-linux-host /opt/headless/package/headless-host
 COPY --from=builder /src/.build/release/headless-mcp /opt/headless/package/headless-mcp
 COPY --from=builder /src/.build/release/Headless_HeadlessProtocol.resources /opt/headless/package/Headless_HeadlessProtocol.resources
-RUN chmod 0755 /opt/headless/linux-e2e.sh \
+RUN chmod 0755 /opt/headless/linux-e2e.sh /opt/headless/conformance.sh \
     /opt/headless/package/install-linux.sh \
     /opt/headless/package/headless \
     /opt/headless/package/headless-host \

@@ -0,0 +1,142 @@
+#!/bin/sh
+# One portable scenario run unchanged against WebKit and Chromium. It locks
+# the common HostCore response shapes and consults the declared engine profile
+# only where the contract intentionally differs.
+set -eu
+
+: "${HEADLESS_CONFORMANCE_CLI:?set HEADLESS_CONFORMANCE_CLI to the built headless CLI}"
+: "${HEADLESS_CONFORMANCE_ENGINE:?set HEADLESS_CONFORMANCE_ENGINE to webkit or chromium}"
+: "${HEADLESS_CONFORMANCE_BASE_URL:?set HEADLESS_CONFORMANCE_BASE_URL to the fixture origin}"
+
+CLI="$HEADLESS_CONFORMANCE_CLI"
+ENGINE="$HEADLESS_CONFORMANCE_ENGINE"
+BASE_URL="${HEADLESS_CONFORMANCE_BASE_URL%/}"
+SESSION="conformance-$ENGINE-$$"
+PREFIX="conformance-$ENGINE-$$"
+
+case "$ENGINE" in
+  webkit|chromium) ;;
+  *) echo "conformance: unsupported engine: $ENGINE" >&2; exit 2 ;;
+esac
+
+fail() {
+  echo "cross-engine conformance failed ($ENGINE): $1" >&2
+  exit 1
+}
+
+assert_field() {
+  printf '%s' "$1" | grep -q '"'"$2"'":' || fail "missing JSON field: $2"
+}
+
+assert_value() {
+  printf '%s' "$1" | grep -q "$2" || fail "missing JSON value: $2"
+}
+
+cleanup() {
+  "$CLI" session close "$SESSION" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT INT TERM
+
+CAPABILITIES="$("$CLI" capabilities)"
+assert_value "$CAPABILITIES" '"currentEngine":"'"$ENGINE"'"'
+assert_field "$CAPABILITIES" engines
+assert_value "$CAPABILITIES" '"webkit"'
+assert_value "$CAPABILITIES" '"chromium"'
+
+PING="$("$CLI" status)"
+assert_value "$PING" '"ready":true'
+assert_value "$PING" '"engine":"'"$ENGINE"'"'
+for field in protocolVersion capabilities artifactDirectory recordingAvailable; do
+  assert_field "$PING" "$field"
+done
+
+"$CLI" session create "$SESSION" | grep -q '"session":"'"$SESSION"'"'
+"$CLI" session list | grep -q '"'"$SESSION"'"'
+
+if EMPTY_BACK="$("$CLI" --session "$SESSION" back 2>&1)"; then
+  fail "back without history unexpectedly succeeded"
+fi
+assert_value "$EMPTY_BACK" '"code":"OPERATION_FAILED"'
+
+VISIT="$("$CLI" --session "$SESSION" visit "$BASE_URL/designers/dashboard")"
+for field in url title readyState; do assert_field "$VISIT" "$field"; done
+assert_value "$VISIT" 'Designers Dashboard'
+
+INSPECT="$("$CLI" --session "$SESSION" inspect --context actions --task 'click Continue')"
+for field in url title contextMode task elements contextStats omitted truncated untrustedContent; do
+  assert_field "$INSPECT" "$field"
+done
+assert_value "$INSPECT" '"contextMode":"actions"'
+assert_value "$INSPECT" '"name":"Continue"'
+
+WAITED="$("$CLI" --session "$SESSION" wait --settled --timeout 10000)"
+for field in url title readyState; do assert_field "$WAITED" "$field"; done
+"$CLI" --session "$SESSION" press Escape | grep -q '"pressed":"Escape"'
+
+CAPTURE="$("$CLI" --session "$SESSION" capture-info)"
+for field in engine page trace recording; do assert_field "$CAPTURE" "$field"; done
+assert_value "$CAPTURE" '"engine":"'"$ENGINE"'"'
+assert_value "$CAPTURE" '"active":false'
+
+CONSOLE="$("$CLI" --session "$SESSION" console list --level error)"
+for field in untrustedContent messages returned available; do assert_field "$CONSOLE" "$field"; done
+NETWORK="$("$CLI" --session "$SESSION" network list)"
+for field in untrustedContent requests returned available; do assert_field "$NETWORK" "$field"; done
+QA="$("$CLI" --session "$SESSION" qa report)"
+for field in untrustedContent summary issues events omitted truncated; do assert_field "$QA" "$field"; done
+
+STYLES="$("$CLI" --session "$SESSION" styles get --role region --name 'Diagnostics probe' --property display)"
+for field in ref role name box styles; do assert_field "$STYLES" "$field"; done
+assert_value "$STYLES" '"display":"flex"'
+COOKIES="$("$CLI" --session "$SESSION" cookies list)"
+for field in cookies returned available truncated source; do assert_field "$COOKIES" "$field"; done
+assert_value "$COOKIES" '"name":"qa_session"'
+STORAGE="$("$CLI" --session "$SESSION" storage list)"
+for field in origin stores; do assert_field "$STORAGE" "$field"; done
+assert_value "$STORAGE" '"scope":"local"'
+assert_value "$STORAGE" '"scope":"session"'
+
+PERFORMANCE="$("$CLI" --session "$SESSION" performance get)"
+for field in url timing resources webVitals caveat; do assert_field "$PERFORMANCE" "$field"; done
+ANIMATIONS="$("$CLI" --session "$SESSION" animations list)"
+for field in count animations truncated; do assert_field "$ANIMATIONS" "$field"; done
+
+SCREENSHOT="$("$CLI" --session "$SESSION" screenshot --output "$PREFIX.png")"
+for field in name path kind bytes createdAt; do assert_field "$SCREENSHOT" "$field"; done
+assert_value "$SCREENSHOT" '"name":"'"$PREFIX"'.png"'
+
+"$CLI" --session "$SESSION" flow start | grep -q '"recording":true'
+"$CLI" --session "$SESSION" visit "$BASE_URL/designers/dashboard" >/dev/null
+"$CLI" --session "$SESSION" click --role button --name Continue | grep -q '"clicked"'
+FLOW="$("$CLI" --session "$SESSION" flow stop --output "$PREFIX-flow.json")"
+for field in name path kind bytes createdAt; do assert_field "$FLOW" "$field"; done
+FLOW_RUN="$("$CLI" --session "$SESSION" flow run "$PREFIX-flow.json")"
+assert_value "$FLOW_RUN" '"completed":2'
+assert_field "$FLOW_RUN" input
+
+REPORT="$("$CLI" --session "$SESSION" report create --output "$PREFIX-report.json")"
+for field in name path kind bytes createdAt; do assert_field "$REPORT" "$field"; done
+
+case "$ENGINE" in
+  chromium)
+    assert_value "$PING" '"networkEmulation":true'
+    EMULATION="$("$CLI" --session "$SESSION" network emulate --latency 25)"
+    assert_value "$EMULATION" '"latencyMs":25'
+    ;;
+  webkit)
+    assert_value "$PING" '"networkEmulation":false'
+    if EMULATION="$("$CLI" --session "$SESSION" network emulate --latency 25 2>&1)"; then
+      fail "declared-unsupported network emulation succeeded"
+    fi
+    assert_value "$EMULATION" '"code":"UNSUPPORTED_CAPABILITY"'
+    ;;
+esac
+
+"$CLI" session close "$SESSION" | grep -q '"closed":"'"$SESSION"'"'
+if CLOSED="$("$CLI" --session "$SESSION" inspect 2>&1)"; then
+  fail "closed session remained reachable"
+fi
+assert_value "$CLOSED" '"code":"SESSION_NOT_FOUND"'
+trap - EXIT INT TERM
+
+echo "Cross-engine conformance passed ($ENGINE)"
@@ -59,6 +59,11 @@ headless start | grep -q '"ready":true'
 UNEXPECTED_TCP="$(awk 'NR > 1 && $4 == "0A" && $2 !~ /:A30B$/ { print $2 }' /proc/net/tcp /proc/net/tcp6)"
 test -z "$UNEXPECTED_TCP"
 
+HEADLESS_CONFORMANCE_CLI="$(command -v headless)" \
+HEADLESS_CONFORMANCE_ENGINE=chromium \
+HEADLESS_CONFORMANCE_BASE_URL=http://127.0.0.1:41739 \
+  /opt/headless/conformance.sh
+
 headless session create qa | grep -q '"session":"qa"'
 headless session list | grep -q '"qa"'
 headless --session qa visit http://127.0.0.1:41739/designers/dashboard/ | grep -q 'Designers Dashboard'

@@ -137,6 +137,11 @@ if lsof -nP -a -p "$HOST_PID" -iTCP -sTCP:LISTEN 2>/dev/null | grep -q LISTEN; t
   echo "Headless host opened an unexpected TCP listener" >&2
   fail
 fi
+STEP="cross-engine-conformance"
+HEADLESS_CONFORMANCE_CLI="$CLI" \
+HEADLESS_CONFORMANCE_ENGINE=webkit \
+HEADLESS_CONFORMANCE_BASE_URL="http://127.0.0.1:$PORT" \
+  Tests/conformance.sh
 STEP="session-visit"
 "$CLI" session create qa | grep -q '"session":"qa"'
 "$CLI" session list | grep -q '"qa"'

@@ -17,6 +17,11 @@ JPEG, cookie, recording-navigation, diagnostics, clipboard, and network
 differences. Both engines return `OPERATION_FAILED` for `back` without history,
 share the 65-second tour bound, and enforce the same key-length contract.
 
+Both platform E2E suites execute `Tests/conformance.sh` against their real
+browser. This single scenario checks shared JSON response shapes and verifies
+that an engine returns its declared capability error where behavior differs;
+the platform suites retain only their deeper engine-specific safety coverage.
+
 ## Commands
 
 ```sh

@@ -254,14 +254,21 @@ CI, metadata/sitemap/404. Details: backlog §F.
 
 ## 14. Testing architecture: promote the conformance suite (Phase 1–2)
 
+**Status:** implemented 2026-08-10.
+
 **Decision:** keep the three-layer shape (protocol unit suite, jsdom runtime
 suite, per-platform E2E), and add the missing keystone: a **cross-engine
 conformance runner** — one scenario file executed against both engines
 asserting identical JSON shapes (or declared capability errors), replacing
-today's hand-mirrored `macos-e2e.sh`/`linux-e2e.sh` assertions that have
+today’s hand-mirrored `macos-e2e.sh`/`linux-e2e.sh` assertions that have
 already drifted. The hand-rolled no-XCTest runner is fine (it keeps Linux
 docker runs trivial); don't churn it to a framework.
 
+The portable `Tests/conformance.sh` scenario is invoked by both platform E2E
+suites. It asserts the same response fields for shared behavior and consults
+the generated engine profile only for declared differences, so adding an
+engine or changing a shared response requires updating one executable contract.
+
 ## 15. Distribution architecture (Phase 3, owner-decided)
 
 **Decision (owner, 2026-08-04):** package managers, no hosted service.
@@ -353,6 +360,7 @@ override, the 500-event bound, and truncation reporting.
 | 6 | Windows = stretch via Chromium engine; WSL2/Docker interim | Decided (owner) | 2026-08-04 |
 | 8 | Real CDP input on Linux as capability upgrade | Planned (Phase 4) | 2026-08-04 |
 | 12 | Version unification on git tag | Planned (Phase 3) | 2026-08-04 |
+| 14 | Run one conformance scenario against every engine | Implemented | 2026-08-10 |
 | 15 | Package-manager distribution set | Decided (owner) | 2026-08-04 |
 | 16 | Preserve CLI value boundaries with `--` and shell quoting | Decided | 2026-08-10 |
 | 17 | Keep full MCP surface; annotate its maximum risk | Decided | 2026-08-10 |

@@ -339,8 +339,12 @@ now has direct regression coverage.
 **Done** — the `web` job in `ci.yml` runs `pnpm --filter @headless/web lint`
 and `build`.
 
-**D4. Cross-engine conformance runner** ([#37](https://github.com/LockInTime/headless/issues/37)) (architecture §14) replacing drifted
-hand-mirrored E2E assertions (e.g. `macos-e2e.sh:231` vs `linux-e2e.sh:157`).
+**D4. Cross-engine conformance runner** ([#37](https://github.com/LockInTime/headless/issues/37)) ~~(architecture §14) replacing drifted
+hand-mirrored E2E assertions (e.g. `macos-e2e.sh:231` vs `linux-e2e.sh:157`).~~
+**Done:** one portable scenario now runs unchanged inside both real-engine E2E
+suites. It locks common JSON fields across lifecycle, navigation, inspection,
+diagnostics, capture, flows, reports, and errors, branching only through the
+declared capability matrix for intentional differences.
 
 **D5. Benchmark refresh discipline [exists: benchmark.sh].** ([#38](https://github.com/LockInTime/headless/issues/38)) Emit JSON
 results artifact; re-run with the task-aware flow (BENCHMARK.md:29-31 says