diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cad654..3a6f257 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ Cutting that release is tracked in ### Added +- A single cross-engine conformance scenario now runs against real WKWebView + and Chromium hosts, locking shared response shapes and declared capability + errors without mirrored platform assertions. - A generated WebKit/Chromium capability matrix now declares exhaustive command support and intentional engine differences; host ping responses include the active engine profile. diff --git a/apps/headless/Dockerfile.linux b/apps/headless/Dockerfile.linux index a0735eb..9cce5e4 100644 --- a/apps/headless/Dockerfile.linux +++ b/apps/headless/Dockerfile.linux @@ -41,12 +41,13 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* COPY Tests/Fixtures /opt/headless/fixtures COPY Tests/linux-e2e.sh /opt/headless/linux-e2e.sh +COPY Tests/conformance.sh /opt/headless/conformance.sh COPY install-linux.sh /opt/headless/package/install-linux.sh COPY --from=builder /src/.build/release/headless /opt/headless/package/headless COPY --from=builder /src/.build/release/headless-linux-host /opt/headless/package/headless-host COPY --from=builder /src/.build/release/headless-mcp /opt/headless/package/headless-mcp COPY --from=builder /src/.build/release/Headless_HeadlessProtocol.resources /opt/headless/package/Headless_HeadlessProtocol.resources -RUN chmod 0755 /opt/headless/linux-e2e.sh \ +RUN chmod 0755 /opt/headless/linux-e2e.sh /opt/headless/conformance.sh \ /opt/headless/package/install-linux.sh \ /opt/headless/package/headless \ /opt/headless/package/headless-host \ diff --git a/apps/headless/Tests/conformance.sh b/apps/headless/Tests/conformance.sh new file mode 100755 index 0000000..4331bc8 --- /dev/null +++ b/apps/headless/Tests/conformance.sh @@ -0,0 +1,142 @@ +#!/bin/sh +# One portable scenario run unchanged against WebKit and Chromium. It locks +# the common HostCore response shapes and consults the declared engine profile +# only where the contract intentionally differs. +set -eu + +: "${HEADLESS_CONFORMANCE_CLI:?set HEADLESS_CONFORMANCE_CLI to the built headless CLI}" +: "${HEADLESS_CONFORMANCE_ENGINE:?set HEADLESS_CONFORMANCE_ENGINE to webkit or chromium}" +: "${HEADLESS_CONFORMANCE_BASE_URL:?set HEADLESS_CONFORMANCE_BASE_URL to the fixture origin}" + +CLI="$HEADLESS_CONFORMANCE_CLI" +ENGINE="$HEADLESS_CONFORMANCE_ENGINE" +BASE_URL="${HEADLESS_CONFORMANCE_BASE_URL%/}" +SESSION="conformance-$ENGINE-$$" +PREFIX="conformance-$ENGINE-$$" + +case "$ENGINE" in + webkit|chromium) ;; + *) echo "conformance: unsupported engine: $ENGINE" >&2; exit 2 ;; +esac + +fail() { + echo "cross-engine conformance failed ($ENGINE): $1" >&2 + exit 1 +} + +assert_field() { + printf '%s' "$1" | grep -q '"'"$2"'":' || fail "missing JSON field: $2" +} + +assert_value() { + printf '%s' "$1" | grep -q "$2" || fail "missing JSON value: $2" +} + +cleanup() { + "$CLI" session close "$SESSION" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +CAPABILITIES="$("$CLI" capabilities)" +assert_value "$CAPABILITIES" '"currentEngine":"'"$ENGINE"'"' +assert_field "$CAPABILITIES" engines +assert_value "$CAPABILITIES" '"webkit"' +assert_value "$CAPABILITIES" '"chromium"' + +PING="$("$CLI" status)" +assert_value "$PING" '"ready":true' +assert_value "$PING" '"engine":"'"$ENGINE"'"' +for field in protocolVersion capabilities artifactDirectory recordingAvailable; do + assert_field "$PING" "$field" +done + +"$CLI" session create "$SESSION" | grep -q '"session":"'"$SESSION"'"' +"$CLI" session list | grep -q '"'"$SESSION"'"' + +if EMPTY_BACK="$("$CLI" --session "$SESSION" back 2>&1)"; then + fail "back without history unexpectedly succeeded" +fi +assert_value "$EMPTY_BACK" '"code":"OPERATION_FAILED"' + +VISIT="$("$CLI" --session "$SESSION" visit "$BASE_URL/designers/dashboard")" +for field in url title readyState; do assert_field "$VISIT" "$field"; done +assert_value "$VISIT" 'Designers Dashboard' + +INSPECT="$("$CLI" --session "$SESSION" inspect --context actions --task 'click Continue')" +for field in url title contextMode task elements contextStats omitted truncated untrustedContent; do + assert_field "$INSPECT" "$field" +done +assert_value "$INSPECT" '"contextMode":"actions"' +assert_value "$INSPECT" '"name":"Continue"' + +WAITED="$("$CLI" --session "$SESSION" wait --settled --timeout 10000)" +for field in url title readyState; do assert_field "$WAITED" "$field"; done +"$CLI" --session "$SESSION" press Escape | grep -q '"pressed":"Escape"' + +CAPTURE="$("$CLI" --session "$SESSION" capture-info)" +for field in engine page trace recording; do assert_field "$CAPTURE" "$field"; done +assert_value "$CAPTURE" '"engine":"'"$ENGINE"'"' +assert_value "$CAPTURE" '"active":false' + +CONSOLE="$("$CLI" --session "$SESSION" console list --level error)" +for field in untrustedContent messages returned available; do assert_field "$CONSOLE" "$field"; done +NETWORK="$("$CLI" --session "$SESSION" network list)" +for field in untrustedContent requests returned available; do assert_field "$NETWORK" "$field"; done +QA="$("$CLI" --session "$SESSION" qa report)" +for field in untrustedContent summary issues events omitted truncated; do assert_field "$QA" "$field"; done + +STYLES="$("$CLI" --session "$SESSION" styles get --role region --name 'Diagnostics probe' --property display)" +for field in ref role name box styles; do assert_field "$STYLES" "$field"; done +assert_value "$STYLES" '"display":"flex"' +COOKIES="$("$CLI" --session "$SESSION" cookies list)" +for field in cookies returned available truncated source; do assert_field "$COOKIES" "$field"; done +assert_value "$COOKIES" '"name":"qa_session"' +STORAGE="$("$CLI" --session "$SESSION" storage list)" +for field in origin stores; do assert_field "$STORAGE" "$field"; done +assert_value "$STORAGE" '"scope":"local"' +assert_value "$STORAGE" '"scope":"session"' + +PERFORMANCE="$("$CLI" --session "$SESSION" performance get)" +for field in url timing resources webVitals caveat; do assert_field "$PERFORMANCE" "$field"; done +ANIMATIONS="$("$CLI" --session "$SESSION" animations list)" +for field in count animations truncated; do assert_field "$ANIMATIONS" "$field"; done + +SCREENSHOT="$("$CLI" --session "$SESSION" screenshot --output "$PREFIX.png")" +for field in name path kind bytes createdAt; do assert_field "$SCREENSHOT" "$field"; done +assert_value "$SCREENSHOT" '"name":"'"$PREFIX"'.png"' + +"$CLI" --session "$SESSION" flow start | grep -q '"recording":true' +"$CLI" --session "$SESSION" visit "$BASE_URL/designers/dashboard" >/dev/null +"$CLI" --session "$SESSION" click --role button --name Continue | grep -q '"clicked"' +FLOW="$("$CLI" --session "$SESSION" flow stop --output "$PREFIX-flow.json")" +for field in name path kind bytes createdAt; do assert_field "$FLOW" "$field"; done +FLOW_RUN="$("$CLI" --session "$SESSION" flow run "$PREFIX-flow.json")" +assert_value "$FLOW_RUN" '"completed":2' +assert_field "$FLOW_RUN" input + +REPORT="$("$CLI" --session "$SESSION" report create --output "$PREFIX-report.json")" +for field in name path kind bytes createdAt; do assert_field "$REPORT" "$field"; done + +case "$ENGINE" in + chromium) + assert_value "$PING" '"networkEmulation":true' + EMULATION="$("$CLI" --session "$SESSION" network emulate --latency 25)" + assert_value "$EMULATION" '"latencyMs":25' + ;; + webkit) + assert_value "$PING" '"networkEmulation":false' + if EMULATION="$("$CLI" --session "$SESSION" network emulate --latency 25 2>&1)"; then + fail "declared-unsupported network emulation succeeded" + fi + assert_value "$EMULATION" '"code":"UNSUPPORTED_CAPABILITY"' + ;; +esac + +"$CLI" session close "$SESSION" | grep -q '"closed":"'"$SESSION"'"' +if CLOSED="$("$CLI" --session "$SESSION" inspect 2>&1)"; then + fail "closed session remained reachable" +fi +assert_value "$CLOSED" '"code":"SESSION_NOT_FOUND"' +trap - EXIT INT TERM + +echo "Cross-engine conformance passed ($ENGINE)" diff --git a/apps/headless/Tests/linux-e2e.sh b/apps/headless/Tests/linux-e2e.sh index 62ef480..47ca8c3 100755 --- a/apps/headless/Tests/linux-e2e.sh +++ b/apps/headless/Tests/linux-e2e.sh @@ -59,6 +59,11 @@ headless start | grep -q '"ready":true' UNEXPECTED_TCP="$(awk 'NR > 1 && $4 == "0A" && $2 !~ /:A30B$/ { print $2 }' /proc/net/tcp /proc/net/tcp6)" test -z "$UNEXPECTED_TCP" +HEADLESS_CONFORMANCE_CLI="$(command -v headless)" \ +HEADLESS_CONFORMANCE_ENGINE=chromium \ +HEADLESS_CONFORMANCE_BASE_URL=http://127.0.0.1:41739 \ + /opt/headless/conformance.sh + headless session create qa | grep -q '"session":"qa"' headless session list | grep -q '"qa"' headless --session qa visit http://127.0.0.1:41739/designers/dashboard/ | grep -q 'Designers Dashboard' diff --git a/apps/headless/Tests/macos-e2e.sh b/apps/headless/Tests/macos-e2e.sh index cb6be36..4684ec2 100755 --- a/apps/headless/Tests/macos-e2e.sh +++ b/apps/headless/Tests/macos-e2e.sh @@ -137,6 +137,11 @@ if lsof -nP -a -p "$HOST_PID" -iTCP -sTCP:LISTEN 2>/dev/null | grep -q LISTEN; t echo "Headless host opened an unexpected TCP listener" >&2 fail fi +STEP="cross-engine-conformance" +HEADLESS_CONFORMANCE_CLI="$CLI" \ +HEADLESS_CONFORMANCE_ENGINE=webkit \ +HEADLESS_CONFORMANCE_BASE_URL="http://127.0.0.1:$PORT" \ + Tests/conformance.sh STEP="session-visit" "$CLI" session create qa | grep -q '"session":"qa"' "$CLI" session list | grep -q '"qa"' diff --git a/apps/headless/docs/P2.md b/apps/headless/docs/P2.md index dcf8fb2..c953754 100644 --- a/apps/headless/docs/P2.md +++ b/apps/headless/docs/P2.md @@ -17,6 +17,11 @@ JPEG, cookie, recording-navigation, diagnostics, clipboard, and network differences. Both engines return `OPERATION_FAILED` for `back` without history, share the 65-second tour bound, and enforce the same key-length contract. +Both platform E2E suites execute `Tests/conformance.sh` against their real +browser. This single scenario checks shared JSON response shapes and verifies +that an engine returns its declared capability error where behavior differs; +the platform suites retain only their deeper engine-specific safety coverage. + ## Commands ```sh diff --git a/docs/roadmap/architecture-decisions.md b/docs/roadmap/architecture-decisions.md index a1886b2..27edb27 100644 --- a/docs/roadmap/architecture-decisions.md +++ b/docs/roadmap/architecture-decisions.md @@ -254,14 +254,21 @@ CI, metadata/sitemap/404. Details: backlog §F. ## 14. Testing architecture: promote the conformance suite (Phase 1–2) +**Status:** implemented 2026-08-10. + **Decision:** keep the three-layer shape (protocol unit suite, jsdom runtime suite, per-platform E2E), and add the missing keystone: a **cross-engine conformance runner** — one scenario file executed against both engines asserting identical JSON shapes (or declared capability errors), replacing -today's hand-mirrored `macos-e2e.sh`/`linux-e2e.sh` assertions that have +today’s hand-mirrored `macos-e2e.sh`/`linux-e2e.sh` assertions that have already drifted. The hand-rolled no-XCTest runner is fine (it keeps Linux docker runs trivial); don't churn it to a framework. +The portable `Tests/conformance.sh` scenario is invoked by both platform E2E +suites. It asserts the same response fields for shared behavior and consults +the generated engine profile only for declared differences, so adding an +engine or changing a shared response requires updating one executable contract. + ## 15. Distribution architecture (Phase 3, owner-decided) **Decision (owner, 2026-08-04):** package managers, no hosted service. @@ -353,6 +360,7 @@ override, the 500-event bound, and truncation reporting. | 6 | Windows = stretch via Chromium engine; WSL2/Docker interim | Decided (owner) | 2026-08-04 | | 8 | Real CDP input on Linux as capability upgrade | Planned (Phase 4) | 2026-08-04 | | 12 | Version unification on git tag | Planned (Phase 3) | 2026-08-04 | +| 14 | Run one conformance scenario against every engine | Implemented | 2026-08-10 | | 15 | Package-manager distribution set | Decided (owner) | 2026-08-04 | | 16 | Preserve CLI value boundaries with `--` and shell quoting | Decided | 2026-08-10 | | 17 | Keep full MCP surface; annotate its maximum risk | Decided | 2026-08-10 | diff --git a/docs/roadmap/improvements-backlog.md b/docs/roadmap/improvements-backlog.md index 50f4584..766d6f2 100644 --- a/docs/roadmap/improvements-backlog.md +++ b/docs/roadmap/improvements-backlog.md @@ -339,8 +339,12 @@ now has direct regression coverage. **Done** — the `web` job in `ci.yml` runs `pnpm --filter @headless/web lint` and `build`. -**D4. Cross-engine conformance runner** ([#37](https://github.com/LockInTime/headless/issues/37)) (architecture §14) replacing drifted -hand-mirrored E2E assertions (e.g. `macos-e2e.sh:231` vs `linux-e2e.sh:157`). +**D4. Cross-engine conformance runner** ([#37](https://github.com/LockInTime/headless/issues/37)) ~~(architecture §14) replacing drifted +hand-mirrored E2E assertions (e.g. `macos-e2e.sh:231` vs `linux-e2e.sh:157`).~~ +**Done:** one portable scenario now runs unchanged inside both real-engine E2E +suites. It locks common JSON fields across lifecycle, navigation, inspection, +diagnostics, capture, flows, reports, and errors, branching only through the +declared capability matrix for intentional differences. **D5. Benchmark refresh discipline [exists: benchmark.sh].** ([#38](https://github.com/LockInTime/headless/issues/38)) Emit JSON results artifact; re-run with the task-aware flow (BENCHMARK.md:29-31 says