test(space-conformance): reference PDS matrix (calibration)

packages/space-conformance-matrix/README.md

@@ -0,0 +1,54 @@
+# @getcirrus/space-conformance-matrix
+
+Runs [`@getcirrus/space-conformance`](../space-conformance) against the
+**atproto reference PDS** — the implementation every space must interoperate
+with — so the suite is calibrated, not just self-consistent.
+
+Private, test-only. Nothing is published from here.
+
+## Why
+
+A conformance suite that only ever runs against its own author's
+implementation proves nothing: it can quietly encode one PDS's habits as if
+they were the protocol. This package is the control. It boots the reference
+PDS in-process with [`@atproto/dev-env`](https://www.npmjs.com/package/@atproto/dev-env)
+(a real PLC plus the reference `@atproto/pds`, backed by SQLite — not a mock),
+creates an operator account, and runs the full check catalog against it.
+
+- A `must` check the reference **passes** is genuinely universal.
+- A `must` check the reference **fails** is either a real gap in the alpha
+  reference or an over-specification in our check. Each such case is pinned and
+  cited in [`test/reference-matrix.test.ts`](test/reference-matrix.test.ts),
+  so "the reference fails this" is a reviewed fact, not a surprise.
+
+The test pins the **whole partition**: which checks pass, which diverge, which
+skip. If a passing check regresses, CI fails. If a pinned divergence starts
+_passing_ — the reference caught up, or a check quietly weakened — CI also
+fails, forcing the list to be re-examined. The matrix is only worth having if
+it cannot silently rot.
+
+## Pinned reference divergences (alpha `0.0.0-spaces-alpha-20260818163953`)
+
+| Check                          | Why the reference diverges                                                                                                                                                                                                                                                                                                                                     |
+| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `discovery.space-host-service` | Reference is multi-tenant did:plc; it serves no DID document at `${origin}/.well-known/did.json`, so the space-host service entry can't be read there.                                                                                                                                                                                                         |
+| `discovery.verification-key`   | Same: no DID document at the origin.                                                                                                                                                                                                                                                                                                                           |
+| `blobs.space-blob-not-public`  | Reference refuses an unreferenced upload on public `sync.getBlob`, but a space-record reference promotes the blob to public availability as a public reference would — the private write leaks the bytes. Proposal 0016 §blob-sync intends space blobs to be fetched via the credential-gated `space.getBlob`; Cirrus enforces it with a per-space key layout. |
+
+The identity-gated checks (foreign reader/writer DIDs) skip here because the
+reference can't resolve harness identities; they run in-process against the
+`@getcirrus/spaces` fixture adapter, which wires a did:key resolver. The
+operator's own credential flow is still covered end-to-end by
+`credential.self-round-trip`.
+
+## Running
+
+```bash
+pnpm --filter @getcirrus/space-conformance-matrix test
+```
+
+`@getcirrus/space-conformance` must be built first (`pnpm build`), which CI
+does before `pnpm test`. The reference stack pulls `better-sqlite3` (a native
+module) and the `@atproto/lex*` family, which the spaces alpha published with
+leaked `workspace:*` specifiers — both are handled by `pnpm-workspace.yaml`
+(`allowBuilds` and `overrides` respectively).

packages/space-conformance-matrix/package.json

@@ -0,0 +1,17 @@
+{
+	"name": "@getcirrus/space-conformance-matrix",
+	"version": "0.0.0",
+	"private": true,
+	"type": "module",
+	"description": "Runs @getcirrus/space-conformance against the atproto reference PDS (via @atproto/dev-env) so the suite is calibrated against the implementation every space must interoperate with.",
+	"scripts": {
+		"test": "vitest run",
+		"check": "tsc --noEmit -p tsconfig.json"
+	},
+	"devDependencies": {
+		"@atproto/dev-env": "0.0.0-spaces-alpha-20260818163953",
+		"@getcirrus/space-conformance": "workspace:*",
+		"typescript": "^5.9.3",
+		"vitest": "4.1.0-beta.1"
+	}
+}

packages/space-conformance-matrix/src/reference.ts

@@ -0,0 +1,49 @@
+/**
+ * Boots the atproto reference PDS in-process (via @atproto/dev-env) and
+ * returns everything the conformance suite needs to point at it: the origin,
+ * an operator account DID, and a full-access session token.
+ *
+ * dev-env starts a real PLC plus the reference `@atproto/pds` (the same code
+ * every hosted atproto PDS runs), backed by SQLite — so this is the reference
+ * implementation, not a mock. Accounts are did:plc and the PDS is
+ * multi-tenant, which is exactly why it is a good calibration target: the
+ * suite must not smuggle in assumptions specific to a single-tenant did:web
+ * host like Cirrus.
+ */
+
+import { TestNetworkNoAppView } from "@atproto/dev-env";
+
+export interface ReferencePds {
+	/** Public origin of the reference PDS, e.g. http://localhost:2583 */
+	origin: string;
+	/** The operator account's DID (did:plc). */
+	operatorDid: string;
+	/** A full-access session token for the operator account. */
+	operatorToken: string;
+	/** Tear down the PLC and PDS servers. */
+	close(): Promise<void>;
+}
+
+export async function startReferencePds(): Promise<ReferencePds> {
+	const network = await TestNetworkNoAppView.create();
+	try {
+		const seed = network.getSeedClient();
+		// The handle is cosmetic — only the DID and token drive the suite.
+		// `.test` is dev-env's user domain; the reference rejects longer or
+		// reserved labels, so keep it short and generic.
+		const operator = await seed.createAccount("operator", {
+			handle: "alice.test",
+			email: "operator@conformance.test",
+			password: "conformance-operator-pw",
+		});
+		return {
+			origin: network.pds.url,
+			operatorDid: operator.did,
+			operatorToken: operator.accessJwt,
+			close: () => network.close(),
+		};
+	} catch (err) {
+		await network.close();
+		throw err;
+	}
+}

packages/space-conformance-matrix/test/reference-matrix.test.ts

@@ -0,0 +1,176 @@
+/**
+ * The reference matrix: run the conformance suite against the atproto
+ * reference PDS and pin, exactly, how the reference behaves on every check.
+ *
+ * Why this exists. A conformance suite that only ever runs against its own
+ * author's implementation proves nothing — it can encode Cirrus's habits as
+ * if they were the protocol. Running the same checks against the reference
+ * PDS (the implementation every space must interoperate with) is the control:
+ *
+ *   - A `must` check that the reference *passes* is genuinely universal.
+ *   - A `must` check that the reference *fails* is either a real gap in the
+ *     alpha reference or an over-specification in our check. Each such case is
+ *     named below with a citation, so "the reference fails this" is a recorded,
+ *     reviewed fact rather than a surprise.
+ *
+ * The assertion pins the whole partition. If a pinned-passing check regresses,
+ * CI fails. If a pinned-divergence check starts *passing* — the reference
+ * caught up, or our check quietly weakened — CI also fails, forcing this list
+ * to be re-examined rather than silently drifting. That two-way pin is the
+ * point: the matrix is only worth having if it can't rot.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+	filterCatalog,
+	runChecks,
+	type RunReport,
+} from "@getcirrus/space-conformance";
+import { fullCatalog } from "@getcirrus/space-conformance/full";
+import { startReferencePds, type ReferencePds } from "../src/reference.js";
+
+/**
+ * Checks the reference PDS is known to fail, with the reason it diverges from
+ * the behaviour the check requires. Every entry is a deliberate, cited
+ * statement about the alpha reference — not a check we distrust.
+ *
+ * The pin is on status, not on the reason: a divergence starting to *pass* is
+ * caught (the partition assertion fails), but one that keeps failing for a new
+ * reason is not. That blind spot is narrow — three checks with distinctive
+ * failure modes — so the reason lives in the comment, and a change of reason
+ * is expected to be noticed in review of whatever moved the reference.
+ */
+const EXPECTED_DIVERGENCES: Record<string, string> = {
+	// The reference is a multi-tenant did:plc host: accounts resolve through
+	// PLC and it serves no DID document at `${origin}/.well-known/did.json`,
+	// so a check that reads the space-host service entry from that document
+	// cannot evaluate it. On Cirrus (single-tenant did:web) it applies and
+	// passes. Proposal 0016 §space-authority.
+	"discovery.space-host-service":
+		"reference is multi-tenant did:plc; no DID document at the origin",
+	"discovery.verification-key":
+		"reference is multi-tenant did:plc; no DID document at the origin",
+	// The reference alpha refuses an *unreferenced* uploaded blob on the
+	// public sync.getBlob (the check's first probe passes there), but a
+	// reference from a space record promotes the blob to public availability
+	// exactly as a public-record reference would — so the private write is
+	// what leaks the bytes to anyone who learns the CID. The protocol intends
+	// space blobs to be fetched via the credential-gated space.getBlob
+	// (proposal 0016 §blob-sync); Cirrus enforces that with a separate
+	// per-space blob key layout, which is exactly the gate this check guards.
+	"blobs.space-blob-not-public":
+		"reference alpha promotes space-referenced blobs to the public sync.getBlob",
+};
+
+/**
+ * Checks that need harness identities the reference cannot resolve (foreign
+ * reader/writer DIDs). They are exercised in-process by the @getcirrus/spaces
+ * fixture adapter, which wires a did:key resolver; here they skip. The
+ * operator's own credential flow is still covered end-to-end by
+ * credential.self-round-trip, which needs no foreign identity.
+ */
+const IDENTITY_GATED = new Set([
+	"credential.round-trip",
+	"credential.cross-space-refused",
+	"delegation.replay-refused",
+	"delegation.wrong-audience-refused",
+	"host.member-list-gates",
+	"host.delete-space-tombstone",
+]);
+
+describe("conformance suite vs the atproto reference PDS", () => {
+	let ref: ReferencePds;
+	let report: RunReport;
+
+	beforeAll(async () => {
+		ref = await startReferencePds();
+		const catalog = filterCatalog(fullCatalog, {
+			// A full reference PDS: an operator session, the public blob
+			// endpoints, and its own getDelegationToken (the self credential
+			// flow). Identities are deliberately withheld — see IDENTITY_GATED.
+			capabilities: ["operator", "pds-blobs", "pds-delegation"],
+			destructive: true,
+		});
+		report = await runChecks({
+			catalog,
+			context: {
+				target: {
+					origin: ref.origin,
+					did: ref.operatorDid,
+					implementation: "atproto-reference-pds",
+				},
+				fetch,
+				operator: {
+					oauth: false,
+					async authorize(init) {
+						init.headers.set("Authorization", `Bearer ${ref.operatorToken}`);
+					},
+				},
+			},
+			suiteVersion: "reference-matrix",
+			alphaBuild: "0.0.0-spaces-alpha-20260818163953",
+		});
+	}, 120_000);
+
+	afterAll(async () => {
+		await ref?.close();
+	});
+
+	const expectedStatus = (id: string): string =>
+		id in EXPECTED_DIVERGENCES
+			? "fail"
+			: IDENTITY_GATED.has(id)
+				? "skipped"
+				: "pass";
+
+	it("behaves exactly as pinned on every check", () => {
+		const wrong = report.results.filter(
+			(r) => r.status !== expectedStatus(r.id),
+		);
+
+		expect(
+			wrong.map(
+				(r) =>
+					`${r.id}: ${r.status} — ${r.detail}${
+						r.id in EXPECTED_DIVERGENCES
+							? ` [divergence no longer holds: ${EXPECTED_DIVERGENCES[r.id]}]`
+							: ""
+					}`,
+			),
+		).toEqual([]);
+	});
+
+	it("passes every interop-critical must check the reference supports", () => {
+		// The load-bearing ones, named so a regression reads clearly. These are
+		// the write, sync, policy and credential mechanics that two independent
+		// implementations must agree on to interoperate at all.
+		const byId = Object.fromEntries(
+			report.results.map((r) => [r.id, r.status]),
+		);
+		for (const id of [
+			"writes.create-and-read",
+			"writes.duplicate-rejected",
+			"writes.applywrites-atomic",
+			"credential.self-round-trip",
+			"sync.oplog-folds-to-commit",
+			"sync.getrepo-two-roots",
+			"host.listrepos-requires-credential",
+			"simplespace.unsupported-policy-rejected",
+			"simplespace.getspace-reflects-config",
+		]) {
+			expect(byId[id], `${id} against the reference`).toBe("pass");
+		}
+	});
+
+	it("every pinned divergence is a real must/should check, not a typo", () => {
+		// Guard the pin against bit-rot: an id in EXPECTED_DIVERGENCES that no
+		// longer exists in the catalog would silently do nothing.
+		const ids = new Set(fullCatalog.map((c) => c.id));
+		for (const id of Object.keys(EXPECTED_DIVERGENCES)) {
+			expect(ids.has(id), `${id} is not a known check`).toBe(true);
+		}
+		for (const id of IDENTITY_GATED) {
+			expect(ids.has(id), `${id} is not a known check`).toBe(true);
+		}
+	});
+});

packages/space-conformance-matrix/tsconfig.json

@@ -0,0 +1,10 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"moduleResolution": "bundler",
+		"resolveJsonModule": true,
+		"noEmit": true,
+		"lib": ["es2022", "dom"]
+	},
+	"include": ["src", "test", "vitest.config.ts"]
+}

packages/space-conformance-matrix/vitest.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+	test: {
+		// The reference PDS harness (@atproto/dev-env) runs a real Node HTTP
+		// server backed by better-sqlite3, so this suite runs in Node, not the
+		// workers pool the other Cirrus packages use.
+		environment: "node",
+		// Booting the reference PLC + PDS and running the full catalog is well
+		// under a minute, but give it headroom for a cold CI runner.
+		testTimeout: 120_000,
+		hookTimeout: 120_000,
+	},
+});