Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions apps/site/src/styles/surfaces/specification.css
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,18 @@

/* ─── Left rail (Fumadocs sidebar slot → spec-rail.tsx) ──────────────────── */

/* Below the rail's in-flow width the sidebar becomes a fixed drawer
(#nd-sidebar-mobile, `invisible fixed`), but fumadocs' grid still reserves
its `sidebar` area from --fd-sidebar-width. On a 390px viewport that spent
181px on a rail that is not in flow and left the article 169px, which
clipped the h1 mid-word. Zero the reserved column; the drawer and its
Open Sidebar trigger are unaffected. */
@media (max-width: 768px) {
[data-pdpp-doc-theme] #nd-docs-layout {
--fd-sidebar-width: 0px;
}
}

[data-pdpp-doc-theme] #nd-sidebar {
--fd-sidebar-width: var(--spacing-rail);
font-family: var(--font-sans);
Expand Down
57 changes: 57 additions & 0 deletions docs/reference/stream-evidence-inventory.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| messages | checkpoint_window | manual_as_of | — | true | — | — |
| projects | full_inventory | manual_as_of | — | true | — | — |

## polyfill/apple_contacts

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| address_books | full_inventory | scheduled_window | — | true | — | — |
| contacts | full_inventory | scheduled_window | — | true | — | — |
| contact_groups | full_inventory | scheduled_window | — | true | — | — |

## polyfill/apple-health

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
Expand Down Expand Up @@ -112,6 +120,20 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| message_bodies | checkpoint_window | scheduled_window | — | true | messages | — |
| attachments | parent_detail_accounting | scheduled_window | — | true | — | — |

## polyfill/google-calendar

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| calendars | full_inventory | scheduled_window | — | true | — | — |
| events | checkpoint_window | scheduled_window | — | true | — | — |

## polyfill/google-contacts

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| people | checkpoint_window | scheduled_window | — | true | — | — |
| contact_groups | full_inventory | scheduled_window | — | true | — | — |

## polyfill/google-maps

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
Expand All @@ -132,6 +154,16 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| location_history | snapshot_import_receipt | manual_as_of | — | true | — | — |
| youtube_watch_history | snapshot_import_receipt | manual_as_of | — | true | — | — |
| search_history | snapshot_import_receipt | manual_as_of | — | true | — | — |
| photos | snapshot_import_receipt | manual_as_of | — | true | — | — |

## polyfill/groupme

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| groups | full_inventory | scheduled_window | — | true | — | — |
| group_messages | checkpoint_window | scheduled_window | — | true | — | — |
| direct_messages | checkpoint_window | scheduled_window | — | true | — | — |
| direct_chat_messages | checkpoint_window | scheduled_window | — | true | — | — |

## polyfill/heb

Expand All @@ -151,6 +183,15 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| messages | snapshot_import_receipt | manual_as_of | — | true | — | — |
| participants | snapshot_import_receipt | manual_as_of | — | false | — | — |
| attachments | parent_detail_accounting | manual_as_of | — | false | — | — |

## polyfill/jellyfin

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| libraries | full_inventory | scheduled_window | — | true | — | — |
| items | full_inventory | scheduled_window | — | true | — | — |

## polyfill/linkedin

Expand All @@ -175,6 +216,12 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| profile | singleton_presence | manual_as_of | — | true | — | — |
| posts | checkpoint_window | manual_as_of | — | true | — | — |

## polyfill/netflix-export

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| viewing_activity | snapshot_import_receipt | manual_as_of | — | false | — | — |

## polyfill/notion

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
Expand Down Expand Up @@ -241,6 +288,16 @@ One row per declared manifest stream, across `packages/polyfill-connectors/manif
| top_artists | full_inventory | manual_as_of | — | true | — | — |
| recently_played | checkpoint_window | manual_as_of | — | true | — | — |

## polyfill/steam

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
| --- | --- | --- | --- | --- | --- | --- |
| profile | full_inventory | scheduled_window | — | false | — | — |
| owned_games | full_inventory | scheduled_window | — | false | — | — |
| recently_played_games | full_inventory | scheduled_window | — | false | — | — |
| friends | full_inventory | scheduled_window | — | false | — | — |
| steam_level | singleton_presence | scheduled_window | — | false | — | — |

## polyfill/strava

| stream | coverage_strategy | freshness_strategy | coverage_policy | required | state_stream | availability.state |
Expand Down
3 changes: 3 additions & 0 deletions packages/polyfill-connectors/bin/register-all.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ const CONNECTORS = [
"google_takeout",
"google_maps",
"google_maps_data_portability",
"google_calendar",
"google_contacts",
"twitter_archive",
"imessage",
"strava",
Expand All @@ -60,6 +62,7 @@ const CONNECTORS = [
"codex",
"apple_health",
"ical",
"apple_contacts",
// 'pocket' intentionally excluded — Mozilla shut Pocket down 2025-07-08; the
// shipped manifest is now public_listing.listed=false /
// status=deprecated_upstream / recommended_mode=manual. See
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright The PDP-Connect Contributors
// SPDX-License-Identifier: Apache-2.0

import assert from "node:assert/strict";
import { test } from "node:test";
import {
type BoundedReadableResponse,
describeBoundedReadRejection,
readBoundedText,
} from "./bounded-response-read.ts";

function streamOf(chunks: string[]): ReadableStream<Uint8Array> {
return new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(new TextEncoder().encode(chunk));
}
controller.close();
},
});
}

function responseWith(headers: Record<string, string>, chunks: string[]): BoundedReadableResponse {
return {
headers: { get: (name: string) => headers[name.toLowerCase()] ?? null },
body: streamOf(chunks),
};
}

test("readBoundedText: normal payload under the cap reads through unchanged", async () => {
const res = responseWith({ "content-length": "5" }, ["hello"]);
const outcome = await readBoundedText(res, 1024);
assert.deepEqual(outcome, { kind: "ok", text: "hello" });
});

test("readBoundedText: normal payload with no Content-Length header still reads through under the cap", async () => {
const res = responseWith({}, ["hello", " world"]);
const outcome = await readBoundedText(res, 1024);
assert.deepEqual(outcome, { kind: "ok", text: "hello world" });
});

test("readBoundedText: declared Content-Length exceeding the cap is rejected BEFORE the body is read", async () => {
// `stream.locked` flips to true only once `getReader()` is called on it.
// Checking it after the call proves the fast Content-Length rejection
// returned before the code ever touched the body stream — a real
// ReadableStream, no cast needed.
const body = streamOf(["x"]);
const res: BoundedReadableResponse = {
headers: { get: (name: string) => (name.toLowerCase() === "content-length" ? "999999999" : null) },
body,
};
const outcome = await readBoundedText(res, 10);
assert.equal(outcome.kind, "content_length_exceeded");
assert.equal(body.locked, false, "getReader() must not be called once Content-Length alone exceeds the cap");
if (outcome.kind === "content_length_exceeded") {
assert.equal(outcome.declaredBytes, 999_999_999);
assert.equal(outcome.maxBytes, 10);
}
});

test("readBoundedText: missing Content-Length with an oversized stream is caught by the streaming cap", async () => {
// No Content-Length header at all — the only guard that can catch this
// is the streaming byte-count cap enforced while consuming the body.
const res = responseWith({}, ["a".repeat(20)]);
const outcome = await readBoundedText(res, 10);
assert.equal(outcome.kind, "content_length_missing_stream_exceeded");
if (outcome.kind === "content_length_missing_stream_exceeded") {
assert.equal(outcome.maxBytes, 10);
}
});

test("readBoundedText: a lying (understated) Content-Length with an oversized stream is caught by the streaming cap", async () => {
// Content-Length claims 5 bytes (under the cap, so the upfront check
// passes) but the actual stream delivers far more — the streaming guard
// is authoritative regardless of what the header declared.
const res = responseWith({ "content-length": "5" }, ["a".repeat(50)]);
const outcome = await readBoundedText(res, 10);
assert.equal(outcome.kind, "content_length_understated_stream_exceeded");
if (outcome.kind === "content_length_understated_stream_exceeded") {
assert.equal(outcome.declaredBytes, 5);
assert.equal(outcome.maxBytes, 10);
}
});

test("readBoundedText: a malformed Content-Length header is treated as absent, not trusted", async () => {
const res = responseWith({ "content-length": "not-a-number" }, ["hello"]);
const outcome = await readBoundedText(res, 1024);
assert.deepEqual(outcome, { kind: "ok", text: "hello" });
});

test("readBoundedText: a negative Content-Length header is treated as absent, not trusted", async () => {
const res = responseWith({ "content-length": "-5" }, ["hello"]);
const outcome = await readBoundedText(res, 1024);
assert.deepEqual(outcome, { kind: "ok", text: "hello" });
});

test("readBoundedText: exactly at the cap is accepted (boundary is inclusive)", async () => {
const res = responseWith({}, ["a".repeat(10)]);
const outcome = await readBoundedText(res, 10);
assert.equal(outcome.kind, "ok");
if (outcome.kind === "ok") {
assert.equal(outcome.text.length, 10);
}
});

test("readBoundedText: one byte over the cap is rejected", async () => {
const res = responseWith({}, ["a".repeat(11)]);
const outcome = await readBoundedText(res, 10);
assert.notEqual(outcome.kind, "ok");
});

test("readBoundedText: null body reads as empty text", async () => {
const res: BoundedReadableResponse = { headers: { get: () => null }, body: null };
const outcome = await readBoundedText(res, 1024);
assert.deepEqual(outcome, { kind: "ok", text: "" });
});

test("readBoundedText: rejects across multiple chunks, not just a single oversized chunk", async () => {
const res = responseWith({}, ["a".repeat(6), "b".repeat(6)]);
const outcome = await readBoundedText(res, 10);
assert.notEqual(outcome.kind, "ok");
});

test("describeBoundedReadRejection: produces a size-only message with no body content", () => {
const msg = describeBoundedReadRejection({ kind: "content_length_exceeded", declaredBytes: 999, maxBytes: 100 });
assert.match(msg, /999/);
assert.match(msg, /100/);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright The PDP-Connect Contributors
// SPDX-License-Identifier: Apache-2.0

/**
* Bounded-size response body reader for the Apple Contacts CardDAV client.
*
* Every authenticated request this connector makes reads an XML multistatus
* body (PROPFIND/REPORT) that embeds vCards, which in turn can embed a
* base64 PHOTO property. None of those three layers has a protocol-enforced
* size ceiling: a misbehaving or compromised CardDAV server (or a
* man-in-the-middle on a redirect hop that slipped past origin validation)
* could return an arbitrarily large body and force this connector to
* allocate unbounded memory before any content is even inspected.
*
* This module is the single choke point every response body passes through.
* It is deliberately NOT a generic "bounded fetch" package — Apple Contacts
* is the only consumer today (per the standing rule: don't build a shared
* abstraction before a second consumer exists). If a second CardDAV-ish
* connector shows up, promote this to `src/` then.
*
* Two independent guards, because either one alone is insufficient:
* 1. `Content-Length`, when present, is checked BEFORE reading a single
* body byte — the fast rejection path for a server that discloses an
* oversized body up front.
* 2. A streaming byte-count cap is enforced while consuming the body
* regardless of what `Content-Length` claimed (or omitted) — this is
* the ONLY guard that catches a missing or dishonest Content-Length
* (a server that under-reports the header, or omits it and chunks
* indefinitely). The stream is aborted as soon as the cap is crossed,
* so memory usage is bounded by `maxBytes` even against a hostile body.
*/

export type BoundedReadOutcome =
| { kind: "ok"; text: string }
| { kind: "content_length_exceeded"; declaredBytes: number; maxBytes: number }
| { kind: "content_length_missing_stream_exceeded"; maxBytes: number }
| { kind: "content_length_understated_stream_exceeded"; declaredBytes: number; maxBytes: number };

export interface BoundedReadableResponse {
body: ReadableStream<Uint8Array> | null;
headers: { get: (name: string) => string | null };
}

const CONTENT_LENGTH_DIGITS_RE = /^\d+$/;

/** Parse a `Content-Length` header value. Returns `null` for anything that
* is not a non-negative integer (missing, empty, non-numeric, negative,
* or a value with trailing garbage) — treated identically to "absent" by
* the caller, which is the safe direction (falls through to the streaming
* guard rather than trusting a malformed declaration). */
function parseContentLength(raw: string | null): number | null {
if (raw === null) {
return null;
}
const trimmed = raw.trim();
if (!CONTENT_LENGTH_DIGITS_RE.test(trimmed)) {
return null;
}
const value = Number(trimmed);
return Number.isSafeInteger(value) ? value : null;
}

/**
* Read a response body as text, enforcing `maxBytes` two ways: an upfront
* `Content-Length` check (when the header is present and parses cleanly),
* and a streaming byte-count cap that is authoritative regardless of what
* the header said. Never buffers more than `maxBytes` (+ one chunk's worth
* of overrun before the cap trips, since chunk boundaries aren't caller
* controlled) before returning a rejection.
*/
export async function readBoundedText(res: BoundedReadableResponse, maxBytes: number): Promise<BoundedReadOutcome> {
const declaredBytes = parseContentLength(res.headers.get("content-length"));
if (declaredBytes !== null && declaredBytes > maxBytes) {
return { kind: "content_length_exceeded", declaredBytes, maxBytes };
}

if (!res.body) {
return { kind: "ok", text: "" };
}

const reader = res.body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) {
break;
}
total += value.length;
if (total > maxBytes) {
return declaredBytes === null
? { kind: "content_length_missing_stream_exceeded", maxBytes }
: { kind: "content_length_understated_stream_exceeded", declaredBytes, maxBytes };
}
chunks.push(value);
}
} finally {
// Release the reader lock and cancel any remaining backpressure so a
// rejected (oversized) body doesn't keep pulling bytes off the wire.
reader.releaseLock();
await res.body.cancel().catch((): undefined => undefined);
}

const buffer = Buffer.concat(chunks.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength)));
return { kind: "ok", text: buffer.toString("utf8") };
}

/** Human-readable summary for a rejected outcome, safe to include in a
* thrown error message — carries no response body content, only sizes. */
export function describeBoundedReadRejection(outcome: Exclude<BoundedReadOutcome, { kind: "ok" }>): string {
switch (outcome.kind) {
case "content_length_exceeded":
return `declared Content-Length ${String(outcome.declaredBytes)} exceeds cap ${String(outcome.maxBytes)}`;
case "content_length_missing_stream_exceeded":
return `response body exceeded cap ${String(outcome.maxBytes)} with no Content-Length header`;
case "content_length_understated_stream_exceeded":
return `response body exceeded cap ${String(outcome.maxBytes)} (declared Content-Length ${String(outcome.declaredBytes)} understated the real size)`;
default:
return "response body exceeded the size cap";
}
}
Loading
Loading